From 14f54ac6af052b3dff8e31d5c9eb579094f129ab Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 23 Feb 2023 17:01:09 -0600 Subject: [PATCH 001/384] Update value_counts with new behavior (#12835) This PR updates value_counts behavior to match pandas-2.x, the result name will be count (or proportion if normalize=True is passed), and the index will be named after the original object name. This PR also fixes two dtype APIs that are breaking changes on pandas side. --- python/cudf/cudf/api/types.py | 5 ++--- python/cudf/cudf/core/dataframe.py | 9 ++++++++- python/cudf/cudf/core/series.py | 20 ++++++++++---------- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/python/cudf/cudf/api/types.py b/python/cudf/cudf/api/types.py index 62f8377a323..ffe89e3e779 100644 --- a/python/cudf/cudf/api/types.py +++ b/python/cudf/cudf/api/types.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. """Define common type operations.""" @@ -244,7 +244,6 @@ def _union_categoricals( is_datetime64_dtype = pd_types.is_datetime64_dtype is_datetime64_ns_dtype = pd_types.is_datetime64_ns_dtype is_datetime64tz_dtype = pd_types.is_datetime64tz_dtype -is_extension_type = pd_types.is_extension_type is_extension_array_dtype = pd_types.is_extension_array_dtype is_float_dtype = _wrap_pandas_is_dtype_api(pd_types.is_float_dtype) is_int64_dtype = pd_types.is_int64_dtype @@ -263,7 +262,7 @@ def _union_categoricals( is_named_tuple = pd_types.is_named_tuple is_iterator = pd_types.is_iterator is_bool = pd_types.is_bool -is_categorical = pd_types.is_categorical +is_categorical = pd_types.is_categorical_dtype is_complex = pd_types.is_complex is_float = pd_types.is_float is_hashable = pd_types.is_hashable diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index d43621d3d36..19f19cd2cb0 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -7211,12 +7211,18 @@ def value_counts( >>> df = cudf.DataFrame({'num_legs': [2, 4, 4, 6], ... 'num_wings': [2, 0, 0, 0]}, ... index=['falcon', 'dog', 'cat', 'ant']) + >>> df + num_legs num_wings + falcon 2 2 + dog 4 0 + cat 4 0 + ant 6 0 >>> df.value_counts() num_legs num_wings 4 0 2 2 2 1 6 0 1 - dtype: int64 + Name: count, dtype: int64 """ if subset: diff = set(subset) - set(self._data) @@ -7238,6 +7244,7 @@ def value_counts( # Pandas always returns MultiIndex even if only one column. if not isinstance(result.index, MultiIndex): result.index = MultiIndex._from_data(result._index._data) + result.name = "proportion" if normalize else "count" return result diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 60655c5a6f9..7838e9409a2 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -2946,7 +2946,7 @@ def value_counts( 3.0 3 2.0 2 1.0 1 - dtype: int32 + Name: count, dtype: int32 The order of the counts can be changed by passing ``ascending=True``: @@ -2954,7 +2954,7 @@ def value_counts( 1.0 1 2.0 2 3.0 3 - dtype: int32 + Name: count, dtype: int32 With ``normalize`` set to True, returns the relative frequency by dividing all values by the sum of values. @@ -2963,7 +2963,7 @@ def value_counts( 3.0 0.500000 2.0 0.333333 1.0 0.166667 - dtype: float32 + Name: proportion, dtype: float32 To include ``NA`` value counts, pass ``dropna=False``: @@ -2983,24 +2983,24 @@ def value_counts( 2.0 2 2 1.0 1 - dtype: int32 + Name: count, dtype: int32 >>> s = cudf.Series([3, 1, 2, 3, 4, np.nan]) >>> s.value_counts(bins=3) (2.0, 3.0] 2 (0.996, 2.0] 2 (3.0, 4.0] 1 - dtype: int32 + Name: count, dtype: int32 """ if bins is not None: series_bins = cudf.cut(self, bins, include_lowest=True) - + result_name = "proportion" if normalize else "count" if dropna and self.null_count == len(self): return Series( [], dtype=np.int32, - name=self.name, - index=cudf.Index([], dtype=self.dtype), + name=result_name, + index=cudf.Index([], dtype=self.dtype, name=self.name), ) if bins is not None: @@ -3009,7 +3009,7 @@ def value_counts( else: res = self.groupby(self, dropna=dropna).count(dropna=dropna) - res.index.name = None + res.index.name = self.name if sort: res = res.sort_values(ascending=ascending) @@ -3024,7 +3024,7 @@ def value_counts( res.index._column, res.index.categories.dtype ) res.index = int_index - + res.name = result_name return res @_cudf_nvtx_annotate From 7d62d4e1638322076198fe341aa84ed73498c10d Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 24 Feb 2023 16:57:48 -0600 Subject: [PATCH 002/384] Drop inplace parameter in categorical methods (#12846) This PR drops `inplace` parameters in categorical methods, these are also removed as part of pandas-2.0 --- python/cudf/cudf/core/column/categorical.py | 213 ++------------------ python/cudf/cudf/tests/test_categorical.py | 153 +++----------- 2 files changed, 41 insertions(+), 325 deletions(-) diff --git a/python/cudf/cudf/core/column/categorical.py b/python/cudf/cudf/core/column/categorical.py index 52f7c0b957f..a44d63cea23 100644 --- a/python/cudf/cudf/core/column/categorical.py +++ b/python/cudf/cudf/core/column/categorical.py @@ -2,7 +2,6 @@ from __future__ import annotations -import warnings from collections import abc from functools import cached_property from typing import TYPE_CHECKING, Any, Mapping, Optional, Sequence, Tuple, cast @@ -130,28 +129,14 @@ def ordered(self) -> bool: """ return self._column.ordered - def as_ordered(self, inplace: bool = False) -> Optional[SeriesOrIndex]: + def as_ordered(self) -> Optional[SeriesOrIndex]: """ Set the Categorical to be ordered. - Parameters - ---------- - inplace : bool, default False - Whether or not to add the categories inplace - or return a copy of this categorical with - added categories. - - .. deprecated:: 23.02 - - The `inplace` parameter is is deprecated and - will be removed in a future version of cudf. - Setting categories as ordered will always - return a new Categorical object. - Returns ------- Categorical - Ordered Categorical or None if inplace. + Ordered Categorical. Examples -------- @@ -177,47 +162,13 @@ def as_ordered(self, inplace: bool = False) -> Optional[SeriesOrIndex]: 6 10 dtype: category Categories (3, int64): [1 < 2 < 10] - >>> s.cat.as_ordered(inplace=True) - >>> s - 0 10 - 1 1 - 2 1 - 3 2 - 4 10 - 5 2 - 6 10 - dtype: category - Categories (3, int64): [1 < 2 < 10] """ - if inplace: - warnings.warn( - "The inplace parameter is deprecated and will be removed in a " - "future release. set_ordered will always return a new Series " - "in the future.", - FutureWarning, - ) - return self._return_or_inplace( - self._column.as_ordered(), inplace=inplace - ) + return self._return_or_inplace(self._column.as_ordered()) - def as_unordered(self, inplace: bool = False) -> Optional[SeriesOrIndex]: + def as_unordered(self) -> Optional[SeriesOrIndex]: """ Set the Categorical to be unordered. - Parameters - ---------- - inplace : bool, default False - Whether or not to set the ordered attribute - in-place or return a copy of this - categorical with ordered set to False. - - .. deprecated:: 23.02 - - The `inplace` parameter is is deprecated and - will be removed in a future version of cudf. - Setting categories as unordered will always - return a new Categorical object. - Returns ------- Categorical @@ -258,32 +209,11 @@ def as_unordered(self, inplace: bool = False) -> Optional[SeriesOrIndex]: 6 10 dtype: category Categories (3, int64): [1, 2, 10] - >>> s.cat.as_unordered(inplace=True) - >>> s - 0 10 - 1 1 - 2 1 - 3 2 - 4 10 - 5 2 - 6 10 - dtype: category - Categories (3, int64): [1, 2, 10] """ - if inplace: - warnings.warn( - "The inplace parameter is deprecated and will be removed in a " - "future release. set_ordered will always return a new Series " - "in the future.", - FutureWarning, - ) - return self._return_or_inplace( - self._column.as_unordered(), inplace=inplace - ) - def add_categories( - self, new_categories: Any, inplace: bool = False - ) -> Optional[SeriesOrIndex]: + return self._return_or_inplace(self._column.as_unordered()) + + def add_categories(self, new_categories: Any) -> Optional[SeriesOrIndex]: """ Add new categories. @@ -295,23 +225,11 @@ def add_categories( ---------- new_categories : category or list-like of category The new categories to be included. - inplace : bool, default False - Whether or not to add the categories inplace - or return a copy of this categorical with - added categories. - - .. deprecated:: 23.04 - - The `inplace` parameter is is deprecated and - will be removed in a future version of cudf. - Adding categories will always return a - new Categorical object. Returns ------- cat - Categorical with new categories added or - None if inplace. + Categorical with new categories added. Examples -------- @@ -332,21 +250,8 @@ def add_categories( 1 2 dtype: category Categories (2, int64): [1, 2] - >>> s.cat.add_categories([0, 3, 4], inplace=True) - >>> s - 0 1 - 1 2 - dtype: category - Categories (5, int64): [1, 2, 0, 3, 4] """ - if inplace: - warnings.warn( - "The `inplace` parameter in cudf.Series.cat.add_categories " - "is deprecated and will be removed in a future version of " - "cudf. Adding categories will always return a new " - "Categorical object.", - FutureWarning, - ) + old_categories = self._column.categories new_categories = column.as_column( new_categories, @@ -376,12 +281,11 @@ def add_categories( if not out_col._categories_equal(new_categories): out_col = out_col._set_categories(new_categories) - return self._return_or_inplace(out_col, inplace=inplace) + return self._return_or_inplace(out_col) def remove_categories( self, removals: Any, - inplace: bool = False, ) -> Optional[SeriesOrIndex]: """ Remove the specified categories. @@ -394,23 +298,11 @@ def remove_categories( ---------- removals : category or list-like of category The categories which should be removed. - inplace : bool, default False - Whether or not to remove the categories - inplace or return a copy of this categorical - with removed categories. - - .. deprecated:: 23.04 - - The `inplace` parameter is is deprecated and - will be removed in a future version of cudf. - Removing categories will always return a - new Categorical object. Returns ------- cat - Categorical with removed categories or None - if inplace. + Categorical with removed categories Examples -------- @@ -446,27 +338,7 @@ def remove_categories( 6 10 dtype: category Categories (3, int64): [1, 2, 10] - >>> s.cat.remove_categories([10], inplace=True) - >>> s - 0 - 1 1 - 2 1 - 3 2 - 4 - 5 2 - 6 - dtype: category - Categories (2, int64): [1, 2] """ - if inplace: - warnings.warn( - "The `inplace` parameter in " - "cudf.Series.cat.remove_categories is deprecated and " - "will be removed in a future version of cudf. " - "Removing categories will always return a new " - "Categorical object.", - FutureWarning, - ) cats = self.categories.to_series() removals = cudf.Series(removals, dtype=cats.dtype) @@ -483,14 +355,13 @@ def remove_categories( if not out_col._categories_equal(new_categories): out_col = out_col._set_categories(new_categories) - return self._return_or_inplace(out_col, inplace=inplace) + return self._return_or_inplace(out_col) def set_categories( self, new_categories: Any, ordered: bool = False, rename: bool = False, - inplace: bool = False, ) -> Optional[SeriesOrIndex]: """ Set the categories to the specified new_categories. @@ -525,23 +396,11 @@ def set_categories( Whether or not the `new_categories` should be considered as a rename of the old categories or as reordered categories. - inplace : bool, default False - Whether or not to reorder the categories in-place - or return a copy of this categorical with - reordered categories. - - .. deprecated:: 23.04 - - The `inplace` parameter is is deprecated and - will be removed in a future version of cudf. - Setting categories will always return a - new Categorical object. Returns ------- cat Categorical with reordered categories - or None if inplace. Examples -------- @@ -565,37 +424,18 @@ def set_categories( 5 10 dtype: category Categories (2, int64): [1, 10] - >>> s.cat.set_categories([1, 10], inplace=True) - >>> s - 0 1 - 1 1 - 2 - 3 10 - 4 - 5 10 - dtype: category - Categories (2, int64): [1, 10] """ - if inplace: - warnings.warn( - "The `inplace` parameter in cudf.Series.cat.set_categories is " - "deprecated and will be removed in a future version of cudf. " - "Setting categories will always return a new Categorical " - "object.", - FutureWarning, - ) + return self._return_or_inplace( self._column.set_categories( new_categories=new_categories, ordered=ordered, rename=rename - ), - inplace=inplace, + ) ) def reorder_categories( self, new_categories: Any, ordered: bool = False, - inplace: bool = False, ) -> Optional[SeriesOrIndex]: """ Reorder categories as specified in new_categories. @@ -611,23 +451,11 @@ def reorder_categories( Whether or not the categorical is treated as a ordered categorical. If not given, do not change the ordered information. - inplace : bool, default False - Whether or not to reorder the categories - inplace or return a copy of this categorical - with reordered categories. - - .. deprecated:: 23.04 - - The `inplace` parameter is is deprecated and - will be removed in a future version of cudf. - Reordering categories will always return a - new Categorical object. Returns ------- cat - Categorical with reordered categories or - None if inplace. + Categorical with reordered categories Raises ------ @@ -664,18 +492,9 @@ def reorder_categories( ValueError: items in new_categories are not the same as in old categories """ - if inplace: - warnings.warn( - "The `inplace` parameter in " - "cudf.Series.cat.reorder_categories is deprecated " - "and will be removed in a future version of cudf. " - "Reordering categories will always return a new " - "Categorical object.", - FutureWarning, - ) + return self._return_or_inplace( self._column.reorder_categories(new_categories, ordered=ordered), - inplace=inplace, ) diff --git a/python/cudf/cudf/tests/test_categorical.py b/python/cudf/cudf/tests/test_categorical.py index 496039ca2f8..6a705e2fa63 100644 --- a/python/cudf/cudf/tests/test_categorical.py +++ b/python/cudf/cudf/tests/test_categorical.py @@ -11,30 +11,14 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_110, PANDAS_GE_134 +from cudf.core._compat import PANDAS_GE_110 from cudf.testing._utils import ( NUMERIC_TYPES, assert_eq, assert_exceptions_equal, - expect_warning_if, ) -@contextmanager -def _hide_deprecated_pandas_categorical_inplace_warnings(function_name): - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", - ( - "The `inplace` parameter in " - f"pandas.Categorical.{function_name} is deprecated and will " - "be removed in a future version." - ), - category=FutureWarning, - ) - yield - - @contextmanager def _hide_cudf_safe_casting_warning(): with warnings.catch_warnings(): @@ -363,8 +347,7 @@ def test_categorical_set_categories_preserves_order(): ) -@pytest.mark.parametrize("inplace", [True, False]) -def test_categorical_as_ordered(pd_str_cat, inplace): +def test_categorical_as_ordered(pd_str_cat): pd_sr = pd.Series(pd_str_cat.copy().set_ordered(False)) cd_sr = cudf.Series(pd_str_cat.copy().set_ordered(False)) @@ -372,23 +355,15 @@ def test_categorical_as_ordered(pd_str_cat, inplace): assert cd_sr.cat.ordered is False assert cd_sr.cat.ordered == pd_sr.cat.ordered - # pandas internally uses a deprecated call to set_ordered(inplace=inplace) - # inside as_ordered. - with pytest.warns(FutureWarning): - pd_sr_1 = pd_sr.cat.as_ordered(inplace=inplace) - with expect_warning_if(inplace, FutureWarning): - cd_sr_1 = cd_sr.cat.as_ordered(inplace=inplace) - if inplace: - pd_sr_1 = pd_sr - cd_sr_1 = cd_sr + pd_sr_1 = pd_sr.cat.as_ordered() + cd_sr_1 = cd_sr.cat.as_ordered() assert cd_sr_1.cat.ordered is True assert cd_sr_1.cat.ordered == pd_sr_1.cat.ordered assert str(cd_sr_1) == str(pd_sr_1) -@pytest.mark.parametrize("inplace", [True, False]) -def test_categorical_as_unordered(pd_str_cat, inplace): +def test_categorical_as_unordered(pd_str_cat): pd_sr = pd.Series(pd_str_cat.copy().set_ordered(True)) cd_sr = cudf.Series(pd_str_cat.copy().set_ordered(True)) @@ -396,15 +371,8 @@ def test_categorical_as_unordered(pd_str_cat, inplace): assert cd_sr.cat.ordered is True assert cd_sr.cat.ordered == pd_sr.cat.ordered - # pandas internally uses a deprecated call to set_ordered(inplace=inplace) - # inside as_unordered. - with pytest.warns(FutureWarning): - pd_sr_1 = pd_sr.cat.as_unordered(inplace=inplace) - with expect_warning_if(inplace, FutureWarning): - cd_sr_1 = cd_sr.cat.as_unordered(inplace=inplace) - if inplace: - pd_sr_1 = pd_sr - cd_sr_1 = cd_sr + pd_sr_1 = pd_sr.cat.as_unordered() + cd_sr_1 = cd_sr.cat.as_unordered() assert cd_sr_1.cat.ordered is False assert cd_sr_1.cat.ordered == pd_sr_1.cat.ordered @@ -413,22 +381,7 @@ def test_categorical_as_unordered(pd_str_cat, inplace): @pytest.mark.parametrize("from_ordered", [True, False]) @pytest.mark.parametrize("to_ordered", [True, False]) -@pytest.mark.parametrize( - "inplace", - [ - pytest.param( - True, - marks=pytest.mark.skipif( - condition=not PANDAS_GE_134, - reason="https://github.com/pandas-dev/pandas/issues/43232", - ), - ), - False, - ], -) -def test_categorical_reorder_categories( - pd_str_cat, from_ordered, to_ordered, inplace -): +def test_categorical_reorder_categories(pd_str_cat, from_ordered, to_ordered): pd_sr = pd.Series(pd_str_cat.copy().set_ordered(from_ordered)) cd_sr = cudf.Series(pd_str_cat.copy().set_ordered(from_ordered)) @@ -437,39 +390,19 @@ def test_categorical_reorder_categories( assert str(pd_sr) == str(cd_sr) - kwargs = dict(ordered=to_ordered, inplace=inplace) + kwargs = dict( + ordered=to_ordered, + ) - with _hide_deprecated_pandas_categorical_inplace_warnings( - "reorder_categories" - ): - pd_sr_1 = pd_sr.cat.reorder_categories(list("cba"), **kwargs) - if inplace: - with pytest.warns(FutureWarning): - cd_sr_1 = cd_sr.cat.reorder_categories(list("cba"), **kwargs) - pd_sr_1 = pd_sr - cd_sr_1 = cd_sr - else: - cd_sr_1 = cd_sr.cat.reorder_categories(list("cba"), **kwargs) + pd_sr_1 = pd_sr.cat.reorder_categories(list("cba"), **kwargs) + cd_sr_1 = cd_sr.cat.reorder_categories(list("cba"), **kwargs) assert_eq(pd_sr_1, cd_sr_1) assert str(cd_sr_1) == str(pd_sr_1) -@pytest.mark.parametrize( - "inplace", - [ - pytest.param( - True, - marks=pytest.mark.skipif( - condition=not PANDAS_GE_134, - reason="https://github.com/pandas-dev/pandas/issues/43232", - ), - ), - False, - ], -) -def test_categorical_add_categories(pd_str_cat, inplace): +def test_categorical_add_categories(pd_str_cat): pd_sr = pd.Series(pd_str_cat.copy()) cd_sr = cudf.Series(pd_str_cat.copy()) @@ -478,18 +411,8 @@ def test_categorical_add_categories(pd_str_cat, inplace): assert str(pd_sr) == str(cd_sr) - with _hide_deprecated_pandas_categorical_inplace_warnings( - "add_categories" - ): - pd_sr_1 = pd_sr.cat.add_categories(["d"], inplace=inplace) - - if inplace: - with pytest.warns(FutureWarning): - cd_sr_1 = cd_sr.cat.add_categories(["d"], inplace=inplace) - pd_sr_1 = pd_sr - cd_sr_1 = cd_sr - else: - cd_sr_1 = cd_sr.cat.add_categories(["d"], inplace=inplace) + pd_sr_1 = pd_sr.cat.add_categories(["d"]) + cd_sr_1 = cd_sr.cat.add_categories(["d"]) assert "d" in pd_sr_1.cat.categories.to_list() assert "d" in cd_sr_1.cat.categories.to_pandas().to_list() @@ -497,20 +420,7 @@ def test_categorical_add_categories(pd_str_cat, inplace): assert_eq(pd_sr_1, cd_sr_1) -@pytest.mark.parametrize( - "inplace", - [ - pytest.param( - True, - marks=pytest.mark.skipif( - condition=not PANDAS_GE_134, - reason="https://github.com/pandas-dev/pandas/issues/43232", - ), - ), - False, - ], -) -def test_categorical_remove_categories(pd_str_cat, inplace): +def test_categorical_remove_categories(pd_str_cat): pd_sr = pd.Series(pd_str_cat.copy()) cd_sr = cudf.Series(pd_str_cat.copy()) @@ -519,18 +429,8 @@ def test_categorical_remove_categories(pd_str_cat, inplace): assert str(pd_sr) == str(cd_sr) - with _hide_deprecated_pandas_categorical_inplace_warnings( - "remove_categories" - ): - pd_sr_1 = pd_sr.cat.remove_categories(["a"], inplace=inplace) - - if inplace: - with pytest.warns(FutureWarning): - cd_sr_1 = cd_sr.cat.remove_categories(["a"], inplace=inplace) - pd_sr_1 = pd_sr - cd_sr_1 = cd_sr - else: - cd_sr_1 = cd_sr.cat.remove_categories(["a"], inplace=inplace) + pd_sr_1 = pd_sr.cat.remove_categories(["a"]) + cd_sr_1 = cd_sr.cat.remove_categories(["a"]) assert "a" not in pd_sr_1.cat.categories.to_list() assert "a" not in cd_sr_1.cat.categories.to_pandas().to_list() @@ -538,15 +438,12 @@ def test_categorical_remove_categories(pd_str_cat, inplace): assert_eq(pd_sr_1, cd_sr_1) # test using ordered operators - with _hide_deprecated_pandas_categorical_inplace_warnings( - "remove_categories" - ) as _, pytest.warns(FutureWarning) as _: - assert_exceptions_equal( - lfunc=cd_sr.to_pandas().cat.remove_categories, - rfunc=cd_sr.cat.remove_categories, - lfunc_args_and_kwargs=([["a", "d"]], {"inplace": inplace}), - rfunc_args_and_kwargs=([["a", "d"]], {"inplace": inplace}), - ) + assert_exceptions_equal( + lfunc=cd_sr.to_pandas().cat.remove_categories, + rfunc=cd_sr.cat.remove_categories, + lfunc_args_and_kwargs=([["a", "d"]], {}), + rfunc_args_and_kwargs=([["a", "d"]], {}), + ) def test_categorical_dataframe_slice_copy(): From d1b1ea80a88053fae05f07f7805c96614902b70b Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 24 Feb 2023 17:10:07 -0600 Subject: [PATCH 003/384] [REVIEW] Raise error when `numeric_only=True` for non-numeric Series (#12843) This PR raises an error when numeric_only=True for rank if the Series is of non-numeric dtype. --- python/cudf/cudf/core/indexed_frame.py | 11 +++++++++-- python/cudf/cudf/tests/test_rank.py | 21 +++++++++++++-------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 2992cb005e5..159cc318789 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -4771,7 +4771,7 @@ def rank( self, axis=0, method="average", - numeric_only=None, + numeric_only=False, na_option="keep", ascending=True, pct=False, @@ -4794,7 +4794,7 @@ def rank( * max: highest rank in the group * first: ranks assigned in order they appear in the array * dense: like 'min', but rank always increases by 1 between groups. - numeric_only : bool, optional + numeric_only : bool, default False For DataFrame objects, rank only numeric columns if set to True. na_option : {'keep', 'top', 'bottom'}, default 'keep' How to rank NaN values: @@ -4829,6 +4829,13 @@ def rank( source = self if numeric_only: + if isinstance( + source, cudf.Series + ) and not _is_non_decimal_numeric_dtype(self.dtype): + raise TypeError( + "Series.rank does not allow numeric_only=True with " + "non-numeric dtype." + ) numeric_cols = ( name for name in self._data.names diff --git a/python/cudf/cudf/tests/test_rank.py b/python/cudf/cudf/tests/test_rank.py index 9bd67309ece..0aa3d53f962 100644 --- a/python/cudf/cudf/tests/test_rank.py +++ b/python/cudf/cudf/tests/test_rank.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. from itertools import chain, combinations_with_replacement, product @@ -55,13 +55,18 @@ def test_rank_all_arguments( assert_eq(gdf["col1"].rank(**kwargs), pdf["col1"].rank(**kwargs)) assert_eq(gdf["col2"].rank(**kwargs), pdf["col2"].rank(**kwargs)) if numeric_only: - with pytest.warns(FutureWarning): - expect = pdf["str"].rank(**kwargs) - got = gdf["str"].rank(**kwargs) - assert expect.empty == got.empty - expected = pdf.select_dtypes(include=np.number) - else: - expected = pdf.copy(deep=True) + assert_exceptions_equal( + lfunc=pdf["str"].rank, + rfunc=gdf["str"].rank, + lfunc_args_and_kwargs=( + [], + kwargs, + ), + rfunc_args_and_kwargs=( + [], + kwargs, + ), + ) actual = gdf.rank(**kwargs) expected = pdf.rank(**kwargs) From 63177336c224c35163a8f0a97456a15c0626abf0 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 28 Feb 2023 10:53:03 -0600 Subject: [PATCH 004/384] Drop is_monotonic (#12853) This PR drops support for `Series.is_monotonic` & `Index.is_monotonic`. Instead, the alternative will be `.is_monotonic_increasing`. --- docs/cudf/source/api_docs/index_objects.rst | 1 - python/cudf/cudf/core/_base_index.py | 19 --------- python/cudf/cudf/core/single_column_frame.py | 23 +--------- python/cudf/cudf/tests/test_dataframe.py | 2 - python/cudf/cudf/tests/test_monotonic.py | 45 ++------------------ 5 files changed, 5 insertions(+), 85 deletions(-) diff --git a/docs/cudf/source/api_docs/index_objects.rst b/docs/cudf/source/api_docs/index_objects.rst index 0a6e3c169f0..03eb6d68538 100644 --- a/docs/cudf/source/api_docs/index_objects.rst +++ b/docs/cudf/source/api_docs/index_objects.rst @@ -25,7 +25,6 @@ Properties Index.has_duplicates Index.duplicated Index.hasnans - Index.is_monotonic Index.is_monotonic_increasing Index.is_monotonic_decreasing Index.is_unique diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index 8f8f2afc734..80ebf88245f 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -3,7 +3,6 @@ from __future__ import annotations import pickle -import warnings from functools import cached_property from typing import Any, Set, TypeVar @@ -189,24 +188,6 @@ def _clean_nulls_from_index(self): """ raise NotImplementedError - @property - def is_monotonic(self): - """Return boolean if values in the object are monotonic_increasing. - - This property is an alias for :attr:`is_monotonic_increasing`. - - Returns - ------- - bool - """ - warnings.warn( - "is_monotonic is deprecated and will be removed in a future " - "version. Use is_monotonic_increasing instead.", - FutureWarning, - ) - - return self.is_monotonic_increasing - @property def is_monotonic_increasing(self): """Return boolean if values in the object are monotonically increasing. diff --git a/python/cudf/cudf/core/single_column_frame.py b/python/cudf/cudf/core/single_column_frame.py index afd06ea3629..46cf49c62e0 100644 --- a/python/cudf/cudf/core/single_column_frame.py +++ b/python/cudf/cudf/core/single_column_frame.py @@ -1,9 +1,9 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. + """Base class for Frame types that only have a single column.""" from __future__ import annotations -import warnings from typing import Any, Dict, Optional, Tuple, TypeVar, Union import cupy @@ -223,25 +223,6 @@ def is_unique(self): """ return self._column.is_unique - @property # type: ignore - @_cudf_nvtx_annotate - def is_monotonic(self): - """Return boolean if values in the object are monotonically increasing. - - This property is an alias for :attr:`is_monotonic_increasing`. - - Returns - ------- - bool - """ - warnings.warn( - "is_monotonic is deprecated and will be removed in a future " - "version. Use is_monotonic_increasing instead.", - FutureWarning, - ) - - return self.is_monotonic_increasing - @property # type: ignore @_cudf_nvtx_annotate def is_monotonic_increasing(self): diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 13f312f6f0c..a727644e42f 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -2518,8 +2518,6 @@ def test_unary_operators(func, pdf, gdf): def test_is_monotonic(gdf): pdf = pd.DataFrame({"x": [1, 2, 3]}, index=[3, 1, 2]) gdf = cudf.DataFrame.from_pandas(pdf) - with pytest.warns(FutureWarning): - assert not gdf.index.is_monotonic assert not gdf.index.is_monotonic_increasing assert not gdf.index.is_monotonic_decreasing diff --git a/python/cudf/cudf/tests/test_monotonic.py b/python/cudf/cudf/tests/test_monotonic.py index f4e8b80342a..93c202c3138 100644 --- a/python/cudf/cudf/tests/test_monotonic.py +++ b/python/cudf/cudf/tests/test_monotonic.py @@ -1,7 +1,8 @@ -# Copyright (c) 2019-2022, NVIDIA CORPORATION. +# Copyright (c) 2019-2023, NVIDIA CORPORATION. """ -Tests related to is_unique and is_monotonic attributes +Tests related to is_unique, is_monotonic_increasing & +is_monotonic_decreasing attributes """ import numpy as np import pandas as pd @@ -30,11 +31,6 @@ def test_range_index(testrange): ) assert index.is_unique == index_pd.is_unique - with pytest.warns(FutureWarning): - expect = index_pd.is_monotonic - with pytest.warns(FutureWarning): - got = index.is_monotonic - assert got == expect assert index.is_monotonic_increasing == index_pd.is_monotonic_increasing assert index.is_monotonic_decreasing == index_pd.is_monotonic_decreasing @@ -58,11 +54,6 @@ def test_generic_index(testlist): index_pd = pd.Index(testlist) assert index.is_unique == index_pd.is_unique - with pytest.warns(FutureWarning): - expect = index_pd.is_monotonic - with pytest.warns(FutureWarning): - got = index.is_monotonic - assert got == expect assert index.is_monotonic_increasing == index_pd.is_monotonic_increasing assert index.is_monotonic_decreasing == index_pd.is_monotonic_decreasing @@ -82,11 +73,6 @@ def test_string_index(testlist): index_pd = pd.Index(testlist) assert index.is_unique == index_pd.is_unique - with pytest.warns(FutureWarning): - expect = index_pd.is_monotonic - with pytest.warns(FutureWarning): - got = index.is_monotonic - assert got == expect assert index.is_monotonic_increasing == index_pd.is_monotonic_increasing assert index.is_monotonic_decreasing == index_pd.is_monotonic_decreasing @@ -102,11 +88,6 @@ def test_categorical_index(testlist): index_pd = pd.CategoricalIndex(raw_cat) assert index.is_unique == index_pd.is_unique - with pytest.warns(FutureWarning): - expect = index_pd.is_monotonic - with pytest.warns(FutureWarning): - got = index.is_monotonic - assert got == expect assert index.is_monotonic_increasing == index_pd.is_monotonic_increasing assert index.is_monotonic_decreasing == index_pd.is_monotonic_decreasing @@ -147,11 +128,6 @@ def test_datetime_index(testlist): index_pd = pd.DatetimeIndex(testlist) assert index.is_unique == index_pd.is_unique - with pytest.warns(FutureWarning): - expect = index_pd.is_monotonic - with pytest.warns(FutureWarning): - got = index.is_monotonic - assert got == expect assert index.is_monotonic_increasing == index_pd.is_monotonic_increasing assert index.is_monotonic_decreasing == index_pd.is_monotonic_decreasing @@ -174,11 +150,6 @@ def test_series(testlist): series_pd = pd.Series(testlist) assert series.is_unique == series_pd.is_unique - with pytest.warns(FutureWarning): - expect = series_pd.index.is_monotonic - with pytest.warns(FutureWarning): - got = series.index.is_monotonic - assert got == expect assert series.is_monotonic_increasing == series_pd.is_monotonic_increasing assert series.is_monotonic_decreasing == series_pd.is_monotonic_decreasing @@ -203,11 +174,6 @@ def test_multiindex(): gdf = cudf.from_pandas(pdf) assert pdf.index.is_unique == gdf.index.is_unique - with pytest.warns(FutureWarning): - expect = pdf.index.is_monotonic - with pytest.warns(FutureWarning): - got = gdf.index.is_monotonic - assert got == expect assert ( pdf.index.is_monotonic_increasing == gdf.index.is_monotonic_increasing ) @@ -242,11 +208,6 @@ def test_multiindex_tuples(testarr): index_pd = pd.MultiIndex.from_tuples(tuples, names=testarr[1]) assert index.is_unique == index_pd.is_unique - with pytest.warns(FutureWarning): - expect = index_pd.is_monotonic - with pytest.warns(FutureWarning): - got = index.is_monotonic - assert got == expect assert index.is_monotonic_increasing == index_pd.is_monotonic_increasing assert index.is_monotonic_decreasing == index_pd.is_monotonic_decreasing From 5af05836c2df4ae92514ebd696e1cd18aa80296e Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 8 Mar 2023 13:39:35 -0600 Subject: [PATCH 005/384] [REVIEW] Drop `datetime_is_numeric` parameter from `describe` (#12890) This PR removes support for `datetime_is_numeric` parameter in `describe`. --- python/cudf/cudf/core/dataframe.py | 13 +------------ python/cudf/cudf/core/series.py | 9 --------- python/cudf/cudf/tests/test_dataframe.py | 19 +++++++------------ python/cudf/cudf/tests/test_series.py | 14 +++++--------- python/cudf/cudf/utils/docutils.py | 9 --------- 5 files changed, 13 insertions(+), 51 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 83da93d9ae1..d9900f3adc3 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -4929,21 +4929,11 @@ def describe( percentiles=None, include=None, exclude=None, - datetime_is_numeric=False, ): """{docstring}""" if not include and not exclude: - default_include = [np.number] - if datetime_is_numeric: - default_include.append("datetime") - else: - warnings.warn( - "`datetime_is_numeric` is deprecated. Specify " - "`datetime_is_numeric=True` to silence this " - "warning and adopt the future behavior now.", - FutureWarning, - ) + default_include = [np.number, "datetime"] data_to_describe = self.select_dtypes(include=default_include) if data_to_describe._num_columns == 0: data_to_describe = self @@ -4964,7 +4954,6 @@ def describe( describe_series_list = [ data_to_describe[col].describe( percentiles=percentiles, - datetime_is_numeric=datetime_is_numeric, ) for col in data_to_describe._column_names ] diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index bd5569c042c..d486851176a 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -6,7 +6,6 @@ import inspect import pickle import textwrap -import warnings from collections import abc from shutil import get_terminal_size from typing import Any, Dict, MutableMapping, Optional, Set, Tuple, Union @@ -3108,17 +3107,9 @@ def describe( percentiles=None, include=None, exclude=None, - datetime_is_numeric=False, ): """{docstring}""" - if not datetime_is_numeric: - warnings.warn( - "`datetime_is_numeric` is deprecated and will be removed in " - "a future release. Specify `datetime_is_numeric=True` to " - "silence this warning and adopt the future behavior now.", - FutureWarning, - ) if percentiles is not None: if not all(0 <= x <= 1 for x in percentiles): raise ValueError( diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 5db338e66cf..df235d48a30 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -3654,8 +3654,8 @@ def test_dataframe_describe_exclude(): df["x"] = df.x.astype("int64") df["y"] = np.random.normal(10, 1, data_length) pdf = df.to_pandas() - with pytest.warns(FutureWarning): - gdf_results = df.describe(exclude=["float"]) + + gdf_results = df.describe(exclude=["float"]) pdf_results = pdf.describe(exclude=["float"]) assert_eq(gdf_results, pdf_results) @@ -3670,8 +3670,7 @@ def test_dataframe_describe_include(): df["x"] = df.x.astype("int64") df["y"] = np.random.normal(10, 1, data_length) pdf = df.to_pandas() - with pytest.warns(FutureWarning): - gdf_results = df.describe(include=["int"]) + gdf_results = df.describe(include=["int"]) pdf_results = pdf.describe(include=["int"]) assert_eq(gdf_results, pdf_results) @@ -3685,8 +3684,7 @@ def test_dataframe_describe_default(): df["x"] = np.random.normal(10, 1, data_length) df["y"] = np.random.normal(10, 1, data_length) pdf = df.to_pandas() - with pytest.warns(FutureWarning): - gdf_results = df.describe() + gdf_results = df.describe() pdf_results = pdf.describe() assert_eq(pdf_results, gdf_results) @@ -3703,8 +3701,7 @@ def test_series_describe_include_all(): df["animal"] = np.random.choice(["dog", "cat", "bird"], data_length) pdf = df.to_pandas() - with pytest.warns(FutureWarning): - gdf_results = df.describe(include="all") + gdf_results = df.describe(include="all") pdf_results = pdf.describe(include="all") assert_eq(gdf_results[["x", "y"]], pdf_results[["x", "y"]]) @@ -3725,8 +3722,7 @@ def test_dataframe_describe_percentiles(): df["x"] = np.random.normal(10, 1, data_length) df["y"] = np.random.normal(10, 1, data_length) pdf = df.to_pandas() - with pytest.warns(FutureWarning): - gdf_results = df.describe(percentiles=sample_percentiles) + gdf_results = df.describe(percentiles=sample_percentiles) pdf_results = pdf.describe(percentiles=sample_percentiles) assert_eq(pdf_results, gdf_results) @@ -4053,8 +4049,7 @@ def test_empty_dataframe_describe(): gdf = cudf.from_pandas(pdf) expected = pdf.describe() - with pytest.warns(FutureWarning): - actual = gdf.describe() + actual = gdf.describe() assert_eq(expected, actual) diff --git a/python/cudf/cudf/tests/test_series.py b/python/cudf/cudf/tests/test_series.py index ce519a445ba..f08295d228d 100644 --- a/python/cudf/cudf/tests/test_series.py +++ b/python/cudf/cudf/tests/test_series.py @@ -408,8 +408,7 @@ def test_series_size(data): def test_series_describe_numeric(dtype): ps = pd.Series([0, 1, 2, 3, 1, 2, 3], dtype=dtype) gs = cudf.from_pandas(ps) - with pytest.warns(FutureWarning): - actual = gs.describe() + actual = gs.describe() expected = ps.describe() assert_eq(expected, actual, check_dtype=True) @@ -426,9 +425,8 @@ def test_series_describe_datetime(dtype): # Treating datetimes as categoricals is deprecated in pandas and will # be removed in future. Future behavior is treating datetime as numeric. - expected = ps.describe(datetime_is_numeric=True) - with pytest.warns(FutureWarning): - actual = gs.describe() + expected = ps.describe() + actual = gs.describe() assert_eq(expected.astype("str"), actual) @@ -439,8 +437,7 @@ def test_series_describe_timedelta(dtype): gs = cudf.from_pandas(ps) expected = ps.describe() - with pytest.warns(FutureWarning): - actual = gs.describe() + actual = gs.describe() assert_eq(actual, expected.astype("str")) @@ -465,8 +462,7 @@ def test_series_describe_other_types(ps): gs = cudf.from_pandas(ps) expected = ps.describe() - with pytest.warns(FutureWarning): - actual = gs.describe() + actual = gs.describe() if len(ps) == 0: assert_eq(expected.fillna("a").astype("str"), actual.fillna("a")) diff --git a/python/cudf/cudf/utils/docutils.py b/python/cudf/cudf/utils/docutils.py index 5a7b8bae980..1a9c4b54aa9 100644 --- a/python/cudf/cudf/utils/docutils.py +++ b/python/cudf/cudf/utils/docutils.py @@ -126,15 +126,6 @@ def wrapper(func): exclude pandas categorical columns, use ``'category'`` - None (default) : The result will exclude nothing. - datetime_is_numeric : bool, default False - For DataFrame input, this also controls whether datetime columns - are included by default. - - .. deprecated:: 23.04 - - `datetime_is_numeric` is deprecated and will be removed in - a future version of cudf. - Returns ------- output_frame : Series or DataFrame From 531f52cde5b014d6533119968ee4ef7edda5b313 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 8 Mar 2023 13:55:54 -0600 Subject: [PATCH 006/384] Drop `names`, `dtype` in `Index.copy` and `dtype`, `levels`, `codes` in `MultiIndex.copy` (#12898) This PR removes `dtype` in Index & `MultiIndex.copy`, and `names` in Index.copy --- python/cudf/cudf/core/index.py | 75 +++-------------------- python/cudf/cudf/core/multiindex.py | 72 +--------------------- python/cudf/cudf/tests/test_index.py | 58 ++++++------------ python/cudf/cudf/tests/test_multiindex.py | 15 +---- 4 files changed, 30 insertions(+), 190 deletions(-) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index cd882aba297..2203d103204 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -302,7 +302,7 @@ def __contains__(self, item): return item in range(self._start, self._stop, self._step) @_cudf_nvtx_annotate - def copy(self, name=None, deep=False, dtype=None, names=None): + def copy(self, name=None, deep=False): """ Make a copy of this object. @@ -311,44 +311,11 @@ def copy(self, name=None, deep=False, dtype=None, names=None): name : object optional (default: None), name of index deep : Bool (default: False) Ignored for RangeIndex - dtype : numpy dtype optional (default: None) - Target dtype for underlying range data - - .. deprecated:: 23.02 - - The `dtype` parameter is deprecated and will be removed in - a future version of cudf. Use the `astype` method instead. - - names : list-like optional (default: False) - Kept compatibility with MultiIndex. Should not be used. - - .. deprecated:: 23.04 - - The parameter `names` is deprecated and will be removed in - a future version of cudf. Use the `name` parameter instead. Returns ------- - New RangeIndex instance with same range, casted to new dtype + New RangeIndex instance with same range """ - if dtype is not None: - warnings.warn( - "parameter dtype is deprecated and will be removed in a " - "future version. Use the astype method instead.", - FutureWarning, - ) - - if names is not None: - warnings.warn( - "parameter names is deprecated and will be removed in a " - "future version. Use the name parameter instead.", - FutureWarning, - ) - - dtype = self.dtype if dtype is None else dtype - - if not np.issubdtype(dtype, np.signedinteger): - raise ValueError(f"Expected Signed Integer Type, Got {dtype}") name = self.name if name is None else name @@ -1140,7 +1107,7 @@ def equals(self, other, **kwargs): return False @_cudf_nvtx_annotate - def copy(self, name=None, deep=False, dtype=None, names=None): + def copy(self, name=None, deep=False): """ Make a copy of this object. @@ -1151,45 +1118,17 @@ def copy(self, name=None, deep=False, dtype=None, names=None): deep : bool, default True Make a deep copy of the data. With ``deep=False`` the original data is used - dtype : numpy dtype, default None - Target datatype to cast into, use original dtype when None - - .. deprecated:: 23.02 - - The `dtype` parameter is deprecated and will be removed in - a future version of cudf. Use the `astype` method instead. - - names : list-like, default False - Kept compatibility with MultiIndex. Should not be used. - - .. deprecated:: 23.04 - - The parameter `names` is deprecated and will be removed in - a future version of cudf. Use the `name` parameter instead. Returns ------- - New index instance, casted to new dtype + New index instance. """ - if dtype is not None: - warnings.warn( - "parameter dtype is deprecated and will be removed in a " - "future version. Use the astype method instead.", - FutureWarning, - ) - - if names is not None: - warnings.warn( - "parameter names is deprecated and will be removed in a " - "future version. Use the name parameter instead.", - FutureWarning, - ) - dtype = self.dtype if dtype is None else dtype name = self.name if name is None else name - col = self._values.astype(dtype) - return _index_from_data({name: col.copy(True) if deep else col}) + return _index_from_data( + {name: self._values.copy(True) if deep else self._values} + ) @_cudf_nvtx_annotate @doc_apply(_index_astype_docstring) diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 1f26371f797..2951a362e73 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -5,7 +5,6 @@ import itertools import numbers import pickle -import warnings from collections import abc from functools import cached_property from numbers import Integral @@ -318,9 +317,6 @@ def name(self, value): def copy( self, names=None, - dtype=None, - levels=None, - codes=None, deep=False, name=None, ): @@ -334,36 +330,12 @@ def copy( ---------- names : sequence of objects, optional (default None) Names for each of the index levels. - dtype : object, optional (default None) - MultiIndex dtype, only supports None or object type - - .. deprecated:: 23.02 - - The `dtype` parameter is deprecated and will be removed in - a future version of cudf. Use the `astype` method instead. - - levels : sequence of arrays, optional (default None) - The unique labels for each level. Original values used if None. - - .. deprecated:: 23.02 - - The `levels` parameter is deprecated and will be removed in - a future version of cudf. - - codes : sequence of arrays, optional (default None) - Integers for each level designating which label at each location. - Original values used if None. - - .. deprecated:: 23.02 - - The `codes` parameter is deprecated and will be removed in - a future version of cudf. - deep : Bool (default False) If True, `._data`, `._levels`, `._codes` will be copied. Ignored if `levels` or `codes` are specified. name : object, optional (default None) - To keep consistent with `Index.copy`, should not be used. + Kept for compatibility with 1-dimensional Index. Should not + be used. Returns ------- @@ -401,46 +373,6 @@ def copy( """ - # TODO: Update message when set_levels is implemented. - # https://github.com/rapidsai/cudf/issues/12307 - if levels is not None: - warnings.warn( - "parameter levels is deprecated and will be removed in a " - "future version.", - FutureWarning, - ) - - # TODO: Update message when set_codes is implemented. - # https://github.com/rapidsai/cudf/issues/12308 - if codes is not None: - warnings.warn( - "parameter codes is deprecated and will be removed in a " - "future version.", - FutureWarning, - ) - - if dtype is not None: - warnings.warn( - "parameter dtype is deprecated and will be removed in a " - "future version. Use the astype method instead.", - FutureWarning, - ) - - dtype = object if dtype is None else dtype - if not pd.core.dtypes.common.is_object_dtype(dtype): - raise TypeError("Dtype for MultiIndex only supports object type.") - - # ._data needs to be rebuilt - if levels is not None or codes is not None: - if self._levels is None or self._codes is None: - self._compute_levels_and_codes() - levels = self._levels if levels is None else levels - codes = self._codes if codes is None else codes - names = self.names if names is None else names - - mi = MultiIndex(levels=levels, codes=codes, names=names, copy=deep) - return mi - mi = MultiIndex._from_data(self._data.copy(deep=deep)) if self._levels is not None: mi._levels = [s.copy(deep) for s in self._levels] diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index f0b74ce70e7..d4ce348fa78 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -25,7 +25,6 @@ NUMERIC_TYPES, OTHER_TYPES, SIGNED_INTEGER_TYPES, - SIGNED_TYPES, UNSIGNED_TYPES, _create_pandas_series, assert_column_memory_eq, @@ -307,90 +306,69 @@ def test_set_index_as_property(): @pytest.mark.parametrize("name", ["x"]) -@pytest.mark.parametrize("dtype", SIGNED_INTEGER_TYPES) -def test_index_copy_range(name, dtype, deep=True): +def test_index_copy_range(name, deep=True): cidx = cudf.RangeIndex(1, 5) pidx = cidx.to_pandas() - with pytest.warns(FutureWarning): - pidx_copy = pidx.copy(name=name, deep=deep, dtype=dtype) - with pytest.warns(FutureWarning): - cidx_copy = cidx.copy(name=name, deep=deep, dtype=dtype) + pidx_copy = pidx.copy(name=name, deep=deep) + cidx_copy = cidx.copy(name=name, deep=deep) assert_eq(pidx_copy, cidx_copy) @pytest.mark.parametrize("name", ["x"]) -@pytest.mark.parametrize("dtype,", ["datetime64[ns]", "int64"]) -def test_index_copy_datetime(name, dtype, deep=True): +def test_index_copy_datetime(name, deep=True): cidx = cudf.DatetimeIndex(["2001", "2002", "2003"]) pidx = cidx.to_pandas() - with pytest.warns(FutureWarning): - pidx_copy = pidx.copy(name=name, deep=deep, dtype=dtype) - with pytest.warns(FutureWarning): - cidx_copy = cidx.copy(name=name, deep=deep, dtype=dtype) + pidx_copy = pidx.copy(name=name, deep=deep) + cidx_copy = cidx.copy(name=name, deep=deep) assert_eq(pidx_copy, cidx_copy) @pytest.mark.parametrize("name", ["x"]) -@pytest.mark.parametrize("dtype", ["category", "object"]) -def test_index_copy_string(name, dtype, deep=True): +def test_index_copy_string(name, deep=True): cidx = cudf.StringIndex(["a", "b", "c"]) pidx = cidx.to_pandas() - with pytest.warns(FutureWarning): - pidx_copy = pidx.copy(name=name, deep=deep, dtype=dtype) - with pytest.warns(FutureWarning): - cidx_copy = cidx.copy(name=name, deep=deep, dtype=dtype) + pidx_copy = pidx.copy(name=name, deep=deep) + cidx_copy = cidx.copy(name=name, deep=deep) assert_eq(pidx_copy, cidx_copy) @pytest.mark.parametrize("name", ["x"]) -@pytest.mark.parametrize( - "dtype", - NUMERIC_TYPES + ["datetime64[ns]", "timedelta64[ns]"] + OTHER_TYPES, -) -def test_index_copy_integer(name, dtype, deep=True): +def test_index_copy_integer(name, deep=True): """Test for NumericIndex Copy Casts""" cidx = cudf.Index([1, 2, 3]) pidx = cidx.to_pandas() - with pytest.warns(FutureWarning): - pidx_copy = pidx.copy(name=name, deep=deep, dtype=dtype) - with pytest.warns(FutureWarning): - cidx_copy = cidx.copy(name=name, deep=deep, dtype=dtype) + pidx_copy = pidx.copy(name=name, deep=deep) + cidx_copy = cidx.copy(name=name, deep=deep) assert_eq(pidx_copy, cidx_copy) @pytest.mark.parametrize("name", ["x"]) -@pytest.mark.parametrize("dtype", SIGNED_TYPES) -def test_index_copy_float(name, dtype, deep=True): +def test_index_copy_float(name, deep=True): """Test for NumericIndex Copy Casts""" cidx = cudf.Index([1.0, 2.0, 3.0]) pidx = cidx.to_pandas() - with pytest.warns(FutureWarning): - pidx_copy = pidx.copy(name=name, deep=deep, dtype=dtype) - with pytest.warns(FutureWarning): - cidx_copy = cidx.copy(name=name, deep=deep, dtype=dtype) + pidx_copy = pidx.copy(name=name, deep=deep) + cidx_copy = cidx.copy(name=name, deep=deep) assert_eq(pidx_copy, cidx_copy) @pytest.mark.parametrize("name", ["x"]) -@pytest.mark.parametrize("dtype", NUMERIC_TYPES + ["category"]) -def test_index_copy_category(name, dtype, deep=True): +def test_index_copy_category(name, deep=True): cidx = cudf.core.index.CategoricalIndex([1, 2, 3]) pidx = cidx.to_pandas() - with pytest.warns(FutureWarning): - pidx_copy = pidx.copy(name=name, deep=deep, dtype=dtype) - with pytest.warns(FutureWarning): - cidx_copy = cidx.copy(name=name, deep=deep, dtype=dtype) + pidx_copy = pidx.copy(name=name, deep=deep) + cidx_copy = cidx.copy(name=name, deep=deep) assert_column_memory_ne(cidx._values, cidx_copy._values) assert_eq(pidx_copy, cidx_copy) diff --git a/python/cudf/cudf/tests/test_multiindex.py b/python/cudf/cudf/tests/test_multiindex.py index 14e3a2a1b9b..d1da63e1d74 100644 --- a/python/cudf/cudf/tests/test_multiindex.py +++ b/python/cudf/cudf/tests/test_multiindex.py @@ -700,15 +700,8 @@ def test_multiindex_equals(): } ], ) -@pytest.mark.parametrize( - "levels", - [[["2000-01-01", "2000-01-02", "2000-01-03"], ["A", "B", "C"]], None], -) -@pytest.mark.parametrize( - "codes", [[[0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0]], None] -) @pytest.mark.parametrize("names", [["X", "Y"]]) -def test_multiindex_copy_sem(data, levels, codes, names): +def test_multiindex_copy_sem(data, names): """Test semantic equality for MultiIndex.copy""" gdf = cudf.DataFrame(data) pdf = gdf.to_pandas() @@ -717,12 +710,10 @@ def test_multiindex_copy_sem(data, levels, codes, names): pdf = pdf.groupby(["Date", "Symbol"], sort=True).mean() gmi = gdf.index - with expect_warning_if(levels is not None or codes is not None): - gmi_copy = gmi.copy(levels=levels, codes=codes, names=names) + gmi_copy = gmi.copy(names=names) pmi = pdf.index - with expect_warning_if(levels is not None or codes is not None): - pmi_copy = pmi.copy(levels=levels, codes=codes, names=names) + pmi_copy = pmi.copy(names=names) for glv, plv in zip(gmi_copy.levels, pmi_copy.levels): assert all(glv.values_host == plv.values) From 7ec76b71e8d9f42693eb23f99f2362d9e3aa4a04 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 8 Mar 2023 17:13:51 -0600 Subject: [PATCH 007/384] Drop `kind` parameter from `Index.get_slice_bound` (#12856) This PR drops `kind` parameter from `Index.get_slice_bound` to match pandas-2.0 API. --- python/cudf/cudf/core/_base_index.py | 3 +- python/cudf/cudf/core/column/column.py | 12 ++----- python/cudf/cudf/core/index.py | 20 ++--------- python/cudf/cudf/tests/test_monotonic.py | 44 ++++++++---------------- 4 files changed, 21 insertions(+), 58 deletions(-) diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index f72c0f8b1be..88763b8a011 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -1412,7 +1412,7 @@ def rename(self, name, inplace=False): out.name = name return out - def get_slice_bound(self, label, side, kind=None): + def get_slice_bound(self, label, side): """ Calculate slice bound that corresponds to given label. Returns leftmost (one-past-the-rightmost if ``side=='right'``) position @@ -1422,7 +1422,6 @@ def get_slice_bound(self, label, side, kind=None): ---------- label : object side : {'left', 'right'} - kind : {'ix', 'loc', 'getitem'} Returns ------- diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 40921b71db5..31cc5a4327f 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -883,7 +883,7 @@ def is_monotonic_decreasing(self) -> bool: ascending=[False], null_position=None ) - def get_slice_bound(self, label: ScalarLike, side: str, kind: str) -> int: + def get_slice_bound(self, label: ScalarLike, side: str) -> int: """ Calculate slice bound that corresponds to given label. Returns leftmost (one-past-the-rightmost if ``side=='right'``) position @@ -893,22 +893,14 @@ def get_slice_bound(self, label: ScalarLike, side: str, kind: str) -> int: ---------- label : Scalar side : {'left', 'right'} - kind : {'ix', 'loc', 'getitem'} """ - if kind not in {"ix", "loc", "getitem", None}: - raise ValueError( - f"Invalid value for ``kind`` parameter," - f" must be either one of the following: " - f"{'ix', 'loc', 'getitem', None}, but found: {kind}" - ) + if side not in {"left", "right"}: raise ValueError( "Invalid value for side kwarg," " must be either 'left' or 'right': %s" % (side,) ) - # TODO: Handle errors/missing keys correctly - # Not currently using `kind` argument. if side == "left": return self.find_first_value(label, closest=True) elif side == "right": diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 363fa37f394..145563dce61 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -495,7 +495,7 @@ def is_monotonic_decreasing(self): return self._step < 0 or len(self) <= 1 @_cudf_nvtx_annotate - def get_slice_bound(self, label, side, kind=None): + def get_slice_bound(self, label, side): """ Calculate slice bound that corresponds to given label. Returns leftmost (one-past-the-rightmost if ``side=='right'``) position @@ -506,20 +506,12 @@ def get_slice_bound(self, label, side, kind=None): label : int A valid value in the ``RangeIndex`` side : {'left', 'right'} - kind : Unused - To keep consistency with other index types. Returns ------- int Index of label. """ - if kind is not None: - warnings.warn( - "'kind' argument in get_slice_bound is deprecated and will be " - "removed in a future version.", - FutureWarning, - ) if side not in {"left", "right"}: raise ValueError(f"Unrecognized side parameter: {side}") @@ -1388,14 +1380,8 @@ def notna(self): notnull = notna @_cudf_nvtx_annotate - def get_slice_bound(self, label, side, kind=None): - if kind is not None: - warnings.warn( - "'kind' argument in get_slice_bound is deprecated and will be " - "removed in a future version.", - FutureWarning, - ) - return self._values.get_slice_bound(label, side, kind) + def get_slice_bound(self, label, side): + return self._values.get_slice_bound(label, side) def _is_numeric(self): return False diff --git a/python/cudf/cudf/tests/test_monotonic.py b/python/cudf/cudf/tests/test_monotonic.py index 93c202c3138..e68024f03d4 100644 --- a/python/cudf/cudf/tests/test_monotonic.py +++ b/python/cudf/cudf/tests/test_monotonic.py @@ -17,7 +17,7 @@ RangeIndex, StringIndex, ) -from cudf.testing._utils import assert_eq, expect_warning_if +from cudf.testing._utils import assert_eq @pytest.mark.parametrize("testrange", [(10, 20, 1), (0, -10, -1), (5, 5, 1)]) @@ -222,15 +222,12 @@ def test_multiindex_tuples(testarr): ], ) @pytest.mark.parametrize("side", ["left", "right"]) -@pytest.mark.parametrize("kind", ["loc", "getitem", None]) -def test_get_slice_bound(testlist, side, kind): +def test_get_slice_bound(testlist, side): index = GenericIndex(testlist) index_pd = pd.Index(testlist) for label in testlist: - with pytest.warns(FutureWarning): - expect = index_pd.get_slice_bound(label, side, kind) - with expect_warning_if(kind is not None, FutureWarning): - got = index.get_slice_bound(label, side, kind) + expect = index_pd.get_slice_bound(label, side) + got = index.get_slice_bound(label, side) assert got == expect @@ -240,16 +237,13 @@ def test_get_slice_bound(testlist, side, kind): [[-1, 0, 5, 10, 11], [-1, 0, 1, 2], [2, 3, 4, 5], [-1, 0, 1], [2, 3, 4]], ) @pytest.mark.parametrize("side", ["left", "right"]) -@pytest.mark.parametrize("kind", ["getitem", "loc"]) -def test_rangeindex_get_slice_bound_basic(bounds, indices, side, kind): +def test_rangeindex_get_slice_bound_basic(bounds, indices, side): start, stop = bounds pd_index = pd.RangeIndex(start, stop) cudf_index = RangeIndex(start, stop) for idx in indices: - with pytest.warns(FutureWarning): - expect = pd_index.get_slice_bound(idx, side, kind) - with expect_warning_if(kind is not None, FutureWarning): - got = cudf_index.get_slice_bound(idx, side, kind) + expect = pd_index.get_slice_bound(idx, side) + got = cudf_index.get_slice_bound(idx, side) assert expect == got @@ -262,31 +256,25 @@ def test_rangeindex_get_slice_bound_basic(bounds, indices, side, kind): [3, 8, 13, 18, 20, 15, 10, 5, -1, 0, 19, 21, 6, 11, 17], ) @pytest.mark.parametrize("side", ["left", "right"]) -@pytest.mark.parametrize("kind", ["getitem", "loc"]) -def test_rangeindex_get_slice_bound_step(bounds, label, side, kind): +def test_rangeindex_get_slice_bound_step(bounds, label, side): start, stop, step = bounds pd_index = pd.RangeIndex(start, stop, step) cudf_index = RangeIndex(start, stop, step) - with pytest.warns(FutureWarning): - expect = pd_index.get_slice_bound(label, side, kind) - with expect_warning_if(kind is not None, FutureWarning): - got = cudf_index.get_slice_bound(label, side, kind) + expect = pd_index.get_slice_bound(label, side) + got = cudf_index.get_slice_bound(label, side) assert expect == got @pytest.mark.parametrize("label", [1, 3, 5, 7, 9, 11]) @pytest.mark.parametrize("side", ["left", "right"]) -@pytest.mark.parametrize("kind", ["loc", "getitem", None]) -def test_get_slice_bound_missing(label, side, kind): +def test_get_slice_bound_missing(label, side): mylist = [2, 4, 6, 8, 10] index = GenericIndex(mylist) index_pd = pd.Index(mylist) - with pytest.warns(FutureWarning): - expect = index_pd.get_slice_bound(label, side, kind) - with expect_warning_if(kind is not None, FutureWarning): - got = index.get_slice_bound(label, side, kind) + expect = index_pd.get_slice_bound(label, side) + got = index.get_slice_bound(label, side) assert got == expect @@ -299,10 +287,8 @@ def test_get_slice_bound_missing_str(label, side): mylist = ["b", "d", "f"] index = GenericIndex(mylist) index_pd = pd.Index(mylist) - with pytest.warns(FutureWarning): - got = index.get_slice_bound(label, side, "getitem") - with pytest.warns(FutureWarning): - expect = index_pd.get_slice_bound(label, side, "getitem") + got = index.get_slice_bound(label, side) + expect = index_pd.get_slice_bound(label, side) assert got == expect From 58b9acb28b3ed640f52c4899f26c4094da1eb352 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 10 Mar 2023 13:29:57 -0600 Subject: [PATCH 008/384] [REVIEW] Update `numeric_only` behavior in reduction APIs (#12847) - [x] This PR removes the deprecation of `numeric_only=None` and defaults to `numeric_only=False`. - [x] Removes `level` parameter from reduction APIs to match pandas-2.0 - [x] Change `axis` defaults to match pandas-2.0 APIs. --- python/cudf/cudf/core/dataframe.py | 60 ++++--- python/cudf/cudf/core/frame.py | 155 ++++++++----------- python/cudf/cudf/core/series.py | 18 +-- python/cudf/cudf/core/single_column_frame.py | 14 +- python/cudf/cudf/tests/test_dataframe.py | 36 ++--- python/cudf/cudf/tests/test_stats.py | 38 ++--- 6 files changed, 129 insertions(+), 192 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 32a0a8ca510..978109917b6 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -5780,7 +5780,7 @@ def _prepare_for_rowwise_op(self, method, skipna): return coerced, mask, common_dtype @_cudf_nvtx_annotate - def count(self, axis=0, level=None, numeric_only=False, **kwargs): + def count(self, axis=0, numeric_only=False): """ Count ``non-NA`` cells for each column or row. @@ -5793,7 +5793,7 @@ def count(self, axis=0, level=None, numeric_only=False, **kwargs): Notes ----- - Parameters currently not supported are `axis`, `level`, `numeric_only`. + Parameters currently not supported are `axis`, `numeric_only`. Examples -------- @@ -5831,12 +5831,9 @@ def _reduce( self, op, axis=None, - level=None, - numeric_only=None, + numeric_only=False, **kwargs, ): - if level is not None: - raise NotImplementedError("level parameter is not implemented yet") source = self if numeric_only: @@ -5872,33 +5869,28 @@ def _reduce( "skew", ) - if numeric_only is None and op in numeric_ops: - warnings.warn( - f"The default value of numeric_only in DataFrame.{op} " - "is deprecated. In a future version, it will default " - "to False. In addition, specifying " - "'numeric_only=None' is deprecated. Select only valid " - "columns or specify the value of numeric_only to " - "silence this warning.", - FutureWarning, - ) - numeric_cols = ( - name + if op in numeric_ops: + if numeric_only: + try: + result = [ + getattr(source._data[col], op)(**kwargs) + for col in source._data.names + ] + except AttributeError: + raise NotImplementedError( + f"Not all column dtypes support op {op}" + ) + elif any( + not is_numeric_dtype(self._data[name]) for name in self._data.names - if is_numeric_dtype(self._data[name]) - ) - source = self._get_columns_by_label(numeric_cols) - if source.empty: - return Series(index=cudf.StringIndex([])) - try: - result = [ - getattr(source._data[col], op)(**kwargs) - for col in source._data.names - ] - except AttributeError: + ): raise TypeError( - f"Not all column dtypes support op {op}" + "Non numeric columns passed with " + "`numeric_only=False`, pass `numeric_only=True` " + f"to perform DataFrame.{op}" ) + else: + raise else: raise @@ -6024,14 +6016,14 @@ def mode(self, axis=0, numeric_only=False, dropna=True): return df @_cudf_nvtx_annotate - def all(self, axis=0, bool_only=None, skipna=True, level=None, **kwargs): + def all(self, axis=0, bool_only=None, skipna=True, **kwargs): obj = self.select_dtypes(include="bool") if bool_only else self - return super(DataFrame, obj).all(axis, skipna, level, **kwargs) + return super(DataFrame, obj).all(axis, skipna, **kwargs) @_cudf_nvtx_annotate - def any(self, axis=0, bool_only=None, skipna=True, level=None, **kwargs): + def any(self, axis=0, bool_only=None, skipna=True, **kwargs): obj = self.select_dtypes(include="bool") if bool_only else self - return super(DataFrame, obj).any(axis, skipna, level, **kwargs) + return super(DataFrame, obj).any(axis, skipna, **kwargs) @_cudf_nvtx_annotate def _apply_cupy_method_axis_1(self, method, *args, **kwargs): diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index ea6a6de0b2b..aaee223e854 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -1885,10 +1885,9 @@ def _reduce(self, *args, **kwargs): @_cudf_nvtx_annotate def min( self, - axis=None, + axis=0, skipna=True, - level=None, - numeric_only=None, + numeric_only=False, **kwargs, ): """ @@ -1900,35 +1899,32 @@ def min( Axis for the function to be applied on. skipna: bool, default True Exclude NA/null values when computing the result. - level: int or level name, default None - If the axis is a MultiIndex (hierarchical), count along a - particular level, collapsing into a Series. - numeric_only: bool, default None - Include only float, int, boolean columns. If None, will attempt to - use everything, then use only numeric data. + numeric_only: bool, default False + If True, includes only float, int, boolean columns. + If False, will raise error in-case there are + non-numeric columns. Returns ------- Series - Notes - ----- - Parameters currently not supported are `level`, `numeric_only`. - Examples -------- >>> import cudf >>> df = cudf.DataFrame({'a': [1, 2, 3, 4], 'b': [7, 8, 9, 10]}) - >>> df.min() + >>> min_series = df.min() + >>> min_series a 1 b 7 dtype: int64 + + >>> min_series.min() + 1 """ return self._reduce( "min", axis=axis, skipna=skipna, - level=level, numeric_only=numeric_only, **kwargs, ) @@ -1936,9 +1932,8 @@ def min( @_cudf_nvtx_annotate def max( self, - axis=None, + axis=0, skipna=True, - level=None, numeric_only=None, **kwargs, ): @@ -1951,12 +1946,10 @@ def max( Axis for the function to be applied on. skipna: bool, default True Exclude NA/null values when computing the result. - level: int or level name, default None - If the axis is a MultiIndex (hierarchical), count along a - particular level, collapsing into a Series. numeric_only: bool, default None - Include only float, int, boolean columns. If None, will attempt to - use everything, then use only numeric data. + If True, includes only float, int, boolean columns. + If False, will raise error in-case there are + non-numeric columns. Returns ------- @@ -1979,7 +1972,6 @@ def max( "max", axis=axis, skipna=skipna, - level=level, numeric_only=numeric_only, **kwargs, ) @@ -1990,8 +1982,7 @@ def sum( axis=None, skipna=True, dtype=None, - level=None, - numeric_only=None, + numeric_only=False, min_count=0, **kwargs, ): @@ -2006,6 +1997,10 @@ def sum( Exclude NA/null values when computing the result. dtype: data type Data type to cast the result to. + numeric_only : bool, default False + If True, includes only float, int, boolean columns. + If False, will raise error in-case there are + non-numeric columns. min_count: int, default 0 The required number of valid values to perform the operation. If fewer than min_count non-NA values are present the result @@ -2018,10 +2013,6 @@ def sum( ------- Series - Notes - ----- - Parameters currently not supported are `level`, `numeric_only`. - Examples -------- >>> import cudf @@ -2036,7 +2027,6 @@ def sum( axis=axis, skipna=skipna, dtype=dtype, - level=level, numeric_only=numeric_only, min_count=min_count, **kwargs, @@ -2048,8 +2038,7 @@ def product( axis=None, skipna=True, dtype=None, - level=None, - numeric_only=None, + numeric_only=False, min_count=0, **kwargs, ): @@ -2064,6 +2053,10 @@ def product( Exclude NA/null values when computing the result. dtype: data type Data type to cast the result to. + numeric_only : bool, default False + If True, includes only float, int, boolean columns. + If False, will raise error in-case there are + non-numeric columns. min_count: int, default 0 The required number of valid values to perform the operation. If fewer than min_count non-NA values are present the result @@ -2076,10 +2069,6 @@ def product( ------- Series - Notes - ----- - Parameters currently not supported are level`, `numeric_only`. - Examples -------- >>> import cudf @@ -2097,7 +2086,6 @@ def product( axis=axis, skipna=skipna, dtype=dtype, - level=level, numeric_only=numeric_only, min_count=min_count, **kwargs, @@ -2107,9 +2095,7 @@ def product( prod = product @_cudf_nvtx_annotate - def mean( - self, axis=None, skipna=True, level=None, numeric_only=None, **kwargs - ): + def mean(self, axis=0, skipna=True, numeric_only=False, **kwargs): """ Return the mean of the values for the requested axis. @@ -2119,13 +2105,10 @@ def mean( Axis for the function to be applied on. skipna : bool, default True Exclude NA/null values when computing the result. - level : int or level name, default None - If the axis is a MultiIndex (hierarchical), count along a - particular level, collapsing into a Series. - numeric_only : bool, default None - Include only float, int, boolean columns. If None, will attempt to - use everything, then use only numeric data. Not implemented for - Series. + numeric_only : bool, default False + If True, includes only float, int, boolean columns. + If False, will raise error in-case there are + non-numeric columns. **kwargs Additional keyword arguments to be passed to the function. @@ -2146,7 +2129,6 @@ def mean( "mean", axis=axis, skipna=skipna, - level=level, numeric_only=numeric_only, **kwargs, ) @@ -2156,9 +2138,8 @@ def std( self, axis=None, skipna=True, - level=None, ddof=1, - numeric_only=None, + numeric_only=False, **kwargs, ): """ @@ -2177,16 +2158,15 @@ def std( ddof: int, default 1 Delta Degrees of Freedom. The divisor used in calculations is N - ddof, where N represents the number of elements. + numeric_only : bool, default False + If True, includes only float, int, boolean columns. + If False, will raise error in-case there are + non-numeric columns. Returns ------- Series - Notes - ----- - Parameters currently not supported are `level` and - `numeric_only` - Examples -------- >>> import cudf @@ -2201,7 +2181,6 @@ def std( "std", axis=axis, skipna=skipna, - level=level, ddof=ddof, numeric_only=numeric_only, **kwargs, @@ -2212,9 +2191,8 @@ def var( self, axis=None, skipna=True, - level=None, ddof=1, - numeric_only=None, + numeric_only=False, **kwargs, ): """ @@ -2233,16 +2211,15 @@ def var( ddof: int, default 1 Delta Degrees of Freedom. The divisor used in calculations is N - ddof, where N represents the number of elements. + numeric_only : bool, default False + If True, includes only float, int, boolean columns. + If False, will raise error in-case there are + non-numeric columns. Returns ------- scalar - Notes - ----- - Parameters currently not supported are `level` and - `numeric_only` - Examples -------- >>> import cudf @@ -2256,16 +2233,13 @@ def var( "var", axis=axis, skipna=skipna, - level=level, ddof=ddof, numeric_only=numeric_only, **kwargs, ) @_cudf_nvtx_annotate - def kurtosis( - self, axis=None, skipna=True, level=None, numeric_only=None, **kwargs - ): + def kurtosis(self, axis=0, skipna=True, numeric_only=False, **kwargs): """ Return Fisher's unbiased kurtosis of a sample. @@ -2278,15 +2252,15 @@ def kurtosis( Axis for the function to be applied on. skipna: bool, default True Exclude NA/null values when computing the result. + numeric_only : bool, default False + If True, includes only float, int, boolean columns. + If False, will raise error in-case there are + non-numeric columns. Returns ------- Series or scalar - Notes - ----- - Parameters currently not supported are `level` and `numeric_only` - Examples -------- **Series** @@ -2312,7 +2286,6 @@ def kurtosis( "kurtosis", axis=axis, skipna=skipna, - level=level, numeric_only=numeric_only, **kwargs, ) @@ -2321,9 +2294,7 @@ def kurtosis( kurt = kurtosis @_cudf_nvtx_annotate - def skew( - self, axis=None, skipna=True, level=None, numeric_only=None, **kwargs - ): + def skew(self, axis=0, skipna=True, numeric_only=False, **kwargs): """ Return unbiased Fisher-Pearson skew of a sample. @@ -2331,6 +2302,10 @@ def skew( ---------- skipna: bool, default True Exclude NA/null values when computing the result. + numeric_only : bool, default False + If True, includes only float, int, boolean columns. + If False, will raise error in-case there are + non-numeric columns. Returns ------- @@ -2338,8 +2313,7 @@ def skew( Notes ----- - Parameters currently not supported are `axis`, `level` and - `numeric_only` + Parameter currently not supported is `axis` Examples -------- @@ -2373,13 +2347,12 @@ def skew( "skew", axis=axis, skipna=skipna, - level=level, numeric_only=numeric_only, **kwargs, ) @_cudf_nvtx_annotate - def all(self, axis=0, skipna=True, level=None, **kwargs): + def all(self, axis=0, skipna=True, **kwargs): """ Return whether all elements are True in DataFrame. @@ -2398,7 +2371,7 @@ def all(self, axis=0, skipna=True, level=None, **kwargs): Notes ----- - Parameters currently not supported are `axis`, `bool_only`, `level`. + Parameters currently not supported are `axis`, `bool_only`. Examples -------- @@ -2413,12 +2386,11 @@ def all(self, axis=0, skipna=True, level=None, **kwargs): "all", axis=axis, skipna=skipna, - level=level, **kwargs, ) @_cudf_nvtx_annotate - def any(self, axis=0, skipna=True, level=None, **kwargs): + def any(self, axis=0, skipna=True, **kwargs): """ Return whether any elements is True in DataFrame. @@ -2437,7 +2409,7 @@ def any(self, axis=0, skipna=True, level=None, **kwargs): Notes ----- - Parameters currently not supported are `axis`, `bool_only`, `level`. + Parameters currently not supported are `axis`, `bool_only`. Examples -------- @@ -2452,7 +2424,6 @@ def any(self, axis=0, skipna=True, level=None, **kwargs): "any", axis=axis, skipna=skipna, - level=level, **kwargs, ) @@ -2486,25 +2457,26 @@ def sum_of_squares(self, dtype=None): return self._reduce("sum_of_squares", dtype=dtype) @_cudf_nvtx_annotate - def median( - self, axis=None, skipna=True, level=None, numeric_only=None, **kwargs - ): + def median(self, axis=0, skipna=True, numeric_only=False, **kwargs): """ Return the median of the values for the requested axis. Parameters ---------- + axis : {index (0), columns (1)} + Axis for the function to be applied on. For Series this + parameter is unused and defaults to 0. skipna : bool, default True Exclude NA/null values when computing the result. + numeric_only : bool, default False + If True, includes only float, int, boolean columns. + If False, will raise error in-case there are + non-numeric columns. Returns ------- scalar - Notes - ----- - Parameters currently not supported are `level` and `numeric_only`. - Examples -------- >>> import cudf @@ -2524,7 +2496,6 @@ def median( "median", axis=axis, skipna=skipna, - level=level, numeric_only=numeric_only, **kwargs, ) diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index d486851176a..041e5aa07b9 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -1850,20 +1850,20 @@ def between(self, left, right, inclusive="both") -> Series: return self._from_data({self.name: lmask & rmask}, self._index) @_cudf_nvtx_annotate - def all(self, axis=0, bool_only=None, skipna=True, level=None, **kwargs): + def all(self, axis=0, bool_only=None, skipna=True, **kwargs): if bool_only not in (None, True): raise NotImplementedError( "The bool_only parameter is not supported for Series." ) - return super().all(axis, skipna, level, **kwargs) + return super().all(axis, skipna, **kwargs) @_cudf_nvtx_annotate - def any(self, axis=0, bool_only=None, skipna=True, level=None, **kwargs): + def any(self, axis=0, bool_only=None, skipna=True, **kwargs): if bool_only not in (None, True): raise NotImplementedError( "The bool_only parameter is not supported for Series." ) - return super().any(axis, skipna, level, **kwargs) + return super().any(axis, skipna, **kwargs) @_cudf_nvtx_annotate def to_pandas(self, index=True, nullable=False, **kwargs): @@ -2460,7 +2460,7 @@ def apply(self, func, convert_dtype=True, args=(), **kwargs): # Stats # @_cudf_nvtx_annotate - def count(self, level=None, **kwargs): + def count(self): """ Return number of non-NA/null observations in the Series @@ -2469,10 +2469,6 @@ def count(self, level=None, **kwargs): int Number of non-null values in the Series. - Notes - ----- - Parameters currently not supported is `level`. - Examples -------- >>> import cudf @@ -2480,10 +2476,6 @@ def count(self, level=None, **kwargs): >>> ser.count() 5 """ - - if level is not None: - raise NotImplementedError("level parameter is not implemented yet") - return self.valid_count @_cudf_nvtx_annotate diff --git a/python/cudf/cudf/core/single_column_frame.py b/python/cudf/cudf/core/single_column_frame.py index da48bc0b5a9..9e380e63ae0 100644 --- a/python/cudf/cudf/core/single_column_frame.py +++ b/python/cudf/cudf/core/single_column_frame.py @@ -15,6 +15,7 @@ _is_scalar_or_zero_d_array, is_bool_dtype, is_integer_dtype, + is_numeric_dtype, ) from cudf.core.column import ColumnBase, as_column from cudf.core.frame import Frame @@ -41,19 +42,16 @@ def _reduce( self, op, axis=None, - level=None, - numeric_only=None, + numeric_only=False, **kwargs, ): if axis not in (None, 0): raise NotImplementedError("axis parameter is not implemented yet") - if level is not None: - raise NotImplementedError("level parameter is not implemented yet") - - if numeric_only: - raise NotImplementedError( - f"Series.{op} does not implement numeric_only" + if numeric_only and not is_numeric_dtype(self._column): + raise TypeError( + f"Series.{op} does not allow numeric_only={numeric_only} " + "with non-numeric dtypes." ) try: return getattr(self._column, op)(**kwargs) diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index df235d48a30..fdb6790187e 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -8338,8 +8338,8 @@ def test_describe_misc_include(df, include): def test_describe_misc_exclude(df, exclude): pdf = df.to_pandas() - expected = pdf.describe(exclude=exclude, datetime_is_numeric=True) - actual = df.describe(exclude=exclude, datetime_is_numeric=True) + expected = pdf.describe(exclude=exclude) + actual = df.describe(exclude=exclude) for col in expected.columns: if expected[col].dtype == np.dtype("object"): @@ -9703,19 +9703,15 @@ def test_dataframe_pct_change(data, periods, fill_method): assert_eq(expected, actual) -def test_mean_timeseries(): +@pytest.mark.parametrize("numeric_only", [True, False]) +def test_mean_timeseries(numeric_only): gdf = cudf.datasets.timeseries() + if not numeric_only: + gdf = gdf.select_dtypes(include="number") pdf = gdf.to_pandas() - expected = pdf.mean(numeric_only=True) - actual = gdf.mean(numeric_only=True) - - assert_eq(expected, actual) - - with pytest.warns(FutureWarning): - expected = pdf.mean() - with pytest.warns(FutureWarning): - actual = gdf.mean() + expected = pdf.mean(numeric_only=numeric_only) + actual = gdf.mean(numeric_only=numeric_only) assert_eq(expected, actual) @@ -9730,19 +9726,15 @@ def test_mean_timeseries(): } ], ) -def test_std_different_dtypes(data): +@pytest.mark.parametrize("numeric_only", [True, False]) +def test_std_different_dtypes(data, numeric_only): gdf = cudf.DataFrame(data) + if not numeric_only: + gdf = gdf.select_dtypes(include="number") pdf = gdf.to_pandas() - expected = pdf.std(numeric_only=True) - actual = gdf.std(numeric_only=True) - - assert_eq(expected, actual) - - with pytest.warns(FutureWarning): - expected = pdf.std() - with pytest.warns(FutureWarning): - actual = gdf.std() + expected = pdf.std(numeric_only=numeric_only) + actual = gdf.std(numeric_only=numeric_only) assert_eq(expected, actual) diff --git a/python/cudf/cudf/tests/test_stats.py b/python/cudf/cudf/tests/test_stats.py index 6478fbaad95..6ca64fdcfa3 100644 --- a/python/cudf/cudf/tests/test_stats.py +++ b/python/cudf/cudf/tests/test_stats.py @@ -593,30 +593,26 @@ def test_cov_corr_invalid_dtypes(gsr): ], ) @pytest.mark.parametrize("null_flag", [False, True]) -def test_kurtosis_df(data, null_flag): +@pytest.mark.parametrize("numeric_only", [False, True]) +def test_kurtosis_df(data, null_flag, numeric_only): + if not numeric_only: + data = data.select_dtypes(include="number") pdata = data.to_pandas() if null_flag and len(data) > 2: data.iloc[[0, 2]] = None pdata.iloc[[0, 2]] = None - with pytest.warns(FutureWarning): - got = data.kurtosis() + got = data.kurtosis(numeric_only=numeric_only) got = got if np.isscalar(got) else got.to_numpy() - with pytest.warns(FutureWarning): - expected = pdata.kurtosis() - np.testing.assert_array_almost_equal(got, expected) - with pytest.warns(FutureWarning): - got = data.kurt() - got = got if np.isscalar(got) else got.to_numpy() - with pytest.warns(FutureWarning): - expected = pdata.kurt() + expected = pdata.kurtosis(numeric_only=numeric_only) np.testing.assert_array_almost_equal(got, expected) - got = data.kurt(numeric_only=True) + got = data.kurt(numeric_only=numeric_only) got = got if np.isscalar(got) else got.to_numpy() - expected = pdata.kurt(numeric_only=True) + + expected = pdata.kurt(numeric_only=numeric_only) np.testing.assert_array_almost_equal(got, expected) @@ -629,21 +625,17 @@ def test_kurtosis_df(data, null_flag): ], ) @pytest.mark.parametrize("null_flag", [False, True]) -def test_skew_df(data, null_flag): +@pytest.mark.parametrize("numeric_only", [False, True]) +def test_skew_df(data, null_flag, numeric_only): + if not numeric_only: + data = data.select_dtypes(include="number") pdata = data.to_pandas() if null_flag and len(data) > 2: data.iloc[[0, 2]] = None pdata.iloc[[0, 2]] = None - with pytest.warns(FutureWarning): - got = data.skew() - with pytest.warns(FutureWarning): - expected = pdata.skew() - got = got if np.isscalar(got) else got.to_numpy() - np.testing.assert_array_almost_equal(got, expected) - - got = data.skew(numeric_only=True) - expected = pdata.skew(numeric_only=True) + got = data.skew(numeric_only=numeric_only) + expected = pdata.skew(numeric_only=numeric_only) got = got if np.isscalar(got) else got.to_numpy() np.testing.assert_array_almost_equal(got, expected) From e115ba593dc168e765fc442dabfd170367b0042c Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 10 Mar 2023 17:37:25 -0600 Subject: [PATCH 009/384] [REVIEW] Drop `DataFrame.append` and `Series.append` (#12839) This PR removes `DataFrame.append` & `Series.append` to match pandas-2.0 API. Test usages are now replaced with `.concat` API calls. --- docs/cudf/source/api_docs/dataframe.rst | 1 - docs/cudf/source/api_docs/series.rst | 1 - python/cudf/cudf/core/dataframe.py | 140 ------------------- python/cudf/cudf/core/indexed_frame.py | 22 --- python/cudf/cudf/core/series.py | 76 ---------- python/cudf/cudf/tests/test_dataframe.py | 168 +++++++++++++---------- python/cudf/cudf/tests/test_series.py | 57 ++++---- 7 files changed, 119 insertions(+), 346 deletions(-) diff --git a/docs/cudf/source/api_docs/dataframe.rst b/docs/cudf/source/api_docs/dataframe.rst index dfe1b2a9b9b..5643f9cff48 100644 --- a/docs/cudf/source/api_docs/dataframe.rst +++ b/docs/cudf/source/api_docs/dataframe.rst @@ -231,7 +231,6 @@ Combining / comparing / joining / merging .. autosummary:: :toctree: api/ - DataFrame.append DataFrame.assign DataFrame.join DataFrame.merge diff --git a/docs/cudf/source/api_docs/series.rst b/docs/cudf/source/api_docs/series.rst index 9cd0770431c..4c0af814f85 100644 --- a/docs/cudf/source/api_docs/series.rst +++ b/docs/cudf/source/api_docs/series.rst @@ -218,7 +218,6 @@ Combining / comparing / joining / merging .. autosummary:: :toctree: api/ - Series.append Series.update Time Series-related diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 978109917b6..9cfdf46826f 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -6624,146 +6624,6 @@ def iterrows(self): "if you wish to iterate over each row." ) - @_cudf_nvtx_annotate - def append( - self, other, ignore_index=False, verify_integrity=False, sort=False - ): - """ - Append rows of `other` to the end of caller, returning a new object. - Columns in `other` that are not in the caller are added as new columns. - - Parameters - ---------- - other : DataFrame or Series/dict-like object, or list of these - The data to append. - ignore_index : bool, default False - If True, do not use the index labels. - sort : bool, default False - Sort columns ordering if the columns of - `self` and `other` are not aligned. - verify_integrity : bool, default False - This Parameter is currently not supported. - - Returns - ------- - DataFrame - - See Also - -------- - cudf.concat : General function to concatenate DataFrame or - objects. - - Notes - ----- - If a list of dict/series is passed and the keys are all contained in - the DataFrame's index, the order of the columns in the resulting - DataFrame will be unchanged. - Iteratively appending rows to a cudf DataFrame can be more - computationally intensive than a single concatenate. A better - solution is to append those rows to a list and then concatenate - the list with the original DataFrame all at once. - `verify_integrity` parameter is not supported yet. - - Examples - -------- - >>> import cudf - >>> df = cudf.DataFrame([[1, 2], [3, 4]], columns=list('AB')) - >>> df - A B - 0 1 2 - 1 3 4 - >>> df2 = cudf.DataFrame([[5, 6], [7, 8]], columns=list('AB')) - >>> df2 - A B - 0 5 6 - 1 7 8 - >>> df.append(df2) - A B - 0 1 2 - 1 3 4 - 0 5 6 - 1 7 8 - - With `ignore_index` set to True: - - >>> df.append(df2, ignore_index=True) - A B - 0 1 2 - 1 3 4 - 2 5 6 - 3 7 8 - - The following, while not recommended methods for generating DataFrames, - show two ways to generate a DataFrame from multiple data sources. - Less efficient: - - >>> df = cudf.DataFrame(columns=['A']) - >>> for i in range(5): - ... df = df.append({'A': i}, ignore_index=True) - >>> df - A - 0 0 - 1 1 - 2 2 - 3 3 - 4 4 - - More efficient than above: - - >>> cudf.concat([cudf.DataFrame([i], columns=['A']) for i in range(5)], - ... ignore_index=True) - A - 0 0 - 1 1 - 2 2 - 3 3 - 4 4 - """ - if isinstance(other, dict): - if not ignore_index: - raise TypeError("Can only append a dict if ignore_index=True") - other = DataFrame(other) - elif isinstance(other, Series): - if other.name is None and not ignore_index: - raise TypeError( - "Can only append a Series if ignore_index=True " - "or if the Series has a name" - ) - - current_cols = self._data.to_pandas_index() - combined_columns = other.index.to_pandas() - if len(current_cols): - if cudf.utils.dtypes.is_mixed_with_object_dtype( - current_cols, combined_columns - ): - raise TypeError( - "cudf does not support mixed types, please type-cast " - "the column index of dataframe and index of series " - "to same dtypes." - ) - - combined_columns = current_cols.union( - combined_columns, sort=False - ) - - if sort: - combined_columns = combined_columns.sort_values() - - other = other.reindex(combined_columns, copy=False).to_frame().T - if not current_cols.equals(combined_columns): - self = self.reindex(columns=combined_columns) - elif ( - isinstance(other, list) - and other - and not isinstance(other[0], DataFrame) - ): - other = DataFrame(other) - cols = self._data.to_pandas_index() - if (cols.get_indexer(other._data.to_pandas_index()) >= 0).all(): - other = other.reindex(columns=cols) - - return super()._append(other, ignore_index, verify_integrity, sort) - @_cudf_nvtx_annotate @copy_docstring(reshape.pivot) def pivot(self, index, columns, values=None): diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 159cc318789..074bd554601 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -3438,28 +3438,6 @@ def repeat(self, repeats, axis=None): self._index_names, ) - def _append( - self, other, ignore_index=False, verify_integrity=False, sort=None - ): - # Note: Do not remove this function until pandas does. This warning is - # to clean up cudf but to match a deprecation in pandas - warnings.warn( - "The append method is deprecated and will be removed in a future " - "version. Use cudf.concat instead.", - FutureWarning, - ) - if verify_integrity not in (None, False): - raise NotImplementedError( - "verify_integrity parameter is not supported yet." - ) - - if is_list_like(other): - to_concat = [self, *other] - else: - to_concat = [self, other] - - return cudf.concat(to_concat, ignore_index=ignore_index, sort=sort) - def astype(self, dtype, copy=False, errors="raise", **kwargs): """Cast the object to the given dtype. diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 041e5aa07b9..f12bd183676 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -841,82 +841,6 @@ def to_dict(self, into: type[dict] = dict) -> dict: """ return self.to_pandas().to_dict(into=into) - @_cudf_nvtx_annotate - def append(self, to_append, ignore_index=False, verify_integrity=False): - """Append values from another ``Series`` or array-like object. - If ``ignore_index=True``, the index is reset. - - Parameters - ---------- - to_append : Series or list/tuple of Series - Series to append with self. - ignore_index : boolean, default False. - If True, do not use the index. - verify_integrity : bool, default False - This Parameter is currently not supported. - - Returns - ------- - Series - A new concatenated series - - See Also - -------- - cudf.concat : General function to concatenate DataFrame or - Series objects. - - Examples - -------- - >>> import cudf - >>> s1 = cudf.Series([1, 2, 3]) - >>> s2 = cudf.Series([4, 5, 6]) - >>> s1 - 0 1 - 1 2 - 2 3 - dtype: int64 - >>> s2 - 0 4 - 1 5 - 2 6 - dtype: int64 - >>> s1.append(s2) - 0 1 - 1 2 - 2 3 - 0 4 - 1 5 - 2 6 - dtype: int64 - - >>> s3 = cudf.Series([4, 5, 6], index=[3, 4, 5]) - >>> s3 - 3 4 - 4 5 - 5 6 - dtype: int64 - >>> s1.append(s3) - 0 1 - 1 2 - 2 3 - 3 4 - 4 5 - 5 6 - dtype: int64 - - With `ignore_index` set to True: - - >>> s1.append(s2, ignore_index=True) - 0 1 - 1 2 - 2 3 - 3 4 - 4 5 - 5 6 - dtype: int64 - """ - return super()._append(to_append, ignore_index, verify_integrity) - @_cudf_nvtx_annotate def reindex(self, *args, **kwargs): """ diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index af95b95ed68..3828a1ac10c 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -22,7 +22,12 @@ from packaging import version import cudf -from cudf.core._compat import PANDAS_GE_134, PANDAS_GE_150, PANDAS_LT_140 +from cudf.core._compat import ( + PANDAS_GE_134, + PANDAS_GE_150, + PANDAS_GE_200, + PANDAS_LT_140, +) from cudf.core.buffer.spill_manager import get_global_manager from cudf.core.column import column from cudf.testing import _utils as utils @@ -239,7 +244,7 @@ def test_series_from_cupy_scalars(): @pytest.mark.parametrize("a", [[1, 2, 3], [1, 10, 30]]) @pytest.mark.parametrize("b", [[4, 5, 6], [-11, -100, 30]]) -def test_append_index(a, b): +def test_concat_index(a, b): df = pd.DataFrame() df["a"] = a @@ -249,19 +254,14 @@ def test_append_index(a, b): gdf["a"] = a gdf["b"] = b - # Check the default index after appending two columns(Series) - with pytest.warns(FutureWarning, match="append method is deprecated"): - expected = df.a.append(df.b) - with pytest.warns(FutureWarning, match="append method is deprecated"): - actual = gdf.a.append(gdf.b) + expected = pd.concat([df.a, df.b]) + actual = cudf.concat([gdf.a, gdf.b]) assert len(expected) == len(actual) assert_eq(expected.index, actual.index) - with pytest.warns(FutureWarning, match="append method is deprecated"): - expected = df.a.append(df.b, ignore_index=True) - with pytest.warns(FutureWarning, match="append method is deprecated"): - actual = gdf.a.append(gdf.b, ignore_index=True) + expected = pd.concat([df.a, df.b], ignore_index=True) + actual = cudf.concat([gdf.a, gdf.b], ignore_index=True) assert len(expected) == len(actual) assert_eq(expected.index, actual.index) @@ -281,7 +281,8 @@ def test_append_index(a, b): pytest.param( {}, marks=pytest_xfail( - reason="https://github.com/rapidsai/cudf/issues/11080" + condition=not PANDAS_GE_150, + reason="https://github.com/rapidsai/cudf/issues/11080", ), ), pytest.param( @@ -1539,7 +1540,8 @@ def test_concat_different_column_dataframe(df1_d, df2_d): pdf1 = pd.DataFrame(df1_d) pdf2 = pd.DataFrame(df2_d) - # pandas warns when trying to concatenate any empty float columns (or float + # pandas(lower than pandas 2.0 only) warns when trying to + # concatenate any empty float columns (or float # columns with all None values) with any non-empty bool columns. def is_invalid_concat(left, right): return ( @@ -1548,7 +1550,7 @@ def is_invalid_concat(left, right): and right.count() == 0 ) - cond = any( + cond = (not PANDAS_GE_200) and any( is_invalid_concat(pdf1[colname], pdf2[colname]) or is_invalid_concat(pdf2[colname], pdf1[colname]) for colname in set(pdf1) & set(pdf2) @@ -7312,22 +7314,37 @@ def test_series_keys(ps): ) @pytest.mark.parametrize("sort", [False, True]) @pytest.mark.parametrize("ignore_index", [True, False]) -def test_dataframe_append_dataframe(df, other, sort, ignore_index): +def test_dataframe_concat_dataframe(df, other, sort, ignore_index): pdf = df other_pd = other gdf = cudf.from_pandas(df) other_gd = cudf.from_pandas(other) - with pytest.warns(FutureWarning, match="append method is deprecated"): - expected = pdf.append(other_pd, sort=sort, ignore_index=ignore_index) - with pytest.warns(FutureWarning, match="append method is deprecated"): - actual = gdf.append(other_gd, sort=sort, ignore_index=ignore_index) + expected = pd.concat([pdf, other_pd], sort=sort, ignore_index=ignore_index) + actual = cudf.concat([gdf, other_gd], sort=sort, ignore_index=ignore_index) + + # In empty dataframe cases, Pandas & cudf differ in columns + # creation, pandas creates RangeIndex(0, 0) + # whereas cudf creates an empty Index([], dtype="object"). + check_column_type = ( + False if len(expected.columns) == len(df.columns) == 0 else True + ) if expected.shape != df.shape: - assert_eq(expected.fillna(-1), actual.fillna(-1), check_dtype=False) + assert_eq( + expected.fillna(-1), + actual.fillna(-1), + check_dtype=False, + check_column_type=check_column_type, + ) else: - assert_eq(expected, actual, check_index_type=not gdf.empty) + assert_eq( + expected, + actual, + check_index_type=not gdf.empty, + check_column_type=check_column_type, + ) @pytest_unmark_spilling @@ -7372,20 +7389,18 @@ def test_dataframe_append_dataframe(df, other, sort, ignore_index): ], ) @pytest.mark.parametrize("sort", [False, True]) -def test_dataframe_append_series_dict(df, other, sort): +def test_dataframe_concat_series(df, other, sort): pdf = df - other_pd = other - gdf = cudf.from_pandas(df) - if isinstance(other, pd.Series): - other_gd = cudf.from_pandas(other) + + if isinstance(other, dict): + other_pd = pd.Series(other) else: - other_gd = other + other_pd = other + other_gd = cudf.from_pandas(other_pd) - with pytest.warns(FutureWarning, match="append method is deprecated"): - expected = pdf.append(other_pd, ignore_index=True, sort=sort) - with pytest.warns(FutureWarning, match="append method is deprecated"): - actual = gdf.append(other_gd, ignore_index=True, sort=sort) + expected = pd.concat([pdf, other_pd], ignore_index=True, sort=sort) + actual = cudf.concat([gdf, other_gd], ignore_index=True, sort=sort) if expected.shape != df.shape: # Ignore the column type comparison because pandas incorrectly @@ -7402,20 +7417,18 @@ def test_dataframe_append_series_dict(df, other, sort): assert_eq(expected, actual, check_index_type=not gdf.empty) -def test_dataframe_append_series_mixed_index(): +def test_dataframe_concat_series_mixed_index(): df = cudf.DataFrame({"first": [], "d": []}) + pdf = df.to_pandas() + sr = cudf.Series([1, 2, 3, 4]) + psr = sr.to_pandas() - with pytest.raises( - TypeError, - match=re.escape( - "cudf does not support mixed types, please type-cast " - "the column index of dataframe and index of series " - "to same dtypes." - ), - ): - with pytest.warns(FutureWarning, match="append method is deprecated"): - df.append(sr, ignore_index=True) + assert_eq( + cudf.concat([df, sr], ignore_index=True), + pd.concat([pdf, psr], ignore_index=True), + check_dtype=False, + ) @pytest_unmark_spilling @@ -7540,24 +7553,40 @@ def test_dataframe_append_series_mixed_index(): ) @pytest.mark.parametrize("sort", [False, True]) @pytest.mark.parametrize("ignore_index", [True, False]) -def test_dataframe_append_dataframe_lists(df, other, sort, ignore_index): +def test_dataframe_concat_dataframe_lists(df, other, sort, ignore_index): pdf = df other_pd = other gdf = cudf.from_pandas(df) - other_gd = [ - cudf.from_pandas(o) if isinstance(o, pd.DataFrame) else o - for o in other - ] + other_gd = [cudf.from_pandas(o) for o in other] + + expected = pd.concat( + [pdf] + other_pd, sort=sort, ignore_index=ignore_index + ) + actual = cudf.concat( + [gdf] + other_gd, sort=sort, ignore_index=ignore_index + ) + + # In some cases, Pandas creates an empty Index([], dtype="object") for + # columns whereas cudf creates a RangeIndex(0, 0). + check_column_type = ( + False if len(expected.columns) == len(df.columns) == 0 else True + ) - with pytest.warns(FutureWarning, match="append method is deprecated"): - expected = pdf.append(other_pd, sort=sort, ignore_index=ignore_index) - with pytest.warns(FutureWarning, match="append method is deprecated"): - actual = gdf.append(other_gd, sort=sort, ignore_index=ignore_index) if expected.shape != df.shape: - assert_eq(expected.fillna(-1), actual.fillna(-1), check_dtype=False) + assert_eq( + expected.fillna(-1), + actual.fillna(-1), + check_dtype=False, + check_column_type=check_column_type, + ) else: - assert_eq(expected, actual, check_index_type=not gdf.empty) + assert_eq( + expected, + actual, + check_index_type=not gdf.empty, + check_column_type=check_column_type, + ) @pytest.mark.parametrize( @@ -7633,20 +7662,19 @@ def test_dataframe_ffill(df): ) @pytest.mark.parametrize("sort", [False, True]) @pytest.mark.parametrize("ignore_index", [True, False]) -def test_dataframe_append_lists(df, other, sort, ignore_index): +def test_dataframe_concat_lists(df, other, sort, ignore_index): pdf = df - other_pd = other + other_pd = [pd.DataFrame(o) for o in other] gdf = cudf.from_pandas(df) - other_gd = [ - cudf.from_pandas(o) if isinstance(o, pd.DataFrame) else o - for o in other - ] + other_gd = [cudf.from_pandas(o) for o in other_pd] - with pytest.warns(FutureWarning, match="append method is deprecated"): - expected = pdf.append(other_pd, sort=sort, ignore_index=ignore_index) - with pytest.warns(FutureWarning, match="append method is deprecated"): - actual = gdf.append(other_gd, sort=sort, ignore_index=ignore_index) + expected = pd.concat( + [pdf] + other_pd, sort=sort, ignore_index=ignore_index + ) + actual = cudf.concat( + [gdf] + other_gd, sort=sort, ignore_index=ignore_index + ) if expected.shape != df.shape: assert_eq( @@ -7659,17 +7687,13 @@ def test_dataframe_append_lists(df, other, sort, ignore_index): assert_eq(expected, actual, check_index_type=not gdf.empty) -def test_dataframe_append_error(): +def test_dataframe_concat_series_without_name(): df = cudf.DataFrame({"a": [1, 2, 3]}) - ps = cudf.Series([1, 2, 3]) + pdf = df.to_pandas() + gs = cudf.Series([1, 2, 3]) + ps = gs.to_pandas() - with pytest.raises( - TypeError, - match="Can only append a Series if ignore_index=True " - "or if the Series has a name", - ): - with pytest.warns(FutureWarning, match="append method is deprecated"): - df.append(ps) + assert_eq(pd.concat([pdf, ps]), cudf.concat([df, gs])) def test_cudf_arrow_array_error(): diff --git a/python/cudf/cudf/tests/test_series.py b/python/cudf/cudf/tests/test_series.py index 682fccda8dc..719dee308b9 100644 --- a/python/cudf/cudf/tests/test_series.py +++ b/python/cudf/cudf/tests/test_series.py @@ -95,17 +95,16 @@ def test_series_init_dict_lists(data): ], ) @pytest.mark.parametrize("ignore_index", [True, False]) -def test_series_append_basic(data, others, ignore_index): +def test_series_concat_basic(data, others, ignore_index): psr = pd.Series(data) gsr = cudf.Series(data) other_ps = pd.Series(others) other_gs = cudf.Series(others) - with pytest.warns(FutureWarning): - expected = psr.append(other_ps, ignore_index=ignore_index) - with pytest.warns(FutureWarning): - actual = gsr.append(other_gs, ignore_index=ignore_index) + expected = pd.concat([psr, other_ps], ignore_index=ignore_index) + actual = cudf.concat([gsr, other_gs], ignore_index=ignore_index) + assert_eq(expected, actual) @@ -142,17 +141,15 @@ def test_series_append_basic(data, others, ignore_index): ], ) @pytest.mark.parametrize("ignore_index", [True, False]) -def test_series_append_basic_str(data, others, ignore_index): +def test_series_concat_basic_str(data, others, ignore_index): psr = pd.Series(data) gsr = cudf.Series(data) other_ps = pd.Series(others) other_gs = cudf.Series(others) - with pytest.warns(FutureWarning): - expected = psr.append(other_ps, ignore_index=ignore_index) - with pytest.warns(FutureWarning): - actual = gsr.append(other_gs, ignore_index=ignore_index) + expected = pd.concat([psr, other_ps], ignore_index=ignore_index) + actual = cudf.concat([gsr, other_gs], ignore_index=ignore_index) assert_eq(expected, actual) @@ -195,21 +192,20 @@ def test_series_append_basic_str(data, others, ignore_index): ], ) @pytest.mark.parametrize("ignore_index", [True, False]) -def test_series_append_series_with_index(data, others, ignore_index): +def test_series_concat_series_with_index(data, others, ignore_index): psr = pd.Series(data) gsr = cudf.Series(data) other_ps = others other_gs = cudf.from_pandas(others) - with pytest.warns(FutureWarning): - expected = psr.append(other_ps, ignore_index=ignore_index) - with pytest.warns(FutureWarning): - actual = gsr.append(other_gs, ignore_index=ignore_index) + expected = pd.concat([psr, other_ps], ignore_index=ignore_index) + actual = cudf.concat([gsr, other_gs], ignore_index=ignore_index) + assert_eq(expected, actual) -def test_series_append_error_mixed_types(): +def test_series_concat_error_mixed_types(): gsr = cudf.Series([1, 2, 3, 4]) other = cudf.Series(["a", "b", "c", "d"]) @@ -218,16 +214,14 @@ def test_series_append_error_mixed_types(): match="cudf does not support mixed types, please type-cast " "both series to same dtypes.", ): - with pytest.warns(FutureWarning): - gsr.append(other) + cudf.concat([gsr, other]) with pytest.raises( TypeError, match="cudf does not support mixed types, please type-cast " "both series to same dtypes.", ): - with pytest.warns(FutureWarning): - gsr.append([gsr, other, gsr, other]) + cudf.concat([gsr, gsr, other, gsr, other]) @pytest.mark.parametrize( @@ -278,35 +272,32 @@ def test_series_append_error_mixed_types(): ], ) @pytest.mark.parametrize("ignore_index", [True, False]) -def test_series_append_list_series_with_index(data, others, ignore_index): +def test_series_concat_list_series_with_index(data, others, ignore_index): psr = pd.Series(data) gsr = cudf.Series(data) other_ps = others other_gs = [cudf.from_pandas(obj) for obj in others] - with pytest.warns(FutureWarning): - expected = psr.append(other_ps, ignore_index=ignore_index) - with pytest.warns(FutureWarning): - actual = gsr.append(other_gs, ignore_index=ignore_index) + expected = pd.concat([psr] + other_ps, ignore_index=ignore_index) + actual = cudf.concat([gsr] + other_gs, ignore_index=ignore_index) + assert_eq(expected, actual) -def test_series_append_existing_buffers(): +def test_series_concat_existing_buffers(): a1 = np.arange(10, dtype=np.float64) gs = cudf.Series(a1) # Add new buffer a2 = cudf.Series(np.arange(5)) - with pytest.warns(FutureWarning): - gs = gs.append(a2) + gs = cudf.concat([gs, a2]) assert len(gs) == 15 np.testing.assert_equal(gs.to_numpy(), np.hstack([a1, a2.to_numpy()])) # Ensure appending to previous buffer a3 = cudf.Series(np.arange(3)) - with pytest.warns(FutureWarning): - gs = gs.append(a3) + gs = cudf.concat([gs, a3]) assert len(gs) == 18 a4 = np.hstack([a1, a2.to_numpy(), a3.to_numpy()]) np.testing.assert_equal(gs.to_numpy(), a4) @@ -314,13 +305,11 @@ def test_series_append_existing_buffers(): # Appending different dtype a5 = cudf.Series(np.array([1, 2, 3], dtype=np.int32)) a6 = cudf.Series(np.array([4.5, 5.5, 6.5], dtype=np.float64)) - with pytest.warns(FutureWarning): - gs = a5.append(a6) + gs = cudf.concat([a5, a6]) np.testing.assert_equal( gs.to_numpy(), np.hstack([a5.to_numpy(), a6.to_numpy()]) ) - with pytest.warns(FutureWarning): - gs = cudf.Series(a6).append(a5) + gs = cudf.concat([cudf.Series(a6), a5]) np.testing.assert_equal( gs.to_numpy(), np.hstack([a6.to_numpy(), a5.to_numpy()]) ) From 4a87cbde1df22eb1029269491757ce173648cf74 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 13 Mar 2023 14:58:10 -0500 Subject: [PATCH 010/384] Drop `na_sentinel` from `factorize` (#12924) This PR drops support for `na_sentinel` in factorize APIs, to match with pandas-2.0 --- python/cudf/cudf/core/algorithms.py | 52 ++------------------ python/cudf/cudf/core/multiindex.py | 8 +-- python/cudf/cudf/core/single_column_frame.py | 11 +---- python/cudf/cudf/tests/test_series.py | 23 --------- 4 files changed, 7 insertions(+), 87 deletions(-) diff --git a/python/cudf/cudf/core/algorithms.py b/python/cudf/cudf/core/algorithms.py index 7012496434a..50ec4b774ee 100644 --- a/python/cudf/cudf/core/algorithms.py +++ b/python/cudf/cudf/core/algorithms.py @@ -11,9 +11,7 @@ from cudf.core.series import Series -def factorize( - values, sort=False, na_sentinel=None, use_na_sentinel=None, size_hint=None -): +def factorize(values, sort=False, use_na_sentinel=True, size_hint=None): """Encode the input values as integer labels Parameters @@ -22,14 +20,6 @@ def factorize( The data to be factorized. sort : bool, default True Sort uniques and shuffle codes to maintain the relationship. - na_sentinel : number, default -1 - Value to indicate missing category. - - .. deprecated:: 23.04 - - The na_sentinel argument is deprecated and will be removed in - a future version of cudf. Specify use_na_sentinel as - either True or False. use_na_sentinel : bool, default True If True, the sentinel -1 will be used for NA values. If False, NA values will be encoded as non-negative @@ -83,51 +73,19 @@ def factorize( >>> uniques Float64Index([, 1.0, 2.0], dtype='float64') """ - # TODO: Drop `na_sentinel` in the next release immediately after - # pandas 2.0 upgrade. - if na_sentinel is not None and use_na_sentinel is not None: - raise ValueError( - "Cannot specify both `na_sentinel` and `use_na_sentile`; " - f"got `na_sentinel={na_sentinel}` and " - f"`use_na_sentinel={use_na_sentinel}`" - ) return_cupy_array = isinstance(values, cp.ndarray) values = Series(values) - if na_sentinel is None: - na_sentinel = ( - -1 - if use_na_sentinel is None or use_na_sentinel - else Scalar(None, dtype=values.dtype) - ) - else: - if na_sentinel is None: - msg = ( - "Specifying `na_sentinel=None` is deprecated, specify " - "`use_na_sentinel=False` instead." - ) - elif na_sentinel == -1: - msg = ( - "Specifying `na_sentinel=-1` is deprecated, specify " - "`use_na_sentinel=True` instead." - ) - else: - msg = ( - "Specifying the specific value to use for `na_sentinel` is " - "deprecated and will be removed in a future version of cudf. " - "Specify `use_na_sentinel=True` to use the sentinel value -1, " - "and `use_na_sentinel=False` to encode NA values.", - ) - warnings.warn(msg, FutureWarning) - if size_hint: warnings.warn("size_hint is not applicable for cudf.factorize") - if use_na_sentinel is None or use_na_sentinel: + if use_na_sentinel: + na_sentinel = Scalar(-1) cats = values._column.dropna() else: + na_sentinel = Scalar(None, dtype=values.dtype) cats = values._column cats = cats.unique().astype(values.dtype) @@ -136,7 +94,7 @@ def factorize( cats, _ = cats.sort_by_values() labels = values._column._label_encoding( - cats=cats, na_sentinel=Scalar(na_sentinel) + cats=cats, na_sentinel=na_sentinel ).values return labels, cats.values if return_cupy_array else Index(cats) diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index d9d1aecb9d6..4cd3f0b3837 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -671,13 +671,7 @@ def _compute_levels_and_codes(self): codes = {} for name, col in self._data.items(): - with warnings.catch_warnings(): - # TODO: Remove this filter when - # `na_sentinel` is removed from `factorize`. - # This is a filter to not let the warnings from - # `factorize` show up in other parts of public APIs. - warnings.simplefilter("ignore") - code, cats = cudf.Series._from_data({None: col}).factorize() + code, cats = cudf.Series._from_data({None: col}).factorize() codes[name] = code.astype(np.int64) levels.append(cudf.Series(cats, name=None)) diff --git a/python/cudf/cudf/core/single_column_frame.py b/python/cudf/cudf/core/single_column_frame.py index 9e380e63ae0..1ffb48fe19e 100644 --- a/python/cudf/cudf/core/single_column_frame.py +++ b/python/cudf/cudf/core/single_column_frame.py @@ -249,21 +249,13 @@ def __cuda_array_interface__(self): return self._column.__cuda_array_interface__ @_cudf_nvtx_annotate - def factorize(self, sort=False, na_sentinel=None, use_na_sentinel=None): + def factorize(self, sort=False, use_na_sentinel=True): """Encode the input values as integer labels. Parameters ---------- sort : bool, default True Sort uniques and shuffle codes to maintain the relationship. - na_sentinel : number, default -1 - Value to indicate missing category. - - .. deprecated:: 23.04 - - The na_sentinel argument is deprecated and will be removed in - a future version of cudf. Specify use_na_sentinel as - either True or False. use_na_sentinel : bool, default True If True, the sentinel -1 will be used for NA values. If False, NA values will be encoded as non-negative @@ -290,7 +282,6 @@ def factorize(self, sort=False, na_sentinel=None, use_na_sentinel=None): return cudf.core.algorithms.factorize( self, sort=sort, - na_sentinel=na_sentinel, use_na_sentinel=use_na_sentinel, ) diff --git a/python/cudf/cudf/tests/test_series.py b/python/cudf/cudf/tests/test_series.py index 719dee308b9..cb50e21094a 100644 --- a/python/cudf/cudf/tests/test_series.py +++ b/python/cudf/cudf/tests/test_series.py @@ -459,29 +459,6 @@ def test_series_describe_other_types(ps): assert_eq(expected.astype("str"), actual) -@pytest.mark.parametrize( - "data", - [ - [1, 2, 3, 2, 1], - [1, 2, None, 3, 1, 1], - [], - ["a", "b", "c", None, "z", "a"], - ], -) -@pytest.mark.parametrize("na_sentinel", [99999, 11, -1, 0]) -def test_series_factorize(data, na_sentinel): - gsr = cudf.Series(data) - psr = gsr.to_pandas() - - with pytest.warns(FutureWarning): - expected_labels, expected_cats = psr.factorize(na_sentinel=na_sentinel) - with pytest.warns(FutureWarning): - actual_labels, actual_cats = gsr.factorize(na_sentinel=na_sentinel) - - assert_eq(expected_labels, actual_labels.get()) - assert_eq(expected_cats.values, actual_cats.to_pandas().values) - - @pytest.mark.parametrize( "data", [ From d1377a580656526119a33cabb898becd15cf152a Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 13 Mar 2023 16:06:56 -0500 Subject: [PATCH 011/384] Add information about `Index.is_*` method deprecation (#12909) This PR adds additional information for the following Index APIs to match with pandas 2.0: is_numeric is_boolean is_integer is_floating is_object is_categorical is_interval --- python/cudf/cudf/core/_base_index.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index 88763b8a011..1d0a30b556d 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -9,6 +9,7 @@ import pandas as pd import cudf +import warnings from cudf._lib.copying import _gather_map_is_valid, gather from cudf._lib.stream_compaction import ( apply_boolean_mask, @@ -858,6 +859,7 @@ def is_numeric(self): >>> idx.is_numeric() False """ + # TODO: Only remove this deprecation after pandas removes this API. warnings.warn( f"{type(self).__name__}.is_numeric is deprecated. " "Use cudf.api.types.is_any_real_numeric_dtype instead", @@ -902,6 +904,7 @@ def is_boolean(self): >>> idx.is_boolean() False """ + # TODO: Only remove this deprecation after pandas removes this API. warnings.warn( f"{type(self).__name__}.is_boolean is deprecated. " "Use cudf.api.types.is_bool_dtype instead", @@ -946,6 +949,7 @@ def is_integer(self): >>> idx.is_integer() False """ + # TODO: Only remove this deprecation after pandas removes this API. warnings.warn( f"{type(self).__name__}.is_integer is deprecated. " "Use cudf.api.types.is_integer_dtype instead", @@ -997,6 +1001,7 @@ def is_floating(self): >>> idx.is_floating() False """ + # TODO: Only remove this deprecation after pandas removes this API. warnings.warn( f"{type(self).__name__}.is_floating is deprecated. " "Use cudf.api.types.is_float_dtype instead", @@ -1042,6 +1047,7 @@ def is_object(self): >>> idx.is_object() False """ + # TODO: Only remove this deprecation after pandas removes this API. warnings.warn( f"{type(self).__name__}.is_object is deprecated. " "Use cudf.api.types.is_object_dtype instead", @@ -1094,6 +1100,7 @@ def is_categorical(self): >>> s.index.is_categorical() False """ + # TODO: Only remove this deprecation after pandas removes this API. warnings.warn( f"{type(self).__name__}.is_categorical is deprecated. " "Use cudf.api.types.is_categorical_dtype instead", @@ -1140,6 +1147,7 @@ def is_interval(self): >>> idx.is_interval() False """ + # TODO: Only remove this deprecation after pandas removes this API. warnings.warn( f"{type(self).__name__}.is_interval is deprecated. " "Use cudf.api.types.is_interval_dtype instead", From 48c1016b6882c79c2fdece65e55ff82a76e67e63 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 31 Mar 2023 13:53:15 -0500 Subject: [PATCH 012/384] [REVIEW] Miscellaneous pytest fixes for pandas-2.0 (#12962) This PR contains miscellaneous fixes in pytests. The changes in pytests are due to suttle changes in behaviors from the pandas-2.0 side. --- python/cudf/cudf/tests/test_parquet.py | 2 +- python/cudf/cudf/tests/test_reshape.py | 4 +++- python/cudf/cudf/tests/test_stats.py | 17 +++++------------ python/cudf/cudf/tests/test_string.py | 12 ++++++++++-- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index c24ff080033..aad163736c2 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -2454,7 +2454,7 @@ def test_parquet_writer_decimal(decimal_type, data): buff = BytesIO() gdf.to_parquet(buff) - got = pd.read_parquet(buff, use_nullable_dtypes=True) + got = pd.read_parquet(buff, dtype_backend="numpy_nullable") assert_eq(gdf.to_pandas(nullable=True), got) diff --git a/python/cudf/cudf/tests/test_reshape.py b/python/cudf/cudf/tests/test_reshape.py index 78e95fdbd81..bf2c1a32b64 100644 --- a/python/cudf/cudf/tests/test_reshape.py +++ b/python/cudf/cudf/tests/test_reshape.py @@ -76,7 +76,9 @@ def test_melt(nulls, num_id_vars, num_value_vars, num_rows, dtype): expect = pd.melt(frame=pdf, id_vars=id_vars, value_vars=value_vars) # pandas' melt makes the 'variable' column of 'object' type (string) # cuDF's melt makes it Categorical because it doesn't support strings - expect["variable"] = expect["variable"].astype("category") + expect["variable"] = expect["variable"].astype( + got["variable"].dtype.to_pandas() + ) assert_eq(expect, got) diff --git a/python/cudf/cudf/tests/test_stats.py b/python/cudf/cudf/tests/test_stats.py index 6ca64fdcfa3..126a90e580c 100644 --- a/python/cudf/cudf/tests/test_stats.py +++ b/python/cudf/cudf/tests/test_stats.py @@ -290,25 +290,18 @@ def test_kurtosis_series(data, null_flag): ], ) @pytest.mark.parametrize("null_flag", [False, True]) -def test_skew_series(data, null_flag): +@pytest.mark.parametrize("numeric_only", [False, True]) +def test_skew_series(data, null_flag, numeric_only): pdata = data.to_pandas() if null_flag and len(data) > 2: data.iloc[[0, 2]] = None pdata.iloc[[0, 2]] = None - got = data.skew() - expected = pdata.skew() - got = got if np.isscalar(got) else got.to_numpy() - np.testing.assert_array_almost_equal(got, expected) - - got = data.skew(numeric_only=False) - expected = pdata.skew(numeric_only=False) - got = got if np.isscalar(got) else got.to_numpy() - np.testing.assert_array_almost_equal(got, expected) + got = data.skew(numeric_only=numeric_only) + expected = pdata.skew(numeric_only=numeric_only) - with pytest.raises(NotImplementedError): - data.skew(numeric_only=True) + assert_eq(got, expected) @pytest.mark.parametrize("dtype", params_dtypes) diff --git a/python/cudf/cudf/tests/test_string.py b/python/cudf/cudf/tests/test_string.py index 10208611f13..693c9ef8044 100644 --- a/python/cudf/cudf/tests/test_string.py +++ b/python/cudf/cudf/tests/test_string.py @@ -1848,7 +1848,11 @@ def test_string_count(data, pat, flags): ps.str.count(pat=pat, flags=flags), check_dtype=False, ) - assert_eq(as_index(gs).str.count(pat=pat), pd.Index(ps).str.count(pat=pat)) + assert_eq( + cudf.Index(gs).str.count(pat=pat), + pd.Index(ps).str.count(pat=pat), + exact=False, + ) @pytest.mark.parametrize( @@ -2214,7 +2218,11 @@ def test_string_str_rindex(data, sub, er): if er is None: assert_eq(ps.str.rindex(sub), gs.str.rindex(sub), check_dtype=False) - assert_eq(pd.Index(ps).str.rindex(sub), as_index(gs).str.rindex(sub)) + assert_eq( + pd.Index(ps).str.rindex(sub), + as_index(gs).str.rindex(sub), + exact=False, + ) try: ps.str.rindex(sub) From dd15a19516df71040a227d581505658626e2e308 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Fri, 7 Apr 2023 09:48:16 -0700 Subject: [PATCH 013/384] Add get_indexer --- python/cudf/cudf/core/_base_index.py | 5 +- python/cudf/cudf/core/index.py | 98 +++++--- python/cudf/cudf/core/multiindex.py | 81 ++++++- python/cudf/cudf/tests/test_index.py | 335 +++++++++++++++++++++++---- 4 files changed, 427 insertions(+), 92 deletions(-) diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index 1d0a30b556d..8d448b99ac6 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -91,7 +91,10 @@ def size(self): def values(self): raise NotImplementedError - def get_loc(self, key, method=None, tolerance=None): + def get_indexer(self, target, method=None, limit=None, tolerance=None): + raise NotImplementedError + + def get_loc(self, key): raise NotImplementedError def __getitem__(self, key): diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 209276215c8..bf57a8f115f 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -575,19 +575,9 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): ) @_cudf_nvtx_annotate - def get_loc(self, key, method=None, tolerance=None): - # We should not actually remove this code until we have implemented the - # get_indexers method as an alternative, see - # https://github.com/rapidsai/cudf/issues/12312 - if method is not None: - warnings.warn( - f"Passing method to {self.__class__.__name__}.get_loc is " - "deprecated and will raise in a future version.", - FutureWarning, - ) - + def get_indexer(self, target, method=None, limit=None, tolerance=None): # Given an actual integer, - idx = (key - self._start) / self._step + idx = (target - self._start) / self._step idx_int_upper_bound = (self._stop - self._start) // self._step if method is None: if tolerance is not None: @@ -597,17 +587,17 @@ def get_loc(self, key, method=None, tolerance=None): ) if idx > idx_int_upper_bound or idx < 0: - raise KeyError(key) + raise KeyError(target) - idx_int = (key - self._start) // self._step + idx_int = (target - self._start) // self._step if idx_int != idx: - raise KeyError(key) + raise KeyError(target) return idx_int if (method == "ffill" and idx < 0) or ( method == "bfill" and idx > idx_int_upper_bound ): - raise KeyError(key) + raise KeyError(target) round_method = { "ffill": math.floor, @@ -615,9 +605,16 @@ def get_loc(self, key, method=None, tolerance=None): "nearest": round, }[method] if tolerance is not None and (abs(idx) * self._step > tolerance): - raise KeyError(key) + raise KeyError(target) return np.clip(round_method(idx), 0, idx_int_upper_bound, dtype=int) + @_cudf_nvtx_annotate + def get_loc(self, key): + # Given an actual integer, + if is_scalar(key): + key = [key] + return self.get_indexer(key) + @_cudf_nvtx_annotate def _union(self, other, sort=None): if isinstance(other, RangeIndex): @@ -1128,12 +1125,12 @@ def astype(self, dtype, copy: bool = True): return _index_from_data(super().astype({self.name: dtype}, copy)) @_cudf_nvtx_annotate - def get_loc(self, key, method=None, tolerance=None): + def get_indexer(self, target, method=None, limit=None, tolerance=None): """Get integer location, slice or boolean mask for requested label. Parameters ---------- - key : label + target : label method : {None, 'pad'/'fill', 'backfill'/'bfill', 'nearest'}, optional - default: exact matches only. - pad / ffill: find the PREVIOUS index value if no exact match. @@ -1144,7 +1141,7 @@ def get_loc(self, key, method=None, tolerance=None): tolerance : int or float, optional Maximum distance from index value for inexact matches. The value of the index at the matching location must satisfy the equation - ``abs(index[loc] - key) <= tolerance``. + ``abs(index[loc] - target) <= tolerance``. Returns ------- @@ -1168,15 +1165,8 @@ def get_loc(self, key, method=None, tolerance=None): >>> numeric_unique_index.get_loc(3) 2 """ - # We should not actually remove this code until we have implemented the - # get_indexers method as an alternative, see - # https://github.com/rapidsai/cudf/issues/12312 - if method is not None: - warnings.warn( - f"Passing method to {self.__class__.__name__}.get_loc is " - "deprecated and will raise in a future version.", - FutureWarning, - ) + if is_scalar(target): + raise TypeError("Should be a sequence") if tolerance is not None: raise NotImplementedError( "Parameter tolerance is not supported yet." @@ -1204,22 +1194,20 @@ def get_loc(self, key, method=None, tolerance=None): "is specified." ) - key_as_table = cudf.core.frame.Frame( - {"None": as_column(key, length=1)} - ) + target_as_table = cudf.core.frame.Frame({"None": as_column(target)}) lower_bound, upper_bound, sort_inds = _lexsorted_equal_range( - self, key_as_table, is_sorted + self, target_as_table, is_sorted ) if lower_bound == upper_bound: - # Key not found, apply method + # target not found, apply method if method in ("pad", "ffill"): if lower_bound == 0: - raise KeyError(key) + raise KeyError(target) return lower_bound - 1 elif method in ("backfill", "bfill"): if lower_bound == self._data.nrows: - raise KeyError(key) + raise KeyError(target) return lower_bound elif method == "nearest": if lower_bound == self._data.nrows: @@ -1230,11 +1218,11 @@ def get_loc(self, key, method=None, tolerance=None): upper_val = self._column.element_indexing(lower_bound) return ( lower_bound - 1 - if abs(lower_val - key) < abs(upper_val - key) + if abs(lower_val - target) < abs(upper_val - target) else lower_bound ) else: - raise KeyError(key) + raise KeyError(target) if lower_bound + 1 == upper_bound: # Search result is unique, return int. @@ -1255,6 +1243,40 @@ def get_loc(self, key, method=None, tolerance=None): mask[true_inds] = True return mask + @_cudf_nvtx_annotate + def get_loc(self, key): + """Get integer location, slice or boolean mask for requested label. + + Parameters + ---------- + key : label + + Returns + ------- + int or slice or boolean mask + - If result is unique, return integer index + - If index is monotonic, loc is returned as a slice object + - Otherwise, a boolean mask is returned + + Examples + -------- + >>> unique_index = cudf.Index(list('abc')) + >>> unique_index.get_loc('b') + 1 + >>> monotonic_index = cudf.Index(list('abbc')) + >>> monotonic_index.get_loc('b') + slice(1, 3, None) + >>> non_monotonic_index = cudf.Index(list('abcb')) + >>> non_monotonic_index.get_loc('b') + array([False, True, False, True]) + >>> numeric_unique_index = cudf.Index([1, 2, 3]) + >>> numeric_unique_index.get_loc(3) + 2 + """ + if is_scalar(key): + key = [key] + return self.get_indexer(target=key) + @_cudf_nvtx_annotate def __repr__(self): max_seq_items = get_option("max_seq_items") or len(self) diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 4cd3f0b3837..f533cff7c12 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1642,7 +1642,7 @@ def _level_index_from_level(self, level): return level @_cudf_nvtx_annotate - def get_loc(self, key, method=None, tolerance=None): + def get_indexer(self, target, method=None, tolerance=None): """ Get location for a label or a tuple of labels. @@ -1650,7 +1650,7 @@ def get_loc(self, key, method=None, tolerance=None): Parameters ---------- - key : label or tuple of labels (one for each level) + target : label or tuple of labels (one for each level) method : None Returns @@ -1712,24 +1712,26 @@ def get_loc(self, key, method=None, tolerance=None): self.is_monotonic_increasing or self.is_monotonic_decreasing ) is_unique = self.is_unique - key = (key,) if not isinstance(key, tuple) else key + target = (target,) if not isinstance(target, tuple) else target - # Handle partial key search. If length of `key` is less than `nlevels`, - # Only search levels up to `len(key)` level. - key_as_table = cudf.core.frame.Frame( - {i: column.as_column(k, length=1) for i, k in enumerate(key)} + # Handle partial target search. If length of `target` is less than `nlevels`, + # Only search levels up to `len(target)` level. + target_as_table = cudf.core.frame.Frame( + {i: column.as_column(k, length=1) for i, k in enumerate(target)} ) partial_index = self.__class__._from_data( - data=self._data.select_by_index(slice(key_as_table._num_columns)) + data=self._data.select_by_index( + slice(target_as_table._num_columns) + ) ) ( lower_bound, upper_bound, sort_inds, - ) = _lexsorted_equal_range(partial_index, key_as_table, is_sorted) + ) = _lexsorted_equal_range(partial_index, target_as_table, is_sorted) if lower_bound == upper_bound: - raise KeyError(key) + raise KeyError(target) if is_unique and lower_bound + 1 == upper_bound: # Indices are unique (Pandas constraint), search result is unique, @@ -1755,6 +1757,65 @@ def get_loc(self, key, method=None, tolerance=None): mask[true_inds] = True return mask + @_cudf_nvtx_annotate + def get_loc(self, key): + """ + Get location for a label or a tuple of labels. + + The location is returned as an integer/slice or boolean mask. + + Parameters + ---------- + key : label or tuple of labels (one for each level) + method : None + + Returns + ------- + loc : int, slice object or boolean mask + - If index is unique, search result is unique, return a single int. + - If index is monotonic, index is returned as a slice object. + - Otherwise, cudf attempts a best effort to convert the search + result into a slice object, and will return a boolean mask if + failed to do so. Notice this can deviate from Pandas behavior + in some situations. + + Examples + -------- + >>> import cudf + >>> mi = cudf.MultiIndex.from_tuples( + ... [('a', 'd'), ('b', 'e'), ('b', 'f')]) + >>> mi.get_loc('b') + slice(1, 3, None) + >>> mi.get_loc(('b', 'e')) + 1 + >>> non_monotonic_non_unique_idx = cudf.MultiIndex.from_tuples( + ... [('c', 'd'), ('b', 'e'), ('a', 'f'), ('b', 'e')]) + >>> non_monotonic_non_unique_idx.get_loc('b') # differ from pandas + slice(1, 4, 2) + + .. pandas-compat:: + **MultiIndex.get_loc** + + The return types of this function may deviates from the + method provided by Pandas. If the index is neither + lexicographically sorted nor unique, a best effort attempt is made + to coerce the found indices into a slice. For example: + + .. code-block:: + + >>> import pandas as pd + >>> import cudf + >>> x = pd.MultiIndex.from_tuples([ + ... (2, 1, 1), (1, 2, 3), (1, 2, 1), + ... (1, 1, 1), (1, 1, 1), (2, 2, 1), + ... ]) + >>> x.get_loc(1) + array([False, True, True, True, True, False]) + >>> cudf.from_pandas(x).get_loc(1) + slice(1, 5, 1) + """ + return self.get_indexer(target=key) + def _get_reconciled_name_object(self, other) -> MultiIndex: """ If the result of a set operation will be self, diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index f9ad48c48af..312baf2d7c6 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -1967,30 +1967,66 @@ def test_get_loc_single_unique_numeric(idx, key, method): "idx", [pd.RangeIndex(3, 100, 4)], ) -@pytest.mark.parametrize("key", list(range(1, 110, 3))) +@pytest.mark.parametrize( + "key", + [ + list(range(1, 20, 3)), + list(range(20, 35, 3)), + list(range(35, 77, 3)), + list(range(77, 110, 3)), + ], +) @pytest.mark.parametrize("method", [None, "ffill"]) -def test_get_loc_rangeindex(idx, key, method): +def test_get_indexer_rangeindex(idx, key, method): pi = idx gi = cudf.from_pandas(pi) + # if ( + # (any(k not in pi for k in key) and method is None) + # # Get key before the first element is KeyError + # or (key < pi.start and method in "ffill") + # # Get key after the last element is KeyError + # or (key >= pi.stop and method in "bfill") + # ): + # assert_exceptions_equal( + # lfunc=pi.get_indexer, + # rfunc=gi.get_indexer, + # lfunc_args_and_kwargs=([], {"key": key, "method": method}), + # rfunc_args_and_kwargs=([], {"key": key, "method": method}), + # ) + # else: + # with expect_warning_if(method is not None): + expected = pi.get_indexer(key, method=method) + # with expect_warning_if(method is not None): + got = gi.get_indexer(key, method=method) + + assert_eq(expected, got) + + +@pytest.mark.parametrize( + "idx", + [pd.RangeIndex(3, 100, 4)], +) +@pytest.mark.parametrize("key", list(range(1, 110, 3))) +def test_get_loc_rangeindex(idx, key): + pi = idx + gi = cudf.from_pandas(pi) if ( - (key not in pi and method is None) + (key not in pi) # Get key before the first element is KeyError - or (key < pi.start and method in "ffill") + or (key < pi.start) # Get key after the last element is KeyError - or (key >= pi.stop and method in "bfill") + or (key >= pi.stop) ): assert_exceptions_equal( lfunc=pi.get_loc, rfunc=gi.get_loc, - lfunc_args_and_kwargs=([], {"key": key, "method": method}), - rfunc_args_and_kwargs=([], {"key": key, "method": method}), + lfunc_args_and_kwargs=([], {"key": key}), + rfunc_args_and_kwargs=([], {"key": key}), ) else: - with expect_warning_if(method is not None): - expected = pi.get_loc(key, method=method) - with expect_warning_if(method is not None): - got = gi.get_loc(key, method=method) + expected = pi.get_loc(key) + got = gi.get_loc(key) assert_eq(expected, got) @@ -2003,8 +2039,7 @@ def test_get_loc_rangeindex(idx, key, method): ], ) @pytest.mark.parametrize("key", [0, 3, 6, 7]) -@pytest.mark.parametrize("method", [None]) -def test_get_loc_single_duplicate_numeric(idx, key, method): +def test_get_loc_single_duplicate_numeric(idx, key): pi = idx gi = cudf.from_pandas(pi) @@ -2012,14 +2047,61 @@ def test_get_loc_single_duplicate_numeric(idx, key, method): assert_exceptions_equal( lfunc=pi.get_loc, rfunc=gi.get_loc, + lfunc_args_and_kwargs=([], {"key": key}), + rfunc_args_and_kwargs=([], {"key": key}), + ) + else: + expected = pi.get_loc(key) + got = gi.get_loc(key) + + assert_eq(expected, got) + + +@pytest.mark.parametrize( + "idx", + [ + pd.Index([1, 3, 3, 6]), # monotonic + pd.Index([6, 1, 3, 3]), # non-monotonic + ], +) +@pytest.mark.parametrize("key", [0, 3, 6, 7]) +@pytest.mark.parametrize("method", [None]) +def test_get_indexer_single_duplicate_numeric(idx, key, method): + pi = idx + gi = cudf.from_pandas(pi) + + if key not in pi: + assert_exceptions_equal( + lfunc=pi.get_indexer, + rfunc=gi.get_indexer, lfunc_args_and_kwargs=([], {"key": key, "method": method}), rfunc_args_and_kwargs=([], {"key": key, "method": method}), ) else: - with expect_warning_if(method is not None): - expected = pi.get_loc(key, method=method) - with expect_warning_if(method is not None): - got = gi.get_loc(key, method=method) + expected = pi.get_indexer(key, method=method) + got = gi.get_indexer(key, method=method) + + assert_eq(expected, got) + + +@pytest.mark.parametrize( + "idx", [pd.Index(["b", "f", "m", "q"]), pd.Index(["m", "f", "b", "q"])] +) +@pytest.mark.parametrize("key", ["a", "f", "n", "z"]) +def test_get_loc_single_unique_string(idx, key): + pi = idx + gi = cudf.from_pandas(pi) + + if key not in pi: + assert_exceptions_equal( + lfunc=pi.get_loc, + rfunc=gi.get_loc, + lfunc_args_and_kwargs=([], {"key": key}), + rfunc_args_and_kwargs=([], {"key": key}), + ) + else: + expected = pi.get_loc(key) + got = gi.get_loc(key) assert_eq(expected, got) @@ -2029,7 +2111,7 @@ def test_get_loc_single_duplicate_numeric(idx, key, method): ) @pytest.mark.parametrize("key", ["a", "f", "n", "z"]) @pytest.mark.parametrize("method", [None, "ffill", "bfill"]) -def test_get_loc_single_unique_string(idx, key, method): +def test_get_indexer_single_unique_string(idx, key, method): pi = idx gi = cudf.from_pandas(pi) @@ -2043,16 +2125,14 @@ def test_get_loc_single_unique_string(idx, key, method): or (key == "z" and method == "bfill") ): assert_exceptions_equal( - lfunc=pi.get_loc, - rfunc=gi.get_loc, + lfunc=pi.get_indexer, + rfunc=gi.get_indexer, lfunc_args_and_kwargs=([], {"key": key, "method": method}), rfunc_args_and_kwargs=([], {"key": key, "method": method}), ) else: - with expect_warning_if(method is not None): - expected = pi.get_loc(key, method=method) - with expect_warning_if(method is not None): - got = gi.get_loc(key, method=method) + expected = pi.get_indexer(key, method=method) + got = gi.get_indexer(key, method=method) assert_eq(expected, got) @@ -2061,8 +2141,7 @@ def test_get_loc_single_unique_string(idx, key, method): "idx", [pd.Index(["b", "m", "m", "q"]), pd.Index(["m", "f", "m", "q"])] ) @pytest.mark.parametrize("key", ["a", "f", "n", "z"]) -@pytest.mark.parametrize("method", [None]) -def test_get_loc_single_duplicate_string(idx, key, method): +def test_get_loc_single_duplicate_string(idx, key): pi = idx gi = cudf.from_pandas(pi) @@ -2070,14 +2149,35 @@ def test_get_loc_single_duplicate_string(idx, key, method): assert_exceptions_equal( lfunc=pi.get_loc, rfunc=gi.get_loc, + lfunc_args_and_kwargs=([], {"key": key}), + rfunc_args_and_kwargs=([], {"key": key}), + ) + else: + expected = pi.get_loc(key) + got = gi.get_loc(key) + + assert_eq(expected, got) + + +@pytest.mark.parametrize( + "idx", [pd.Index(["b", "m", "m", "q"]), pd.Index(["m", "f", "m", "q"])] +) +@pytest.mark.parametrize("key", ["a", "f", "n", "z"]) +@pytest.mark.parametrize("method", [None]) +def test_get_indexer_single_duplicate_string(idx, key, method): + pi = idx + gi = cudf.from_pandas(pi) + + if key not in pi: + assert_exceptions_equal( + lfunc=pi.get_indexer, + rfunc=gi.get_indexer, lfunc_args_and_kwargs=([], {"key": key, "method": method}), rfunc_args_and_kwargs=([], {"key": key, "method": method}), ) else: - with expect_warning_if(method is not None): - expected = pi.get_loc(key, method=method) - with expect_warning_if(method is not None): - got = gi.get_loc(key, method=method) + expected = pi.get_indexer(key, method=method) + got = gi.get_indexer(key, method=method) assert_eq(expected, got) @@ -2097,8 +2197,7 @@ def test_get_loc_single_duplicate_string(idx, key, method): ], ) @pytest.mark.parametrize("key", [1, (1, 2), (1, 2, 3), (2, 1, 1), (9, 9, 9)]) -@pytest.mark.parametrize("method", [None]) -def test_get_loc_multi_numeric(idx, key, method): +def test_get_loc_multi_numeric(idx, key): pi = idx.sort_values() gi = cudf.from_pandas(pi) @@ -2106,14 +2205,90 @@ def test_get_loc_multi_numeric(idx, key, method): assert_exceptions_equal( lfunc=pi.get_loc, rfunc=gi.get_loc, + lfunc_args_and_kwargs=([], {"key": key}), + rfunc_args_and_kwargs=([], {"key": key}), + ) + else: + expected = pi.get_loc(key) + got = gi.get_loc(key) + + assert_eq(expected, got) + + +@pytest.mark.parametrize( + "idx", + [ + pd.MultiIndex.from_tuples( + [(1, 1, 1), (1, 1, 2), (1, 2, 1), (1, 2, 3), (2, 1, 1), (2, 2, 1)] + ), + pd.MultiIndex.from_tuples( + [(2, 1, 1), (1, 2, 3), (1, 2, 1), (1, 1, 2), (2, 2, 1), (1, 1, 1)] + ), + pd.MultiIndex.from_tuples( + [(1, 1, 1), (1, 1, 2), (1, 1, 2), (1, 2, 3), (2, 1, 1), (2, 2, 1)] + ), + ], +) +@pytest.mark.parametrize("key", [1, (1, 2), (1, 2, 3), (2, 1, 1), (9, 9, 9)]) +@pytest.mark.parametrize("method", [None]) +def test_get_indexer_multi_numeric(idx, key, method): + pi = idx.sort_values() + gi = cudf.from_pandas(pi) + + if key not in pi: + assert_exceptions_equal( + lfunc=pi.get_indexer, + rfunc=gi.get_indexer, lfunc_args_and_kwargs=([], {"key": key, "method": method}), rfunc_args_and_kwargs=([], {"key": key, "method": method}), ) else: - with expect_warning_if(method is not None): - expected = pi.get_loc(key, method=method) - with expect_warning_if(method is not None): - got = gi.get_loc(key, method=method) + expected = pi.get_indexer(key, method=method) + got = gi.get_indexer(key, method=method) + + assert_eq(expected, got) + + +@pytest.mark.parametrize( + "idx", + [ + pd.MultiIndex.from_tuples( + [(2, 1, 1), (1, 2, 3), (1, 2, 1), (1, 1, 1), (1, 1, 1), (2, 2, 1)] + ) + ], +) +@pytest.mark.parametrize( + "key, result", + [ + (1, slice(1, 5, 1)), # deviates + ((1, 2), slice(1, 3, 1)), + ((1, 2, 3), slice(1, 2, None)), + ((2, 1, 1), slice(0, 1, None)), + ((9, 9, 9), None), + ], +) +def test_get_loc_multi_numeric_deviate(idx, key, result): + pi = idx + gi = cudf.from_pandas(pi) + + with expect_warning_if( + isinstance(key, tuple), pd.errors.PerformanceWarning + ): + key_flag = key not in pi + + if key_flag: + with expect_warning_if( + isinstance(key, tuple), pd.errors.PerformanceWarning + ): + assert_exceptions_equal( + lfunc=pi.get_loc, + rfunc=gi.get_loc, + lfunc_args_and_kwargs=([], {"key": key}), + rfunc_args_and_kwargs=([], {"key": key}), + ) + else: + expected = result + got = gi.get_loc(key) assert_eq(expected, got) @@ -2137,7 +2312,7 @@ def test_get_loc_multi_numeric(idx, key, method): ], ) @pytest.mark.parametrize("method", [None]) -def test_get_loc_multi_numeric_deviate(idx, key, result, method): +def test_get_indexer_multi_numeric_deviate(idx, key, result, method): pi = idx gi = cudf.from_pandas(pi) @@ -2222,8 +2397,7 @@ def test_get_loc_multi_numeric_deviate(idx, key, result, method): @pytest.mark.parametrize( "key", ["a", ("a", "a"), ("a", "b", "c"), ("b", "c", "a"), ("z", "z", "z")] ) -@pytest.mark.parametrize("method", [None]) -def test_get_loc_multi_string(idx, key, method): +def test_get_loc_multi_string(idx, key): pi = idx.sort_values() gi = cudf.from_pandas(pi) @@ -2231,14 +2405,89 @@ def test_get_loc_multi_string(idx, key, method): assert_exceptions_equal( lfunc=pi.get_loc, rfunc=gi.get_loc, + lfunc_args_and_kwargs=([], {"key": key}), + rfunc_args_and_kwargs=([], {"key": key}), + ) + else: + expected = pi.get_loc(key) + got = gi.get_loc(key) + + assert_eq(expected, got) + + +@pytest.mark.parametrize( + "idx", + [ + pd.MultiIndex.from_tuples( + [ + ("a", "a", "a"), + ("a", "a", "b"), + ("a", "b", "a"), + ("a", "b", "c"), + ("b", "a", "a"), + ("b", "c", "a"), + ] + ), + pd.MultiIndex.from_tuples( + [ + ("a", "a", "b"), + ("a", "b", "c"), + ("b", "a", "a"), + ("a", "a", "a"), + ("a", "b", "a"), + ("b", "c", "a"), + ] + ), + pd.MultiIndex.from_tuples( + [ + ("a", "a", "a"), + ("a", "b", "c"), + ("b", "a", "a"), + ("a", "a", "b"), + ("a", "b", "a"), + ("b", "c", "a"), + ] + ), + pd.MultiIndex.from_tuples( + [ + ("a", "a", "a"), + ("a", "a", "b"), + ("a", "a", "b"), + ("a", "b", "c"), + ("b", "a", "a"), + ("b", "c", "a"), + ] + ), + pd.MultiIndex.from_tuples( + [ + ("a", "a", "b"), + ("b", "a", "a"), + ("b", "a", "a"), + ("a", "a", "a"), + ("a", "b", "a"), + ("b", "c", "a"), + ] + ), + ], +) +@pytest.mark.parametrize( + "key", ["a", ("a", "a"), ("a", "b", "c"), ("b", "c", "a"), ("z", "z", "z")] +) +@pytest.mark.parametrize("method", [None]) +def test_get_indexer_multi_string(idx, key, method): + pi = idx.sort_values() + gi = cudf.from_pandas(pi) + + if key not in pi: + assert_exceptions_equal( + lfunc=pi.get_indexer, + rfunc=gi.get_indexer, lfunc_args_and_kwargs=([], {"key": key, "method": method}), rfunc_args_and_kwargs=([], {"key": key, "method": method}), ) else: - with expect_warning_if(method is not None): - expected = pi.get_loc(key, method=method) - with expect_warning_if(method is not None): - got = gi.get_loc(key, method=method) + expected = pi.get_indexer(key, method=method) + got = gi.get_indexer(key, method=method) assert_eq(expected, got) From 6dce4ef93c0e4e40c764922a0370417cb78150b6 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 7 Apr 2023 12:05:38 -0500 Subject: [PATCH 014/384] Fix ufunc tests (#13083) Pandas 2.0 introduced support for ufuncs when the two columns are indexed. This PR updates the pytests accordingly. --- python/cudf/cudf/core/indexed_frame.py | 8 -------- python/cudf/cudf/tests/test_array_ufunc.py | 18 ++++++++++++++---- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 074bd554601..43085b297b0 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -3330,14 +3330,6 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): fname = ufunc.__name__ if ret is not None: - # pandas bitwise operations return bools if indexes are misaligned. - if "bitwise" in fname: - reflect = self is not inputs[0] - other = inputs[0] if reflect else inputs[1] - if isinstance(other, self.__class__) and not self.index.equals( - other.index - ): - ret = ret.astype(bool) return ret # Attempt to dispatch all other functions to cupy. diff --git a/python/cudf/cudf/tests/test_array_ufunc.py b/python/cudf/cudf/tests/test_array_ufunc.py index ac77c6b89f3..8f4ae7c23a0 100644 --- a/python/cudf/cudf/tests/test_array_ufunc.py +++ b/python/cudf/cudf/tests/test_array_ufunc.py @@ -10,7 +10,7 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_150 +from cudf.core._compat import PANDAS_GE_150, PANDAS_GE_200 from cudf.testing._utils import assert_eq, set_random_null_mask_inplace _UFUNCS = [ @@ -165,6 +165,16 @@ def test_ufunc_series(request, ufunc, has_nulls, indexed): ) ) + request.applymarker( + pytest.mark.xfail( + condition=PANDAS_GE_200 + and fname.startswith("bitwise") + and indexed + and has_nulls, + reason="https://github.com/pandas-dev/pandas/issues/52500", + ) + ) + N = 100 # Avoid zeros in either array to skip division by 0 errors. Also limit the # scale to avoid issues with overflow, etc. We use ints because some @@ -342,8 +352,8 @@ def test_ufunc_dataframe(request, ufunc, has_nulls, indexed): request.applymarker( pytest.mark.xfail( condition=( - indexed - and fname + not PANDAS_GE_200 + and indexed in { "add", "arctan2", @@ -379,7 +389,7 @@ def test_ufunc_dataframe(request, ufunc, has_nulls, indexed): } ), reason=( - "pandas does not currently support misaligned " + "pandas<2.0 does not currently support misaligned " "indexes in DataFrames" ), ) From 192e2045ffff4366b501e6a1b3d781cc30a4cdf6 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 7 Apr 2023 12:06:19 -0500 Subject: [PATCH 015/384] [REVIEW] datetime and timedelta improvements (#12934) This PR fixes 1046 pytest failures that are related to `datetime64` & `timedelta64` types. This PR(`time_2.0`): ```bash = 990 failed, 86109 passed, 2034 skipped, 995 xfailed, 165 xpassed in 546.70s (0:09:06) = ``` on `pandas_2.0_feature_branch`: ```bash == 2036 failed, 85423 passed, 2034 skipped, 860 xfailed in 720.53s (0:12:00) === ``` --- python/cudf/cudf/core/column/datetime.py | 63 ++++-- python/cudf/cudf/core/column/timedelta.py | 191 ++++++++++-------- python/cudf/cudf/core/index.py | 22 +- python/cudf/cudf/core/tools/datetimes.py | 12 +- python/cudf/cudf/tests/test_binops.py | 2 +- python/cudf/cudf/tests/test_dataframe.py | 19 +- python/cudf/cudf/tests/test_datetime.py | 42 ++-- python/cudf/cudf/tests/test_parquet.py | 23 ++- python/cudf/cudf/tests/test_timedelta.py | 34 +++- python/cudf/cudf/tests/test_udf_masked_ops.py | 13 +- python/cudf/cudf/utils/dtypes.py | 10 +- 11 files changed, 283 insertions(+), 148 deletions(-) diff --git a/python/cudf/cudf/core/column/datetime.py b/python/cudf/cudf/core/column/datetime.py index 14aa7bdd84b..107ebfbbcc3 100644 --- a/python/cudf/cudf/core/column/datetime.py +++ b/python/cudf/cudf/core/column/datetime.py @@ -9,9 +9,9 @@ from typing import Any, Mapping, Sequence, cast import numpy as np -import pandas as pd import cudf +import pandas as pd from cudf import _lib as libcudf from cudf._typing import ( ColumnBinaryOperand, @@ -21,6 +21,7 @@ ScalarLike, ) from cudf.api.types import is_datetime64_dtype, is_scalar, is_timedelta64_dtype +from cudf.core._compat import PANDAS_GE_200 from cudf.core.buffer import Buffer, cuda_array_interface_wrapper from cudf.core.column import ColumnBase, as_column, column, string from cudf.core.column.timedelta import _unit_to_nanoseconds_conversion @@ -200,9 +201,16 @@ def to_pandas( # Workaround until following issue is fixed: # https://issues.apache.org/jira/browse/ARROW-9772 - # Pandas supports only `datetime64[ns]`, hence the cast. + if PANDAS_GE_200: + host_values = self.fillna("NaT").values_host + else: + # Pandas<2.0 supports only `datetime64[ns]`, hence the cast. + host_values = ( + self.astype("datetime64[ns]").fillna("NaT").values_host + ) + return pd.Series( - self.astype("datetime64[ns]").fillna("NaT").values_host, + host_values, copy=False, index=index, ) @@ -243,19 +251,30 @@ def normalize_binop_value(self, other: DatetimeLikeScalar) -> ScalarLike: if isinstance(other, np.datetime64): if np.isnat(other): - return cudf.Scalar(None, dtype=self.dtype) + other_time_unit = cudf.utils.dtypes.get_time_unit(other) + if other_time_unit not in {"s", "ms", "ns", "us"}: + other_time_unit = "ns" + + return cudf.Scalar( + None, dtype=f"datetime64[{other_time_unit}]" + ) other = other.astype(self.dtype) return cudf.Scalar(other) elif isinstance(other, np.timedelta64): other_time_unit = cudf.utils.dtypes.get_time_unit(other) + if np.isnat(other): + return cudf.Scalar( + None, + dtype="timedelta64[ns]" + if other_time_unit not in {"s", "ms", "ns", "us"} + else other.dtype, + ) + if other_time_unit not in {"s", "ms", "ns", "us"}: other = other.astype("timedelta64[s]") - if np.isnat(other): - return cudf.Scalar(None, dtype=other.dtype) - return cudf.Scalar(other) elif isinstance(other, str): try: @@ -352,7 +371,7 @@ def mean( skipna=skipna, min_count=min_count, dtype=dtype ), unit=self.time_unit, - ) + ).as_unit(self.time_unit) def std( self, @@ -366,12 +385,12 @@ def std( skipna=skipna, min_count=min_count, dtype=dtype, ddof=ddof ) * _unit_to_nanoseconds_conversion[self.time_unit], - ) + ).as_unit(self.time_unit) def median(self, skipna: bool = None) -> pd.Timestamp: return pd.Timestamp( self.as_numerical.median(skipna=skipna), unit=self.time_unit - ) + ).as_unit(self.time_unit) def quantile( self, @@ -387,7 +406,9 @@ def quantile( return_scalar=return_scalar, ) if return_scalar: - return pd.Timestamp(result, unit=self.time_unit) + return pd.Timestamp(result, unit=self.time_unit).as_unit( + self.time_unit + ) return result.astype(self.dtype) def _binaryop(self, other: ColumnBinaryOperand, op: str) -> ColumnBase: @@ -396,7 +417,9 @@ def _binaryop(self, other: ColumnBinaryOperand, op: str) -> ColumnBase: if other is NotImplemented: return NotImplemented if isinstance(other, cudf.DateOffset): - return other._datetime_binop(self, op, reflect=reflect) + return other._datetime_binop(self, op, reflect=reflect).astype( + self.dtype + ) # We check this on `other` before reflection since we already know the # dtype of `self`. @@ -441,7 +464,11 @@ def _binaryop(self, other: ColumnBinaryOperand, op: str) -> ColumnBase: if out_dtype is None: return NotImplemented - return libcudf.binaryop.binaryop(lhs, rhs, op, out_dtype) + result_col = libcudf.binaryop.binaryop(lhs, rhs, op, out_dtype) + if out_dtype != cudf.dtype(np.bool_) and op == "__add__": + return result_col # .astype(lhs.dtype) + else: + return result_col def fillna( self, fill_value: Any = None, method: str = None, dtype: Dtype = None @@ -525,7 +552,15 @@ def infer_format(element: str, **kwargs) -> str: fmt = _guess_datetime_format(element, **kwargs) if fmt is not None: - return fmt + if ".%f" in fmt: + # For context read: + # https://github.com/pandas-dev/pandas/issues/52418 + # We cannot rely on format containing only %f + # c++/libcudf expects .%3f, .%6f, .%9f + # Logic below handles those cases well. + pass + else: + return fmt element_parts = element.split(".") if len(element_parts) != 2: diff --git a/python/cudf/cudf/core/column/timedelta.py b/python/cudf/cudf/core/column/timedelta.py index e7979fa4d27..29fe448db75 100644 --- a/python/cudf/cudf/core/column/timedelta.py +++ b/python/cudf/cudf/core/column/timedelta.py @@ -17,6 +17,7 @@ from cudf.core.column import ColumnBase, column, string from cudf.utils.dtypes import np_to_pa_dtype from cudf.utils.utils import _fillna_natwise +from cudf.core._compat import PANDAS_GE_200 _dtype_to_format_conversion = { "timedelta64[ns]": "%D days %H:%M:%S", @@ -149,9 +150,16 @@ def to_pandas( # Workaround until following issue is fixed: # https://issues.apache.org/jira/browse/ARROW-9772 - # Pandas supports only `timedelta64[ns]`, hence the cast. + if PANDAS_GE_200: + host_values = self.fillna("NaT").values_host + else: + # Pandas<2.0 supports only `timedelta64[ns]`, hence the cast. + host_values = ( + self.astype("timedelta64[ns]").fillna("NaT").values_host + ) + pd_series = pd.Series( - self.astype("timedelta64[ns]").fillna("NaT").values_host, + host_values, copy=False, ) @@ -213,16 +221,21 @@ def _binaryop(self, other: ColumnBinaryOperand, op: str) -> ColumnBase: def normalize_binop_value(self, other) -> ColumnBinaryOperand: if isinstance(other, (ColumnBase, cudf.Scalar)): return other - if isinstance(other, datetime.timedelta): - other = np.timedelta64(other) - elif isinstance(other, pd.Timestamp): + if isinstance(other, pd.Timestamp): other = other.to_datetime64() elif isinstance(other, pd.Timedelta): other = other.to_timedelta64() + elif isinstance(other, datetime.timedelta): + other = np.timedelta64(other) if isinstance(other, np.timedelta64): other_time_unit = cudf.utils.dtypes.get_time_unit(other) if np.isnat(other): - return cudf.Scalar(None, dtype=self.dtype) + return cudf.Scalar( + None, + dtype="timedelta64[ns]" + if other_time_unit not in {"s", "ms", "ns", "us"} + else self.dtype, + ) if other_time_unit not in {"s", "ms", "ns", "us"}: common_dtype = "timedelta64[s]" @@ -259,9 +272,8 @@ def fillna( col: ColumnBase = self if is_scalar(fill_value): if isinstance(fill_value, np.timedelta64): - dtype = determine_out_dtype(self.dtype, fill_value.dtype) + dtype = self.dtype fill_value = fill_value.astype(dtype) - col = col.astype(dtype) if not isinstance(fill_value, cudf.Scalar): fill_value = cudf.Scalar(fill_value, dtype=dtype) else: @@ -311,12 +323,12 @@ def mean(self, skipna=None, dtype: Dtype = np.float64) -> pd.Timedelta: return pd.Timedelta( self.as_numerical.mean(skipna=skipna, dtype=dtype), unit=self.time_unit, - ) + ).as_unit(self.time_unit) def median(self, skipna: bool = None) -> pd.Timedelta: return pd.Timedelta( self.as_numerical.median(skipna=skipna), unit=self.time_unit - ) + ).as_unit(self.time_unit) def isin(self, values: Sequence) -> ColumnBase: return cudf.core.tools.datetimes._isin_datetimelike(self, values) @@ -335,7 +347,9 @@ def quantile( return_scalar=return_scalar, ) if return_scalar: - return pd.Timedelta(result, unit=self.time_unit) + return pd.Timedelta(result, unit=self.time_unit).as_unit( + self.time_unit + ) return result.astype(self.dtype) def sum( @@ -352,7 +366,7 @@ def sum( skipna=skipna, min_count=min_count, dtype=dtype ), unit=self.time_unit, - ) + ).as_unit(self.time_unit) def std( self, @@ -366,7 +380,7 @@ def std( skipna=skipna, min_count=min_count, ddof=ddof, dtype=dtype ), unit=self.time_unit, - ) + ).as_unit(self.time_unit) def components(self, index=None) -> "cudf.DataFrame": """ @@ -397,79 +411,72 @@ def components(self, index=None) -> "cudf.DataFrame": 4 37 13 12 14 234 0 0 """ # noqa: E501 - return cudf.DataFrame( - data={ - "days": self - // cudf.Scalar( - np.timedelta64(_unit_to_nanoseconds_conversion["D"], "ns") - ), - "hours": ( - self - % cudf.Scalar( - np.timedelta64( - _unit_to_nanoseconds_conversion["D"], "ns" - ) - ) - ) - // cudf.Scalar( - np.timedelta64(_unit_to_nanoseconds_conversion["h"], "ns") - ), - "minutes": ( - self - % cudf.Scalar( - np.timedelta64( - _unit_to_nanoseconds_conversion["h"], "ns" - ) - ) - ) - // cudf.Scalar( - np.timedelta64(_unit_to_nanoseconds_conversion["m"], "ns") - ), - "seconds": ( - self - % cudf.Scalar( - np.timedelta64( - _unit_to_nanoseconds_conversion["m"], "ns" - ) - ) - ) - // cudf.Scalar( - np.timedelta64(_unit_to_nanoseconds_conversion["s"], "ns") - ), - "milliseconds": ( - self - % cudf.Scalar( - np.timedelta64( - _unit_to_nanoseconds_conversion["s"], "ns" - ) - ) + date_meta = { + "seconds": ["m", "s"], + "milliseconds": ["s", "ms"], + "microseconds": ["ms", "us"], + "nanoseconds": ["us", "ns"], + } + data = { + "days": self + // cudf.Scalar( + np.timedelta64( + _unit_to_nanoseconds_conversion["D"], "ns" + ).astype(self.dtype) + ), + "hours": ( + self + % cudf.Scalar( + np.timedelta64( + _unit_to_nanoseconds_conversion["D"], "ns" + ).astype(self.dtype) ) - // cudf.Scalar( - np.timedelta64(_unit_to_nanoseconds_conversion["ms"], "ns") - ), - "microseconds": ( - self - % cudf.Scalar( - np.timedelta64( - _unit_to_nanoseconds_conversion["ms"], "ns" - ) - ) + ) + // cudf.Scalar( + np.timedelta64( + _unit_to_nanoseconds_conversion["h"], "ns" + ).astype(self.dtype) + ), + "minutes": ( + self + % cudf.Scalar( + np.timedelta64( + _unit_to_nanoseconds_conversion["h"], "ns" + ).astype(self.dtype) ) - // cudf.Scalar( - np.timedelta64(_unit_to_nanoseconds_conversion["us"], "ns") - ), - "nanoseconds": ( - self - % cudf.Scalar( - np.timedelta64( - _unit_to_nanoseconds_conversion["us"], "ns" - ) - ) + ) + // cudf.Scalar( + np.timedelta64( + _unit_to_nanoseconds_conversion["m"], "ns" + ).astype(self.dtype) + ), + } + keys_list = iter(date_meta.keys()) + for name in keys_list: + value = date_meta[name] + data[name] = ( + self + % cudf.Scalar( + np.timedelta64( + _unit_to_nanoseconds_conversion[value[0]], "ns" + ).astype(self.dtype) ) - // cudf.Scalar( - np.timedelta64(_unit_to_nanoseconds_conversion["ns"], "ns") - ), - }, + ) // cudf.Scalar( + np.timedelta64( + _unit_to_nanoseconds_conversion[value[1]], "ns" + ).astype(self.dtype) + ) + if self._time_unit == value[1]: + break + + for name in keys_list: + res_col = cudf.core.column.full(len(self), 0, dtype="int64") + if self.nullable: + res_col = res_col.set_mask(self.mask) + data[name] = res_col + + return cudf.DataFrame( + data=data, index=index, ) @@ -483,7 +490,9 @@ def days(self) -> "cudf.core.column.NumericalColumn": NumericalColumn """ return self // cudf.Scalar( - np.timedelta64(_unit_to_nanoseconds_conversion["D"], "ns") + np.timedelta64(_unit_to_nanoseconds_conversion["D"], "ns").astype( + self.dtype + ) ) @property @@ -503,7 +512,9 @@ def seconds(self) -> "cudf.core.column.NumericalColumn": return ( self % cudf.Scalar( - np.timedelta64(_unit_to_nanoseconds_conversion["D"], "ns") + np.timedelta64( + _unit_to_nanoseconds_conversion["D"], "ns" + ).astype(self.dtype) ) ) // cudf.Scalar( np.timedelta64(_unit_to_nanoseconds_conversion["s"], "ns") @@ -524,7 +535,10 @@ def microseconds(self) -> "cudf.core.column.NumericalColumn": # division operation to extract the number of microseconds. return ( - self % np.timedelta64(_unit_to_nanoseconds_conversion["s"], "ns") + self + % np.timedelta64( + _unit_to_nanoseconds_conversion["s"], "ns" + ).astype(self.dtype) ) // cudf.Scalar( np.timedelta64(_unit_to_nanoseconds_conversion["us"], "ns") ) @@ -544,6 +558,11 @@ def nanoseconds(self) -> "cudf.core.column.NumericalColumn": # performing division operation to extract the number # of nanoseconds. + if self._time_unit != "ns": + res_col = cudf.core.column.full(len(self), 0, dtype="int64") + if self.nullable: + res_col = res_col.set_mask(self.mask) + return cast("cudf.core.column.NumericalColumn", res_col) return ( self % cudf.Scalar( diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 209276215c8..0ce4ccfa00e 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -64,6 +64,7 @@ numeric_normalize_types, ) from cudf.utils.utils import _cudf_nvtx_annotate, search_range +from cudf.core._compat import PANDAS_GE_200 T = TypeVar("T", bound="Frame") @@ -2289,7 +2290,10 @@ def isocalendar(self): @_cudf_nvtx_annotate def to_pandas(self, nullable=False): - nanos = self._values.astype("datetime64[ns]") + if PANDAS_GE_200: + nanos = self._values + else: + nanos = self._values.astype("datetime64[ns]") return pd.DatetimeIndex(nanos.to_pandas(), name=self.name) @_cudf_nvtx_annotate @@ -2507,7 +2511,9 @@ def days(self): """ Number of days for each element. """ - return as_index(arbitrary=self._values.days, name=self.name) + return as_index( + arbitrary=self._values.days, name=self.name, dtype="int32" + ) @property # type: ignore @_cudf_nvtx_annotate @@ -2515,7 +2521,9 @@ def seconds(self): """ Number of seconds (>= 0 and less than 1 day) for each element. """ - return as_index(arbitrary=self._values.seconds, name=self.name) + return as_index( + arbitrary=self._values.seconds, name=self.name, dtype="int32" + ) @property # type: ignore @_cudf_nvtx_annotate @@ -2523,7 +2531,9 @@ def microseconds(self): """ Number of microseconds (>= 0 and less than 1 second) for each element. """ - return as_index(arbitrary=self._values.microseconds, name=self.name) + return as_index( + arbitrary=self._values.microseconds, name=self.name, dtype="int32" + ) @property # type: ignore @_cudf_nvtx_annotate @@ -2532,7 +2542,9 @@ def nanoseconds(self): Number of nanoseconds (>= 0 and less than 1 microsecond) for each element. """ - return as_index(arbitrary=self._values.nanoseconds, name=self.name) + return as_index( + arbitrary=self._values.nanoseconds, name=self.name, dtype="int32" + ) @property # type: ignore @_cudf_nvtx_annotate diff --git a/python/cudf/cudf/core/tools/datetimes.py b/python/cudf/cudf/core/tools/datetimes.py index 92ef49e92d9..e5a03d76721 100644 --- a/python/cudf/cudf/core/tools/datetimes.py +++ b/python/cudf/cudf/core/tools/datetimes.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2022, NVIDIA CORPORATION. +# Copyright (c) 2019-2023, NVIDIA CORPORATION. import math import re @@ -55,7 +55,7 @@ def to_datetime( format=None, exact=True, unit="ns", - infer_datetime_format=False, + infer_datetime_format=True, origin="unix", cache=True, ): @@ -90,7 +90,7 @@ def to_datetime( origin(unix epoch start). Example, with unit='ms' and origin='unix' (the default), this would calculate the number of milliseconds to the unix epoch start. - infer_datetime_format : bool, default False + infer_datetime_format : bool, default True If True and no `format` is given, attempt to infer the format of the datetime strings, and if it can be inferred, switch to a faster method of parsing them. In some cases this can increase the parsing @@ -130,6 +130,12 @@ def to_datetime( f"{['ignore', 'raise', 'coerce', 'warn']}, found: " f"{errors}" ) + if infer_datetime_format in {None, False}: + warnings.warn( + "`infer_datetime_format` is deprecated and will " + "be removed in a future version of cudf.", + FutureWarning, + ) if arg is None: return None diff --git a/python/cudf/cudf/tests/test_binops.py b/python/cudf/cudf/tests/test_binops.py index 7d01f89eada..8b9a25fa865 100644 --- a/python/cudf/cudf/tests/test_binops.py +++ b/python/cudf/cudf/tests/test_binops.py @@ -1721,7 +1721,7 @@ def test_datetime_dateoffset_binaryop( date_col, n_periods, frequency, dtype, op ): gsr = cudf.Series(date_col, dtype=dtype) - psr = gsr.to_pandas() # converts to nanos + psr = gsr.to_pandas() kwargs = {frequency: n_periods} diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 90bc7ad8414..f696ad2fe4d 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -2643,6 +2643,12 @@ def test_from_arrow_chunked_arrays(nelem, nchunks, data_type): pa_chunk_array = pa.chunked_array(np_list_data) expect = pd.Series(pa_chunk_array.to_pandas()) + if cudf.api.types.is_datetime64_dtype( + data_type + ) or cudf.api.types.is_timedelta64_dtype(data_type): + # Workaround for an Arrow Bug: + # https://github.com/apache/arrow/issues/34462 + expect = expect.astype(data_type) got = cudf.Series(pa_chunk_array) assert_eq(expect, got) @@ -2657,6 +2663,12 @@ def test_from_arrow_chunked_arrays(nelem, nchunks, data_type): ) expect = pa_table.to_pandas() + if cudf.api.types.is_datetime64_dtype( + data_type + ) or cudf.api.types.is_timedelta64_dtype(data_type): + # Workaround for an Arrow Bug: + # https://github.com/apache/arrow/issues/34462 + expect = expect.astype(data_type) got = cudf.DataFrame.from_arrow(pa_table) assert_eq(expect, got) @@ -3929,9 +3941,6 @@ def test_all(data): got = gdata.all(bool_only=True) expected = pdata.all(bool_only=True) assert_eq(got, expected) - else: - with pytest.raises(NotImplementedError): - gdata.all(level="a") got = gdata.all() expected = pdata.all() @@ -3990,9 +3999,6 @@ def test_any(data, axis): got = gdata.any(bool_only=True) expected = pdata.any(bool_only=True) assert_eq(got, expected) - else: - with pytest.raises(NotImplementedError): - gdata.any(level="a") got = gdata.any(axis=axis) expected = pdata.any(axis=axis) @@ -5187,7 +5193,6 @@ def test_rowwise_ops_nullable_int_dtypes(op, expected): def test_rowwise_ops_datetime_dtypes(data, op, skipna): gdf = cudf.DataFrame(data) - pdf = gdf.to_pandas() with expect_warning_if( diff --git a/python/cudf/cudf/tests/test_datetime.py b/python/cudf/cudf/tests/test_datetime.py index 1211938ff10..5f76ed81cc8 100644 --- a/python/cudf/cudf/tests/test_datetime.py +++ b/python/cudf/cudf/tests/test_datetime.py @@ -191,8 +191,8 @@ def test_dt_series(data, field): pd_data = pd.Series(data.copy()) gdf_data = Series(pd_data) base = getattr(pd_data.dt, field) - test = getattr(gdf_data.dt, field).to_pandas().astype("int64") - assert_eq(base, test) + test = getattr(gdf_data.dt, field) + assert_eq(base, test, check_dtype=False) @pytest.mark.parametrize("data", [data1(), data2()]) @@ -200,7 +200,7 @@ def test_dt_series(data, field): def test_dt_index(data, field): pd_data = data.copy() gdf_data = DatetimeIndex(pd_data) - assert_eq(getattr(gdf_data, field), getattr(pd_data, field)) + assert_eq(getattr(gdf_data, field), getattr(pd_data, field), exact=False) def test_setitem_datetime(): @@ -614,8 +614,7 @@ def test_datetime_dataframe(): ], ) @pytest.mark.parametrize("dayfirst", [True, False]) -@pytest.mark.parametrize("infer_datetime_format", [True, False]) -def test_cudf_to_datetime(data, dayfirst, infer_datetime_format): +def test_cudf_to_datetime(data, dayfirst): pd_data = data if isinstance(pd_data, (pd.Series, pd.DataFrame, pd.Index)): gd_data = cudf.from_pandas(pd_data) @@ -625,14 +624,24 @@ def test_cudf_to_datetime(data, dayfirst, infer_datetime_format): else: gd_data = pd_data - expected = pd.to_datetime( - pd_data, dayfirst=dayfirst, infer_datetime_format=infer_datetime_format - ) - actual = cudf.to_datetime( - gd_data, dayfirst=dayfirst, infer_datetime_format=infer_datetime_format - ) - - assert_eq(actual, expected) + expected = pd.to_datetime(pd_data, dayfirst=dayfirst) + actual = cudf.to_datetime(gd_data, dayfirst=dayfirst) + + # TODO: Remove typecast to `ns` and following if/else + # workaround after following issue is fixed: + # https://github.com/pandas-dev/pandas/issues/52449 + + if actual is not None and expected is not None: + assert_eq( + actual.astype(pd_data.dtype) + if pd_data is not None + and hasattr(pd_data, "dtype") + and cudf.api.types.is_datetime_dtype(pd_data.dtype) + else actual.astype("datetime64[ns]"), + expected, + ) + else: + assert_eq(actual, expected) @pytest.mark.parametrize( @@ -722,7 +731,11 @@ def test_to_datetime_units(data, unit): expected = pd.to_datetime(pd_data, unit=unit) actual = cudf.to_datetime(gd_data, unit=unit) - assert_eq(actual, expected) + # TODO: Remove typecast to `ns` after following + # issue is fixed: + # https://github.com/pandas-dev/pandas/issues/52449 + + assert_eq(actual.astype("datetime64[ns]"), expected) @pytest.mark.parametrize( @@ -896,6 +909,7 @@ def test_str_to_datetime_error(): np.datetime64("2005-02-25"), np.datetime64("2005-02-25T03:30"), np.datetime64("nat"), + # TODO: https://github.com/pandas-dev/pandas/issues/52295 ], ) @pytest.mark.parametrize("data_dtype", DATETIME_TYPES) diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index aad163736c2..fe692a87ca8 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -20,7 +20,7 @@ from pyarrow import fs as pa_fs, parquet as pq import cudf -from cudf.core._compat import PANDAS_LT_153 +from cudf.core._compat import PANDAS_LT_153, PANDAS_GE_200 from cudf.io.parquet import ( ParquetDatasetWriter, ParquetWriter, @@ -286,6 +286,16 @@ def test_parquet_reader_basic(parquet_file, columns, engine): if "col_category" in got.columns: got = got.drop(columns=["col_category"]) + if PANDAS_GE_200 and columns is None: + # https://github.com/pandas-dev/pandas/issues/52412 + assert expect["col_datetime64[ms]"].dtype == np.dtype("datetime64[ns]") + assert expect["col_datetime64[us]"].dtype == np.dtype("datetime64[ns]") + expect["col_datetime64[ms]"] = expect["col_datetime64[ms]"].astype( + "datetime64[ms]" + ) + expect["col_datetime64[us]"] = expect["col_datetime64[us]"].astype( + "datetime64[us]" + ) assert_eq(expect, got, check_categorical=False) @@ -1432,7 +1442,16 @@ def test_parquet_writer_int96_timestamps(tmpdir, pdf, gdf): expect = pdf got = pd.read_parquet(gdf_fname) - + if PANDAS_GE_200: + # https://github.com/pandas-dev/pandas/issues/52412 + assert got["col_datetime64[ms]"].dtype == np.dtype("datetime64[ns]") + assert got["col_datetime64[us]"].dtype == np.dtype("datetime64[ns]") + got["col_datetime64[ms]"] = got["col_datetime64[ms]"].astype( + "datetime64[ms]" + ) + got["col_datetime64[us]"] = got["col_datetime64[us]"].astype( + "datetime64[us]" + ) # verify INT96 timestamps were converted back to the same data. assert_eq(expect, got, check_categorical=False) diff --git a/python/cudf/cudf/tests/test_timedelta.py b/python/cudf/cudf/tests/test_timedelta.py index 4b1e8cf1027..7f501373be3 100644 --- a/python/cudf/cudf/tests/test_timedelta.py +++ b/python/cudf/cudf/tests/test_timedelta.py @@ -11,6 +11,7 @@ import cudf from cudf.testing import _utils as utils from cudf.testing._utils import assert_eq, assert_exceptions_equal +from cudf.core._compat import PANDAS_GE_200 _TIMEDELTA_DATA = [ [1000000, 200000, 3000000], @@ -528,7 +529,13 @@ def test_timedelta_series_mod_with_scalar_zero(reverse): datetime.timedelta(seconds=768), datetime.timedelta(microseconds=7), np.timedelta64(4, "s"), - np.timedelta64("nat", "s"), + pytest.param( + np.timedelta64("nat", "s"), + marks=pytest.mark.xfail( + strict=False, + reason="https://github.com/pandas-dev/pandas/issues/52295", + ), + ), np.timedelta64(1, "s"), np.timedelta64(1, "ms"), np.timedelta64(1, "us"), @@ -686,38 +693,41 @@ def test_timedelta_dt_components(data, dtype): @pytest.mark.parametrize( "data", - _TIMEDELTA_DATA, + _TIMEDELTA_DATA_NON_OVERFLOW, + # TODO-PANDAS-2.0: Replace above with `_TIMEDELTA_DATA` + # after the following issue is fixed: + # https://github.com/pandas-dev/pandas/issues/52386 ) @pytest.mark.parametrize("dtype", utils.TIMEDELTA_TYPES) def test_timedelta_dt_properties(data, dtype): gsr = cudf.Series(data, dtype=dtype) psr = gsr.to_pandas() - def local_assert(expected, actual): + def local_assert(expected, actual, **kwargs): if gsr.isnull().any(): - assert_eq(expected, actual.astype("float")) + assert_eq(expected, actual.astype("float"), **kwargs) else: - assert_eq(expected, actual) + assert_eq(expected, actual, **kwargs) expected_days = psr.dt.days actual_days = gsr.dt.days - local_assert(expected_days, actual_days) + local_assert(expected_days, actual_days, check_dtype=False) expected_seconds = psr.dt.seconds actual_seconds = gsr.dt.seconds - local_assert(expected_seconds, actual_seconds) + local_assert(expected_seconds, actual_seconds, check_dtype=False) expected_microseconds = psr.dt.microseconds actual_microseconds = gsr.dt.microseconds - local_assert(expected_microseconds, actual_microseconds) + local_assert(expected_microseconds, actual_microseconds, check_dtype=False) expected_nanoseconds = psr.dt.nanoseconds actual_nanoseconds = gsr.dt.nanoseconds - local_assert(expected_nanoseconds, actual_nanoseconds) + local_assert(expected_nanoseconds, actual_nanoseconds, check_dtype=False) @pytest.mark.parametrize( @@ -1315,7 +1325,11 @@ def test_numeric_to_timedelta(data, dtype, timedelta_dtype): psr = sr.to_pandas() actual = sr.astype(timedelta_dtype) - expected = pd.Series(psr.to_numpy().astype(timedelta_dtype)) + + if PANDAS_GE_200: + expected = psr.astype(timedelta_dtype) + else: + expected = pd.Series(psr.to_numpy().astype(timedelta_dtype)) assert_eq(expected, actual) diff --git a/python/cudf/cudf/tests/test_udf_masked_ops.py b/python/cudf/cudf/tests/test_udf_masked_ops.py index ab0205df677..3c827b4f242 100644 --- a/python/cudf/cudf/tests/test_udf_masked_ops.py +++ b/python/cudf/cudf/tests/test_udf_masked_ops.py @@ -184,7 +184,18 @@ def func(row): ) gdf["a"] = gdf["a"].astype(dtype_l) gdf["b"] = gdf["b"].astype(dtype_r) - run_masked_udf_test(func, gdf, check_dtype=False) + + pdf = gdf.to_pandas(nullable=True) + + expect = op(pdf["a"], pdf["b"]) + obtain = gdf.apply(func, axis=1) + assert_eq(expect, obtain, check_dtype=False) + # TODO: After the following pandas issue is + # fixed, uncomment the following line and delete + # through `to_pandas(nullable=True)` statement. + # https://github.com/pandas-dev/pandas/issues/52411 + + # run_masked_udf_test(func, gdf, check_dtype=False) @pytest.mark.parametrize("op", comparison_ops) diff --git a/python/cudf/cudf/utils/dtypes.py b/python/cudf/cudf/utils/dtypes.py index c7a8c8b4096..9fbc099b1a1 100644 --- a/python/cudf/cudf/utils/dtypes.py +++ b/python/cudf/cudf/utils/dtypes.py @@ -269,14 +269,14 @@ def to_cudf_compatible_scalar(val, dtype=None): # the string value directly (cudf.DeviceScalar will DTRT) return val - if isinstance(val, datetime.datetime): - val = np.datetime64(val) - elif isinstance(val, datetime.timedelta): - val = np.timedelta64(val) - elif isinstance(val, pd.Timestamp): + if isinstance(val, pd.Timestamp): val = val.to_datetime64() elif isinstance(val, pd.Timedelta): val = val.to_timedelta64() + elif isinstance(val, datetime.datetime): + val = np.datetime64(val) + elif isinstance(val, datetime.timedelta): + val = np.timedelta64(val) val = _maybe_convert_to_default_type( cudf.api.types.pandas_dtype(type(val)) From 60c257aad843bcd262f42d116a908e06de3aac32 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 7 Apr 2023 16:15:44 -0500 Subject: [PATCH 016/384] Fix MultiIndex construction in pandas 2.0 (#13092) This PR removes a `MultiIndex` construction workaround that retains correct dtypes of each level. Thus fixing 19 pytests: ```bash = 907 failed, 86196 passed, 2034 skipped, 992 xfailed, 165 xpassed in 536.13s (0:08:56) = ``` On `pandas_2.0_feature_branch`: ```bash = 926 failed, 86177 passed, 2034 skipped, 992 xfailed, 165 xpassed in 545.17s (0:09:05) = ``` --- python/cudf/cudf/core/column_accessor.py | 40 +++++++++++++----------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/python/cudf/cudf/core/column_accessor.py b/python/cudf/cudf/core/column_accessor.py index 707eda3f5e6..d1bfa4dd55d 100644 --- a/python/cudf/cudf/core/column_accessor.py +++ b/python/cudf/cudf/core/column_accessor.py @@ -18,10 +18,10 @@ ) import pandas as pd -from packaging.version import Version import cudf from cudf.core import column +from cudf.core._compat import PANDAS_GE_200 if TYPE_CHECKING: from cudf.core.column import ColumnBase @@ -247,24 +247,28 @@ def _clear_cache(self): def to_pandas_index(self) -> pd.Index: """Convert the keys of the ColumnAccessor to a Pandas Index object.""" if self.multiindex and len(self.level_names) > 0: - # Using `from_frame()` instead of `from_tuples` - # prevents coercion of values to a different type - # (e.g., ''->NaT) - with warnings.catch_warnings(): - # Specifying `dtype="object"` here and passing that to - # `from_frame` is deprecated in pandas, but we cannot remove - # that without also losing compatibility with other current - # pandas behaviors like the NaT inference above. For now we - # must catch the warnings internally, but we will need to - # remove this when we implement compatibility with pandas 2.0, - # which will remove these compatibility layers. - assert Version(pd.__version__) < Version("2.0.0") - warnings.simplefilter("ignore") - result = pd.MultiIndex.from_frame( - pd.DataFrame( - self.names, columns=self.level_names, dtype="object" - ), + if PANDAS_GE_200: + result = pd.MultiIndex.from_tuples( + self.names, + names=self.level_names, ) + else: + # Using `from_frame()` instead of `from_tuples` + # prevents coercion of values to a different type + # (e.g., ''->NaT) + with warnings.catch_warnings(): + # Specifying `dtype="object"` here and passing that to + # `from_frame` is deprecated in pandas, but we cannot + # remove that without also losing compatibility with other + # current pandas behaviors like the NaT inference above. + warnings.simplefilter("ignore") + result = pd.MultiIndex.from_frame( + pd.DataFrame( + self.names, + columns=self.level_names, + dtype="object", + ), + ) else: result = pd.Index(self.names, name=self.name, tupleize_cols=False) return result From be199680e1bbdfea789e3d44400ab8b1b1940588 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 13 Apr 2023 20:13:14 -0500 Subject: [PATCH 017/384] [REVIEW] Enable `numeric_only` for row-wise ops (#13090) This PR enables numeric_only for row-wise ops to be on parity with pandas-2.0. --- python/cudf/cudf/core/dataframe.py | 38 ++---- python/cudf/cudf/core/frame.py | 4 +- python/cudf/cudf/tests/test_array_ufunc.py | 2 +- python/cudf/cudf/tests/test_dataframe.py | 151 ++++++++------------- 4 files changed, 77 insertions(+), 118 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 416a2047fb2..d9f9f65a9d1 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -5738,7 +5738,7 @@ def make_false_column_like_self(): # Stats # @_cudf_nvtx_annotate - def _prepare_for_rowwise_op(self, method, skipna): + def _prepare_for_rowwise_op(self, method, skipna, numeric_only): """Prepare a DataFrame for CuPy-based row-wise operations.""" if method not in _cupy_nan_methods_map and any( @@ -5752,26 +5752,23 @@ def _prepare_for_rowwise_op(self, method, skipna): ) raise ValueError(msg) - is_pure_dt = all(is_datetime_dtype(dt) for dt in self.dtypes) - - if not is_pure_dt: + if numeric_only: filtered = self.select_dtypes(include=[np.number, np.bool_]) else: filtered = self.copy(deep=False) - common_dtype = find_common_type(filtered.dtypes) + is_pure_dt = all(is_datetime_dtype(dt) for dt in filtered.dtypes) - if filtered._num_columns < self._num_columns: - # When we update our pandas compatibility target to 2.0, pandas - # will stop supporting numeric_only=None and users will have to - # specify True/False. At that time we should also top our implicit - # removal of non-numeric columns here. - assert Version(pd.__version__) < Version("2.0.0") - msg = ( - "Row-wise operations currently only support int, float " - "and bool dtypes. Non numeric columns are ignored." + common_dtype = find_common_type(filtered.dtypes) + if ( + not numeric_only + and is_string_dtype(common_dtype) + and any(not is_string_dtype(dt) for dt in filtered.dtypes) + ): + raise TypeError( + f"Cannot perform row-wise {method} across mixed-dtype columns," + " try type-casting all the columns to same dtype." ) - warnings.warn(msg) if not skipna and any(col.nullable for col in filtered._columns): mask = DataFrame( @@ -5857,7 +5854,7 @@ def _reduce( ) source = self._get_columns_by_label(numeric_cols) if source.empty: - return Series(index=cudf.StringIndex([])) + return Series(index=self.index) axis = source._get_axis_from_axis_arg(axis) @@ -6063,12 +6060,7 @@ def _apply_cupy_method_axis_1(self, method, *args, **kwargs): "Row-wise operations currently do not support `level`." ) - numeric_only = kwargs.pop("numeric_only", None) - if numeric_only not in (None, True): - raise NotImplementedError( - "Row-wise operations currently do not " - "support `numeric_only=False`." - ) + numeric_only = kwargs.pop("numeric_only", False) min_count = kwargs.pop("min_count", None) if min_count not in (None, 0): @@ -6088,7 +6080,7 @@ def _apply_cupy_method_axis_1(self, method, *args, **kwargs): kwargs.pop("cast_to_int", None) prepared, mask, common_dtype = self._prepare_for_rowwise_op( - method, skipna + method, skipna, numeric_only ) for col in prepared._data.names: if prepared._data[col].nullable: diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index a07bc60922d..03ac9b6688b 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -1945,7 +1945,7 @@ def max( self, axis=0, skipna=True, - numeric_only=None, + numeric_only=False, **kwargs, ): """ @@ -1957,7 +1957,7 @@ def max( Axis for the function to be applied on. skipna: bool, default True Exclude NA/null values when computing the result. - numeric_only: bool, default None + numeric_only: bool, default False If True, includes only float, int, boolean columns. If False, will raise error in-case there are non-numeric columns. diff --git a/python/cudf/cudf/tests/test_array_ufunc.py b/python/cudf/cudf/tests/test_array_ufunc.py index 8f4ae7c23a0..2daf942d4b0 100644 --- a/python/cudf/cudf/tests/test_array_ufunc.py +++ b/python/cudf/cudf/tests/test_array_ufunc.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. import operator import warnings diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index f696ad2fe4d..bed9b81b803 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -4890,7 +4890,9 @@ def test_df_constructor_dtype(dtype): { "a": [1, 2, 3, 4], "b": [7, np.NaN, 9, 10], - "c": [np.NaN, np.NaN, np.NaN, np.NaN], + "c": cudf.Series( + [np.NaN, np.NaN, np.NaN, np.NaN], nan_as_null=False + ), "d": cudf.Series([None, None, None, None], dtype="int64"), "e": [100, None, 200, None], "f": cudf.Series([10, None, np.NaN, 11], nan_as_null=False), @@ -4910,38 +4912,34 @@ def test_df_constructor_dtype(dtype): "op", ["max", "min", "sum", "product", "mean", "var", "std"] ) @pytest.mark.parametrize("skipna", [True, False]) -def test_rowwise_ops(data, op, skipna): +@pytest.mark.parametrize("numeric_only", [True, False]) +def test_rowwise_ops(data, op, skipna, numeric_only): gdf = data pdf = gdf.to_pandas() - kwargs = {"axis": 1, "skipna": skipna} + kwargs = {"axis": 1, "skipna": skipna, "numeric_only": numeric_only} if op in ("var", "std"): kwargs["ddof"] = 0 - with expect_warning_if( - not all( - ( - (pdf[column].count() == 0) - if skipna - else (pdf[column].notna().count() == 0) - ) - or cudf.api.types.is_numeric_dtype(pdf[column].dtype) - or cudf.api.types.is_bool_dtype(pdf[column].dtype) - for column in pdf + if not numeric_only and not all( + ( + (pdf[column].count() == 0) + if skipna + else (pdf[column].notna().count() == 0) ) + or cudf.api.types.is_numeric_dtype(pdf[column].dtype) + or cudf.api.types.is_bool_dtype(pdf[column].dtype) + for column in pdf ): + with pytest.raises(TypeError): + expected = getattr(pdf, op)(**kwargs) + with pytest.raises(TypeError): + got = getattr(gdf, op)(**kwargs) + else: expected = getattr(pdf, op)(**kwargs) - with expect_warning_if( - not all( - cudf.api.types.is_numeric_dtype(gdf[column].dtype) - or cudf.api.types.is_bool_dtype(gdf[column].dtype) - for column in gdf - ), - UserWarning, - ): got = getattr(gdf, op)(**kwargs) - assert_eq(expected, got, check_exact=False) + assert_eq(expected, got, check_dtype=False) @pytest.mark.parametrize( @@ -4971,67 +4969,18 @@ def test_rowwise_ops_nullable_dtypes_all_null(op): @pytest.mark.parametrize( - "op,expected", + "op", [ - ( - "max", - cudf.Series( - [10.0, None, np.NaN, 2234.0, None, np.NaN], - dtype="float64", - nan_as_null=False, - ), - ), - ( - "min", - cudf.Series( - [10.0, None, np.NaN, 13.0, None, np.NaN], - dtype="float64", - nan_as_null=False, - ), - ), - ( - "sum", - cudf.Series( - [20.0, None, np.NaN, 2247.0, None, np.NaN], - dtype="float64", - nan_as_null=False, - ), - ), - ( - "product", - cudf.Series( - [100.0, None, np.NaN, 29042.0, None, np.NaN], - dtype="float64", - nan_as_null=False, - ), - ), - ( - "mean", - cudf.Series( - [10.0, None, np.NaN, 1123.5, None, np.NaN], - dtype="float64", - nan_as_null=False, - ), - ), - ( - "var", - cudf.Series( - [0.0, None, np.NaN, 1233210.25, None, np.NaN], - dtype="float64", - nan_as_null=False, - ), - ), - ( - "std", - cudf.Series( - [0.0, None, np.NaN, 1110.5, None, np.NaN], - dtype="float64", - nan_as_null=False, - ), - ), + "max", + "min", + "sum", + "product", + "mean", + "var", + "std", ], ) -def test_rowwise_ops_nullable_dtypes_partial_null(op, expected): +def test_rowwise_ops_nullable_dtypes_partial_null(op): gdf = cudf.DataFrame( { "a": [10, 11, 12, 13, 14, 15], @@ -5044,10 +4993,12 @@ def test_rowwise_ops_nullable_dtypes_partial_null(op, expected): if op in ("var", "std"): got = getattr(gdf, op)(axis=1, ddof=0, skipna=False) + expected = getattr(gdf.to_pandas(), op)(axis=1, ddof=0, skipna=False) else: got = getattr(gdf, op)(axis=1, skipna=False) + expected = getattr(gdf.to_pandas(), op)(axis=1, skipna=False) - assert_eq(got.null_count, expected.null_count) + assert_eq(got.null_count, 2) assert_eq(got, expected) @@ -5190,23 +5141,39 @@ def test_rowwise_ops_nullable_int_dtypes(op, expected): ) @pytest.mark.parametrize("op", ["max", "min"]) @pytest.mark.parametrize("skipna", [True, False]) -def test_rowwise_ops_datetime_dtypes(data, op, skipna): +@pytest.mark.parametrize("numeric_only", [True, False]) +def test_rowwise_ops_datetime_dtypes(data, op, skipna, numeric_only): gdf = cudf.DataFrame(data) pdf = gdf.to_pandas() - with expect_warning_if( - not all(cudf.api.types.is_datetime64_dtype(dt) for dt in gdf.dtypes), - UserWarning, - ): - got = getattr(gdf, op)(axis=1, skipna=skipna) - with expect_warning_if( - not all(pd.api.types.is_datetime64_dtype(dt) for dt in gdf.dtypes), - FutureWarning, + if not numeric_only and not all( + cudf.api.types.is_datetime64_dtype(dt) for dt in gdf.dtypes ): - expected = getattr(pdf, op)(axis=1, skipna=skipna) + with pytest.raises(TypeError): + got = getattr(gdf, op)( + axis=1, skipna=skipna, numeric_only=numeric_only + ) + with pytest.raises(TypeError): + expected = getattr(pdf, op)( + axis=1, skipna=skipna, numeric_only=numeric_only + ) + else: + got = getattr(gdf, op)( + axis=1, skipna=skipna, numeric_only=numeric_only + ) + expected = getattr(pdf, op)( + axis=1, skipna=skipna, numeric_only=numeric_only + ) + if got.dtype == cudf.dtype( + "datetime64[us]" + ) and expected.dtype == np.dtype("datetime64[ns]"): + # Workaround for a PANDAS-BUG: + # https://github.com/pandas-dev/pandas/issues/52524 + assert_eq(got.astype("datetime64[ns]"), expected) + else: - assert_eq(got, expected) + assert_eq(got, expected, check_dtype=False) @pytest.mark.parametrize( From 8ff4861685087996b2fd411f2224c0df26687595 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 14 Apr 2023 21:46:14 -0500 Subject: [PATCH 018/384] [REVIEW] Fix `DataFrame.__getitem__` to work with `pandas-2.0` (#13139) This PR updates `DataFrame.__getitem__` to be able to work with pandas-2.0. For which, we conditionally pass `dtype` to `pandas.Series` constructor so that we don't get a warning in `<2.0` versions. This PR also fixes 76 pytests: ``` = 907 failed, 86353 passed, 2034 skipped, 992 xfailed, 165 xpassed in 504.93s (0:08:24) = ``` on `pandas_2.0_feature_branch`: ``` = 983 failed, 86277 passed, 2034 skipped, 992 xfailed, 165 xpassed in 515.47s (0:08:35) = ``` --- python/cudf/cudf/core/dataframe.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index efb3def1eac..760fcef826c 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -32,7 +32,6 @@ import pandas as pd import pyarrow as pa from nvtx import annotate -from packaging.version import Version from pandas._config import get_option from pandas.core.dtypes.common import is_float, is_integer from pandas.io.formats import console @@ -104,6 +103,7 @@ _cudf_nvtx_annotate, _external_only_api, ) +from cudf.core._compat import PANDAS_GE_200 T = TypeVar("T", bound="DataFrame") @@ -1167,13 +1167,13 @@ def __getitem__(self, arg): elif can_convert_to_column(arg): mask = arg if is_list_like(mask): - # An explicit dtype is needed to avoid pandas warnings from - # empty sets of columns. This shouldn't be needed in pandas - # 2.0, we don't need to specify a dtype when we know we're not - # trying to match any columns so the default is fine. dtype = None - if len(mask) == 0: - assert Version(pd.__version__) < Version("2.0.0") + if len(mask) == 0 and not PANDAS_GE_200: + # An explicit dtype is needed to avoid pandas + # warnings from empty sets of columns. This + # shouldn't be needed in pandas 2.0, we don't + # need to specify a dtype when we know we're not + # trying to match any columns so the default is fine. dtype = "float64" mask = pd.Series(mask, dtype=dtype) if mask.dtype == "bool": From bd38d702955331450e81be1ee83f982c15d04f77 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 18 Apr 2023 10:05:35 -0500 Subject: [PATCH 019/384] Drop backfill and pad in GroupBy (#13156) This PR drops support for `pad` and `backfill` in `GroupBy`. This PR: ``` = 881 failed, 86383 passed, 2034 skipped, 956 xfailed, 165 xpassed in 522.05s (0:08:42) = ``` On `pandas_2.0_feature_branch`: ``` = 911 failed, 86389 passed, 2034 skipped, 956 xfailed, 165 xpassed in 521.12s (0:08:41) = ``` --- docs/cudf/source/api_docs/groupby.rst | 2 - python/cudf/cudf/core/groupby/groupby.py | 56 +++--------------------- python/cudf/cudf/tests/test_groupby.py | 21 ++++----- 3 files changed, 14 insertions(+), 65 deletions(-) diff --git a/docs/cudf/source/api_docs/groupby.rst b/docs/cudf/source/api_docs/groupby.rst index 550a0ef1c89..26dd9bb354b 100644 --- a/docs/cudf/source/api_docs/groupby.rst +++ b/docs/cudf/source/api_docs/groupby.rst @@ -42,7 +42,6 @@ Computations / descriptive stats :toctree: api/ GroupBy.bfill - GroupBy.backfill GroupBy.count GroupBy.cumcount GroupBy.cummax @@ -63,7 +62,6 @@ Computations / descriptive stats GroupBy.ngroup GroupBy.nth GroupBy.nunique - GroupBy.pad GroupBy.prod GroupBy.shift GroupBy.size diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 5e98db0d575..d137651679d 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -1945,28 +1945,6 @@ def _scan_fill(self, method: str, limit: int) -> DataFrameOrSeries: result = self._mimic_pandas_order(result) return result._copy_type_metadata(values) - def pad(self, limit=None): - """Forward fill NA values. - - .. deprecated:: 23.06 - `pad` is deprecated, use `ffill` instead. - - Parameters - ---------- - limit : int, default None - Unsupported - """ - - if limit is not None: - raise NotImplementedError("Does not support limit param yet.") - - warnings.warn( - "pad is deprecated and will be removed in a future version. " - "Use ffill instead.", - FutureWarning, - ) - return self._scan_fill("ffill", limit) - def ffill(self, limit=None): """Forward fill NA values. @@ -1981,27 +1959,6 @@ def ffill(self, limit=None): return self._scan_fill("ffill", limit) - def backfill(self, limit=None): - """Backward fill NA values. - - .. deprecated:: 23.06 - `backfill` is deprecated, use `bfill` instead. - - Parameters - ---------- - limit : int, default None - Unsupported - """ - if limit is not None: - raise NotImplementedError("Does not support limit param yet.") - - warnings.warn( - "backfill is deprecated and will be removed in a future version. " - "Use bfill instead.", - FutureWarning, - ) - return self._scan_fill("bfill", limit) - def bfill(self, limit=None): """Backward fill NA values. @@ -2030,11 +1987,11 @@ def fillna( ---------- value : scalar, dict Value to use to fill the holes. Cannot be specified with method. - method : {'backfill', 'bfill', 'pad', 'ffill', None}, default None + method : { 'bfill', 'ffill', None}, default None Method to use for filling holes in reindexed Series - - pad/ffill: propagate last valid observation forward to next valid - - backfill/bfill: use next valid observation to fill gap + - ffill: propagate last valid observation forward to next valid + - bfill: use next valid observation to fill gap axis : {0 or 'index', 1 or 'columns'} Unsupported inplace : bool, default False @@ -2064,11 +2021,8 @@ def fillna( raise ValueError("Cannot specify both 'value' and 'method'.") if method is not None: - if method not in {"pad", "ffill", "backfill", "bfill"}: - raise ValueError( - "Method can only be of 'pad', 'ffill'," - "'backfill', 'bfill'." - ) + if method not in {"ffill", "bfill"}: + raise ValueError("Method can only be of 'ffill', 'bfill'.") return getattr(self, method, limit)() values = self.obj.__class__._from_data( diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index 24db7008804..472a3fa5976 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -2499,7 +2499,7 @@ def test_groupby_various_by_fillna(by, data, args): @pytest.mark.parametrize("nelem", [10, 100, 1000]) -@pytest.mark.parametrize("method", ["pad", "ffill", "backfill", "bfill"]) +@pytest.mark.parametrize("method", ["ffill", "bfill"]) def test_groupby_fillna_method(nelem, method): t = rand_dataframe( dtypes_meta=[ @@ -2538,8 +2538,7 @@ def test_groupby_fillna_method(nelem, method): gdf = cudf.from_pandas(pdf) expect = pdf.groupby(key_col).fillna(method=method) - with expect_warning_if(method in {"pad", "backfill"}): - got = gdf.groupby(key_col).fillna(method=method) + got = gdf.groupby(key_col).fillna(method=method) assert_groupby_results_equal( expect[value_cols], got[value_cols], sort=False @@ -2879,19 +2878,17 @@ def test_groupby_transform_maintain_index(by): ], ) @pytest.mark.parametrize("periods", [-5, -2, 0, 2, 5]) -@pytest.mark.parametrize("fill_method", ["ffill", "bfill", "pad", "backfill"]) +@pytest.mark.parametrize("fill_method", ["ffill", "bfill"]) def test_groupby_pct_change(data, gkey, periods, fill_method): gdf = cudf.DataFrame(data) pdf = gdf.to_pandas() - with expect_warning_if(fill_method in ("pad", "backfill")): - actual = gdf.groupby(gkey).pct_change( - periods=periods, fill_method=fill_method - ) - with expect_warning_if(fill_method in ("pad", "backfill")): - expected = pdf.groupby(gkey).pct_change( - periods=periods, fill_method=fill_method - ) + actual = gdf.groupby(gkey).pct_change( + periods=periods, fill_method=fill_method + ) + expected = pdf.groupby(gkey).pct_change( + periods=periods, fill_method=fill_method + ) assert_eq(expected, actual) From 81565cfa490348892b7e883faf78a7000fa2ada0 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 18 Apr 2023 10:06:47 -0500 Subject: [PATCH 020/384] [REVIEW] Add `no_default` and adapt `Series.reset_index` to differentiate `None` for `name` parameter (#13152) In `pandas-2.0` the behavior for `name` parameter has changed to actually name a column `0` if no value is passed to `name`. But if `name=None`, the column will be named `None` too: ```python In [1]: import pandas as pd In [2]: s = pd.Series([10, 11, 23], index=[2, 3, 5]) In [3]: s Out[3]: 2 10 3 11 5 23 dtype: int64 In [4]: s.reset_index() Out[4]: index 0 0 2 10 1 3 11 2 5 23 In [5]: s.reset_index(name=None) Out[5]: index None 0 2 10 1 3 11 2 5 23 ``` To achieve the same behavior in `cudf`, we had to introduce `no_default` value(which is same as pandas's `no_default` value). This also fixes 18 pytests: ``` = 965 failed, 86325 passed, 2044 skipped, 992 xfailed, 165 xpassed in 508.32s (0:08:28) = ``` On `pandas_2.0_feature_branch`: ``` = 983 failed, 86277 passed, 2034 skipped, 992 xfailed, 165 xpassed in 541.87s (0:09:01) = ``` --- python/cudf/cudf/api/extensions/__init__.py | 4 +++- python/cudf/cudf/core/series.py | 7 +++++-- python/cudf/cudf/tests/test_series.py | 17 ++++++----------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/python/cudf/cudf/api/extensions/__init__.py b/python/cudf/cudf/api/extensions/__init__.py index eeb5dcdb32a..c51fa5dc7ca 100644 --- a/python/cudf/cudf/api/extensions/__init__.py +++ b/python/cudf/cudf/api/extensions/__init__.py @@ -1,12 +1,14 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. from cudf.api.extensions.accessor import ( register_dataframe_accessor, register_index_accessor, register_series_accessor, ) +from pandas.api.extensions import no_default __all__ = [ + "no_default", "register_dataframe_accessor", "register_index_accessor", "register_series_accessor", diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index d1a5922f0a5..8fafa97bd47 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -79,6 +79,7 @@ to_cudf_compatible_scalar, ) from cudf.utils.utils import _cudf_nvtx_annotate +from cudf.api.extensions import no_default def _format_percentile_names(percentiles): @@ -996,7 +997,9 @@ def reindex(self, *args, **kwargs): """, ) ) - def reset_index(self, level=None, drop=False, name=None, inplace=False): + def reset_index( + self, level=None, drop=False, name=no_default, inplace=False + ): if not drop and inplace: raise TypeError( "Cannot reset_index inplace on a Series " @@ -1004,7 +1007,7 @@ def reset_index(self, level=None, drop=False, name=None, inplace=False): ) data, index = self._reset_index(level=level, drop=drop) if not drop: - if name is None: + if name is no_default: name = 0 if self.name is None else self.name data[name] = data.pop(self.name) return cudf.core.dataframe.DataFrame._from_data(data, index) diff --git a/python/cudf/cudf/tests/test_series.py b/python/cudf/cudf/tests/test_series.py index b9b40b9744c..85e5397c7c1 100644 --- a/python/cudf/cudf/tests/test_series.py +++ b/python/cudf/cudf/tests/test_series.py @@ -23,6 +23,7 @@ expect_warning_if, gen_rand, ) +from cudf.api.extensions import no_default def _series_na_data(): @@ -1407,7 +1408,7 @@ def test_nullable_bool_dtype_series(data, bool_dtype): @pytest.mark.parametrize("level", [None, 0, "l0", 1, ["l0", 1]]) @pytest.mark.parametrize("drop", [True, False]) @pytest.mark.parametrize("original_name", [None, "original_ser"]) -@pytest.mark.parametrize("name", [None, "ser"]) +@pytest.mark.parametrize("name", [None, "ser", no_default]) @pytest.mark.parametrize("inplace", [True, False]) def test_reset_index(level, drop, inplace, original_name, name): midx = pd.MultiIndex.from_tuples( @@ -1422,10 +1423,8 @@ def test_reset_index(level, drop, inplace, original_name, name): "test_reset_index_dup_level_name_exceptions" ) - with expect_warning_if(name is None and not drop): - expect = ps.reset_index( - level=level, drop=drop, name=name, inplace=inplace - ) + expect = ps.reset_index(level=level, drop=drop, name=name, inplace=inplace) + got = gs.reset_index(level=level, drop=drop, name=name, inplace=inplace) if inplace: expect = ps @@ -1450,10 +1449,7 @@ def test_reset_index_dup_level_name(level, drop, inplace, original_name, name): "test_reset_index_dup_level_name_exceptions" ) - with expect_warning_if(name is None and not drop): - expect = ps.reset_index( - level=level, drop=drop, inplace=inplace, name=name - ) + expect = ps.reset_index(level=level, drop=drop, inplace=inplace, name=name) got = gs.reset_index(level=level, drop=drop, inplace=inplace, name=name) if inplace: expect = ps @@ -1479,8 +1475,7 @@ def test_reset_index_named(drop, inplace, original_name, name): "test_reset_index_dup_level_name_exceptions" ) - with expect_warning_if(name is None and not drop): - expect = ps.reset_index(drop=drop, inplace=inplace, name=name) + expect = ps.reset_index(drop=drop, inplace=inplace, name=name) got = gs.reset_index(drop=drop, inplace=inplace, name=name) if inplace: From 199787d25b5e61f135a66bf1fdea9648d796cdae Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 18 Apr 2023 10:50:25 -0500 Subject: [PATCH 021/384] Fix `is_string_dtype` to adapt to `pandas-2.0` changes (#13141) With `pandas-2.0`, `pd.api.types.is_string_dtype(obj)` is going to perform a data-introspection to determine the true dtype of the underlying data. This path won't work for gpu objects, hence this PR adds special handling for GPU objects before we hit `pd.api.types.is_string_dtype(obj)` API. This PR fixes 56 pytests: ``` = 927 failed, 86333 passed, 2034 skipped, 992 xfailed, 165 xpassed in 506.69s (0:08:26) = ``` On `pandas_2.0_feature_branch`: ``` = 983 failed, 86277 passed, 2034 skipped, 992 xfailed, 165 xpassed in 557.07s (0:09:17) = ``` --- python/cudf/cudf/api/types.py | 21 ++++++++++++++------- python/cudf/cudf/tests/test_api_types.py | 7 ++++--- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/python/cudf/cudf/api/types.py b/python/cudf/cudf/api/types.py index 06e383f2275..c112132adb6 100644 --- a/python/cudf/cudf/api/types.py +++ b/python/cudf/cudf/api/types.py @@ -104,13 +104,20 @@ def is_string_dtype(obj): Whether or not the array or dtype is of the string dtype. """ return ( - pd.api.types.is_string_dtype(obj) - # Reject all cudf extension types. - and not is_categorical_dtype(obj) - and not is_decimal_dtype(obj) - and not is_list_dtype(obj) - and not is_struct_dtype(obj) - and not is_interval_dtype(obj) + ( + isinstance(obj, (cudf.Index, cudf.Series)) + and obj.dtype == cudf.dtype("O") + ) + or (isinstance(obj, cudf.core.column.StringColumn)) + or ( + pd.api.types.is_string_dtype(obj) + # Reject all cudf extension types. + and not is_categorical_dtype(obj) + and not is_decimal_dtype(obj) + and not is_list_dtype(obj) + and not is_struct_dtype(obj) + and not is_interval_dtype(obj) + ) ) diff --git a/python/cudf/cudf/tests/test_api_types.py b/python/cudf/cudf/tests/test_api_types.py index c2cd78f88a0..04c2aa0b263 100644 --- a/python/cudf/cudf/tests/test_api_types.py +++ b/python/cudf/cudf/tests/test_api_types.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2022, NVIDIA CORPORATION. +# Copyright (c) 2018-2023, NVIDIA CORPORATION. import numpy as np import pandas as pd @@ -6,6 +6,7 @@ from pandas.api import types as pd_types import cudf +from cudf.core._compat import PANDAS_GE_200 from cudf.api import types @@ -497,8 +498,8 @@ def test_is_integer(obj, expect): (pd.Series(dtype="int"), False), (pd.Series(dtype="float"), False), (pd.Series(dtype="complex"), False), - (pd.Series(dtype="str"), True), - (pd.Series(dtype="unicode"), True), + (pd.Series(dtype="str"), not PANDAS_GE_200), + (pd.Series(dtype="unicode"), not PANDAS_GE_200), (pd.Series(dtype="datetime64[s]"), False), (pd.Series(dtype="timedelta64[s]"), False), (pd.Series(dtype="category"), False), From 47492da6d930b5ae80a45b4422ea422fa3c03621 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 18 Apr 2023 19:37:35 -0500 Subject: [PATCH 022/384] Handle pandas warnings for pad and backfill (#13168) This PR adds pytest handling for wanring incase of pad and backfill. --- python/cudf/cudf/tests/test_dataframe.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index bb544343a74..4af24434cfb 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -7572,7 +7572,8 @@ def test_dataframe_concat_dataframe_lists(df, other, sort, ignore_index): def test_dataframe_bfill(df, alias): gdf = cudf.from_pandas(df) - actual = getattr(df, alias)() + with expect_warning_if(PANDAS_GE_200 and alias == "backfill"): + actual = getattr(df, alias)() with expect_warning_if(alias == "backfill"): expected = getattr(gdf, alias)() assert_eq(expected, actual) @@ -7589,7 +7590,8 @@ def test_dataframe_bfill(df, alias): def test_dataframe_ffill(df, alias): gdf = cudf.from_pandas(df) - actual = getattr(df, alias)() + with expect_warning_if(PANDAS_GE_200 and alias == "pad"): + actual = getattr(df, alias)() with expect_warning_if(alias == "pad"): expected = getattr(gdf, alias)() assert_eq(expected, actual) From fbe184863af6170e2d482302e3fd2e12286bb443 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 19 Apr 2023 10:34:18 -0500 Subject: [PATCH 023/384] [REVIEW] Fix datetime pytests & raise errors for timezone un-aware typecasting (#13164) This PR fixes some of the `to_datetime` related pytests and also raises error while constructing a time-zone un-aware type to datetime types. This PR fixes 62 pytests: ``` = 745 failed, 87877 passed, 2044 skipped, 956 xfailed, 165 xpassed in 492.06s (0:08:12) = ``` On `pandas_2.0_feature_branch`: ``` = 807 failed, 87819 passed, 2044 skipped, 956 xfailed, 165 xpassed in 488.43s (0:08:08) = ``` --- python/cudf/cudf/core/column/datetime.py | 14 ++++-- python/cudf/cudf/core/column/string.py | 4 ++ python/cudf/cudf/tests/test_datetime.py | 60 +++++++++++++++--------- 3 files changed, 51 insertions(+), 27 deletions(-) diff --git a/python/cudf/cudf/core/column/datetime.py b/python/cudf/cudf/core/column/datetime.py index 107ebfbbcc3..2c49f17f21c 100644 --- a/python/cudf/cudf/core/column/datetime.py +++ b/python/cudf/cudf/core/column/datetime.py @@ -580,11 +580,15 @@ def infer_format(element: str, **kwargs) -> str: raise ValueError("Unable to infer the timestamp format from the data") if len(second_parts) > 1: - # "Z" indicates Zulu time(widely used in aviation) - Which is - # UTC timezone that currently cudf only supports. Having any other - # unsupported timezone will let the code fail below - # with a ValueError. - second_parts.remove("Z") + if "Z" in second_parts: + # "Z" indicates Zulu time(widely used in aviation) - Which is + # UTC timezone that currently cudf only supports. Having any other + # unsupported timezone will let the code fail below + # with a ValueError. + raise NotImplementedError( + "cuDF does not yet support timezone-aware datetimes" + ) + second_part = "".join(second_parts[1:]) if len(second_part) > 1: diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index d9a6c6c4cd6..19e30eeeb89 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -5526,6 +5526,10 @@ def as_datetime_column( self.apply_boolean_mask(self.notnull()).element_indexing(0) ) + if format.endswith("%z"): + raise NotImplementedError( + "cuDF does not yet support timezone-aware datetimes" + ) return self._as_datetime_or_timedelta_column(out_dtype, format) def as_timedelta_column( diff --git a/python/cudf/cudf/tests/test_datetime.py b/python/cudf/cudf/tests/test_datetime.py index 26dd4f69dbd..10b23745fbd 100644 --- a/python/cudf/cudf/tests/test_datetime.py +++ b/python/cudf/cudf/tests/test_datetime.py @@ -10,6 +10,7 @@ import pytest import cudf +import warnings import cudf.testing.dataset_generator as dataset_generator from cudf import DataFrame, Series from cudf.core._compat import PANDAS_GE_150, PANDAS_LT_140 @@ -680,12 +681,14 @@ def test_to_datetime_errors(data): else: gd_data = pd_data - assert_exceptions_equal( - pd.to_datetime, - cudf.to_datetime, - ([pd_data],), - ([gd_data],), - ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + assert_exceptions_equal( + pd.to_datetime, + cudf.to_datetime, + ([pd_data],), + ([gd_data],), + ) def test_to_datetime_not_implemented(): @@ -785,14 +788,19 @@ def test_to_datetime_format(data, format, infer_datetime_format): else: gd_data = pd_data - expected = pd.to_datetime( - pd_data, format=format, infer_datetime_format=infer_datetime_format - ) - actual = cudf.to_datetime( - gd_data, format=format, infer_datetime_format=infer_datetime_format - ) + with expect_warning_if(True, UserWarning): + expected = pd.to_datetime( + pd_data, format=format, infer_datetime_format=infer_datetime_format + ) + with expect_warning_if(not infer_datetime_format): + actual = cudf.to_datetime( + gd_data, format=format, infer_datetime_format=infer_datetime_format + ) + # TODO: Remove typecast to `ns` after following + # issue is fixed: + # https://github.com/pandas-dev/pandas/issues/52449 - assert_eq(actual, expected) + assert_eq(actual.astype("datetime64[ns]"), expected) def test_datetime_can_cast_safely(): @@ -847,7 +855,11 @@ def test_datetime_scalar_timeunit_cast(timeunit): gs = Series(testscalar) ps = pd.Series(testscalar) - assert_eq(ps, gs) + # TODO: Remove typecast to `ns` after following + # issue is fixed: + # https://github.com/pandas-dev/pandas/issues/52449 + + assert_eq(ps, gs.astype("datetime64[ns]")) gdf = DataFrame() gdf["a"] = np.arange(5) @@ -857,6 +869,11 @@ def test_datetime_scalar_timeunit_cast(timeunit): pdf["a"] = np.arange(5) pdf["b"] = testscalar + assert gdf["b"].dtype == cudf.dtype("datetime64[s]") + # TODO: Remove typecast to `ns` after following + # issue is fixed: + # https://github.com/pandas-dev/pandas/issues/52449 + gdf["b"] = gdf["b"].astype("datetime64[ns]") assert_eq(pdf, gdf) @@ -1267,10 +1284,6 @@ def test_datetime_reductions(data, op, dtype): @pytest.mark.parametrize( "data", [ - np.datetime_as_string( - np.arange("2002-10-27T04:30", 4 * 60, 60, dtype="M8[m]"), - timezone="UTC", - ), np.datetime_as_string( np.arange("2002-10-27T04:30", 10 * 60, 1, dtype="M8[m]"), timezone="UTC", @@ -1294,10 +1307,13 @@ def test_datetime_infer_format(data, dtype): sr = cudf.Series(data) psr = pd.Series(data) - expected = psr.astype(dtype) - actual = sr.astype(dtype) - - assert_eq(expected, actual) + assert_exceptions_equal( + lfunc=psr.astype, + rfunc=sr.astype, + lfunc_args_and_kwargs=([], {"dtype": dtype}), + rfunc_args_and_kwargs=([], {"dtype": dtype}), + check_exception_type=False, + ) def test_dateoffset_instance_subclass_check(): From 615828d4e47da49c029b68de4f8357f81773cce1 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 19 Apr 2023 10:39:20 -0500 Subject: [PATCH 024/384] [REVIEW] Fix pytests where empty column indexes are compared (#13166) This PR fixes pytests where empty column object comparisons fail, this is because of the following inconsistency between pandas & cudf: ```python In [1]: import pandas as pd In [2]: import cudf In [3]: pd.DataFrame().columns Out[3]: RangeIndex(start=0, stop=0, step=1) In [4]: cudf.DataFrame().columns Out[4]: Index([], dtype='object') In [5]: pd.DataFrame().columns.dtype Out[5]: dtype('int64') In [6]: cudf.DataFrame().columns.dtype Out[6]: dtype('O') ``` This PR fixes 28 failures: ``` = 779 failed, 87847 passed, 2044 skipped, 956 xfailed, 165 xpassed in 483.17s (0:08:03) = ``` On `pandas_2.0_feature_branch`: ``` = 807 failed, 87819 passed, 2044 skipped, 956 xfailed, 165 xpassed in 488.43s (0:08:08) = ``` --- python/cudf/cudf/tests/test_concat.py | 27 ++++++++++----- python/cudf/cudf/tests/test_dataframe.py | 44 ++++++++++++++++++++---- python/cudf/cudf/tests/test_groupby.py | 12 ++++++- python/cudf/cudf/tests/test_parquet.py | 9 +++-- 4 files changed, 74 insertions(+), 18 deletions(-) diff --git a/python/cudf/cudf/tests/test_concat.py b/python/cudf/cudf/tests/test_concat.py index 910f0b9cf86..925a522399d 100644 --- a/python/cudf/cudf/tests/test_concat.py +++ b/python/cudf/cudf/tests/test_concat.py @@ -8,9 +8,13 @@ import cudf as gd from cudf.api.types import is_categorical_dtype -from cudf.core._compat import PANDAS_GE_150, PANDAS_LT_140 +from cudf.core._compat import PANDAS_GE_150, PANDAS_LT_140, PANDAS_GE_200 from cudf.core.dtypes import Decimal32Dtype, Decimal64Dtype, Decimal128Dtype -from cudf.testing._utils import assert_eq, assert_exceptions_equal +from cudf.testing._utils import ( + assert_eq, + assert_exceptions_equal, + expect_warning_if, +) def make_frames(index=None, nulls="none"): @@ -365,7 +369,7 @@ def test_pandas_concat_compatibility_axis1_eq_index(): ps1 = s1.to_pandas() ps2 = s2.to_pandas() - with pytest.warns(FutureWarning): + with expect_warning_if(not PANDAS_GE_200): assert_exceptions_equal( lfunc=pd.concat, rfunc=gd.concat, @@ -596,7 +600,12 @@ def test_concat_empty_dataframes(df, other, ignore_index): actual[key] = col.fillna(-1) assert_eq(expected, actual, check_dtype=False, check_index_type=True) else: - assert_eq(expected, actual, check_index_type=not gdf.empty) + assert_eq( + expected, + actual, + check_index_type=not gdf.empty, + check_column_type=not PANDAS_GE_200, + ) @pytest.mark.parametrize("ignore_index", [True, False]) @@ -1084,10 +1093,12 @@ def test_concat_join_no_overlapping_columns_empty_df_basic( ignore_index=ignore_index, axis=axis, ) - # TODO: change `check_index_type` to `True` - # after following bug from pandas is fixed: - # https://github.com/pandas-dev/pandas/issues/46675 - assert_eq(expected, actual, check_index_type=False) + assert_eq( + expected, + actual, + check_index_type=PANDAS_GE_150, + check_column_type=not PANDAS_GE_200, + ) @pytest.mark.parametrize("ignore_index", [True, False]) diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 4af24434cfb..66453dd544d 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -6325,12 +6325,22 @@ def test_dataframe_init_1d_list(data, columns): expect = pd.DataFrame(data, columns=columns) actual = cudf.DataFrame(data, columns=columns) - assert_eq(expect, actual, check_index_type=len(data) != 0) + assert_eq( + expect, + actual, + check_index_type=len(data) != 0, + check_column_type=not PANDAS_GE_200 and len(data) == 0, + ) expect = pd.DataFrame(data, columns=None) actual = cudf.DataFrame(data, columns=None) - assert_eq(expect, actual, check_index_type=len(data) != 0) + assert_eq( + expect, + actual, + check_index_type=len(data) != 0, + check_column_type=not PANDAS_GE_200 and len(data) == 0, + ) @pytest.mark.parametrize( @@ -7190,7 +7200,11 @@ def test_dataframe_from_dict_cp_np_arrays( def test_dataframe_keys(df): gdf = cudf.from_pandas(df) - assert_eq(df.keys(), gdf.keys()) + assert_eq( + df.keys(), + gdf.keys(), + exact=not (PANDAS_GE_200 and len(gdf.columns) == 0), + ) @pytest.mark.parametrize( @@ -7662,7 +7676,12 @@ def test_dataframe_concat_lists(df, other, sort, ignore_index): check_column_type=not gdf.empty, ) else: - assert_eq(expected, actual, check_index_type=not gdf.empty) + assert_eq( + expected, + actual, + check_index_type=not gdf.empty, + check_column_type=PANDAS_GE_200 and len(gdf.columns) != 0, + ) def test_dataframe_concat_series_without_name(): @@ -7943,6 +7962,7 @@ def test_dataframe_init_with_columns(data, columns): gdf, check_index_type=len(pdf.index) != 0, check_dtype=not (pdf.empty and len(pdf.columns)), + check_column_type=not PANDAS_GE_200, ) @@ -8023,7 +8043,12 @@ def test_dataframe_init_from_series_list(data, ignore_dtype, columns): check_index_type=True, ) else: - assert_eq(expected, actual, check_index_type=True) + assert_eq( + expected, + actual, + check_index_type=True, + check_column_type=not PANDAS_GE_200, + ) @pytest_unmark_spilling @@ -8114,7 +8139,7 @@ def test_dataframe_init_from_series_list_with_index( actual = actual.sort_index(axis=1) assert_eq(expected.fillna(-1), actual.fillna(-1), check_dtype=False) else: - assert_eq(expected, actual) + assert_eq(expected, actual, check_column_type=not PANDAS_GE_200) @pytest.mark.parametrize( @@ -8715,7 +8740,12 @@ def assert_local_eq(actual, df, expected, host_columns): check_index_type=check_index_type, ) else: - assert_eq(expected, actual, check_index_type=check_index_type) + assert_eq( + expected, + actual, + check_index_type=check_index_type, + check_column_type=not PANDAS_GE_200, + ) gdf = cudf.from_pandas(df) host_columns = ( diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index 472a3fa5976..f5ce7ec95a0 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -18,7 +18,7 @@ import cudf from cudf import DataFrame, Series -from cudf.core._compat import PANDAS_GE_150, PANDAS_LT_140 +from cudf.core._compat import PANDAS_GE_150, PANDAS_LT_140, PANDAS_GE_200 from cudf.core.udf.groupby_typing import SUPPORTED_GROUPBY_NUMPY_TYPES from cudf.core.udf.utils import precompiled from cudf.testing._utils import ( @@ -1971,11 +1971,16 @@ def test_groupby_apply_return_series_dataframe(func, args): ) def test_groupby_no_keys(pdf): gdf = cudf.from_pandas(pdf) + if isinstance(pdf, pd.DataFrame): + kwargs = {"check_column_type": not PANDAS_GE_200} + else: + kwargs = {} assert_groupby_results_equal( pdf.groupby([]).max(), gdf.groupby([]).max(), check_dtype=False, check_index_type=False, # Int64Index v/s Float64Index + **kwargs, ) @@ -1985,10 +1990,15 @@ def test_groupby_no_keys(pdf): ) def test_groupby_apply_no_keys(pdf): gdf = cudf.from_pandas(pdf) + if isinstance(pdf, pd.DataFrame): + kwargs = {"check_column_type": not PANDAS_GE_200} + else: + kwargs = {} assert_groupby_results_equal( pdf.groupby([], group_keys=False).apply(lambda x: x.max()), gdf.groupby([]).apply(lambda x: x.max()), check_index_type=False, # Int64Index v/s Float64Index + **kwargs, ) diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index fe692a87ca8..2c3f4176674 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -311,7 +311,7 @@ def test_parquet_reader_empty_pandas_dataframe(tmpdir, engine): expect = expect.reset_index(drop=True) got = got.reset_index(drop=True) - assert_eq(expect, got) + assert_eq(expect, got, check_column_type=not PANDAS_GE_200) @pytest.mark.parametrize("has_null", [False, True]) @@ -2210,7 +2210,12 @@ def run_parquet_index(pdf, index): expected = pd.read_parquet(pandas_buffer) actual = cudf.read_parquet(cudf_buffer) - assert_eq(expected, actual, check_index_type=True) + assert_eq( + expected, + actual, + check_index_type=True, + check_column_type=not PANDAS_GE_200, + ) @pytest.mark.parametrize( From 8e8a1ea40558745c806dd695d8b3472442eb653c Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 19 Apr 2023 13:56:27 -0500 Subject: [PATCH 025/384] [REVIEW] Raise error when there is a binary operation between certain `DataFrame` and `Series` objects (#13138) This PR raises an error when there is a binary operation performed between `DataFrame` & `Series` with unequal `columns` and `index` respectively. This PR fixes 120 pytests: ``` = 833 failed, 86451 passed, 2034 skipped, 968 xfailed, 165 xpassed in 490.86s (0:08:10) = ``` on `pandas_2.0_feature_branch`: ``` = 953 failed, 86307 passed, 2034 skipped, 992 xfailed, 165 xpassed in 511.09s (0:08:31) = ``` --- python/cudf/cudf/core/dataframe.py | 14 ++++++++ python/cudf/cudf/tests/test_dataframe.py | 46 +++++++++++------------- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 760fcef826c..b6de299e387 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -1943,6 +1943,20 @@ def _make_operands_and_index_for_binop( if _is_scalar_or_zero_d_array(other): rhs = {name: other for name in self._data} elif isinstance(other, Series): + if ( + not can_reindex + and fn in cudf.utils.utils._EQUALITY_OPS + and ( + not self._data.to_pandas_index().equals( + other.index.to_pandas() + ) + ) + ): + raise ValueError( + "Can only compare DataFrame & Series objects " + "whose columns & index are same respectively, " + "please reindex." + ) rhs = dict(zip(other.index.values_host, other.values_host)) # For keys in right but not left, perform binops between NaN (not # NULL!) and the right value (result is NaN). diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 66453dd544d..7f22ffc0df2 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -9,7 +9,6 @@ import re import string import textwrap -import warnings from collections import OrderedDict, defaultdict from copy import copy @@ -5338,7 +5337,9 @@ def test_cov_nans(): cudf.Series([4, 2, 3], index=cudf.core.index.RangeIndex(0, 3)), pytest.param( cudf.Series([4, 2, 3, 4, 5], index=["a", "b", "d", "0", "12"]), - marks=pytest_xfail, + marks=pytest.mark.xfail( + not PANDAS_GE_200, reason="works only with pandas 2.0+" + ), ), ], ) @@ -5361,39 +5362,32 @@ def test_cov_nans(): ], ) def test_df_sr_binop(gsr, colnames, op): - # Anywhere that the column names of the DataFrame don't match the index - # names of the Series will trigger a deprecated reindexing. Since this - # behavior is deprecated in pandas, this test is temporarily silencing - # those warnings until cudf updates to pandas 2.0 as its compatibility - # target, at which point a large number of the parametrizations can be - # removed altogether (along with this warnings filter). - with warnings.catch_warnings(): - assert version.parse(pd.__version__) < version.parse("2.0.0") - warnings.filterwarnings( - action="ignore", - category=FutureWarning, - message=( - "Automatic reindexing on DataFrame vs Series comparisons is " - "deprecated" - ), - ) - data = [[3.0, 2.0, 5.0], [3.0, None, 5.0], [6.0, 7.0, np.nan]] - data = dict(zip(colnames, data)) + data = [[3.0, 2.0, 5.0], [3.0, None, 5.0], [6.0, 7.0, np.nan]] + data = dict(zip(colnames, data)) - gsr = gsr.astype("float64") + gsr = gsr.astype("float64") - gdf = cudf.DataFrame(data) - pdf = gdf.to_pandas(nullable=True) + gdf = cudf.DataFrame(data) + pdf = gdf.to_pandas(nullable=True) - psr = gsr.to_pandas(nullable=True) + psr = gsr.to_pandas(nullable=True) + try: expect = op(pdf, psr) + except ValueError: + with pytest.raises(ValueError): + op(gdf, gsr) + with pytest.raises(ValueError): + op(psr, pdf) + with pytest.raises(ValueError): + op(gsr, gdf) + else: got = op(gdf, gsr).to_pandas(nullable=True) - assert_eq(expect, got, check_dtype=False) + assert_eq(expect, got, check_dtype=False, check_like=True) expect = op(psr, pdf) got = op(gsr, gdf).to_pandas(nullable=True) - assert_eq(expect, got, check_dtype=False) + assert_eq(expect, got, check_dtype=False, check_like=True) @pytest_unmark_spilling From 901a9716c18505f7a29749df5bf2f2eece89a49f Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 19 Apr 2023 19:53:57 -0500 Subject: [PATCH 026/384] Fix `datetime64` related inconsistencies in pytests (#13175) This PR fixes `datetime64` related pytest failures where pandas returns `ns` time resolutions for quite a lot of cases, i.e., mostly on the IO APIs side. Fixes 72 pytests: ``` = 484 failed, 88162 passed, 2044 skipped, 932 xfailed, 165 xpassed in 444.53s (0:07:24) = ``` On `pandas_2.0_feature_branch`: ``` = 556 failed, 88090 passed, 2044 skipped, 932 xfailed, 165 xpassed in 456.49s (0:07:36) = ``` --- python/cudf/cudf/tests/test_csv.py | 7 ++++- python/cudf/cudf/tests/test_groupby.py | 32 ++++++++++++++++++++--- python/cudf/cudf/tests/test_index.py | 8 +++++- python/cudf/cudf/tests/test_joining.py | 7 +++++ python/cudf/cudf/tests/test_orc.py | 21 ++++++++++++++- python/cudf/cudf/tests/test_parquet.py | 17 ++++++++++++ python/cudf/cudf/tests/test_resampling.py | 4 ++- python/cudf/cudf/tests/test_string.py | 12 ++++----- 8 files changed, 94 insertions(+), 14 deletions(-) diff --git a/python/cudf/cudf/tests/test_csv.py b/python/cudf/cudf/tests/test_csv.py index 4a7804da62c..b66e6bc74fb 100644 --- a/python/cudf/cudf/tests/test_csv.py +++ b/python/cudf/cudf/tests/test_csv.py @@ -16,7 +16,7 @@ import cudf from cudf import read_csv -from cudf.core._compat import PANDAS_LT_140 +from cudf.core._compat import PANDAS_LT_140, PANDAS_GE_200 from cudf.testing._utils import assert_eq, assert_exceptions_equal @@ -367,6 +367,11 @@ def test_csv_reader_skiprows_skipfooter(tmpdir, pd_mixed_dataframe): assert len(out.columns) == len(df_out.columns) assert len(out) == len(df_out) + if PANDAS_GE_200: + # TODO: Remove typecast to `ns` after following + # issue is fixed: + # https://github.com/pandas-dev/pandas/issues/52449 + out["2"] = out["2"].astype("datetime64[ns]") assert_eq(df_out, out) diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index f5ce7ec95a0..63d98ada905 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -2648,7 +2648,13 @@ def test_groupby_freq_week(label, closed): got = gdf.groupby( cudf.Grouper(key="Publish date", freq="1W", label=label, closed=closed) ).mean() - assert_eq(expect, got, check_like=True, check_dtype=False) + assert_eq( + expect, + got, + check_like=True, + check_dtype=False, + check_index_type=not PANDAS_GE_200, + ) @pytest.mark.parametrize("label", [None, "left", "right"]) @@ -2675,7 +2681,13 @@ def test_groupby_freq_day(label, closed): got = gdf.groupby( cudf.Grouper(key="Publish date", freq="3D", label=label, closed=closed) ).mean() - assert_eq(expect, got, check_like=True, check_dtype=False) + assert_eq( + expect, + got, + check_like=True, + check_dtype=False, + check_index_type=not PANDAS_GE_200, + ) @pytest.mark.parametrize("label", [None, "left", "right"]) @@ -2702,7 +2714,13 @@ def test_groupby_freq_min(label, closed): got = gdf.groupby( cudf.Grouper(key="Publish date", freq="1h", label=label, closed=closed) ).mean() - assert_eq(expect, got, check_like=True, check_dtype=False) + assert_eq( + expect, + got, + check_like=True, + check_dtype=False, + check_index_type=not PANDAS_GE_200, + ) @pytest.mark.parametrize("label", [None, "left", "right"]) @@ -2729,7 +2747,13 @@ def test_groupby_freq_s(label, closed): got = gdf.groupby( cudf.Grouper(key="Publish date", freq="3s", label=label, closed=closed) ).mean() - assert_eq(expect, got, check_like=True, check_dtype=False) + assert_eq( + expect, + got, + check_like=True, + check_dtype=False, + check_index_type=not PANDAS_GE_200, + ) @pytest.mark.parametrize( diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index f9ad48c48af..d5d330d7177 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -1764,7 +1764,13 @@ def test_index_from_arrow(data): arrow_array = pa.Array.from_pandas(pdi) expected_index = pd.Index(arrow_array.to_pandas()) gdi = cudf.Index.from_arrow(arrow_array) - + if PANDAS_GE_200: + # Arrow bug: + # https://github.com/apache/arrow/issues/33321 + # arrow cannot convert non-nanosecond + # resolution to appropriate type in pandas. + # Hence need to type-cast. + expected_index = expected_index.astype(gdi.dtype) assert_eq(expected_index, gdi) diff --git a/python/cudf/cudf/tests/test_joining.py b/python/cudf/cudf/tests/test_joining.py index b197e91882a..c578266ac22 100644 --- a/python/cudf/cudf/tests/test_joining.py +++ b/python/cudf/cudf/tests/test_joining.py @@ -15,6 +15,7 @@ assert_exceptions_equal, expect_warning_if, ) +from cudf.core._compat import PANDAS_GE_200 _JOIN_TYPES = ("left", "inner", "outer", "right", "leftanti", "leftsemi") @@ -785,6 +786,12 @@ def test_join_datetimes_index(dtype): assert gdf["d"].dtype == cudf.dtype(dtype) + if PANDAS_GE_200: + # TODO: Remove typecast to `ns` after following + # issue is fixed: + # https://github.com/pandas-dev/pandas/issues/52449 + gdf = gdf.astype("datetime64[ns]") + assert_join_results_equal(pdf, gdf, how="inner") diff --git a/python/cudf/cudf/tests/test_orc.py b/python/cudf/cudf/tests/test_orc.py index 33095761fde..4701f69d862 100644 --- a/python/cudf/cudf/tests/test_orc.py +++ b/python/cudf/cudf/tests/test_orc.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2022, NVIDIA CORPORATION. +# Copyright (c) 2019-2023, NVIDIA CORPORATION. import datetime import decimal @@ -23,6 +23,7 @@ gen_rand_series, supported_numpy_dtypes, ) +from cudf.core._compat import PANDAS_GE_200 # Removal of these deprecated features is no longer imminent. They will not be # removed until a suitable alternative has been implemented. As a result, we @@ -159,6 +160,12 @@ def test_orc_reader_datetimestamp(datadir, inputfile, use_index): pdf = orcfile.read().to_pandas(date_as_object=False) gdf = cudf.read_orc(path, use_index=use_index) + if PANDAS_GE_200: + # TODO: Remove typecast to `ns` after following + # issue is fixed: + # https://github.com/pandas-dev/pandas/issues/52449 + gdf = gdf.astype("datetime64[ns]") + assert_eq(pdf, gdf, check_categorical=False) @@ -1847,6 +1854,12 @@ def test_orc_reader_negative_timestamp(negative_timestamp_df, engine): with expect_warning_if(engine == "pyarrow", UserWarning): got = cudf.read_orc(buffer, engine=engine) + if PANDAS_GE_200: + # TODO: Remove typecast to `ns` after following + # issue is fixed: + # https://github.com/pandas-dev/pandas/issues/52449 + negative_timestamp_df = negative_timestamp_df.astype("datetime64[ns]") + assert_eq(negative_timestamp_df, got) @@ -1854,6 +1867,12 @@ def test_orc_writer_negative_timestamp(negative_timestamp_df): buffer = BytesIO() negative_timestamp_df.to_orc(buffer) + if PANDAS_GE_200: + # TODO: Remove typecast to `ns` after following + # issue is fixed: + # https://github.com/pandas-dev/pandas/issues/52449 + negative_timestamp_df = negative_timestamp_df.astype("datetime64[ns]") + assert_eq(negative_timestamp_df, pd.read_orc(buffer)) assert_eq(negative_timestamp_df, pyarrow.orc.ORCFile(buffer).read()) diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index 2c3f4176674..ebebd857231 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -664,6 +664,13 @@ def test_parquet_reader_microsecond_timestamps(datadir): expect = pd.read_parquet(fname) got = cudf.read_parquet(fname) + if PANDAS_GE_200: + # TODO: Remove typecast to `ns` after following + # issue is fixed: + # https://github.com/pandas-dev/pandas/issues/52449 + assert got["a"].dtype == cudf.dtype("datetime64[us]") + got = got.astype("datetime64[ns]") + assert_eq(expect, got) @@ -2513,6 +2520,16 @@ def test_parquet_writer_nulls_pandas_read(tmpdir, pdf): got = pd.read_parquet(fname) nullable = num_rows > 0 + if PANDAS_GE_200: + # TODO: Remove typecast to `ns` after following + # issue is fixed: + # https://github.com/pandas-dev/pandas/issues/52449 + gdf["col_datetime64[ms]"] = gdf["col_datetime64[ms]"].astype( + "datetime64[ns]" + ) + gdf["col_datetime64[us]"] = gdf["col_datetime64[us]"].astype( + "datetime64[ns]" + ) assert_eq(gdf.to_pandas(nullable=nullable), got) diff --git a/python/cudf/cudf/tests/test_resampling.py b/python/cudf/cudf/tests/test_resampling.py index f0101803995..ce5b05adff1 100644 --- a/python/cudf/cudf/tests/test_resampling.py +++ b/python/cudf/cudf/tests/test_resampling.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. import numpy as np import pandas as pd @@ -6,6 +6,7 @@ import cudf from cudf.testing._utils import assert_eq +from cudf.core._compat import PANDAS_GE_200 def assert_resample_results_equal(lhs, rhs, **kwargs): @@ -14,6 +15,7 @@ def assert_resample_results_equal(lhs, rhs, **kwargs): rhs.sort_index(), check_dtype=False, check_freq=False, + check_index_type=not PANDAS_GE_200, **kwargs, ) diff --git a/python/cudf/cudf/tests/test_string.py b/python/cudf/cudf/tests/test_string.py index ad47c79a3cf..2af9c70e706 100644 --- a/python/cudf/cudf/tests/test_string.py +++ b/python/cudf/cudf/tests/test_string.py @@ -200,12 +200,12 @@ def test_string_astype(dtype): data = ["True", "False", "True", "False", "False"] elif dtype.startswith("datetime64"): data = [ - "2019-06-04T00:00:00Z", - "2019-06-04T12:12:12Z", - "2019-06-03T00:00:00Z", - "2019-05-04T00:00:00Z", - "2018-06-04T00:00:00Z", - "1922-07-21T01:02:03Z", + "2019-06-04T00:00:00", + "2019-06-04T12:12:12", + "2019-06-03T00:00:00", + "2019-05-04T00:00:00", + "2018-06-04T00:00:00", + "1922-07-21T01:02:03", ] elif dtype == "str" or dtype == "object": data = ["ab", "cd", "ef", "gh", "ij"] From 31e08c97ee166ed0b457c310509ad70c85d150e2 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 20 Apr 2023 15:20:39 -0500 Subject: [PATCH 027/384] Fix `DataFrame.describe` pytests (#13191) https://github.com/rapidsai/cudf/pull/12890 dropped support for `datetime_is_numeric` from `describe` API. This PR cleans-up a remaining pytest that was using this parameter. This PR fixes 20 pytests: ``` = 464 failed, 88182 passed, 2044 skipped, 932 xfailed, 165 xpassed in 440.68s (0:07:20) = ``` On `pandas_2.0_feature_branch`: ``` = 484 failed, 88162 passed, 2044 skipped, 932 xfailed, 165 xpassed in 457.87s (0:07:37) = ``` --- python/cudf/cudf/tests/test_dataframe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 7f22ffc0df2..eb7d6ecbc9c 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -8273,8 +8273,8 @@ def test_dataframe_iterrows_itertuples(): def test_describe_misc_include(df, include): pdf = df.to_pandas() - expected = pdf.describe(include=include, datetime_is_numeric=True) - actual = df.describe(include=include, datetime_is_numeric=True) + expected = pdf.describe(include=include) + actual = df.describe(include=include) for col in expected.columns: if expected[col].dtype == np.dtype("object"): From 27e18c83f4768ec938d5421627302dfcc047c7a8 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 21 Apr 2023 19:27:29 -0500 Subject: [PATCH 028/384] Change default `dtype` for `get_dummies` to `bool` (#13174) This PR changes the default dtype for get_dummies to bool from uint8 to match pandas-2.0: pandas-dev/pandas#48022 --- python/cudf/cudf/_lib/transform.pyx | 2 +- python/cudf/cudf/core/reshape.py | 40 +++++++++++------------ python/cudf/cudf/tests/test_onehot.py | 47 +++++++++++---------------- 3 files changed, 40 insertions(+), 49 deletions(-) diff --git a/python/cudf/cudf/_lib/transform.pyx b/python/cudf/cudf/_lib/transform.pyx index a0a8279b213..d8eb6134042 100644 --- a/python/cudf/cudf/_lib/transform.pyx +++ b/python/cudf/cudf/_lib/transform.pyx @@ -163,7 +163,7 @@ def one_hot_encode(Column input_column, Column categories): move(c_result.second), owner=owner, column_names=[ - x if x is not None else 'null' for x in pylist_categories + x if x is not None else '' for x in pylist_categories ] ) return encodings diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index df1a543c4aa..e1b425cab9f 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -609,7 +609,7 @@ def get_dummies( cats=None, sparse=False, drop_first=False, - dtype="uint8", + dtype="bool", ): """Returns a dataframe whose columns are the one hot encodings of all columns in `df` @@ -640,7 +640,7 @@ def get_dummies( columns. Note this is different from pandas default behavior, which encodes all columns with dtype object or categorical dtype : str, optional - Output dtype, default 'uint8' + Output dtype, default 'bool' Examples -------- @@ -648,15 +648,15 @@ def get_dummies( >>> df = cudf.DataFrame({"a": ["value1", "value2", None], "b": [0, 0, 0]}) >>> cudf.get_dummies(df) b a_value1 a_value2 - 0 0 1 0 - 1 0 0 1 - 2 0 0 0 + 0 0 True False + 1 0 False True + 2 0 False False >>> cudf.get_dummies(df, dummy_na=True) - b a_None a_value1 a_value2 - 0 0 0 1 0 - 1 0 0 0 1 - 2 0 1 0 0 + b a_ a_value1 a_value2 + 0 0 False True False + 1 0 False False True + 2 0 True False False >>> import numpy as np >>> df = cudf.DataFrame({"a":cudf.Series([1, 2, np.nan, None], @@ -669,11 +669,11 @@ def get_dummies( 3 >>> cudf.get_dummies(df, dummy_na=True, columns=["a"]) - a_1.0 a_2.0 a_nan a_null - 0 1 0 0 0 - 1 0 1 0 0 - 2 0 0 1 0 - 3 0 0 0 1 + a_ a_1.0 a_2.0 a_nan + 0 False True False False + 1 False False True False + 2 False False False True + 3 True False False False >>> series = cudf.Series([1, 2, None, 2, 4]) >>> series @@ -684,12 +684,12 @@ def get_dummies( 4 4 dtype: int64 >>> cudf.get_dummies(series, dummy_na=True) - null 1 2 4 - 0 0 1 0 0 - 1 0 0 1 0 - 2 1 0 0 0 - 3 0 0 1 0 - 4 0 0 0 1 + 1 2 4 + 0 False True False False + 1 False False True False + 2 True False False False + 3 False False True False + 4 False False False True """ if cats is None: diff --git a/python/cudf/cudf/tests/test_onehot.py b/python/cudf/cudf/tests/test_onehot.py index d42b0e85d28..17ce145a2c2 100644 --- a/python/cudf/cudf/tests/test_onehot.py +++ b/python/cudf/cudf/tests/test_onehot.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2022, NVIDIA CORPORATION. +# Copyright (c) 2018-2023, NVIDIA CORPORATION. from string import ascii_lowercase @@ -23,19 +23,13 @@ (range(10), [1, 2, 3, 4, 5] * 2), ], ) -def test_get_dummies(data, index): +@pytest.mark.parametrize("dtype", ["bool", "uint8"]) +def test_get_dummies(data, index, dtype): gdf = DataFrame({"x": data}, index=index) pdf = pd.DataFrame({"x": data}, index=index) - encoded_expected = pd.get_dummies(pdf, prefix="test") - encoded_actual = cudf.get_dummies(gdf, prefix="test") - - utils.assert_eq( - encoded_expected, - encoded_actual, - check_dtype=len(data) != 0, - ) - encoded_actual = cudf.get_dummies(gdf, prefix="test", dtype=np.uint8) + encoded_expected = pd.get_dummies(pdf, prefix="test", dtype=dtype) + encoded_actual = cudf.get_dummies(gdf, prefix="test", dtype=dtype) utils.assert_eq( encoded_expected, @@ -63,16 +57,13 @@ def test_onehot_get_dummies_multicol(n_cols): @pytest.mark.parametrize("nan_as_null", [True, False]) @pytest.mark.parametrize("dummy_na", [True, False]) def test_onehost_get_dummies_dummy_na(nan_as_null, dummy_na): - pdf = pd.DataFrame({"a": [0, 1, np.nan]}) - df = DataFrame.from_pandas(pdf, nan_as_null=nan_as_null) + df = cudf.DataFrame({"a": [0, 1, np.nan]}, nan_as_null=nan_as_null) + pdf = df.to_pandas(nullable=nan_as_null) expected = pd.get_dummies(pdf, dummy_na=dummy_na, columns=["a"]) got = cudf.get_dummies(df, dummy_na=dummy_na, columns=["a"]) - if dummy_na and nan_as_null: - got = got.rename(columns={"a_null": "a_nan"})[expected.columns] - - utils.assert_eq(expected, got) + utils.assert_eq(expected, got, check_like=True) @pytest.mark.parametrize( @@ -120,12 +111,12 @@ def test_get_dummies_with_nan(): ) expected = cudf.DataFrame( { - "a_null": [0, 0, 0, 1], - "a_1.0": [1, 0, 0, 0], - "a_2.0": [0, 1, 0, 0], - "a_nan": [0, 0, 1, 0], + "a_": [False, False, False, True], + "a_1.0": [True, False, False, False], + "a_2.0": [False, True, False, False], + "a_nan": [False, False, True, False], }, - dtype="uint8", + dtype="bool", ) actual = cudf.get_dummies(df, dummy_na=True, columns=["a"]) @@ -163,13 +154,13 @@ def test_get_dummies_array_like_with_nan(): ser = cudf.Series([0.1, 2, 3, None, np.nan], nan_as_null=False) expected = cudf.DataFrame( { - "a_null": [0, 0, 0, 1, 0], - "a_0.1": [1, 0, 0, 0, 0], - "a_2.0": [0, 1, 0, 0, 0], - "a_3.0": [0, 0, 1, 0, 0], - "a_nan": [0, 0, 0, 0, 1], + "a_": [False, False, False, True, False], + "a_0.1": [True, False, False, False, False], + "a_2.0": [False, True, False, False, False], + "a_3.0": [False, False, True, False, False], + "a_nan": [False, False, False, False, True], }, - dtype="uint8", + dtype="bool", ) actual = cudf.get_dummies(ser, dummy_na=True, prefix="a", prefix_sep="_") From 6a863854d1029bba61c5b2164be39c3979bf0ae7 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 21 Apr 2023 19:27:47 -0500 Subject: [PATCH 029/384] [REVIEW] Update parameter ordering in `DataFrame.pivot` (#13190) This PR updates parameter ordering in `DataFrame.pivot` to match pandas-2.0. This PR fixes 7 related pytests: ``` = 477 failed, 88169 passed, 2044 skipped, 932 xfailed, 165 xpassed in 438.55s (0:07:18) = ``` On `pandas_2.0_feature_branch`: ``` = 484 failed, 88162 passed, 2044 skipped, 932 xfailed, 165 xpassed in 457.87s (0:07:37) = ``` --- python/cudf/cudf/core/dataframe.py | 3 ++- python/cudf/cudf/core/reshape.py | 11 ++++++----- python/cudf/cudf/tests/test_reshape.py | 10 ++-------- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index b6de299e387..d4d3591a360 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -104,6 +104,7 @@ _external_only_api, ) from cudf.core._compat import PANDAS_GE_200 +from cudf.api.extensions import no_default T = TypeVar("T", bound="DataFrame") @@ -6636,7 +6637,7 @@ def iterrows(self): @_cudf_nvtx_annotate @copy_docstring(reshape.pivot) - def pivot(self, index, columns, values=None): + def pivot(self, *, columns, index=no_default, values=no_default): return cudf.core.reshape.pivot( self, index=index, columns=columns, values=values ) diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index e1b425cab9f..43d683490b8 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -13,6 +13,7 @@ from cudf._typing import Dtype from cudf.core.column import ColumnBase, as_column, column_empty_like from cudf.core.column.categorical import CategoricalColumn +from cudf.api.extensions import no_default _AXIS_MAP = {0: 0, 1: 1, "index": 0, "columns": 1} @@ -905,7 +906,7 @@ def as_tuple(x): ) -def pivot(data, index=None, columns=None, values=None): +def pivot(data, columns=None, index=no_default, values=no_default): """ Return reshaped DataFrame organized by the given index and column values. @@ -915,10 +916,10 @@ def pivot(data, index=None, columns=None, values=None): Parameters ---------- - index : column name, optional - Column used to construct the index of the result. columns : column name, optional Column used to construct the columns of the result. + index : column name, optional + Column used to construct the index of the result. values : column name or list of column names, optional Column(s) whose values are rearranged to produce the result. If not specified, all remaining columns of the DataFrame @@ -957,7 +958,7 @@ def pivot(data, index=None, columns=None, values=None): """ df = data values_is_list = True - if values is None: + if values is no_default: values = df._columns_view( col for col in df._column_names if col not in (index, columns) ) @@ -966,7 +967,7 @@ def pivot(data, index=None, columns=None, values=None): values = [values] values_is_list = False values = df._columns_view(values) - if index is None: + if index is no_default: index = df.index else: index = cudf.core.index.Index(df.loc[:, index]) diff --git a/python/cudf/cudf/tests/test_reshape.py b/python/cudf/cudf/tests/test_reshape.py index bf2c1a32b64..b70d6554c0f 100644 --- a/python/cudf/cudf/tests/test_reshape.py +++ b/python/cudf/cudf/tests/test_reshape.py @@ -382,14 +382,8 @@ def test_pivot_simple(index, column, data): pdf = pd.DataFrame({"index": index, "column": column, "data": data}) gdf = cudf.from_pandas(pdf) - # In pandas 2.0 this will be a failure because pandas will require all of - # these as keyword arguments. Matching that check in cudf is a bit - # cumbersome and not worth the effort to match the warning, so this code - # just catches pandas's warning (rather than updating the signature) so - # that when it starts failing we know to update our impl of pivot. - with pytest.warns(FutureWarning): - expect = pdf.pivot("index", "column") - got = gdf.pivot("index", "column") + expect = pdf.pivot(columns="column", index="index") + got = gdf.pivot(columns="column", index="index") check_index_and_columns = expect.shape != (0, 0) assert_eq( From ea7d18cc5f3640c13cba1bbccdba3d65e588fdbb Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 26 Apr 2023 11:40:01 -0500 Subject: [PATCH 030/384] Fix ceil, floor and round pytests (#13218) A fix for https://github.com/pandas-dev/pandas/issues/52761 has been merged by @mroeschke , this PR xfails the pytests conditionally for `2.0.0` and passes for rest of the versions. This PR fixes 27 pytests: ``` = 404 failed, 88221 passed, 2044 skipped, 959 xfailed, 165 xpassed in 442.21s (0:07:22) = ``` On `pandas_2.0_feature_branch`: ``` = 431 failed, 88221 passed, 2044 skipped, 932 xfailed, 165 xpassed in 456.25s (0:07:36) = ``` --- python/cudf/cudf/core/_compat.py | 1 + python/cudf/cudf/tests/test_datetime.py | 54 ++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/python/cudf/cudf/core/_compat.py b/python/cudf/cudf/core/_compat.py index 6ecbe414ebb..183faa12904 100644 --- a/python/cudf/cudf/core/_compat.py +++ b/python/cudf/cudf/core/_compat.py @@ -9,4 +9,5 @@ PANDAS_LT_140 = PANDAS_VERSION < version.parse("1.4.0") PANDAS_GE_150 = PANDAS_VERSION >= version.parse("1.5.0") PANDAS_LT_153 = PANDAS_VERSION < version.parse("1.5.3") +PANDAS_EQ_200 = PANDAS_VERSION == version.parse("2.0.0") PANDAS_GE_200 = PANDAS_VERSION >= version.parse("2.0.0") diff --git a/python/cudf/cudf/tests/test_datetime.py b/python/cudf/cudf/tests/test_datetime.py index 10b23745fbd..68c9f725aa7 100644 --- a/python/cudf/cudf/tests/test_datetime.py +++ b/python/cudf/cudf/tests/test_datetime.py @@ -13,7 +13,7 @@ import warnings import cudf.testing.dataset_generator as dataset_generator from cudf import DataFrame, Series -from cudf.core._compat import PANDAS_GE_150, PANDAS_LT_140 +from cudf.core._compat import PANDAS_GE_150, PANDAS_LT_140, PANDAS_EQ_200 from cudf.core.index import DatetimeIndex from cudf.testing._utils import ( DATETIME_TYPES, @@ -1906,8 +1906,22 @@ def test_error_values(): @pytest.mark.parametrize( "resolution", ["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"] ) -def test_ceil(data, time_type, resolution): - +def test_ceil(request, data, time_type, resolution): + alias_map = {"L": "ms", "U": "us", "N": "ns"} + request.applymarker( + pytest.mark.xfail( + condition=( + PANDAS_EQ_200 + and resolution in {"L", "ms", "U", "us", "N"} + and np.dtype( + f"datetime64[{alias_map.get(resolution, resolution)}]" + ) + > np.dtype(time_type) + ), + reason="https://github.com/pandas-dev/pandas/issues/52761", + strict=True, + ) + ) gs = cudf.Series(data, dtype=time_type) ps = gs.to_pandas() @@ -1937,7 +1951,22 @@ def test_ceil(data, time_type, resolution): @pytest.mark.parametrize( "resolution", ["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"] ) -def test_floor(data, time_type, resolution): +def test_floor(request, data, time_type, resolution): + alias_map = {"L": "ms", "U": "us", "N": "ns"} + request.applymarker( + pytest.mark.xfail( + condition=( + PANDAS_EQ_200 + and resolution in {"L", "ms", "U", "us", "N"} + and np.dtype( + f"datetime64[{alias_map.get(resolution, resolution)}]" + ) + > np.dtype(time_type) + ), + reason="https://github.com/pandas-dev/pandas/issues/52761", + strict=True, + ) + ) gs = cudf.Series(data, dtype=time_type) ps = gs.to_pandas() @@ -1968,7 +1997,22 @@ def test_floor(data, time_type, resolution): @pytest.mark.parametrize( "resolution", ["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"] ) -def test_round(data, time_type, resolution): +def test_round(request, data, time_type, resolution): + alias_map = {"L": "ms", "U": "us", "N": "ns"} + request.applymarker( + pytest.mark.xfail( + condition=( + PANDAS_EQ_200 + and resolution in {"L", "ms", "U", "us", "N"} + and np.dtype( + f"datetime64[{alias_map.get(resolution, resolution)}]" + ) + > np.dtype(time_type) + ), + reason="https://github.com/pandas-dev/pandas/issues/52761", + strict=True, + ) + ) gs = cudf.Series(data, dtype=time_type) ps = gs.to_pandas() From e355ba46f2a742f1625918854dead3c92553cc68 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 26 Apr 2023 19:08:47 -0700 Subject: [PATCH 031/384] More implementation for get_indexer --- python/cudf/cudf/core/index.py | 199 +++++++++++++++------------ python/cudf/cudf/core/multiindex.py | 134 +++++++++++------- python/cudf/cudf/tests/test_index.py | 177 +++++++----------------- 3 files changed, 245 insertions(+), 265 deletions(-) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 9e41d5ed75e..14f7c91eea0 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -2,7 +2,6 @@ from __future__ import annotations -import math import pickle import warnings from functools import cached_property @@ -576,45 +575,45 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): ) @_cudf_nvtx_annotate - def get_indexer(self, target, method=None, limit=None, tolerance=None): - # Given an actual integer, - idx = (target - self._start) / self._step - idx_int_upper_bound = (self._stop - self._start) // self._step + def get_indexer(self, target, method=None, tolerance=None): if method is None: - if tolerance is not None: - raise ValueError( - "tolerance argument only valid if using pad, " - "backfill or nearest lookups" - ) - - if idx > idx_int_upper_bound or idx < 0: - raise KeyError(target) - - idx_int = (target - self._start) // self._step - if idx_int != idx: - raise KeyError(target) - return idx_int - - if (method == "ffill" and idx < 0) or ( - method == "bfill" and idx > idx_int_upper_bound - ): - raise KeyError(target) - - round_method = { - "ffill": math.floor, - "bfill": math.ceil, - "nearest": round, - }[method] - if tolerance is not None and (abs(idx) * self._step > tolerance): - raise KeyError(target) - return np.clip(round_method(idx), 0, idx_int_upper_bound, dtype=int) + if self.step > 0: + start, stop, step = self.start, self.stop, self.step + else: + # Reversed + reverse = self._range[::-1] + start, stop, step = reverse.start, reverse.stop, reverse.step + + target_array = cupy.asarray(target) + locs = target_array - start + valid = (locs % step == 0) & (locs >= 0) & (target_array < stop) + locs[~valid] = -1 + locs[valid] = locs[valid] / step + + if step != self.step: + # Reversed + locs[valid] = len(self) - 1 - locs[valid] + return locs + else: + return self._as_int_index().get_indexer( + target=target, method=method, tolerance=tolerance + ) @_cudf_nvtx_annotate def get_loc(self, key): # Given an actual integer, - if is_scalar(key): - key = [key] - return self.get_indexer(key) + if not is_scalar(key): + raise TypeError("Should be a sequence") + # Given an actual integer, + idx = (key - self._start) / self._step + idx_int_upper_bound = (self._stop - self._start) // self._step + if idx > idx_int_upper_bound or idx < 0: + raise KeyError(key) + + idx_int = (key - self._start) // self._step + if idx_int != idx: + raise KeyError(key) + return idx_int @_cudf_nvtx_annotate def _union(self, other, sort=None): @@ -1168,10 +1167,10 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): """ if is_scalar(target): raise TypeError("Should be a sequence") - if tolerance is not None: - raise NotImplementedError( - "Parameter tolerance is not supported yet." - ) + # if tolerance is not None: + # raise NotImplementedError( + # "Parameter tolerance is not supported yet." + # ) if method not in { None, "ffill", @@ -1185,6 +1184,9 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): f" or nearest. Got {method}" ) + if not self.is_unique: + raise ValueError("Cannot get index for a non-unique Index.") + is_sorted = ( self.is_monotonic_increasing or self.is_monotonic_decreasing ) @@ -1195,54 +1197,45 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): "is specified." ) - target_as_table = cudf.core.frame.Frame({"None": as_column(target)}) - lower_bound, upper_bound, sort_inds = _lexsorted_equal_range( - self, target_as_table, is_sorted + needle_table = cudf.DataFrame( + {"None": as_column(target), "order": arange(0, len(target))} ) - - if lower_bound == upper_bound: - # target not found, apply method - if method in ("pad", "ffill"): - if lower_bound == 0: - raise KeyError(target) - return lower_bound - 1 - elif method in ("backfill", "bfill"): - if lower_bound == self._data.nrows: - raise KeyError(target) - return lower_bound - elif method == "nearest": - if lower_bound == self._data.nrows: - return lower_bound - 1 - elif lower_bound == 0: - return 0 - lower_val = self._column.element_indexing(lower_bound - 1) - upper_val = self._column.element_indexing(lower_bound) - return ( - lower_bound - 1 - if abs(lower_val - target) < abs(upper_val - target) - else lower_bound - ) - else: - raise KeyError(target) - - if lower_bound + 1 == upper_bound: - # Search result is unique, return int. - return ( - lower_bound - if is_sorted - else sort_inds.element_indexing(lower_bound) + haystack_table = cudf.DataFrame( + {"None": self._column, "order": arange(0, len(self))} + ) + merged_table = haystack_table.merge( + needle_table, on="None", how="outer" + ) + result_series = ( + merged_table.sort_values(by="order_y") + .head(len(target))["order_x"] + .reset_index(drop=True) + ) + if method is None: + result_series = result_series.fillna(-1) + else: + nonexact = result_series.isnull() + result_series[nonexact] = self.searchsorted( + needle_table["None"][nonexact], + side="left" if method in {"pad", "ffill"} else "right", ) - - if is_sorted: - # In monotonic index, lex search result is continuous. A slice for - # the range is returned. - return slice(lower_bound, upper_bound) - - # Not sorted and not unique. Return a boolean mask - mask = cupy.full(self._data.nrows, False) - true_inds = sort_inds.slice(lower_bound, upper_bound).values - mask[true_inds] = True - return mask + if method in {"pad", "ffill"}: + # searchsorted returns "indices into a sorted array such that, + # if the corresponding elements in v were inserted before the + # indices, the order of a would be preserved". + # Thus, we need to subtract 1 to find values to the left. + result_series[nonexact] -= 1 + # This also mapped not found values (values of 0 from + # np.searchsorted) to -1, which conveniently is also our + # sentinel for missing values + else: + # Mark indices to the right of the largest value as not found + result_series[result_series == len(self)] = -1 + if tolerance is not None: + distance = self[result_series] - needle_table["None"] + # return cupy.where(distance <= tolerance, result_series, -1) + return result_series.where(distance <= tolerance, -1).to_cupy() + return result_series.to_cupy() @_cudf_nvtx_annotate def get_loc(self, key): @@ -1275,8 +1268,40 @@ def get_loc(self, key): 2 """ if is_scalar(key): - key = [key] - return self.get_indexer(target=key) + target = [key] + else: + target = key + + is_sorted = ( + self.is_monotonic_increasing or self.is_monotonic_decreasing + ) + + target_as_table = cudf.core.frame.Frame({"None": as_column(target)}) + lower_bound, upper_bound, sort_inds = _lexsorted_equal_range( + self, target_as_table, is_sorted + ) + + if lower_bound == upper_bound: + raise KeyError(target) + + if lower_bound + 1 == upper_bound: + # Search result is unique, return int. + return ( + lower_bound + if is_sorted + else sort_inds.element_indexing(lower_bound) + ) + + if is_sorted: + # In monotonic index, lex search result is continuous. A slice for + # the range is returned. + return slice(lower_bound, upper_bound) + + # Not sorted and not unique. Return a boolean mask + mask = cupy.full(self._data.nrows, False) + true_inds = sort_inds.slice(lower_bound, upper_bound).values + mask[true_inds] = True + return mask @_cudf_nvtx_annotate def __repr__(self): diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index f533cff7c12..01e7df28020 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1702,60 +1702,45 @@ def get_indexer(self, target, method=None, tolerance=None): raise NotImplementedError( "Parameter tolerance is not supported yet." ) - if method is not None: - raise NotImplementedError( - "only the default get_loc method is currently supported for" - " MultiIndex" - ) - - is_sorted = ( - self.is_monotonic_increasing or self.is_monotonic_decreasing - ) - is_unique = self.is_unique - target = (target,) if not isinstance(target, tuple) else target - - # Handle partial target search. If length of `target` is less than `nlevels`, - # Only search levels up to `len(target)` level. - target_as_table = cudf.core.frame.Frame( - {i: column.as_column(k, length=1) for i, k in enumerate(target)} + target = cudf.MultiIndex.from_tuples(target) + needle_table = target.to_frame(index=False) + col_names = list(range(0, self.nlevels)) + needle_table["order"] = needle_table.index + haystack_table = self.copy(deep=True).to_frame(index=False) + haystack_table["order"] = haystack_table.index + merged_table = haystack_table.merge( + needle_table, on=col_names, how="outer" ) - partial_index = self.__class__._from_data( - data=self._data.select_by_index( - slice(target_as_table._num_columns) - ) + result_series = ( + merged_table.sort_values(by="order_y") + .head(len(target))["order_x"] + .reset_index(drop=True) ) - ( - lower_bound, - upper_bound, - sort_inds, - ) = _lexsorted_equal_range(partial_index, target_as_table, is_sorted) - - if lower_bound == upper_bound: - raise KeyError(target) - - if is_unique and lower_bound + 1 == upper_bound: - # Indices are unique (Pandas constraint), search result is unique, - # return int. - return ( - lower_bound - if is_sorted - else sort_inds.element_indexing(lower_bound) + if method is None: + result_series = result_series.fillna(-1) + else: + nonexact = result_series.isnull() + result_series[nonexact] = self.searchsorted( + needle_table[col_names][nonexact], + side="left" if method in {"pad", "ffill"} else "right", ) - - if is_sorted: - # In monotonic index, lex search result is continuous. A slice for - # the range is returned. - return slice(lower_bound, upper_bound) - - true_inds = sort_inds.slice(lower_bound, upper_bound).values - true_inds = _maybe_indices_to_slice(true_inds) - if isinstance(true_inds, slice): - return true_inds - - # Not sorted and not unique. Return a boolean mask - mask = cp.full(self._data.nrows, False) - mask[true_inds] = True - return mask + if method in {"pad", "ffill"}: + # searchsorted returns "indices into a sorted array such that, + # if the corresponding elements in v were inserted before the + # indices, the order of a would be preserved". + # Thus, we need to subtract 1 to find values to the left. + result_series[nonexact] -= 1 + # This also mapped not found values (values of 0 from + # np.searchsorted) to -1, which conveniently is also our + # sentinel for missing values + else: + # Mark indices to the right of the largest value as not found + result_series[result_series == len(self)] = -1 + if tolerance is not None: + distance = self[result_series] - needle_table["None"] + # return cupy.where(distance <= tolerance, result_series, -1) + return result_series.where(distance <= tolerance, -1).to_cupy() + return result_series.to_cupy() @_cudf_nvtx_annotate def get_loc(self, key): @@ -1814,7 +1799,52 @@ def get_loc(self, key): >>> cudf.from_pandas(x).get_loc(1) slice(1, 5, 1) """ - return self.get_indexer(target=key) + is_sorted = ( + self.is_monotonic_increasing or self.is_monotonic_decreasing + ) + is_unique = self.is_unique + key = (key,) if not isinstance(key, tuple) else key + + # Handle partial key search. If length of `key` is less than `nlevels`, + # Only search levels up to `len(key)` level. + key_as_table = cudf.core.frame.Frame( + {i: column.as_column(k, length=1) for i, k in enumerate(key)} + ) + partial_index = self.__class__._from_data( + data=self._data.select_by_index(slice(key_as_table._num_columns)) + ) + ( + lower_bound, + upper_bound, + sort_inds, + ) = _lexsorted_equal_range(partial_index, key_as_table, is_sorted) + + if lower_bound == upper_bound: + raise KeyError(key) + + if is_unique and lower_bound + 1 == upper_bound: + # Indices are unique (Pandas constraint), search result is unique, + # return int. + return ( + lower_bound + if is_sorted + else sort_inds.element_indexing(lower_bound) + ) + + if is_sorted: + # In monotonic index, lex search result is continuous. A slice for + # the range is returned. + return slice(lower_bound, upper_bound) + + true_inds = sort_inds.slice(lower_bound, upper_bound).values + true_inds = _maybe_indices_to_slice(true_inds) + if isinstance(true_inds, slice): + return true_inds + + # Not sorted and not unique. Return a boolean mask + mask = cp.full(self._data.nrows, False) + mask[true_inds] = True + return mask def _get_reconciled_name_object(self, other) -> MultiIndex: """ diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index 823209881c8..50afcf4a902 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -1939,20 +1939,16 @@ def test_index_set_names_error(idx, level, names): "idx", [pd.Index([1, 3, 6]), pd.Index([6, 1, 3])], # monotonic # non-monotonic ) -@pytest.mark.parametrize("key", list(range(0, 8))) +@pytest.mark.parametrize("key", [list(range(0, 8))]) @pytest.mark.parametrize("method", [None, "ffill", "bfill", "nearest"]) -def test_get_loc_single_unique_numeric(idx, key, method): +def test_get_indexer_single_unique_numeric(idx, key, method): pi = idx gi = cudf.from_pandas(pi) if ( - (key not in pi and method is None) # `method` only applicable to monotonic index - or (not pi.is_monotonic_increasing and method is not None) - # Get key before the first element is KeyError - or (key == 0 and method in "ffill") - # Get key after the last element is KeyError - or (key == 7 and method in "bfill") + not pi.is_monotonic_increasing + and method is not None ): assert_exceptions_equal( lfunc=pi.get_loc, @@ -1961,10 +1957,9 @@ def test_get_loc_single_unique_numeric(idx, key, method): rfunc_args_and_kwargs=([], {"key": key, "method": method}), ) else: - with expect_warning_if(method is not None): - expected = pi.get_loc(key, method=method) - with expect_warning_if(method is not None): - got = gi.get_loc(key, method=method) + with expect_warning_if(not PANDAS_GE_200 and method is not None): + expected = pi.get_indexer(key, method=method) + got = gi.get_indexer(key, method=method) assert_eq(expected, got) @@ -1982,29 +1977,18 @@ def test_get_loc_single_unique_numeric(idx, key, method): list(range(77, 110, 3)), ], ) -@pytest.mark.parametrize("method", [None, "ffill"]) -def test_get_indexer_rangeindex(idx, key, method): +@pytest.mark.parametrize("method", [None, "ffill", "bfill"]) +@pytest.mark.parametrize("tolerance", [None, 0, 1, 13, 20]) +def test_get_indexer_rangeindex(idx, key, method, tolerance): pi = idx gi = cudf.from_pandas(pi) - # if ( - # (any(k not in pi for k in key) and method is None) - # # Get key before the first element is KeyError - # or (key < pi.start and method in "ffill") - # # Get key after the last element is KeyError - # or (key >= pi.stop and method in "bfill") - # ): - # assert_exceptions_equal( - # lfunc=pi.get_indexer, - # rfunc=gi.get_indexer, - # lfunc_args_and_kwargs=([], {"key": key, "method": method}), - # rfunc_args_and_kwargs=([], {"key": key, "method": method}), - # ) - # else: - # with expect_warning_if(method is not None): - expected = pi.get_indexer(key, method=method) - # with expect_warning_if(method is not None): - got = gi.get_indexer(key, method=method) + expected = pi.get_indexer( + key, method=method, tolerance=None if method is None else tolerance + ) + got = gi.get_indexer( + key, method=method, tolerance=None if method is None else tolerance + ) assert_eq(expected, got) @@ -2066,17 +2050,17 @@ def test_get_loc_single_duplicate_numeric(idx, key): @pytest.mark.parametrize( "idx", [ - pd.Index([1, 3, 3, 6]), # monotonic - pd.Index([6, 1, 3, 3]), # non-monotonic + pd.Index([-1, 2, 3, 6]), # monotonic + pd.Index([6, 1, 3, 4]), # non-monotonic ], ) -@pytest.mark.parametrize("key", [0, 3, 6, 7]) -@pytest.mark.parametrize("method", [None]) +@pytest.mark.parametrize("key", [[0, 3, 1], [6, 7]]) +@pytest.mark.parametrize("method", [None, "ffill", "bfill"]) def test_get_indexer_single_duplicate_numeric(idx, key, method): pi = idx gi = cudf.from_pandas(pi) - if key not in pi: + if not pi.is_monotonic_increasing and method is not None: assert_exceptions_equal( lfunc=pi.get_indexer, rfunc=gi.get_indexer, @@ -2115,21 +2099,13 @@ def test_get_loc_single_unique_string(idx, key): @pytest.mark.parametrize( "idx", [pd.Index(["b", "f", "m", "q"]), pd.Index(["m", "f", "b", "q"])] ) -@pytest.mark.parametrize("key", ["a", "f", "n", "z"]) +@pytest.mark.parametrize("key", [["a", "f", "n", "z"], ["p", "p", "b"]]) @pytest.mark.parametrize("method", [None, "ffill", "bfill"]) def test_get_indexer_single_unique_string(idx, key, method): pi = idx gi = cudf.from_pandas(pi) - if ( - (key not in pi and method is None) - # `method` only applicable to monotonic index - or (not pi.is_monotonic_increasing and method is not None) - # Get key before the first element is KeyError - or (key == "a" and method == "ffill") - # Get key after the last element is KeyError - or (key == "z" and method == "bfill") - ): + if not pi.is_monotonic_increasing and method is not None: assert_exceptions_equal( lfunc=pi.get_indexer, rfunc=gi.get_indexer, @@ -2166,15 +2142,19 @@ def test_get_loc_single_duplicate_string(idx, key): @pytest.mark.parametrize( - "idx", [pd.Index(["b", "m", "m", "q"]), pd.Index(["m", "f", "m", "q"])] + "idx", [pd.Index(["b", "m", "m", "q"]), pd.Index(["a", "f", "m", "q"])] ) -@pytest.mark.parametrize("key", ["a", "f", "n", "z"]) -@pytest.mark.parametrize("method", [None]) +@pytest.mark.parametrize("key", [["a"], ["f", "n", "z"]]) +@pytest.mark.parametrize("method", [None, "ffill", "bfill"]) def test_get_indexer_single_duplicate_string(idx, key, method): pi = idx gi = cudf.from_pandas(pi) - if key not in pi: + if ( + # `method` only applicable to monotonic index + (not pi.is_monotonic_increasing and method is not None) + or not pi.is_unique + ): assert_exceptions_equal( lfunc=pi.get_indexer, rfunc=gi.get_indexer, @@ -2231,28 +2211,20 @@ def test_get_loc_multi_numeric(idx, key): [(2, 1, 1), (1, 2, 3), (1, 2, 1), (1, 1, 2), (2, 2, 1), (1, 1, 1)] ), pd.MultiIndex.from_tuples( - [(1, 1, 1), (1, 1, 2), (1, 1, 2), (1, 2, 3), (2, 1, 1), (2, 2, 1)] + [(1, 1, 1), (1, 1, 2), (1, 1, 24), (1, 2, 3), (2, 1, 1), (2, 2, 1)] ), ], ) -@pytest.mark.parametrize("key", [1, (1, 2), (1, 2, 3), (2, 1, 1), (9, 9, 9)]) +@pytest.mark.parametrize("key", [[(1, 2, 3)], [(9, 9, 9)]]) @pytest.mark.parametrize("method", [None]) def test_get_indexer_multi_numeric(idx, key, method): pi = idx.sort_values() gi = cudf.from_pandas(pi) - if key not in pi: - assert_exceptions_equal( - lfunc=pi.get_indexer, - rfunc=gi.get_indexer, - lfunc_args_and_kwargs=([], {"key": key, "method": method}), - rfunc_args_and_kwargs=([], {"key": key, "method": method}), - ) - else: - expected = pi.get_indexer(key, method=method) - got = gi.get_indexer(key, method=method) + expected = pi.get_indexer(key, method=method) + got = gi.get_indexer(key, method=method) - assert_eq(expected, got) + assert_eq(expected, got) @pytest.mark.parametrize( @@ -2303,46 +2275,27 @@ def test_get_loc_multi_numeric_deviate(idx, key, result): "idx", [ pd.MultiIndex.from_tuples( - [(2, 1, 1), (1, 2, 3), (1, 2, 1), (1, 1, 1), (1, 1, 1), (2, 2, 1)] + [(2, 1, 1), (1, 2, 3), (1, 2, 1), (1, 1, 10), (1, 1, 1), (2, 2, 1)] ) ], ) @pytest.mark.parametrize( - "key, result", + "key", [ - (1, slice(1, 5, 1)), # deviates - ((1, 2), slice(1, 3, 1)), - ((1, 2, 3), slice(1, 2, None)), - ((2, 1, 1), slice(0, 1, None)), - ((9, 9, 9), None), + ((1, 2, 3),), + ((2, 1, 1),), + ((9, 9, 9),), ], ) -@pytest.mark.parametrize("method", [None]) -def test_get_indexer_multi_numeric_deviate(idx, key, result, method): +@pytest.mark.parametrize("method", [None, "ffill", "bfill"]) +def test_get_indexer_multi_numeric_deviate(idx, key, method): pi = idx gi = cudf.from_pandas(pi) - with expect_warning_if( - isinstance(key, tuple), pd.errors.PerformanceWarning - ): - key_flag = key not in pi - - if key_flag: - with expect_warning_if( - isinstance(key, tuple), pd.errors.PerformanceWarning - ): - assert_exceptions_equal( - lfunc=pi.get_loc, - rfunc=gi.get_loc, - lfunc_args_and_kwargs=([], {"key": key, "method": method}), - rfunc_args_and_kwargs=([], {"key": key, "method": method}), - ) - else: - expected = result - with expect_warning_if(method is not None): - got = gi.get_loc(key, method=method) + expected = pi.get_indexer(key, method=method) + got = gi.get_indexer(key, method=method) - assert_eq(expected, got) + assert_eq(expected, got) @pytest.mark.parametrize( @@ -2454,48 +2407,20 @@ def test_get_loc_multi_string(idx, key): ("b", "c", "a"), ] ), - pd.MultiIndex.from_tuples( - [ - ("a", "a", "a"), - ("a", "a", "b"), - ("a", "a", "b"), - ("a", "b", "c"), - ("b", "a", "a"), - ("b", "c", "a"), - ] - ), - pd.MultiIndex.from_tuples( - [ - ("a", "a", "b"), - ("b", "a", "a"), - ("b", "a", "a"), - ("a", "a", "a"), - ("a", "b", "a"), - ("b", "c", "a"), - ] - ), ], ) @pytest.mark.parametrize( - "key", ["a", ("a", "a"), ("a", "b", "c"), ("b", "c", "a"), ("z", "z", "z")] + "key", [[("a", "b", "c"), ("b", "c", "a")], [("z", "z", "z")]] ) -@pytest.mark.parametrize("method", [None]) +@pytest.mark.parametrize("method", [None, "ffill", "bfill"]) def test_get_indexer_multi_string(idx, key, method): pi = idx.sort_values() gi = cudf.from_pandas(pi) - if key not in pi: - assert_exceptions_equal( - lfunc=pi.get_indexer, - rfunc=gi.get_indexer, - lfunc_args_and_kwargs=([], {"key": key, "method": method}), - rfunc_args_and_kwargs=([], {"key": key, "method": method}), - ) - else: - expected = pi.get_indexer(key, method=method) - got = gi.get_indexer(key, method=method) + expected = pi.get_indexer(key, method=method) + got = gi.get_indexer(key, method=method) - assert_eq(expected, got) + assert_eq(expected, got) @pytest.mark.parametrize( From 569b3e7fd4da778f1e2effadc529ef17387809b1 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 26 Apr 2023 21:10:02 -0500 Subject: [PATCH 032/384] Fix `kurtosis` pytests to support `numeric_only` parameter (#13217) https://github.com/rapidsai/cudf/pull/12847 introduced support for `numeric_only`, this PR cleans up a `kurt` related pytest that was relying on the old behavior. This PR fixes 18 pytests : ``` = 413 failed, 88257 passed, 2044 skipped, 932 xfailed, 165 xpassed in 463.03s (0:07:43) = ``` On `pandas_2.0_feature_branch`: ``` = 431 failed, 88221 passed, 2044 skipped, 932 xfailed, 165 xpassed in 456.25s (0:07:36) = ``` --- python/cudf/cudf/tests/test_stats.py | 55 +++++++++++++--------------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/python/cudf/cudf/tests/test_stats.py b/python/cudf/cudf/tests/test_stats.py index 126a90e580c..12a08bdcefa 100644 --- a/python/cudf/cudf/tests/test_stats.py +++ b/python/cudf/cudf/tests/test_stats.py @@ -233,44 +233,39 @@ def test_misc_quantiles(data, q): @pytest.mark.parametrize( "data", [ - cudf.Series(np.random.normal(-100, 100, 1000)), - cudf.Series(np.random.randint(-50, 50, 1000)), - cudf.Series(np.zeros(100)), - cudf.Series(np.repeat(np.nan, 100)), - cudf.Series(np.array([1.123, 2.343, np.nan, 0.0])), - cudf.Series( - [5, 10, 53, None, np.nan, None, 12, 43, -423], nan_as_null=False - ), - cudf.Series([1.1032, 2.32, 43.4, 13, -312.0], index=[0, 4, 3, 19, 6]), - cudf.Series([]), - cudf.Series([-3]), + {"data": np.random.normal(-100, 100, 1000)}, + {"data": np.random.randint(-50, 50, 1000)}, + {"data": (np.zeros(100))}, + {"data": np.repeat(np.nan, 100)}, + {"data": np.array([1.123, 2.343, np.nan, 0.0])}, + { + "data": [5, 10, 53, None, np.nan, None, 12, 43, -423], + "nan_as_null": False, + }, + {"data": [1.1032, 2.32, 43.4, 13, -312.0], "index": [0, 4, 3, 19, 6]}, + {"data": []}, + {"data": [-3]}, ], ) @pytest.mark.parametrize("null_flag", [False, True]) -def test_kurtosis_series(data, null_flag): - pdata = data.to_pandas() +@pytest.mark.parametrize("numeric_only", [False, True]) +def test_kurtosis_series(data, null_flag, numeric_only): + gs = cudf.Series(**data) + ps = gs.to_pandas() - if null_flag and len(data) > 2: - data.iloc[[0, 2]] = None - pdata.iloc[[0, 2]] = None + if null_flag and len(gs) > 2: + gs.iloc[[0, 2]] = None + ps.iloc[[0, 2]] = None - got = data.kurtosis() - got = got if np.isscalar(got) else got.to_numpy() - expected = pdata.kurtosis() - np.testing.assert_array_almost_equal(got, expected) + got = gs.kurtosis(numeric_only=numeric_only) + expected = ps.kurtosis(numeric_only=numeric_only) - got = data.kurt() - got = got if np.isscalar(got) else got.to_numpy() - expected = pdata.kurt() - np.testing.assert_array_almost_equal(got, expected) + assert_eq(got, expected) - got = data.kurt(numeric_only=False) - got = got if np.isscalar(got) else got.to_numpy() - expected = pdata.kurt(numeric_only=False) - np.testing.assert_array_almost_equal(got, expected) + got = gs.kurt(numeric_only=numeric_only) + expected = ps.kurt(numeric_only=numeric_only) - with pytest.raises(NotImplementedError): - data.kurt(numeric_only=True) + assert_eq(got, expected) @pytest.mark.parametrize( From bbc84f6dee786e117d904da2c523ee35dd921976 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 26 Apr 2023 21:11:02 -0500 Subject: [PATCH 033/384] Fix parquet pytests errors with pandas-2.0 (#13216) Pandas-2.0 fixed the following issue where dtype of the column was being changed even when `np.nan` was not being written to it: ```python In [1]: import pandas as pd In [2]: df = pd.DataFrame({'a':[1, 2, 3]}) In [3]: df.dtypes Out[3]: a int64 dtype: object In [4]: df Out[4]: a 0 1 1 2 2 3 In [7]: df[[False]*3] = np.nan In [8]: df Out[8]: a 0 1 1 2 2 3 In [9]: df.dtypes Out[9]: a int64 dtype: object ``` Bug in pre-2.0: ```python In [1]: import pandas as pd In [2]: df = pd.DataFrame({'a':[1, 2, 3]}) In [3]: df.dtypes Out[3]: a int64 dtype: object In [4]: df Out[4]: a 0 1 1 2 2 3 In [7]: df[[False]*3] = np.nan In [8]: df Out[8]: a 0 1.0 1 2.0 2 3.0 In [9]: df.dtypes Out[9]: a float64 dtype: object ``` `make_pdf` was basically operating correctly with the help of this bug, this PR makes some fixes to the method and the callers to preserve the pytest behaviors. This PR fixes 6 pytests: ``` = 425 failed, 88227 passed, 2044 skipped, 932 xfailed, 165 xpassed in 471.32s (0:07:51) = ``` On `pandas_2.0_feature_branch`: ``` = 431 failed, 88221 passed, 2044 skipped, 932 xfailed, 165 xpassed in 456.25s (0:07:36) = ``` --- python/cudf/cudf/tests/test_parquet.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index ebebd857231..c0d9af6d67d 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -216,10 +216,13 @@ def make_pdf(nrows, ncolumns=1, nvalids=0, dtype=np.int64): ) test_pdf.columns.name = None - # Randomly but reproducibly mark subset of rows as invalid - random.seed(1337) - mask = random.sample(range(nrows), nvalids) - test_pdf[test_pdf.index.isin(mask)] = np.NaN + if nvalids: + # Randomly but reproducibly mark subset of rows as invalid + random.seed(1337) + mask = random.sample(range(nrows), nvalids) + test_pdf[test_pdf.index.isin(mask)] = np.NaN + if dtype: + test_pdf = test_pdf.astype(dtype) return test_pdf @@ -693,7 +696,7 @@ def test_parquet_reader_select_columns(datadir): def test_parquet_reader_invalids(tmpdir): - test_pdf = make_pdf(nrows=1000, nvalids=1000 // 4, dtype=np.int64) + test_pdf = make_pdf(nrows=1000, nvalids=1000 // 4, dtype="Int64") fname = tmpdir.join("invalids.parquet") test_pdf.to_parquet(fname, engine="pyarrow") @@ -701,7 +704,7 @@ def test_parquet_reader_invalids(tmpdir): expect = pd.read_parquet(fname) got = cudf.read_parquet(fname) - assert_eq(expect, got) + assert_eq(expect, got.to_pandas(nullable=True)) def test_parquet_reader_filenotfound(tmpdir): @@ -788,8 +791,8 @@ def create_parquet_source(df, src_type, fname): "src", ["filepath", "pathobj", "bytes_io", "bytes", "url"] ) def test_parquet_reader_multiple_files(tmpdir, src): - test_pdf1 = make_pdf(nrows=1000, nvalids=1000 // 2) - test_pdf2 = make_pdf(nrows=500) + test_pdf1 = make_pdf(nrows=1000, nvalids=1000 // 2, dtype="float64") + test_pdf2 = make_pdf(nrows=500, dtype="float64") expect = pd.concat([test_pdf1, test_pdf2]) src1 = create_parquet_source(test_pdf1, src, tmpdir.join("multi1.parquet")) @@ -1465,8 +1468,8 @@ def test_parquet_writer_int96_timestamps(tmpdir, pdf, gdf): def test_multifile_parquet_folder(tmpdir): - test_pdf1 = make_pdf(nrows=10, nvalids=10 // 2) - test_pdf2 = make_pdf(nrows=20) + test_pdf1 = make_pdf(nrows=10, nvalids=10 // 2, dtype="float64") + test_pdf2 = make_pdf(nrows=20, dtype="float64") expect = pd.concat([test_pdf1, test_pdf2]) tmpdir.mkdir("multi_part") From 3a85f646a961790c4547a54501679dc983db99a7 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 23 May 2023 13:39:09 -0500 Subject: [PATCH 034/384] Fix csv reader pytest & MultiIndex docstring (#13417) Pandas-2.0 moved to a very strict & consistent date format inference, we should plan to move similarly but meanwhile for the pytest to pass, we will need to pass date_format='mixed'. This PR also fixes a miscellaneous issue with MultiIndex.copy docstring. --- python/cudf/cudf/core/multiindex.py | 15 ++++++--------- python/cudf/cudf/tests/test_csv.py | 3 +++ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index dcc6783147b..0498aa474b6 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -348,8 +348,6 @@ def copy( ... codes=[[0, 0, 1, 1], [0, 1, 0, 1]], ... names=['Date', 'Symbol']) >>> idx2 = idx1.copy( - ... levels=[['day1', 'day2'], ['com1', 'com2']], - ... codes=[[0, 0, 1, 1], [0, 1, 0, 1]], ... names=['col1', 'col2']) >>> df.index = idx1 @@ -363,13 +361,12 @@ def copy( >>> df.index = idx2 >>> df - Close - col1 col2 - day1 com1 3400.00 - com2 226.58 - day2 com1 3401.80 - com2 228.91 - + Close + col1 col2 + 2020-08-27 AMZN 3400.00 + MSFT 226.58 + 2020-08-28 AMZN 3401.80 + MSFT 228.91 """ mi = MultiIndex._from_data(self._data.copy(deep=deep)) diff --git a/python/cudf/cudf/tests/test_csv.py b/python/cudf/cudf/tests/test_csv.py index b66e6bc74fb..5bb6de49f10 100644 --- a/python/cudf/cudf/tests/test_csv.py +++ b/python/cudf/cudf/tests/test_csv.py @@ -246,11 +246,14 @@ def test_csv_reader_datetime(parse_dates): parse_dates=parse_dates, dayfirst=True, ) + # Need to used `date_format='mixed'`, + # https://github.com/pandas-dev/pandas/issues/53355 pdf = pd.read_csv( StringIO(buffer), names=["date1", "date2", "bad"], parse_dates=parse_dates, dayfirst=True, + date_format="mixed", ) assert_eq(gdf, pdf) From c1e78b9665fcd8df63d518d0b57983b53c862c31 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 26 May 2023 15:26:10 -0500 Subject: [PATCH 035/384] Deprecate `Groupby.dtypes` (#13453) This PR deprecates `Groupby.dtypes` since it is deprecated in `pandas-2.1` This PR fixes 5 pytests: ``` = 474 failed, 95510 passed, 2044 skipped, 763 xfailed, 300 xpassed in 459.93s (0:07:39) = ``` On `pandas_2.0_feature_branch`: ``` = 479 failed, 95505 passed, 2044 skipped, 763 xfailed, 300 xpassed in 471.66s (0:07:51) = ``` --- python/cudf/cudf/core/_compat.py | 1 + python/cudf/cudf/core/groupby/groupby.py | 19 ++++++++++++++----- python/cudf/cudf/tests/test_groupby.py | 13 +++++++++++-- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/python/cudf/cudf/core/_compat.py b/python/cudf/cudf/core/_compat.py index 183faa12904..bbcde903871 100644 --- a/python/cudf/cudf/core/_compat.py +++ b/python/cudf/cudf/core/_compat.py @@ -11,3 +11,4 @@ PANDAS_LT_153 = PANDAS_VERSION < version.parse("1.5.3") PANDAS_EQ_200 = PANDAS_VERSION == version.parse("2.0.0") PANDAS_GE_200 = PANDAS_VERSION >= version.parse("2.0.0") +PANDAS_GE_210 = PANDAS_VERSION >= version.parse("2.1.0") diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index df91625d7f2..a2d973605ba 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -291,6 +291,9 @@ def dtypes(self): """ Return the dtypes in this group. + .. deprecated:: 23.08 + Use `.dtypes` on base object instead. + Returns ------- pandas.DataFrame @@ -302,17 +305,23 @@ def dtypes(self): >>> df = cudf.DataFrame({'a': [1, 2, 3, 3], 'b': ['x', 'y', 'z', 'a'], ... 'c':[10, 11, 12, 12]}) >>> df.groupby("a").dtypes - b c + a b c a - 1 object int64 - 2 object int64 - 3 object int64 + 1 int64 object int64 + 2 int64 object int64 + 3 int64 object int64 """ + warnings.warn( + f"{type(self).__name__}.dtypes is deprecated and will be " + "removed in a future version. Check the dtypes on the " + "base object instead", + FutureWarning, + ) index = self.grouping.keys.unique().sort_values().to_pandas() return pd.DataFrame( { name: [self.obj._dtypes[name]] * len(index) - for name in self.grouping.values._column_names + for name in self.obj._data.names }, index=index, ) diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index a486bf8ff89..a560196f14b 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -19,7 +19,12 @@ import cudf from cudf import DataFrame, Series -from cudf.core._compat import PANDAS_GE_150, PANDAS_LT_140, PANDAS_GE_200 +from cudf.core._compat import ( + PANDAS_GE_150, + PANDAS_LT_140, + PANDAS_GE_200, + PANDAS_GE_210, +) from cudf.core.udf.groupby_typing import SUPPORTED_GROUPBY_NUMPY_TYPES from cudf.core.udf.utils import precompiled from cudf.testing._utils import ( @@ -3100,8 +3105,12 @@ def test_groupby_dtypes(groups): {"a": [1, 2, 3, 3], "b": ["x", "y", "z", "a"], "c": [10, 11, 12, 12]} ) pdf = df.to_pandas() + with expect_warning_if(PANDAS_GE_210): + expected = pdf.groupby(groups).dtypes + with pytest.warns(FutureWarning): + actual = df.groupby(groups).dtypes - assert_eq(pdf.groupby(groups).dtypes, df.groupby(groups).dtypes) + assert_eq(expected, actual) @pytest.mark.parametrize("index_names", ["a", "b", "c", ["b", "c"]]) From 2dafcfcddf570263e3c244e2e03897cf8e3fc40b Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 26 May 2023 15:39:17 -0500 Subject: [PATCH 036/384] Enforce Groupby.__iter__ deprecation and miscellaneous pytest fixes (#13423) This PR: - [x] Enforces deprecation in `GroupBy.__iter__` - [x] Fixes miscellaneous pytest failures due to already existing differences in cudf vs pandas & nuly introduced `inferred_type` in Index. --- python/cudf/cudf/core/column/column.py | 11 +++++++++- python/cudf/cudf/core/dataframe.py | 9 +++++--- python/cudf/cudf/core/groupby/groupby.py | 12 +++------- .../cudf/cudf/tests/test_column_accessor.py | 13 +++++++++-- python/cudf/cudf/tests/test_dataframe.py | 9 ++++++-- python/cudf/cudf/tests/test_groupby.py | 22 +++++++------------ python/cudf/cudf/tests/test_replace.py | 7 +++--- 7 files changed, 49 insertions(+), 34 deletions(-) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index b08c35d8997..0a87dc144c1 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -2314,7 +2314,16 @@ def as_column( pa_type = np_to_pa_dtype( _maybe_convert_to_default_type("float") ) - + if ( + pa_type is None + and isinstance(arbitrary, pd.Index) + and arbitrary.shape == (0,) + ): + # When an empty `pd.Index` is passed to `pa.array`, + # a type of `null-type` is returned by pyarrow, hence + # we need this workaround to preserve the dtype of + # column being created. + pa_type = np_to_pa_dtype(arbitrary.dtype) data = as_column( pa.array( arbitrary, diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index df2f87c805f..675b870056d 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -5852,6 +5852,7 @@ def _reduce( ): source = self + axis = source._get_axis_from_axis_arg(axis) if numeric_only: numeric_cols = ( name @@ -5860,9 +5861,11 @@ def _reduce( ) source = self._get_columns_by_label(numeric_cols) if source.empty: - return Series(index=self.index) - - axis = source._get_axis_from_axis_arg(axis) + return Series( + index=self._data.to_pandas_index()[:0] + if axis == 0 + else source.index + ) if axis == 0: try: diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index a2d973605ba..9e9b52a7538 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -272,19 +272,13 @@ def __init__( self.grouping = _Grouping(obj, by, level) def __iter__(self): - if isinstance(self._by, list) and len(self._by) == 1: - warnings.warn( - "In a future version of cudf, a length 1 tuple will be " - "returned when iterating over a groupby with a grouper equal " - "to a list of length 1. To avoid this warning, do not supply " - "a list with a single grouper.", - FutureWarning, - ) group_names, offsets, _, grouped_values = self._grouped() if isinstance(group_names, cudf.BaseIndex): group_names = group_names.to_pandas() for i, name in enumerate(group_names): - yield name, grouped_values[offsets[i] : offsets[i + 1]] + yield (name,) if isinstance(self._by, list) and len( + self._by + ) == 1 else name, grouped_values[offsets[i] : offsets[i + 1]] @property def dtypes(self): diff --git a/python/cudf/cudf/tests/test_column_accessor.py b/python/cudf/cudf/tests/test_column_accessor.py index 99d4bdd9910..b983c2dcab9 100644 --- a/python/cudf/cudf/tests/test_column_accessor.py +++ b/python/cudf/cudf/tests/test_column_accessor.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. import pandas as pd @@ -7,6 +7,7 @@ import cudf from cudf.core.column_accessor import ColumnAccessor from cudf.testing._utils import assert_eq +from cudf.core._compat import PANDAS_GE_200 simple_test_data = [ {}, @@ -52,7 +53,15 @@ def test_to_pandas_simple(simple_data): Test that a ColumnAccessor converts to a correct pd.Index """ ca = ColumnAccessor(simple_data) - assert_eq(ca.to_pandas_index(), pd.DataFrame(simple_data).columns) + # We cannot return RangeIndex, while pandas returns RangeIndex. + # Pandas compares `inferred_type` which is `empty` for + # Index([], dtype='object'), and `integer` for RangeIndex() + # to ignore this `inferred_type` comparison, we pass exact=False. + assert_eq( + ca.to_pandas_index(), + pd.DataFrame(simple_data).columns, + exact=not PANDAS_GE_200, + ) def test_to_pandas_multiindex(mi_data): diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index ee1309ef402..5875959b0c2 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -308,7 +308,7 @@ def test_axes(data): actual = csr.axes for e, a in zip(expected, actual): - assert_eq(e, a) + assert_eq(e, a, exact=not PANDAS_GE_200) def test_dataframe_truncate_axis_0(): @@ -4938,7 +4938,12 @@ def test_rowwise_ops(data, op, skipna, numeric_only): expected = getattr(pdf, op)(**kwargs) got = getattr(gdf, op)(**kwargs) - assert_eq(expected, got, check_dtype=False) + assert_eq( + expected, + got, + check_dtype=False, + check_index_type=False if len(got.index) == 0 else True, + ) @pytest.mark.parametrize( diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index a560196f14b..5583b2290ae 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -33,7 +33,6 @@ TIMEDELTA_TYPES, assert_eq, assert_exceptions_equal, - expect_warning_if, ) from cudf.testing.dataset_generator import rand_dataframe @@ -982,8 +981,7 @@ def test_groupby_unsupported_columns(): ) pdf["b"] = pd_cat gdf = cudf.from_pandas(pdf) - with pytest.warns(FutureWarning): - pdg = pdf.groupby("x").sum() + pdg = pdf.groupby("x").sum(numeric_only=True) # cudf does not yet support numeric_only, so our default is False (unlike # pandas, which defaults to inferring and throws a warning about it). gdg = gdf.groupby("x").sum() @@ -1547,15 +1545,11 @@ def test_grouping(grouper): ) gdf = cudf.from_pandas(pdf) - # There's no easy way to validate that the same warning is thrown by both - # cudf and pandas here because it's only thrown upon iteration, so we - # settle for catching warnings on the whole block. - with expect_warning_if(isinstance(grouper, list) and len(grouper) == 1): - for pdf_group, gdf_group in zip( - pdf.groupby(grouper), gdf.groupby(grouper) - ): - assert pdf_group[0] == gdf_group[0] - assert_eq(pdf_group[1], gdf_group[1]) + for pdf_group, gdf_group in zip( + pdf.groupby(grouper), gdf.groupby(grouper) + ): + assert pdf_group[0] == gdf_group[0] + assert_eq(pdf_group[1], gdf_group[1]) @pytest.mark.parametrize("agg", [lambda x: x.count(), "count"]) @@ -3311,8 +3305,8 @@ def test_head_tail_empty(): expected = pdf.groupby(pd.Series(values)).head() got = df.groupby(cudf.Series(values)).head() - assert_eq(expected, got) + assert_eq(expected, got, check_column_type=not PANDAS_GE_200) expected = pdf.groupby(pd.Series(values)).tail() got = df.groupby(cudf.Series(values)).tail() - assert_eq(expected, got) + assert_eq(expected, got, check_column_type=not PANDAS_GE_200) diff --git a/python/cudf/cudf/tests/test_replace.py b/python/cudf/cudf/tests/test_replace.py index 13e44e7cf59..364afacd261 100644 --- a/python/cudf/cudf/tests/test_replace.py +++ b/python/cudf/cudf/tests/test_replace.py @@ -8,7 +8,7 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_134, PANDAS_GE_150 +from cudf.core._compat import PANDAS_GE_134, PANDAS_GE_150, PANDAS_GE_200 from cudf.core.dtypes import Decimal32Dtype, Decimal64Dtype, Decimal128Dtype from cudf.testing._utils import ( INTEGER_TYPES, @@ -1008,8 +1008,9 @@ def test_numeric_series_replace_dtype(series_dtype, replacement): pd.Series(["one", "two", "three"], dtype="category"), {"to_replace": "one", "value": "two", "inplace": True}, marks=pytest.mark.xfail( - condition=not PANDAS_GE_134, - reason="https://github.com/pandas-dev/pandas/issues/43232", + condition=(not PANDAS_GE_134) or (PANDAS_GE_200), + reason="https://github.com/pandas-dev/pandas/issues/43232" + "https://github.com/pandas-dev/pandas/issues/53358", ), ), ( From 16c987e2051e98bd3f714d1ad69ea7bb894eb4e1 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 30 May 2023 08:41:25 -0500 Subject: [PATCH 037/384] Preserve Index and grouped columns in `Groupby.nth` (#13442) In pandas-2.0 `groupby.nth` behavior has changed: https://pandas.pydata.org/docs/whatsnew/v2.0.0.html#dataframegroupby-nth-and-seriesgroupby-nth-now-behave-as-filtrations This PR enables preserving the callers index in the end result and returns grouping columns as part of the result. This PR fixes all 12 pytests in `python/cudf/cudf/tests/test_groupby.py::test_groupby_nth` --- python/cudf/cudf/core/groupby/groupby.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 9e9b52a7538..f79a337373e 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -802,10 +802,21 @@ def nth(self, n): """ Return the nth row from each group. """ - result = self.agg(lambda x: x.nth(n)).sort_index() - sizes = self.size().sort_index() - return result[sizes > n] + self.obj["__groupbynth_order__"] = range(0, len(self.obj)) + # We perform another groupby here to have the grouping columns + # be a part of dataframe columns. + result = self.obj.groupby(self.grouping.keys).agg(lambda x: x.nth(n)) + sizes = self.size().reindex(result.index) + + result = result[sizes > n] + + result._index = self.obj.index.take( + result._data["__groupbynth_order__"] + ) + del result._data["__groupbynth_order__"] + del self.obj._data["__groupbynth_order__"] + return result @_cudf_nvtx_annotate def ngroup(self, ascending=True): From 258bf3df9d0d29068985c43d43597f480165a17f Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 30 May 2023 08:43:47 -0500 Subject: [PATCH 038/384] `Index` class deprecation enforcements (#13204) This PR: - [x] Enforces `Index` related deprecations by removing `Float32Index`, `Float64Index`, `GenericIndex`, `Int8Index`, `Int16Index`, `Int32Index`, `Int64Index`, `StringIndex`, `UInt8Index`, `UInt16Index`, `UInt32Index`, `UInt64Index`. - [x] Cleans up the repr logic to more closely align with pandas for `` value representation incase of `string` dtype. - [x] Fixes docstring and pytests to support the removals of the above classes. This PR also fixes 202 pytests: ```bash = 267 failed, 95670 passed, 2044 skipped, 763 xfailed, 300 xpassed in 442.18s (0:07:22) = ``` On `pandas_2.0_feature_branch`: ```bash = 469 failed, 95464 passed, 2044 skipped, 763 xfailed, 300 xpassed in 469.26s (0:07:49) = ``` --- docs/cudf/source/api_docs/index_objects.rst | 3 - docs/cudf/source/conf.py | 2 +- .../source/developer_guide/library_design.md | 25 +- python/cudf/benchmarks/conftest.py | 6 +- python/cudf/cudf/__init__.py | 24 - python/cudf/cudf/_typing.py | 6 +- python/cudf/cudf/core/_base_index.py | 72 +- python/cudf/cudf/core/algorithms.py | 8 +- python/cudf/cudf/core/column/categorical.py | 4 +- python/cudf/cudf/core/column/methods.py | 4 +- python/cudf/cudf/core/column/string.py | 10 +- python/cudf/cudf/core/dataframe.py | 16 +- python/cudf/cudf/core/dtypes.py | 11 +- python/cudf/cudf/core/frame.py | 8 +- python/cudf/cudf/core/groupby/groupby.py | 3 +- python/cudf/cudf/core/index.py | 674 +++--------------- python/cudf/cudf/core/indexed_frame.py | 2 +- python/cudf/cudf/core/multiindex.py | 14 +- python/cudf/cudf/core/reshape.py | 4 +- python/cudf/cudf/core/series.py | 6 +- python/cudf/cudf/core/single_column_frame.py | 4 +- python/cudf/cudf/testing/testing.py | 26 +- python/cudf/cudf/tests/test_binops.py | 4 +- python/cudf/cudf/tests/test_dataframe.py | 4 +- python/cudf/cudf/tests/test_groupby.py | 7 +- python/cudf/cudf/tests/test_index.py | 109 +-- python/cudf/cudf/tests/test_monotonic.py | 10 +- python/cudf/cudf/tests/test_pack.py | 10 +- python/cudf/cudf/tests/test_pickling.py | 6 +- python/cudf/cudf/tests/test_repr.py | 31 +- python/cudf/cudf/tests/test_serialize.py | 4 +- python/cudf/cudf/tests/test_string.py | 5 +- python/dask_cudf/dask_cudf/backends.py | 8 +- 33 files changed, 284 insertions(+), 846 deletions(-) diff --git a/docs/cudf/source/api_docs/index_objects.rst b/docs/cudf/source/api_docs/index_objects.rst index 2a8d18e9cb7..1b748a8f69f 100644 --- a/docs/cudf/source/api_docs/index_objects.rst +++ b/docs/cudf/source/api_docs/index_objects.rst @@ -149,9 +149,6 @@ Numeric Index :template: autosummary/class_without_autosummary.rst RangeIndex - Int64Index - UInt64Index - Float64Index .. _api.categoricalindex: diff --git a/docs/cudf/source/conf.py b/docs/cudf/source/conf.py index 2d3d2494747..4d9558ecd33 100644 --- a/docs/cudf/source/conf.py +++ b/docs/cudf/source/conf.py @@ -261,7 +261,7 @@ def process_class_docstrings(app, what, name, obj, options, lines): from the processed docstring. """ if what == "class": - if name in {"cudf.RangeIndex", "cudf.Int64Index", "cudf.UInt64Index", "cudf.Float64Index", "cudf.CategoricalIndex", "cudf.IntervalIndex", "cudf.MultiIndex", "cudf.DatetimeIndex", "cudf.TimedeltaIndex", "cudf.TimedeltaIndex"}: + if name in {"cudf.RangeIndex", "cudf.CategoricalIndex", "cudf.IntervalIndex", "cudf.MultiIndex", "cudf.DatetimeIndex", "cudf.TimedeltaIndex", "cudf.TimedeltaIndex"}: cut_index = lines.index('.. rubric:: Attributes') lines[:] = lines[:cut_index] diff --git a/docs/cudf/source/developer_guide/library_design.md b/docs/cudf/source/developer_guide/library_design.md index 16b84476549..e1f91a6417d 100644 --- a/docs/cudf/source/developer_guide/library_design.md +++ b/docs/cudf/source/developer_guide/library_design.md @@ -22,7 +22,7 @@ Finally we tie these pieces together to provide a more holistic view of the proj % class IndexedFrame % class SingleColumnFrame % class BaseIndex -% class GenericIndex +% class Index % class MultiIndex % class RangeIndex % class DataFrame @@ -42,8 +42,8 @@ Finally we tie these pieces together to provide a more holistic view of the proj % BaseIndex <|-- MultiIndex % Frame <|-- MultiIndex % -% BaseIndex <|-- GenericIndex -% SingleColumnFrame <|-- GenericIndex +% BaseIndex <|-- Index +% SingleColumnFrame <|-- Index % % @enduml @@ -89,31 +89,26 @@ While we've highlighted some exceptional cases of Indexes before, let's start wi In practice, `BaseIndex` does have concrete implementations of a small set of methods. However, currently many of these implementations are not applicable to all subclasses and will be eventually be removed. -Almost all indexes are subclasses of `GenericIndex`, a single-columned index with the class hierarchy: +Almost all indexes are subclasses of `Index`, a single-columned index with the class hierarchy: ```python -class GenericIndex(SingleColumnFrame, BaseIndex) +class Index(SingleColumnFrame, BaseIndex) ``` Integer, float, or string indexes are all composed of a single column of data. -Most `GenericIndex` methods are inherited from `Frame`, saving us the trouble of rewriting them. +Most `Index` methods are inherited from `Frame`, saving us the trouble of rewriting them. We now consider the three main exceptions to this model: - A `RangeIndex` is not backed by a column of data, so it inherits directly from `BaseIndex` alone. Wherever possible, its methods have special implementations designed to avoid materializing columns. - Where such an implementation is infeasible, we fall back to converting it to an `Int64Index` first instead. + Where such an implementation is infeasible, we fall back to converting it to an `Index` of `int64` + dtype first instead. - A `MultiIndex` is backed by _multiple_ columns of data. Therefore, its inheritance hierarchy looks like `class MultiIndex(Frame, BaseIndex)`. Some of its more `Frame`-like methods may be inherited, but many others must be reimplemented since in many cases a `MultiIndex` is not expected to behave like a `Frame`. -- Just like in pandas, `Index` itself can never be instantiated. - `pandas.Index` is the parent class for indexes, - but its constructor returns an appropriate subclass depending on the input data type and shape. - Unfortunately, mimicking this behavior requires overriding `__new__`, - which in turn makes shared initialization across inheritance trees much more cumbersome to manage. - To enable sharing constructor logic across different index classes, - we instead define `BaseIndex` as the parent class of all indexes. +- To enable sharing constructor logic across different index classes, + we define `BaseIndex` as the parent class of all indexes. `Index` inherits from `BaseIndex`, but it masquerades as a `BaseIndex` to match pandas. - This class should contain no implementations since it is simply a factory for other indexes. ## The Column layer diff --git a/python/cudf/benchmarks/conftest.py b/python/cudf/benchmarks/conftest.py index 4f2bb96061f..5d0f80189c9 100644 --- a/python/cudf/benchmarks/conftest.py +++ b/python/cudf/benchmarks/conftest.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. """Defines pytest fixtures for all benchmarks. @@ -40,8 +40,8 @@ In addition to the above fixtures, we also provide the following more specialized fixtures: - rangeindex: Since RangeIndex always holds int64 data we cannot conflate - it with index_dtype_int64 (a true Int64Index), and it cannot hold nulls. - As a result, it is provided as a separate fixture. + it with index_dtype_int64 (a true Index with int64 dtype), and it + cannot hold nulls. As a result, it is provided as a separate fixture. """ import os diff --git a/python/cudf/cudf/__init__.py b/python/cudf/cudf/__init__.py index de0f2d67add..c64da9a8ab2 100644 --- a/python/cudf/cudf/__init__.py +++ b/python/cudf/cudf/__init__.py @@ -40,22 +40,10 @@ BaseIndex, CategoricalIndex, DatetimeIndex, - Float32Index, - Float64Index, - GenericIndex, Index, - Int8Index, - Int16Index, - Int32Index, - Int64Index, IntervalIndex, RangeIndex, - StringIndex, TimedeltaIndex, - UInt8Index, - UInt16Index, - UInt32Index, - UInt64Index, interval_range, ) from cudf.core.missing import NA @@ -106,15 +94,8 @@ "DatetimeIndex", "Decimal32Dtype", "Decimal64Dtype", - "Float32Index", - "Float64Index", - "GenericIndex", "Grouper", "Index", - "Int16Index", - "Int32Index", - "Int64Index", - "Int8Index", "IntervalDtype", "IntervalIndex", "ListDtype", @@ -123,13 +104,8 @@ "RangeIndex", "Scalar", "Series", - "StringIndex", "StructDtype", "TimedeltaIndex", - "UInt16Index", - "UInt32Index", - "UInt64Index", - "UInt8Index", "api", "concat", "crosstab", diff --git a/python/cudf/cudf/_typing.py b/python/cudf/cudf/_typing.py index e2ea12a0e4d..79762edbd65 100644 --- a/python/cudf/cudf/_typing.py +++ b/python/cudf/cudf/_typing.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. import sys from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, TypeVar, Union @@ -37,9 +37,7 @@ DataFrameOrSeries = Union["cudf.Series", "cudf.DataFrame"] SeriesOrIndex = Union["cudf.Series", "cudf.core.index.BaseIndex"] -SeriesOrSingleColumnIndex = Union[ - "cudf.Series", "cudf.core.index.GenericIndex" -] +SeriesOrSingleColumnIndex = Union["cudf.Series", "cudf.core.index.Index"] # Groupby aggregation AggType = Union[str, Callable] diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index 7d16824174a..46e7cdfac61 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -58,9 +58,9 @@ >>> import cudf >>> index = cudf.Index([1, 2, 3]) >>> index -Int64Index([1, 2, 3], dtype='int64') +Index([1, 2, 3], dtype='int64') >>> index.astype('float64') -Float64Index([1.0, 2.0, 3.0], dtype='float64') +Index([1.0, 2.0, 3.0], dtype='float64') """ @@ -135,7 +135,7 @@ def get_level_values(self, level): >>> import cudf >>> idx = cudf.Index(["a", "b", "c"]) >>> idx.get_level_values(0) - StringIndex(['a' 'b' 'c'], dtype='object') + Index(['a', 'b', 'c'], dtype='object') """ if level == self.name: @@ -182,7 +182,7 @@ def _clean_nulls_from_index(self): to `` as a preprocessing step to `__repr__` methods. This will involve changing type of Index object - to StringIndex but it is the responsibility of the `__repr__` + to string dtype but it is the responsibility of the `__repr__` methods using this method to replace or handle representation of the actual types correctly. """ @@ -225,7 +225,7 @@ def hasnans(self): >>> import numpy as np >>> index = cudf.Index([1, 2, np.nan, 3, 4], nan_as_null=False) >>> index - Float64Index([1.0, 2.0, nan, 3.0, 4.0], dtype='float64') + Index([1.0, 2.0, nan, 3.0, 4.0], dtype='float64') >>> index.hasnans True @@ -233,7 +233,7 @@ def hasnans(self): >>> index = cudf.Index([1, 2, None, 3, 4]) >>> index - Int64Index([1, 2, , 3, 4], dtype='int64') + Index([1, 2, , 3, 4], dtype='int64') >>> index.hasnans True """ @@ -286,9 +286,9 @@ def set_names(self, names, level=None, inplace=False): >>> import cudf >>> idx = cudf.Index([1, 2, 3, 4]) >>> idx - Int64Index([1, 2, 3, 4], dtype='int64') + Index([1, 2, 3, 4], dtype='int64') >>> idx.set_names('quarter') - Int64Index([1, 2, 3, 4], dtype='int64', name='quarter') + Index([1, 2, 3, 4], dtype='int64', name='quarter') >>> idx = cudf.MultiIndex.from_product([['python', 'cobra'], ... [2018, 2019]]) >>> idx @@ -347,7 +347,7 @@ def union(self, other, sort=None): >>> idx1 = cudf.Index([1, 2, 3, 4]) >>> idx2 = cudf.Index([3, 4, 5, 6]) >>> idx1.union(idx2) - Int64Index([1, 2, 3, 4, 5, 6], dtype='int64') + Index([1, 2, 3, 4, 5, 6], dtype='int64') MultiIndex case @@ -437,7 +437,7 @@ def intersection(self, other, sort=False): >>> idx1 = cudf.Index([1, 2, 3, 4]) >>> idx2 = cudf.Index([3, 4, 5, 6]) >>> idx1.intersection(idx2) - Int64Index([3, 4], dtype='int64') + Index([3, 4], dtype='int64') MultiIndex case @@ -541,9 +541,9 @@ def fillna(self, value, downcast=None): >>> import cudf >>> index = cudf.Index([1, 2, None, 4]) >>> index - Int64Index([1, 2, , 4], dtype='int64') + Index([1, 2, , 4], dtype='int64') >>> index.fillna(3) - Int64Index([1, 2, 3, 4], dtype='int64') + Index([1, 2, 3, 4], dtype='int64') """ if downcast is not None: raise NotImplementedError( @@ -635,13 +635,13 @@ def to_pandas(self, nullable=False): >>> import cudf >>> idx = cudf.Index([-3, 10, 15, 20]) >>> idx - Int64Index([-3, 10, 15, 20], dtype='int64') + Index([-3, 10, 15, 20], dtype='int64') >>> idx.to_pandas() - Int64Index([-3, 10, 15, 20], dtype='int64') + Index([-3, 10, 15, 20], dtype='int64') >>> type(idx.to_pandas()) - + >>> type(idx) - + """ raise NotImplementedError @@ -666,7 +666,7 @@ def isin(self, values): -------- >>> idx = cudf.Index([1,2,3]) >>> idx - Int64Index([1, 2, 3], dtype='int64') + Index([1, 2, 3], dtype='int64') Check whether each index value in a list of values. @@ -736,17 +736,17 @@ def append(self, other): >>> import cudf >>> idx = cudf.Index([1, 2, 10, 100]) >>> idx - Int64Index([1, 2, 10, 100], dtype='int64') + Index([1, 2, 10, 100], dtype='int64') >>> other = cudf.Index([200, 400, 50]) >>> other - Int64Index([200, 400, 50], dtype='int64') + Index([200, 400, 50], dtype='int64') >>> idx.append(other) - Int64Index([1, 2, 10, 100, 200, 400, 50], dtype='int64') + Index([1, 2, 10, 100, 200, 400, 50], dtype='int64') append accepts list of Index objects >>> idx.append([other, other]) - Int64Index([1, 2, 10, 100, 200, 400, 50, 200, 400, 50], dtype='int64') + Index([1, 2, 10, 100, 200, 400, 50, 200, 400, 50], dtype='int64') """ raise NotImplementedError @@ -778,14 +778,14 @@ def difference(self, other, sort=None): >>> import cudf >>> idx1 = cudf.Index([2, 1, 3, 4]) >>> idx1 - Int64Index([2, 1, 3, 4], dtype='int64') + Index([2, 1, 3, 4], dtype='int64') >>> idx2 = cudf.Index([3, 4, 5, 6]) >>> idx2 - Int64Index([3, 4, 5, 6], dtype='int64') + Index([3, 4, 5, 6], dtype='int64') >>> idx1.difference(idx2) - Int64Index([1, 2], dtype='int64') + Index([1, 2], dtype='int64') >>> idx1.difference(idx2, sort=False) - Int64Index([2, 1], dtype='int64') + Index([2, 1], dtype='int64') """ if sort not in {None, False}: raise ValueError( @@ -1231,18 +1231,18 @@ def sort_values( >>> import cudf >>> idx = cudf.Index([10, 100, 1, 1000]) >>> idx - Int64Index([10, 100, 1, 1000], dtype='int64') + Index([10, 100, 1, 1000], dtype='int64') Sort values in ascending order (default behavior). >>> idx.sort_values() - Int64Index([1, 10, 100, 1000], dtype='int64') + Index([1, 10, 100, 1000], dtype='int64') Sort values in descending order, and also get the indices `idx` was sorted by. >>> idx.sort_values(ascending=False, return_indexer=True) - (Int64Index([1000, 100, 10, 1], dtype='int64'), array([3, 1, 0, 2], + (Index([1000, 100, 10, 1], dtype='int64'), array([3, 1, 0, 2], dtype=int32)) Sorting values in a MultiIndex: @@ -1319,7 +1319,7 @@ def join( names=['a', 'b']) >>> rhs = cudf.DataFrame({"a": [1, 4, 3]}).set_index('a').index >>> rhs - Int64Index([1, 4, 3], dtype='int64', name='a') + Index([1, 4, 3], dtype='int64', name='a') >>> lhs.join(rhs, how='inner') MultiIndex([(3, 4), (1, 2)], @@ -1402,12 +1402,12 @@ def rename(self, name, inplace=False): >>> import cudf >>> index = cudf.Index([1, 2, 3], name='one') >>> index - Int64Index([1, 2, 3], dtype='int64', name='one') + Index([1, 2, 3], dtype='int64', name='one') >>> index.name 'one' >>> renamed_index = index.rename('two') >>> renamed_index - Int64Index([1, 2, 3], dtype='int64', name='two') + Index([1, 2, 3], dtype='int64', name='two') >>> renamed_index.name 'two' """ @@ -1501,9 +1501,9 @@ def from_pandas(cls, index, nan_as_null=None): >>> data = [10, 20, 30, np.nan] >>> pdi = pd.Index(data) >>> cudf.Index.from_pandas(pdi) - Float64Index([10.0, 20.0, 30.0, ], dtype='float64') + Index([10.0, 20.0, 30.0, ], dtype='float64') >>> cudf.Index.from_pandas(pdi, nan_as_null=False) - Float64Index([10.0, 20.0, 30.0, nan], dtype='float64') + Index([10.0, 20.0, 30.0, nan], dtype='float64') """ if not isinstance(index, pd.Index): raise TypeError("not a pandas.Index") @@ -1674,7 +1674,7 @@ def take(self, indices, axis=0, allow_fill=True, fill_value=None): -------- >>> idx = cudf.Index(['a', 'b', 'c', 'd', 'e']) >>> idx.take([2, 0, 4, 3]) - StringIndex(['c' 'a' 'e' 'd'], dtype='object') + Index(['c', 'a', 'e', 'd'], dtype='object') """ if axis not in {0, "index"}: @@ -1725,9 +1725,9 @@ def repeat(self, repeats, axis=None): -------- >>> index = cudf.Index([10, 22, 33, 55]) >>> index - Int64Index([10, 22, 33, 55], dtype='int64') + Index([10, 22, 33, 55], dtype='int64') >>> index.repeat(5) - Int64Index([10, 10, 10, 10, 10, 22, 22, 22, 22, 22, 33, + Index([10, 10, 10, 10, 10, 22, 22, 22, 22, 22, 33, 33, 33, 33, 33, 55, 55, 55, 55, 55], dtype='int64') """ diff --git a/python/cudf/cudf/core/algorithms.py b/python/cudf/cudf/core/algorithms.py index 50ec4b774ee..56bb575d6d6 100644 --- a/python/cudf/cudf/core/algorithms.py +++ b/python/cudf/cudf/core/algorithms.py @@ -46,7 +46,7 @@ def factorize(values, sort=False, use_na_sentinel=True, size_hint=None): >>> codes array([0, 1, 1], dtype=int8) >>> uniques - StringIndex(['a' 'c'], dtype='object') + Index(['a' 'c'], dtype='object') When ``use_na_sentinel=True`` (the default), missing values are indicated in the `codes` with the sentinel value ``-1`` and missing values are not @@ -56,7 +56,7 @@ def factorize(values, sort=False, use_na_sentinel=True, size_hint=None): >>> codes array([ 1, -1, 0, 2, 1], dtype=int8) >>> uniques - StringIndex(['a' 'b' 'c'], dtype='object') + Index(['a', 'b', 'c'], dtype='object') If NA is in the values, and we want to include NA in the uniques of the values, it can be achieved by setting ``use_na_sentinel=False``. @@ -66,12 +66,12 @@ def factorize(values, sort=False, use_na_sentinel=True, size_hint=None): >>> codes array([ 0, 1, 0, -1], dtype=int8) >>> uniques - Float64Index([1.0, 2.0], dtype='float64') + Index([1.0, 2.0], dtype='float64') >>> codes, uniques = cudf.factorize(values, use_na_sentinel=False) >>> codes array([1, 2, 1, 0], dtype=int8) >>> uniques - Float64Index([, 1.0, 2.0], dtype='float64') + Index([, 1.0, 2.0], dtype='float64') """ return_cupy_array = isinstance(values, cp.ndarray) diff --git a/python/cudf/cudf/core/column/categorical.py b/python/cudf/cudf/core/column/categorical.py index c026574f8cd..6352f9f1fa0 100644 --- a/python/cudf/cudf/core/column/categorical.py +++ b/python/cudf/cudf/core/column/categorical.py @@ -63,7 +63,7 @@ class CategoricalAccessor(ColumnMethods): dtype: category Categories (3, int64): [1, 2, 3] >>> s.cat.categories - Int64Index([1, 2, 3], dtype='int64') + Index([1, 2, 3], dtype='int64') >>> s.cat.reorder_categories([3,2,1]) 0 1 1 2 @@ -106,7 +106,7 @@ def __init__(self, parent: SeriesOrSingleColumnIndex): super().__init__(parent=parent) @property - def categories(self) -> "cudf.core.index.GenericIndex": + def categories(self) -> "cudf.core.index.Index": """ The categories of this categorical. """ diff --git a/python/cudf/cudf/core/column/methods.py b/python/cudf/cudf/core/column/methods.py index c1b6dad00b7..0e7bcdc296c 100644 --- a/python/cudf/cudf/core/column/methods.py +++ b/python/cudf/cudf/core/column/methods.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. from __future__ import annotations @@ -8,7 +8,7 @@ import cudf -ParentType = Union["cudf.Series", "cudf.core.index.GenericIndex"] +ParentType = Union["cudf.Series", "cudf.core.index.Index"] class ColumnMethods: diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index 2e74ec62204..0205d0ee43b 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -708,9 +708,9 @@ def contains( >>> data = ['Mouse', 'dog', 'house and parrot', '23.0', np.NaN] >>> idx = cudf.Index(data) >>> idx - StringIndex(['Mouse' 'dog' 'house and parrot' '23.0' None], dtype='object') + Index(['Mouse' 'dog' 'house and parrot' '23.0' None], dtype='object') >>> idx.str.contains('23', regex=False) - GenericIndex([False, False, False, True, ], dtype='bool') + Index([False, False, False, True, ], dtype='bool') Returning 'house' or 'dog' when either expression occurs in a string. @@ -2811,7 +2811,7 @@ def partition(self, sep: str = " ", expand: bool = True) -> SeriesOrIndex: >>> idx = cudf.Index(['X 123', 'Y 999']) >>> idx - StringIndex(['X 123' 'Y 999'], dtype='object') + Index(['X 123' 'Y 999'], dtype='object') Which will create a MultiIndex: @@ -2876,7 +2876,7 @@ def rpartition(self, sep: str = " ", expand: bool = True) -> SeriesOrIndex: >>> idx = cudf.Index(['X 123', 'Y 999']) >>> idx - StringIndex(['X 123' 'Y 999'], dtype='object') + Index(['X 123' 'Y 999'], dtype='object') Which will create a MultiIndex: @@ -3542,7 +3542,7 @@ def count(self, pat: str, flags: int = 0) -> SeriesOrIndex: >>> index = cudf.Index(['A', 'A', 'Aaba', 'cat']) >>> index.str.count('a') - Int64Index([0, 0, 2, 1], dtype='int64') + Index([0, 0, 2, 1], dtype='int64') """ # noqa W605 if isinstance(pat, re.Pattern): flags = pat.flags & ~re.U diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 675b870056d..624e378011a 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -1673,7 +1673,7 @@ def _concat( if empty_has_index and num_empty_input_frames == len(objs): out._index = cudf.RangeIndex(result_index_length) elif are_all_range_index and not ignore_index: - out._index = cudf.core.index.GenericIndex._concat( + out._index = cudf.core.index.Index._concat( [o._index for o in objs] ) @@ -3381,7 +3381,7 @@ def rename( if index: if ( any(type(item) == str for item in index.values()) - and type(self.index) != cudf.StringIndex + and type(self.index._values) != cudf.core.column.StringColumn ): raise NotImplementedError( "Implicit conversion of index to " @@ -6606,7 +6606,7 @@ def keys(self): Columns: [0, 1, 2, 3] Index: [] >>> df.keys() - Int64Index([0, 1, 2, 3], dtype='int64') + Index([0, 1, 2, 3], dtype='int64') """ return self._data.to_pandas_index() @@ -7308,14 +7308,14 @@ def from_pandas(obj, nan_as_null=None): >>> pidx = pd.Index([1, 2, 10, 20]) >>> pidx - Int64Index([1, 2, 10, 20], dtype='int64') + Index([1, 2, 10, 20], dtype='int64') >>> gidx = cudf.from_pandas(pidx) >>> gidx - Int64Index([1, 2, 10, 20], dtype='int64') + Index([1, 2, 10, 20], dtype='int64') >>> type(gidx) - + >>> type(pidx) - + Converting a Pandas MultiIndex to cuDF MultiIndex: @@ -7494,7 +7494,7 @@ def _get_union_of_indices(indexes): if len(indexes) == 1: return indexes[0] else: - merged_index = cudf.core.index.GenericIndex._concat(indexes) + merged_index = cudf.core.index.Index._concat(indexes) merged_index = merged_index.drop_duplicates() _, inds = merged_index._values.sort_by_values() return merged_index.take(inds) diff --git a/python/cudf/cudf/core/dtypes.py b/python/cudf/cudf/core/dtypes.py index edd557aad1f..dce595b0843 100644 --- a/python/cudf/cudf/core/dtypes.py +++ b/python/cudf/cudf/core/dtypes.py @@ -162,7 +162,7 @@ def __init__(self, categories=None, ordered: bool = False) -> None: self._ordered = ordered @property - def categories(self) -> "cudf.core.index.GenericIndex": + def categories(self) -> "cudf.core.index.Index": """ An ``Index`` containing the unique categories allowed. @@ -171,7 +171,7 @@ def categories(self) -> "cudf.core.index.GenericIndex": >>> import cudf >>> dtype = cudf.CategoricalDtype(categories=['b', 'a'], ordered=True) >>> dtype.categories - StringIndex(['b' 'a'], dtype='object') + Index(['b', 'a'], dtype='object') """ if self._categories is None: return cudf.core.index.as_index( @@ -238,9 +238,10 @@ def to_pandas(self) -> pd.CategoricalDtype: if self._categories is None: categories = None else: - if isinstance( - self._categories, (cudf.Float32Index, cudf.Float64Index) - ): + if self._categories.dtype in { + cudf.dtype("float32"), + cudf.dtype("float64"), + }: categories = self._categories.dropna().to_pandas() else: categories = self._categories.to_pandas() diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index c7330da5cfa..89b38fad376 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -210,12 +210,12 @@ def size(self): >>> index = cudf.Index([]) >>> index - Float64Index([], dtype='float64') + Index([], dtype='float64') >>> index.size 0 >>> index = cudf.Index([1, 2, 3, 10]) >>> index - Int64Index([1, 2, 3, 10], dtype='int64') + Index([1, 2, 3, 10], dtype='int64') >>> index.size 4 @@ -1289,7 +1289,7 @@ def isna(self): >>> idx = cudf.Index([1, 2, None, np.NaN, 0.32, np.inf]) >>> idx - Float64Index([1.0, 2.0, , , 0.32, Inf], dtype='float64') + Index([1.0, 2.0, , , 0.32, Inf], dtype='float64') >>> idx.isna() array([False, False, True, True, False, False]) """ @@ -1368,7 +1368,7 @@ def notna(self): >>> idx = cudf.Index([1, 2, None, np.NaN, 0.32, np.inf]) >>> idx - Float64Index([1.0, 2.0, , , 0.32, Inf], dtype='float64') + Index([1.0, 2.0, , , 0.32, Inf], dtype='float64') >>> idx.notna() array([ True, True, False, False, True, True]) """ diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index f79a337373e..8e88d994708 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -531,7 +531,8 @@ def agg(self, func): orig_dtypes = tuple(c.dtype for c in columns) # Note: When there are no key columns, the below produces - # a Float64Index, while Pandas returns an Int64Index + # an Index with float64 dtype, while Pandas returns + # an Index with int64 dtype. # (GH: 6945) ( result_columns, diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 783f4012311..c0664d3ca4d 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -9,12 +9,10 @@ from numbers import Number from typing import ( Any, - Dict, List, MutableMapping, Optional, Tuple, - Type, Union, ) @@ -22,6 +20,7 @@ import numpy as np import pandas as pd from pandas._config import get_option +from typing_extensions import Self import cudf from cudf._lib.datetime import extract_quarter, is_leap_year @@ -34,7 +33,6 @@ is_interval_dtype, is_list_like, is_scalar, - is_string_dtype, ) from cudf.core._base_index import BaseIndex, _index_astype_docstring from cudf.core.column import ( @@ -66,8 +64,33 @@ from cudf.core._compat import PANDAS_GE_200 +class IndexMeta(type): + """Custom metaclass for Index that overrides instance/subclass tests.""" + + def __call__(cls, data, *args, **kwargs): + if cls is Index: + return as_index( + arbitrary=data, + *args, + **kwargs, + ) + return super().__call__(data, *args, **kwargs) + + def __instancecheck__(self, instance): + if self is cudf.Index: + return isinstance(instance, BaseIndex) + else: + return False + + def __subclasscheck__(self, subclass): + if self is cudf.Index: + return issubclass(subclass, BaseIndex) + else: + return False + + def _lexsorted_equal_range( - idx: Union[GenericIndex, cudf.MultiIndex], + idx: Union[Index, cudf.MultiIndex], key_as_table: Frame, is_sorted: bool, ) -> Tuple[int, int, Optional[ColumnBase]]: @@ -100,18 +123,13 @@ def _index_from_data(data: MutableMapping, name: Any = None): values = next(iter(data.values())) if isinstance(values, NumericalColumn): - try: - index_class_type: Type[ - Union[GenericIndex, cudf.MultiIndex] - ] = _dtype_to_index[values.dtype.type] - except KeyError: - index_class_type = GenericIndex + index_class_type = Index elif isinstance(values, DatetimeColumn): index_class_type = DatetimeIndex elif isinstance(values, TimeDeltaColumn): index_class_type = TimedeltaIndex elif isinstance(values, StringColumn): - index_class_type = StringIndex + index_class_type = Index elif isinstance(values, CategoricalColumn): index_class_type = CategoricalIndex elif isinstance(values, (IntervalColumn, StructColumn)): @@ -195,8 +213,8 @@ def __init__( self._end = self._start + self._step * (len(self._range) - 1) def _copy_type_metadata( - self: RangeIndex, other: RangeIndex, *, override_dtypes=None - ) -> RangeIndex: + self, other: RangeIndex, *, override_dtypes=None + ) -> Self: # There is no metadata to be copied for RangeIndex since it does not # have an underlying column. return self @@ -564,7 +582,7 @@ def __rmul__(self, other): def _as_int_index(self): # Convert self to an integer index. This method is used to perform ops # that are not defined directly on RangeIndex. - return _dtype_to_index[self.dtype.type]._from_data(self._data) + return cudf.Index._from_data(self._data) @_cudf_nvtx_annotate def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): @@ -770,13 +788,13 @@ def sort_values( @_cudf_nvtx_annotate def _gather(self, gather_map, nullify=False, check_bounds=True): gather_map = cudf.core.column.as_column(gather_map) - return _dtype_to_index[self.dtype.type]._from_columns( + return cudf.Index._from_columns( [self._values.take(gather_map, nullify, check_bounds)], [self.name] ) @_cudf_nvtx_annotate def _apply_boolean_mask(self, boolean_mask): - return _dtype_to_index[self.dtype.type]._from_columns( + return cudf.Index._from_columns( [self._values.apply_boolean_mask(boolean_mask)], [self.name] ) @@ -784,7 +802,7 @@ def repeat(self, repeats, axis=None): return self._as_int_index().repeat(repeats, axis) def _split(self, splits): - return _dtype_to_index[self.dtype.type]._from_columns( + return cudf.Index._from_columns( [self._as_int_index()._split(splits)], [self.name] ) @@ -917,7 +935,7 @@ def __abs__(self): return abs(self._as_int_index()) -class GenericIndex(SingleColumnFrame, BaseIndex): +class Index(SingleColumnFrame, BaseIndex, metaclass=IndexMeta): """ An array of orderable values that represent the indices of another Column @@ -939,21 +957,6 @@ class GenericIndex(SingleColumnFrame, BaseIndex): @_cudf_nvtx_annotate def __init__(self, data, **kwargs): kwargs = _setdefault_name(data, **kwargs) - - # normalize the input - if isinstance(data, cudf.Series): - data = data._column - elif isinstance(data, column.ColumnBase): - data = data - else: - if isinstance(data, (list, tuple)): - if len(data) == 0: - data = np.asarray([], dtype="int64") - else: - data = np.asarray(data) - data = column.as_column(data) - assert isinstance(data, (NumericalColumn, StringColumn)) - name = kwargs.get("name") super().__init__({name: data}) @@ -985,8 +988,8 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): # pandas returns numpy arrays when the outputs are boolean. for i, o in enumerate(out): # We explicitly _do not_ use isinstance here: we want only - # boolean GenericIndexes, not dtype-specific subclasses. - if type(o) is GenericIndex and o.dtype.kind == "b": + # boolean Indexes, not dtype-specific subclasses. + if type(o) is Index and o.dtype.kind == "b": out[i] = o.values return out[0] if ufunc.nout == 1 else tuple(out) @@ -995,14 +998,21 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): @classmethod @_cudf_nvtx_annotate - def _from_data( - cls, data: MutableMapping, name: Any = None - ) -> GenericIndex: + def _from_data(cls, data: MutableMapping, name: Any = None) -> Self: out = super()._from_data(data=data) if name is not None: out.name = name return out + @classmethod + @_cudf_nvtx_annotate + def from_arrow(cls, obj): + try: + return cls(ColumnBase.from_arrow(obj)) + except TypeError: + # Try interpreting object as a MultiIndex before failing. + return cudf.MultiIndex.from_arrow(obj) + def _binaryop( self, other: Frame, @@ -1019,16 +1029,16 @@ def _binaryop( # pandas returns numpy arrays when the outputs are boolean. We # explicitly _do not_ use isinstance here: we want only boolean - # GenericIndexes, not dtype-specific subclasses. - if type(ret) is GenericIndex and ret.dtype.kind == "b": + # Indexes, not dtype-specific subclasses. + if type(ret) is Index and ret.dtype.kind == "b": return ret.values return ret # Override just to make mypy happy. @_cudf_nvtx_annotate def _copy_type_metadata( - self: GenericIndex, other: GenericIndex, *, override_dtypes=None - ) -> GenericIndex: + self, other: Self, *, override_dtypes=None + ) -> Self: return super()._copy_type_metadata( other, override_dtypes=override_dtypes ) @@ -1294,9 +1304,10 @@ def __repr__(self): output = output.replace("nan", cudf._NA_REP) elif preprocess._values.nullable: - output = repr(self._clean_nulls_from_index().to_pandas()) - - if not isinstance(self, StringIndex): + if isinstance(self._values, StringColumn): + output = repr(self.to_pandas(nullable=True)) + else: + output = repr(self._clean_nulls_from_index().to_pandas()) # We should remove all the single quotes # from the output due to the type-cast to # object dtype happening above. @@ -1341,7 +1352,7 @@ def __getitem__(self, index): @_cudf_nvtx_annotate def dtype(self): """ - `dtype` of the underlying values in GenericIndex. + `dtype` of the underlying values in Index. """ return self._values.dtype @@ -1382,19 +1393,21 @@ def get_slice_bound(self, label, side): return self._values.get_slice_bound(label, side) def _is_numeric(self): - return False + return isinstance( + self._values, cudf.core.column.NumericalColumn + ) and self.dtype != cudf.dtype("bool") def _is_boolean(self): - return True + return self.dtype == cudf.dtype("bool") def _is_integer(self): - return False + return cudf.api.types.is_integer_dtype(self.dtype) def _is_floating(self): - return False + return cudf.api.types.is_float_dtype(self.dtype) def _is_object(self): - return False + return isinstance(self._values, cudf.core.column.StringColumn) def _is_categorical(self): return False @@ -1536,333 +1549,19 @@ def isin(self, values): return self._values.isin(values).values - -class NumericIndex(GenericIndex): - """Immutable, ordered and sliceable sequence of labels. - The basic object storing row labels for all cuDF objects. - - Parameters - ---------- - data : array-like (1-dimensional) - dtype : NumPy dtype, - but not used. - copy : bool - Make a copy of input data. - name : object - Name to be stored in the index. - - Returns - ------- - Index - """ - - # Subclasses must define the dtype they are associated with. - _dtype: Union[None, Type[np.number]] = None - + @copy_docstring(StringMethods) # type: ignore + @property @_cudf_nvtx_annotate - def __init__(self, data=None, dtype=None, copy=False, name=None): - warnings.warn( - f"cudf.{self.__class__.__name__} is deprecated and will be " - "removed from cudf in a future version. Use cudf.Index with the " - "appropriate dtype instead.", - FutureWarning, - ) - - dtype = type(self)._dtype - if copy: - data = column.as_column(data, dtype=dtype).copy() - - kwargs = _setdefault_name(data, name=name) - - data = column.as_column(data, dtype=dtype) - - super().__init__(data, **kwargs) - - def _is_numeric(self): - return True - - def _is_boolean(self): - return False - - def _is_integer(self): - return True - - def _is_floating(self): - return False - - def _is_object(self): - return False - - def _is_categorical(self): - return False - - def _is_interval(self): - return False - - -class Int8Index(NumericIndex): - """ - Immutable, ordered and sliceable sequence of labels. - The basic object storing row labels for all cuDF objects. - Int8Index is a special case of Index with purely - integer(``int8``) labels. - - Parameters - ---------- - data : array-like (1-dimensional) - dtype : NumPy dtype, - but not used. - copy : bool - Make a copy of input data. - name : object - Name to be stored in the index. - - Returns - ------- - Int8Index - """ - - _dtype = np.int8 - - -class Int16Index(NumericIndex): - """ - Immutable, ordered and sliceable sequence of labels. - The basic object storing row labels for all cuDF objects. - Int16Index is a special case of Index with purely - integer(``int16``) labels. - - Parameters - ---------- - data : array-like (1-dimensional) - dtype : NumPy dtype, - but not used. - copy : bool - Make a copy of input data. - name : object - Name to be stored in the index. - - Returns - ------- - Int16Index - """ - - _dtype = np.int16 - - -class Int32Index(NumericIndex): - """ - Immutable, ordered and sliceable sequence of labels. - The basic object storing row labels for all cuDF objects. - Int32Index is a special case of Index with purely - integer(``int32``) labels. - - Parameters - ---------- - data : array-like (1-dimensional) - dtype : NumPy dtype, - but not used. - copy : bool - Make a copy of input data. - name : object - Name to be stored in the index. - - Returns - ------- - Int32Index - """ - - _dtype = np.int32 - - -class Int64Index(NumericIndex): - """ - Immutable, ordered and sliceable sequence of labels. - The basic object storing row labels for all cuDF objects. - Int64Index is a special case of Index with purely - integer(``int64``) labels. - - Parameters - ---------- - data : array-like (1-dimensional) - dtype : NumPy dtype, - but not used. - copy : bool - Make a copy of input data. - name : object - Name to be stored in the index. - - Returns - ------- - Int64Index - """ - - _dtype = np.int64 - - -class UInt8Index(NumericIndex): - """ - Immutable, ordered and sliceable sequence of labels. - The basic object storing row labels for all cuDF objects. - UInt8Index is a special case of Index with purely - integer(``uint64``) labels. - - Parameters - ---------- - data : array-like (1-dimensional) - dtype : NumPy dtype, - but not used. - copy : bool - Make a copy of input data. - name : object - Name to be stored in the index. - - Returns - ------- - UInt8Index - """ - - _dtype = np.uint8 - - -class UInt16Index(NumericIndex): - """ - Immutable, ordered and sliceable sequence of labels. - The basic object storing row labels for all cuDF objects. - UInt16Index is a special case of Index with purely - integer(``uint16``) labels. - - Parameters - ---------- - data : array-like (1-dimensional) - dtype : NumPy dtype, - but not used. - copy : bool - Make a copy of input data. - name : object - Name to be stored in the index. - - Returns - ------- - UInt16Index - """ - - _dtype = np.uint16 - - -class UInt32Index(NumericIndex): - """ - Immutable, ordered and sliceable sequence of labels. - The basic object storing row labels for all cuDF objects. - UInt32Index is a special case of Index with purely - integer(``uint32``) labels. - - Parameters - ---------- - data : array-like (1-dimensional) - dtype : NumPy dtype, - but not used. - copy : bool - Make a copy of input data. - name : object - Name to be stored in the index. - - Returns - ------- - UInt32Index - """ - - _dtype = np.uint32 - - -class UInt64Index(NumericIndex): - """ - Immutable, ordered and sliceable sequence of labels. - The basic object storing row labels for all cuDF objects. - UInt64Index is a special case of Index with purely - integer(``uint64``) labels. - - Parameters - ---------- - data : array-like (1-dimensional) - dtype : NumPy dtype, - but not used. - copy : bool - Make a copy of input data. - name : object - Name to be stored in the index. - - Returns - ------- - UInt64Index - """ - - _dtype = np.uint64 - - -class Float32Index(NumericIndex): - """ - Immutable, ordered and sliceable sequence of labels. - The basic object storing row labels for all cuDF objects. - Float32Index is a special case of Index with purely - float(``float32``) labels. - - Parameters - ---------- - data : array-like (1-dimensional) - dtype : NumPy dtype, - but not used. - copy : bool - Make a copy of input data. - name : object - Name to be stored in the index. - - Returns - ------- - Float32Index - """ - - _dtype = np.float32 - - def _is_integer(self): - return False - - def _is_floating(self): - return True - - -class Float64Index(NumericIndex): - """ - Immutable, ordered and sliceable sequence of labels. - The basic object storing row labels for all cuDF objects. - Float64Index is a special case of Index with purely - float(``float64``) labels. - - Parameters - ---------- - data : array-like (1-dimensional) - dtype : NumPy dtype, - but not used. - copy : bool - Make a copy of input data. - name : object - Name to be stored in the index. - - Returns - ------- - Float64Index - """ - - _dtype = np.float64 - - def _is_integer(self): - return False - - def _is_floating(self): - return True + def str(self): + if isinstance(self._values, cudf.core.column.StringColumn): + return StringMethods(parent=self) + else: + raise AttributeError( + "Can only use .str accessor with string values!" + ) -class DatetimeIndex(GenericIndex): +class DatetimeIndex(Index): """ Immutable , ordered and sliceable sequence of datetime64 data, represented internally as int64. @@ -1952,7 +1651,6 @@ def __init__( if copy: data = data.copy() - super().__init__(data, **kwargs) @property # type: ignore @@ -1970,7 +1668,7 @@ def year(self): >>> datetime_index DatetimeIndex(['2000-12-31', '2001-12-31', '2002-12-31'], dtype='datetime64[ns]') >>> datetime_index.year - Int16Index([2000, 2001, 2002], dtype='int16') + Index([2000, 2001, 2002], dtype='int16') """ # noqa: E501 return self._get_dt_field("year") @@ -1989,7 +1687,7 @@ def month(self): >>> datetime_index DatetimeIndex(['2000-01-31', '2000-02-29', '2000-03-31'], dtype='datetime64[ns]') >>> datetime_index.month - Int16Index([1, 2, 3], dtype='int16') + Index([1, 2, 3], dtype='int16') """ # noqa: E501 return self._get_dt_field("month") @@ -2008,7 +1706,7 @@ def day(self): >>> datetime_index DatetimeIndex(['2000-01-01', '2000-01-02', '2000-01-03'], dtype='datetime64[ns]') >>> datetime_index.day - Int16Index([1, 2, 3], dtype='int16') + Index([1, 2, 3], dtype='int16') """ # noqa: E501 return self._get_dt_field("day") @@ -2029,7 +1727,7 @@ def hour(self): '2000-01-01 02:00:00'], dtype='datetime64[ns]') >>> datetime_index.hour - Int16Index([0, 1, 2], dtype='int16') + Index([0, 1, 2], dtype='int16') """ return self._get_dt_field("hour") @@ -2050,7 +1748,7 @@ def minute(self): '2000-01-01 00:02:00'], dtype='datetime64[ns]') >>> datetime_index.minute - Int16Index([0, 1, 2], dtype='int16') + Index([0, 1, 2], dtype='int16') """ return self._get_dt_field("minute") @@ -2071,7 +1769,7 @@ def second(self): '2000-01-01 00:00:02'], dtype='datetime64[ns]') >>> datetime_index.second - Int16Index([0, 1, 2], dtype='int16') + Index([0, 1, 2], dtype='int16') """ return self._get_dt_field("second") @@ -2092,7 +1790,7 @@ def microsecond(self): '2000-01-01 00:00:00.000002'], dtype='datetime64[ns]') >>> datetime_index.microsecond - Int32Index([0, 1, 2], dtype='int32') + Index([0, 1, 2], dtype='int32') """ # noqa: E501 return as_index( ( @@ -2124,7 +1822,7 @@ def nanosecond(self): '2000-01-01 00:00:00.000000002'], dtype='datetime64[ns]') >>> datetime_index.nanosecond - Int16Index([0, 1, 2], dtype='int16') + Index([0, 1, 2], dtype='int16') """ return self._get_dt_field("nanosecond") @@ -2146,7 +1844,7 @@ def weekday(self): '2017-01-08'], dtype='datetime64[ns]') >>> datetime_index.weekday - Int16Index([5, 6, 0, 1, 2, 3, 4, 5, 6], dtype='int16') + Index([5, 6, 0, 1, 2, 3, 4, 5, 6], dtype='int16') """ return self._get_dt_field("weekday") @@ -2168,7 +1866,7 @@ def dayofweek(self): '2017-01-08'], dtype='datetime64[ns]') >>> datetime_index.dayofweek - Int16Index([5, 6, 0, 1, 2, 3, 4, 5, 6], dtype='int16') + Index([5, 6, 0, 1, 2, 3, 4, 5, 6], dtype='int16') """ return self._get_dt_field("weekday") @@ -2191,7 +1889,7 @@ def dayofyear(self): '2017-01-08'], dtype='datetime64[ns]') >>> datetime_index.dayofyear - Int16Index([366, 1, 2, 3, 4, 5, 6, 7, 8], dtype='int16') + Index([366, 1, 2, 3, 4, 5, 6, 7, 8], dtype='int16') """ return self._get_dt_field("day_of_year") @@ -2214,7 +1912,7 @@ def day_of_year(self): '2017-01-08'], dtype='datetime64[ns]') >>> datetime_index.day_of_year - Int16Index([366, 1, 2, 3, 4, 5, 6, 7, 8], dtype='int16') + Index([366, 1, 2, 3, 4, 5, 6, 7, 8], dtype='int16') """ return self._get_dt_field("day_of_year") @@ -2249,7 +1947,7 @@ def quarter(self): Returns ------- - Int8Index + Index Integer indicating which quarter the date belongs to. Examples @@ -2258,7 +1956,7 @@ def quarter(self): >>> gIndex = cudf.DatetimeIndex(["2020-05-31 08:00:00", ... "1999-12-31 18:40:00"]) >>> gIndex.quarter - Int8Index([2, 4], dtype='int8') + Index([2, 4], dtype='int8') """ res = extract_quarter(self._values) return Index(res, dtype="int8") @@ -2303,7 +2001,7 @@ def to_pandas(self, nullable=False): def _get_dt_field(self, field): out_column = self._values.get_dt_field(field) # column.column_empty_like always returns a Column object - # but we need a NumericalColumn for GenericIndex.. + # but we need a NumericalColumn for Index.. # how should this be handled? out_column = column.build_column( data=out_column.base_data, @@ -2515,7 +2213,7 @@ def tz_convert(self, tz): return DatetimeIndex._from_data({self.name: result_col}) -class TimedeltaIndex(GenericIndex): +class TimedeltaIndex(Index): """ Immutable, ordered and sliceable sequence of timedelta64 data, represented internally as int64. @@ -2588,7 +2286,6 @@ def __init__( if copy: data = data.copy() - super().__init__(data, **kwargs) @_cudf_nvtx_annotate @@ -2605,8 +2302,9 @@ def days(self): """ Number of days for each element. """ + # Need to specifically return `int64` to avoid overflow. return as_index( - arbitrary=self._values.days, name=self.name, dtype="int32" + arbitrary=self._values.days, name=self.name, dtype="int64" ) @property # type: ignore @@ -2664,7 +2362,7 @@ def _is_boolean(self): return False -class CategoricalIndex(GenericIndex): +class CategoricalIndex(Index): """ A categorical of orderable values that represent the indices of another Column @@ -2759,7 +2457,6 @@ def __init__( data = data.as_ordered() elif ordered is False and data.ordered is True: data = data.as_unordered() - super().__init__(data, **kwargs) @property # type: ignore @@ -2929,7 +2626,7 @@ def interval_range( return IntervalIndex(interval_col) -class IntervalIndex(GenericIndex): +class IntervalIndex(Index): """ Immutable index of intervals that are closed on the same side. @@ -3043,80 +2740,6 @@ def _is_boolean(self): return False -class StringIndex(GenericIndex): - """String defined indices into another Column - - .. deprecated:: 23.06 - `StringIndex` is deprecated, use `Index` instead. - - Attributes - ---------- - _values: A StringColumn object or NDArray of strings - name: A string - """ - - @_cudf_nvtx_annotate - def __init__(self, values, copy=False, **kwargs): - warnings.warn( - f"cudf.{self.__class__.__name__} is deprecated and will be " - "removed from cudf in a future version. Use cudf.Index with the " - "appropriate dtype instead.", - FutureWarning, - ) - kwargs = _setdefault_name(values, **kwargs) - if isinstance(values, StringColumn): - values = values.copy(deep=copy) - elif isinstance(values, StringIndex): - values = values._values.copy(deep=copy) - else: - values = column.as_column(values, dtype="str") - if not is_string_dtype(values.dtype): - raise ValueError( - "Couldn't create StringIndex from passed in object" - ) - - super().__init__(values, **kwargs) - - @_cudf_nvtx_annotate - def to_pandas(self, nullable=False): - return pd.Index( - self.to_numpy(na_value=None), - name=self.name, - dtype=pd.StringDtype() if nullable else "object", - ) - - @_cudf_nvtx_annotate - def __repr__(self): - return ( - f"{self.__class__.__name__}({self._values.values_host}," - f" dtype='object'" - + ( - f", name={pd.io.formats.printing.default_pprint(self.name)}" - if self.name is not None - else "" - ) - + ")" - ) - - @copy_docstring(StringMethods) # type: ignore - @property - @_cudf_nvtx_annotate - def str(self): - return StringMethods(parent=self) - - def _clean_nulls_from_index(self): - if self._values.has_nulls(): - return self.fillna(cudf._NA_REP) - else: - return self - - def _is_boolean(self): - return False - - def _is_object(self): - return True - - @_cudf_nvtx_annotate def as_index(arbitrary, nan_as_null=None, **kwargs) -> BaseIndex: """Create an Index from an arbitrary object @@ -3137,7 +2760,7 @@ def as_index(arbitrary, nan_as_null=None, **kwargs) -> BaseIndex: result : subclass of Index - CategoricalIndex for Categorical input. - DatetimeIndex for Datetime input. - - GenericIndex for all other inputs. + - Index for all other inputs. """ kwargs = _setdefault_name(arbitrary, **kwargs) if isinstance(arbitrary, cudf.MultiIndex): @@ -3174,119 +2797,12 @@ def as_index(arbitrary, nan_as_null=None, **kwargs) -> BaseIndex: ) -_dtype_to_index: Dict[Any, Type[NumericIndex]] = { - np.int8: Int8Index, - np.int16: Int16Index, - np.int32: Int32Index, - np.int64: Int64Index, - np.uint8: UInt8Index, - np.uint16: UInt16Index, - np.uint32: UInt32Index, - np.uint64: UInt64Index, - np.float32: Float32Index, - np.float64: Float64Index, -} - - def _setdefault_name(values, **kwargs): if kwargs.get("name") is None: kwargs["name"] = getattr(values, "name", None) return kwargs -class IndexMeta(type): - """Custom metaclass for Index that overrides instance/subclass tests.""" - - def __instancecheck__(self, instance): - return isinstance(instance, BaseIndex) - - def __subclasscheck__(self, subclass): - return issubclass(subclass, BaseIndex) - - -class Index(BaseIndex, metaclass=IndexMeta): - """The basic object storing row labels for all cuDF objects. - - Parameters - ---------- - data : array-like (1-dimensional)/ DataFrame - If it is a DataFrame, it will return a MultiIndex - dtype : NumPy dtype (default: object) - If dtype is None, we find the dtype that best fits the data. - copy : bool - Make a copy of input data. - name : object - Name to be stored in the index. - tupleize_cols : bool (default: True) - When True, attempt to create a MultiIndex if possible. - tupleize_cols == False is not yet supported. - nan_as_null : bool, Default True - If ``None``/``True``, converts ``np.nan`` values to - ``null`` values. - If ``False``, leaves ``np.nan`` values as is. - - Returns - ------- - Index - cudf Index - - Warnings - -------- - This class should not be subclassed. It is designed as a factory for - different subclasses of :class:`BaseIndex` depending on the provided input. - If you absolutely must, and if you're intimately familiar with the - internals of cuDF, subclass :class:`BaseIndex` instead. - - Examples - -------- - >>> import cudf - >>> cudf.Index([1, 2, 3], dtype="uint64", name="a") - UInt64Index([1, 2, 3], dtype='uint64', name='a') - - >>> cudf.Index(cudf.DataFrame({"a":[1, 2], "b":[2, 3]})) - MultiIndex([(1, 2), - (2, 3)], - names=['a', 'b']) - """ - - @_cudf_nvtx_annotate - def __new__( - cls, - data=None, - dtype=None, - copy=False, - name=None, - tupleize_cols=True, - nan_as_null=True, - **kwargs, - ): - assert ( - cls is Index - ), "Index cannot be subclassed, extend BaseIndex instead." - if tupleize_cols is not True: - raise NotImplementedError( - "tupleize_cols != True is not yet supported" - ) - - return as_index( - data, - copy=copy, - dtype=dtype, - name=name, - nan_as_null=nan_as_null, - **kwargs, - ) - - @classmethod - @_cudf_nvtx_annotate - def from_arrow(cls, obj): - try: - return cls(ColumnBase.from_arrow(obj)) - except TypeError: - # Try interpreting object as a MultiIndex before failing. - return cudf.MultiIndex.from_arrow(obj) - - @_cudf_nvtx_annotate def _concat_range_index(indexes: List[RangeIndex]) -> BaseIndex: """ diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 40330b45e5b..e406ef14080 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -948,7 +948,7 @@ def _copy_type_metadata( self._index, cudf.core.index.CategoricalIndex ): self._index = cudf.Index( - cast(cudf.core.index.NumericIndex, self._index)._column, + cast("cudf.Index", self._index)._column, name=self._index.name, ) elif isinstance(other._index, cudf.MultiIndex) and not isinstance( diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 0498aa474b6..cdc120935ee 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1000,11 +1000,11 @@ def _concat(cls, objs): obj.columns = colnames source_data = cudf.DataFrame._concat(source_data) - names = [None] * source_data._num_columns - objs = list(filter(lambda o: o.names is not None, objs)) - for o in range(len(objs)): - for i, name in enumerate(objs[o].names): - names[i] = names[i] or name + try: + # Only set names if all objs have the same names + (names,) = {o.names for o in objs} - {None} + except ValueError: + names = [None] * source_data._num_columns return cudf.MultiIndex.from_frame(source_data, names=names) @classmethod @@ -1377,7 +1377,7 @@ def droplevel(self, level=-1): Dropping multiple levels: >>> idx.droplevel(["first", "second"]) - Int64Index([0, 1, 2, 0, 1, 2], dtype='int64', name='third') + Index([0, 1, 2, 0, 1, 2], dtype='int64', name='third') """ mi = self.copy(deep=False) mi._poplevels(level) @@ -1779,7 +1779,7 @@ def _union(self, other, sort=None): # TODO: When to_frame is refactored to return a # deep copy in future, we should push most of the common # logic between MultiIndex._union & BaseIndex._union into - # GenericIndex._union. + # Index._union. other_df = other.copy(deep=True).to_frame(index=False) self_df = self.copy(deep=True).to_frame(index=False) col_names = list(range(0, self.nlevels)) diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index b8164255e6d..d3cd84465ca 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -83,7 +83,7 @@ def _get_combined_index(indexes, intersect: bool = False, sort=None): else: index = indexes[0] if sort is None: - sort = not isinstance(index, cudf.StringIndex) + sort = not index._is_object() for other in indexes[1:]: index = index.union(other, sort=False) @@ -427,7 +427,7 @@ def concat(objs, axis=0, join="outer", ignore_index=False, sort=None): elif typ is cudf.MultiIndex: return cudf.MultiIndex._concat(objs) elif issubclass(typ, cudf.Index): - return cudf.core.index.GenericIndex._concat(objs) + return cudf.core.index.Index._concat(objs) else: raise TypeError(f"cannot concatenate object of type {typ}") diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 34936253bf0..4af8aee171c 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -1421,9 +1421,7 @@ def _concat(cls, objs, axis=0, index=True): if isinstance(objs[0].index, cudf.MultiIndex): index = cudf.MultiIndex._concat([o.index for o in objs]) else: - index = cudf.core.index.GenericIndex._concat( - [o.index for o in objs] - ) + index = cudf.core.index.Index._concat([o.index for o in objs]) names = {obj.name for obj in objs} if len(names) == 1: @@ -3327,7 +3325,7 @@ def keys(self): c 3 dtype: int64 >>> sr.keys() - StringIndex(['a' 'b' 'c'], dtype='object') + Index(['a', 'b', 'c'], dtype='object') """ return self.index diff --git a/python/cudf/cudf/core/single_column_frame.py b/python/cudf/cudf/core/single_column_frame.py index d058d4cee75..27cd1085fa7 100644 --- a/python/cudf/cudf/core/single_column_frame.py +++ b/python/cudf/cudf/core/single_column_frame.py @@ -167,7 +167,7 @@ def from_arrow(cls, array): >>> import cudf >>> import pyarrow as pa >>> cudf.Index.from_arrow(pa.array(["a", "b", None])) - StringIndex(['a' 'b' None], dtype='object') + Index(['a' 'b' None], dtype='object') >>> cudf.Series.from_arrow(pa.array(["a", "b", None])) 0 a 1 b @@ -274,7 +274,7 @@ def factorize(self, sort=False, use_na_sentinel=True): >>> codes array([0, 0, 1], dtype=int8) >>> uniques - StringIndex(['a' 'c'], dtype='object') + Index(['a' 'c'], dtype='object') """ return cudf.core.algorithms.factorize( self, diff --git a/python/cudf/cudf/testing/testing.py b/python/cudf/cudf/testing/testing.py index 484c013f774..0f54391b426 100644 --- a/python/cudf/cudf/testing/testing.py +++ b/python/cudf/cudf/testing/testing.py @@ -65,25 +65,17 @@ def _check_types( if not exact or exact == "equiv": if ( isinstance(left, cudf.RangeIndex) - and isinstance( - right, - ( - cudf.Int8Index, - cudf.Int16Index, - cudf.Int32Index, - cudf.Int64Index, - ), + and ( + isinstance(right, cudf.Index) + and hasattr(right, "dtype") + and right.dtype.kind == "i" ) ) or ( isinstance(right, cudf.RangeIndex) - and isinstance( - left, - ( - cudf.Int8Index, - cudf.Int16Index, - cudf.Int32Index, - cudf.Int64Index, - ), + and ( + isinstance(left, cudf.Index) + and hasattr(left, "dtype") + and left.dtype.kind == "i" ) ): return @@ -324,7 +316,7 @@ def assert_index_equal( exact : bool or {'equiv'}, default 'equiv' Whether to check the Index class, dtype and inferred_type are identical. If 'equiv', then RangeIndex can be substituted - for Int8Index, Int16Index, Int32Index, Int64Index as well. + for Index with an int8/int32/int64 dtype as well. check_names : bool, default True Whether to check the names attribute. check_less_precise : bool or int, default False diff --git a/python/cudf/cudf/tests/test_binops.py b/python/cudf/cudf/tests/test_binops.py index e62f19f7877..c74d1fdd85b 100644 --- a/python/cudf/cudf/tests/test_binops.py +++ b/python/cudf/cudf/tests/test_binops.py @@ -664,11 +664,11 @@ def test_different_shapes_and_columns_with_unaligned_indices(binop): # Test with a RangeIndex pdf1 = pd.DataFrame({"x": [4, 3, 2, 1], "y": [7, 3, 8, 6]}) - # Test with a GenericIndex + # Test with a Index pdf2 = pd.DataFrame( {"x": [1, 2, 3, 7], "y": [4, 5, 6, 7]}, index=[0, 1, 3, 4] ) - # Test with a GenericIndex in a different order + # Test with a Index in a different order pdf3 = pd.DataFrame( {"x": [4, 5, 6, 7], "y": [1, 2, 3, 7], "z": [0, 5, 3, 7]}, index=[0, 3, 5, 3], diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 5875959b0c2..e6f2f9ec448 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -6518,7 +6518,7 @@ def test_dataframe_info_basic(): str_cmp = textwrap.dedent( """\ - StringIndex: 10 entries, a to 1111 + Index: 10 entries, a to 1111 Data columns (total 10 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- @@ -6591,7 +6591,7 @@ def test_dataframe_info_verbose_mem_usage(): str_cmp = textwrap.dedent( """\ - StringIndex: 3 entries, sdfdsf to dsfdf + Index: 3 entries, sdfdsf to dsfdf Data columns (total 2 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index 5583b2290ae..7c610eca88c 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -33,6 +33,7 @@ TIMEDELTA_TYPES, assert_eq, assert_exceptions_equal, + expect_warning_if, ) from cudf.testing.dataset_generator import rand_dataframe @@ -1290,7 +1291,7 @@ def test_groupby_index_type(): df["string_col"] = ["a", "b", "c"] df["counts"] = [1, 2, 3] res = df.groupby(by="string_col").counts.sum() - assert isinstance(res.index, cudf.StringIndex) + assert res.index.dtype == cudf.dtype("object") @pytest.mark.parametrize( @@ -2020,7 +2021,7 @@ def test_groupby_no_keys(pdf): pdf.groupby([]).max(), gdf.groupby([]).max(), check_dtype=False, - check_index_type=False, # Int64Index v/s Float64Index + check_index_type=False, # Int64 v/s Float64 **kwargs, ) @@ -2038,7 +2039,7 @@ def test_groupby_apply_no_keys(pdf): assert_groupby_results_equal( pdf.groupby([], group_keys=False).apply(lambda x: x.max()), gdf.groupby([]).apply(lambda x: x.max()), - check_index_type=False, # Int64Index v/s Float64Index + check_index_type=False, # Int64 v/s Float64 **kwargs, ) diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index 0bfd486ae74..de4c72389cf 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -15,7 +15,7 @@ from cudf.core.index import ( CategoricalIndex, DatetimeIndex, - GenericIndex, + Index, IntervalIndex, RangeIndex, as_index, @@ -204,9 +204,9 @@ def test_pandas_as_index(): gdf_category_index = as_index(pdf_category_index) # Check instance types - assert isinstance(gdf_int_index, GenericIndex) - assert isinstance(gdf_uint_index, GenericIndex) - assert isinstance(gdf_float_index, GenericIndex) + assert isinstance(gdf_int_index, Index) + assert isinstance(gdf_uint_index, Index) + assert isinstance(gdf_float_index, Index) assert isinstance(gdf_datetime_index, DatetimeIndex) assert isinstance(gdf_category_index, CategoricalIndex) @@ -329,7 +329,7 @@ def test_index_copy_datetime(name, deep=True): @pytest.mark.parametrize("name", ["x"]) def test_index_copy_string(name, deep=True): - cidx = cudf.StringIndex(["a", "b", "c"]) + cidx = cudf.Index(["a", "b", "c"]) pidx = cidx.to_pandas() pidx_copy = pidx.copy(name=name, deep=deep) @@ -393,12 +393,12 @@ def test_index_copy_deep(idx, deep, copy_on_write): original_cow_setting = cudf.get_option("copy_on_write") cudf.set_option("copy_on_write", copy_on_write) if ( - isinstance(idx, cudf.StringIndex) + isinstance(idx._values, cudf.core.column.StringColumn) or not deep or (cudf.get_option("copy_on_write") and not deep) ): # StringColumn is immutable hence, deep copies of a - # StringIndex will share the same StringColumn. + # Index with string dtype will share the same StringColumn. # When `copy_on_write` is turned on, Index objects will # have unique column object but they all point to same @@ -1207,91 +1207,48 @@ def test_index_basic(data, dtype, name): @pytest.mark.parametrize("name", [1, "a", None]) @pytest.mark.parametrize("dtype", SIGNED_INTEGER_TYPES) def test_integer_index_apis(data, name, dtype): - with pytest.warns(FutureWarning): - pindex = pd.Int64Index(data, dtype=dtype, name=name) - # Int8Index - with pytest.warns(FutureWarning): - gindex = cudf.Int8Index(data, dtype=dtype, name=name) - - assert_eq(pindex, gindex) - assert gindex.dtype == np.dtype("int8") - - # Int16Index - with pytest.warns(FutureWarning): - gindex = cudf.Int16Index(data, dtype=dtype, name=name) - - assert_eq(pindex, gindex) - assert gindex.dtype == np.dtype("int16") - - # Int32Index - with pytest.warns(FutureWarning): - gindex = cudf.Int32Index(data, dtype=dtype, name=name) - - assert_eq(pindex, gindex) - assert gindex.dtype == np.dtype("int32") + if PANDAS_GE_200: + pindex = pd.Index(data, dtype=dtype, name=name) + else: + with pytest.warns(FutureWarning): + pindex = pd.Int64Index(data, dtype=dtype, name=name) - # Int64Index - with pytest.warns(FutureWarning): - gindex = cudf.Int64Index(data, dtype=dtype, name=name) + gindex = cudf.Index(data, dtype=dtype, name=name) assert_eq(pindex, gindex) - assert gindex.dtype == np.dtype("int64") + assert gindex.dtype == dtype @pytest.mark.parametrize("data", [[1, 2, 3, 4], []]) @pytest.mark.parametrize("name", [1, "a", None]) @pytest.mark.parametrize("dtype", UNSIGNED_TYPES) def test_unsigned_integer_index_apis(data, name, dtype): - with pytest.warns(FutureWarning): - pindex = pd.UInt64Index(data, dtype=dtype, name=name) - # UInt8Index - with pytest.warns(FutureWarning): - gindex = cudf.UInt8Index(data, dtype=dtype, name=name) - - assert_eq(pindex, gindex) - assert gindex.dtype == np.dtype("uint8") - - # UInt16Index - with pytest.warns(FutureWarning): - gindex = cudf.UInt16Index(data, dtype=dtype, name=name) - - assert_eq(pindex, gindex) - assert gindex.dtype == np.dtype("uint16") - - # UInt32Index - with pytest.warns(FutureWarning): - gindex = cudf.UInt32Index(data, dtype=dtype, name=name) - - assert_eq(pindex, gindex) - assert gindex.dtype == np.dtype("uint32") + if PANDAS_GE_200: + pindex = pd.Index(data, dtype=dtype, name=name) + else: + with pytest.warns(FutureWarning): + pindex = pd.UInt64Index(data, dtype=dtype, name=name) - # UInt64Index - with pytest.warns(FutureWarning): - gindex = cudf.UInt64Index(data, dtype=dtype, name=name) + gindex = cudf.Index(data, dtype=dtype, name=name) assert_eq(pindex, gindex) - assert gindex.dtype == np.dtype("uint64") + assert gindex.dtype == dtype @pytest.mark.parametrize("data", [[1, 2, 3, 4], []]) @pytest.mark.parametrize("name", [1, "a", None]) @pytest.mark.parametrize("dtype", FLOAT_TYPES) def test_float_index_apis(data, name, dtype): - with pytest.warns(FutureWarning): - pindex = pd.Float64Index(data, dtype=dtype, name=name) - # Float32Index - with pytest.warns(FutureWarning): - gindex = cudf.Float32Index(data, dtype=dtype, name=name) - - assert_eq(pindex, gindex) - assert gindex.dtype == np.dtype("float32") + if PANDAS_GE_200: + pindex = pd.Index(data, dtype=dtype, name=name) + else: + with pytest.warns(FutureWarning): + pindex = pd.Float64Index(data, dtype=dtype, name=name) - # Float64Index - with pytest.warns(FutureWarning): - gindex = cudf.Float64Index(data, dtype=dtype, name=name) + gindex = cudf.Index(data, dtype=dtype, name=name) assert_eq(pindex, gindex) - assert gindex.dtype == np.dtype("float64") + assert gindex.dtype == dtype @pytest.mark.parametrize("data", [[1, 2, 3, 4], []]) @@ -1591,6 +1548,9 @@ def test_interval_index_from_breaks(closed): [[1, 2, 3, 4], ["yellow", "violet", "pink", "white"]], names=("number1", "color2"), ), + pd.MultiIndex.from_arrays( + [[1, 1, 2, 2], ["red", "blue", "red", "blue"]], + ), ], ) @pytest.mark.parametrize( @@ -1604,6 +1564,9 @@ def test_interval_index_from_breaks(closed): [[1, 2, 3, 4], ["yellow", "violet", "pink", "white"]], names=("number1", "color2"), ), + pd.MultiIndex.from_arrays( + [[1, 1, 2, 2], ["red", "blue", "red", "blue"]], + ), ], ) def test_multiindex_append(data, other): @@ -1726,7 +1689,7 @@ def test_index_fillna(data, fill_value): assert_eq( pdi.fillna(fill_value), gdi.fillna(fill_value), exact=False - ) # Int64Index v/s Float64Index + ) # Int64 v/s Float64 @pytest.mark.parametrize( @@ -1764,7 +1727,7 @@ def test_index_from_arrow(data): arrow_array = pa.Array.from_pandas(pdi) expected_index = pd.Index(arrow_array.to_pandas()) gdi = cudf.Index.from_arrow(arrow_array) - if PANDAS_GE_200: + if PANDAS_GE_200 and gdi.dtype == cudf.dtype("datetime64[s]"): # Arrow bug: # https://github.com/apache/arrow/issues/33321 # arrow cannot convert non-nanosecond diff --git a/python/cudf/cudf/tests/test_monotonic.py b/python/cudf/cudf/tests/test_monotonic.py index 1fcf41389dc..db7e4588e95 100644 --- a/python/cudf/cudf/tests/test_monotonic.py +++ b/python/cudf/cudf/tests/test_monotonic.py @@ -13,7 +13,7 @@ from cudf.core.index import ( CategoricalIndex, DatetimeIndex, - GenericIndex, + Index, RangeIndex, ) from cudf.testing._utils import assert_eq @@ -49,7 +49,7 @@ def test_range_index(testrange): ) def test_generic_index(testlist): - index = GenericIndex(testlist) + index = Index(testlist) index_pd = pd.Index(testlist) assert index.is_unique == index_pd.is_unique @@ -222,7 +222,7 @@ def test_multiindex_tuples(testarr): ) @pytest.mark.parametrize("side", ["left", "right"]) def test_get_slice_bound(testlist, side): - index = GenericIndex(testlist) + index = Index(testlist) index_pd = pd.Index(testlist) for label in testlist: expect = index_pd.get_slice_bound(label, side) @@ -269,7 +269,7 @@ def test_rangeindex_get_slice_bound_step(bounds, label, side): @pytest.mark.parametrize("side", ["left", "right"]) def test_get_slice_bound_missing(label, side): mylist = [2, 4, 6, 8, 10] - index = GenericIndex(mylist) + index = Index(mylist) index_pd = pd.Index(mylist) expect = index_pd.get_slice_bound(label, side) @@ -284,7 +284,7 @@ def test_get_slice_bound_missing_str(label, side): # Slicing for monotonic string indices not yet supported # when missing values are specified (allowed in pandas) mylist = ["b", "d", "f"] - index = GenericIndex(mylist) + index = Index(mylist) index_pd = pd.Index(mylist) got = index.get_slice_bound(label, side) expect = index_pd.get_slice_bound(label, side) diff --git a/python/cudf/cudf/tests/test_pack.py b/python/cudf/cudf/tests/test_pack.py index 9b5a8c19cf5..9011efebedb 100644 --- a/python/cudf/cudf/tests/test_pack.py +++ b/python/cudf/cudf/tests/test_pack.py @@ -18,7 +18,7 @@ import numpy as np import pandas as pd -from cudf import DataFrame, GenericIndex, Series +from cudf import DataFrame, Index, Series from cudf._lib.copying import pack, unpack from cudf.testing._utils import assert_eq @@ -52,7 +52,7 @@ def check_packed_equality(df): assert_packed_frame_equality(df[2:-2]) # sorted sortvaldf = df.sort_values("vals") - assert isinstance(sortvaldf.index, GenericIndex) + assert isinstance(sortvaldf.index, Index) assert_packed_frame_equality(sortvaldf) @@ -120,7 +120,7 @@ def check_packed_unique_pointers(df): assert_packed_frame_unique_pointers(df[2:-2]) # sorted sortvaldf = df.sort_values("vals") - assert isinstance(sortvaldf.index, GenericIndex) + assert isinstance(sortvaldf.index, Index) assert_packed_frame_unique_pointers(sortvaldf) @@ -188,7 +188,7 @@ def check_packed_pickled_equality(df): assert_packed_frame_picklable(df[2:-2]) # sorted sortvaldf = df.sort_values("vals") - assert isinstance(sortvaldf.index, GenericIndex) + assert isinstance(sortvaldf.index, Index) assert_packed_frame_picklable(sortvaldf) # out-of-band buffers = [] @@ -261,7 +261,7 @@ def check_packed_serialized_equality(df): assert_packed_frame_serializable(df[2:-2]) # sorted sortvaldf = df.sort_values("vals") - assert isinstance(sortvaldf.index, GenericIndex) + assert isinstance(sortvaldf.index, Index) assert_packed_frame_serializable(sortvaldf) diff --git a/python/cudf/cudf/tests/test_pickling.py b/python/cudf/cudf/tests/test_pickling.py index 71c1f206a64..69ccb5be860 100644 --- a/python/cudf/cudf/tests/test_pickling.py +++ b/python/cudf/cudf/tests/test_pickling.py @@ -6,7 +6,7 @@ import pandas as pd import pytest -from cudf import DataFrame, GenericIndex, RangeIndex, Series +from cudf import DataFrame, Index, RangeIndex, Series from cudf.core.buffer import as_buffer from cudf.testing._utils import assert_eq @@ -22,7 +22,7 @@ def check_serialization(df): assert_frame_picklable(df[2:-2]) # sorted sortvaldf = df.sort_values("vals") - assert isinstance(sortvaldf.index, (GenericIndex, RangeIndex)) + assert isinstance(sortvaldf.index, (Index, RangeIndex)) assert_frame_picklable(sortvaldf) # out-of-band buffers = [] @@ -80,7 +80,7 @@ def test_memory_usage_dataframe(): def test_pickle_index(): nelem = 10 - idx = GenericIndex(np.arange(nelem), name="a") + idx = Index(np.arange(nelem), name="a") pickled = pickle.dumps(idx) out = pickle.loads(pickled) assert (idx == out).all() diff --git a/python/cudf/cudf/tests/test_repr.py b/python/cudf/cudf/tests/test_repr.py index e7fa401f1ec..7a67fddd87b 100644 --- a/python/cudf/cudf/tests/test_repr.py +++ b/python/cudf/cudf/tests/test_repr.py @@ -302,39 +302,40 @@ def test_dataframe_sliced(gdf, slice, max_seq_items, max_rows): [ ( cudf.Index([1, 2, 3, None]), - "Int64Index([1, 2, 3, ], dtype='int64')", + "Index([1, 2, 3, ], dtype='int64')", ), ( cudf.Index([None, 2.2, 3.324342, None]), - "Float64Index([, 2.2, 3.324342, ], dtype='float64')", + "Index([, 2.2, 3.324342, ], dtype='float64')", ), ( cudf.Index([None, None, None], name="hello"), - "StringIndex([None None None], dtype='object', name='hello')", + "Index([, , ], dtype='object', name='hello')", ), ( cudf.Index([None, None, None], dtype="float", name="hello"), - "Float64Index([, , ], dtype='float64', name='hello')", + "Index([, , ], dtype='float64', name='hello')", ), ( cudf.Index([None], dtype="float64", name="hello"), - "Float64Index([], dtype='float64', name='hello')", + "Index([], dtype='float64', name='hello')", ), ( cudf.Index([None], dtype="int8", name="hello"), - "Int8Index([], dtype='int8', name='hello')", + "Index([], dtype='int8', name='hello')", ), ( cudf.Index([None] * 50, dtype="object"), - "StringIndex([None None None None None None None None " - "None None None None None None\n None None None None None None " - "None None None None None None None None\n None None None None " - "None None None None None None None None None None\n None None " - "None None None None None None], dtype='object')", + "Index([, , , , , , , , , " + ", , ,\n , , , , , , , " + ", , , , ,\n , , , , " + ", , , , , , , ,\n , " + ", , , , , , , , , , " + ",\n , ],\n dtype='object')", ), ( cudf.Index([None] * 20, dtype="uint32"), - "UInt32Index([, , , , , , , , " + "Index([, , , , , , , , " ",\n , , , , , , , , " ",\n , ],\n dtype='uint32')", ), @@ -342,7 +343,7 @@ def test_dataframe_sliced(gdf, slice, max_seq_items, max_rows): cudf.Index( [None, 111, 22, 33, None, 23, 34, 2343, None], dtype="int16" ), - "Int16Index([, 111, 22, 33, , 23, 34, 2343, ], " + "Index([, 111, 22, 33, , 23, 34, 2343, ], " "dtype='int16')", ), ( @@ -482,7 +483,7 @@ def test_dataframe_null_index_repr(df, pandas_special_case): actual_repr = repr(gdf) if pandas_special_case: - # Pandas inconsistently print StringIndex null values + # Pandas inconsistently print Index null values # as `None` at some places and `NaN` at few other places # Whereas cudf is consistent with strings `null` values # to be printed as `None` everywhere. @@ -561,7 +562,7 @@ def test_series_null_index_repr(sr, pandas_special_case): actual_repr = repr(gsr) if pandas_special_case: - # Pandas inconsistently print StringIndex null values + # Pandas inconsistently print Index null values # as `None` at some places and `NaN` at few other places # Whereas cudf is consistent with strings `null` values # to be printed as `None` everywhere. diff --git a/python/cudf/cudf/tests/test_serialize.py b/python/cudf/cudf/tests/test_serialize.py index e7f26e259c6..2fdc3ef441b 100644 --- a/python/cudf/cudf/tests/test_serialize.py +++ b/python/cudf/cudf/tests/test_serialize.py @@ -193,8 +193,8 @@ def test_serialize_range_index(): def test_serialize_generic_index(): - index = cudf.core.index.GenericIndex(cudf.Series(np.arange(10))) - outindex = cudf.core.index.GenericIndex.deserialize(*index.serialize()) + index = cudf.core.index.Index(cudf.Series(np.arange(10))) + outindex = cudf.core.index.Index.deserialize(*index.serialize()) assert_eq(index, outindex) diff --git a/python/cudf/cudf/tests/test_string.py b/python/cudf/cudf/tests/test_string.py index 200bd30cb12..618f94ed25b 100644 --- a/python/cudf/cudf/tests/test_string.py +++ b/python/cudf/cudf/tests/test_string.py @@ -17,7 +17,7 @@ from cudf import concat from cudf.core._compat import PANDAS_GE_150 from cudf.core.column.string import StringColumn -from cudf.core.index import StringIndex, as_index +from cudf.core.index import Index, as_index from cudf.testing._utils import ( DATETIME_TYPES, NUMERIC_TYPES, @@ -1075,8 +1075,7 @@ def test_string_index(): pdf.index = stringIndex gdf.index = stringIndex assert_eq(pdf, gdf) - with pytest.warns(FutureWarning): - stringIndex = StringIndex(["a", "b", "c", "d", "e"], name="name") + stringIndex = Index(["a", "b", "c", "d", "e"], name="name") pdf.index = stringIndex.to_pandas() gdf.index = stringIndex assert_eq(pdf, gdf) diff --git a/python/dask_cudf/dask_cudf/backends.py b/python/dask_cudf/dask_cudf/backends.py index 821ec103204..e44775e56df 100644 --- a/python/dask_cudf/dask_cudf/backends.py +++ b/python/dask_cudf/dask_cudf/backends.py @@ -57,8 +57,8 @@ def _nonempty_index(idx): data = np.array([start, "1970-01-02"], dtype=idx.dtype) values = cudf.core.column.as_column(data) return cudf.core.index.DatetimeIndex(values, name=idx.name) - elif isinstance(idx, cudf.StringIndex): - return cudf.StringIndex(["cat", "dog"], name=idx.name) + elif isinstance(idx._column, cudf.core.column.StringColumn): + return cudf.Index(["cat", "dog"], name=idx.name) elif isinstance(idx, cudf.core.index.CategoricalIndex): key = tuple(idx._data.keys()) assert len(key) == 1 @@ -69,8 +69,8 @@ def _nonempty_index(idx): categories=categories, codes=codes, ordered=ordered ) return cudf.core.index.CategoricalIndex(values, name=idx.name) - elif isinstance(idx, cudf.core.index.GenericIndex): - return cudf.core.index.GenericIndex( + elif isinstance(idx, cudf.core.index.Index): + return cudf.core.index.Index( np.arange(2, dtype=idx.dtype), name=idx.name ) elif isinstance(idx, cudf.core.multiindex.MultiIndex): From 72a663ed43c0f95da36eb55c933a9bf564506b6a Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 30 May 2023 10:21:00 -0700 Subject: [PATCH 039/384] Fix MultiIndex.get_indexer pytest --- python/cudf/cudf/tests/test_index.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index 58872e00394..80707763246 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -2251,10 +2251,15 @@ def test_get_loc_multi_numeric_deviate(idx, key, result): ], ) @pytest.mark.parametrize("method", [None, "ffill", "bfill"]) -def test_get_indexer_multi_numeric_deviate(idx, key, method): +def test_get_indexer_multi_numeric_deviate(request, idx, key, method): pi = idx gi = cudf.from_pandas(pi) - + request.applymarker( + pytest.mark.xfail( + condition=method is not None and key == ((1, 2, 3),), + reason="https://github.com/pandas-dev/pandas/issues/53452", + ) + ) expected = pi.get_indexer(key, method=method) got = gi.get_indexer(key, method=method) From 8791749ec7eabf355ca665695a6122ea5c7f4a05 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 30 May 2023 13:49:07 -0700 Subject: [PATCH 040/384] Complete get_indexer implementation --- python/cudf/cudf/core/index.py | 114 +++++++++++++++++++++------ python/cudf/cudf/core/multiindex.py | 38 +++++---- python/cudf/cudf/tests/test_index.py | 17 ++-- 3 files changed, 119 insertions(+), 50 deletions(-) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index a9db663a19b..a4e716910f1 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -16,6 +16,7 @@ ) import cupy +import operator import numpy as np import pandas as pd from pandas._config import get_option @@ -1174,10 +1175,7 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): """ if is_scalar(target): raise TypeError("Should be a sequence") - # if tolerance is not None: - # raise NotImplementedError( - # "Parameter tolerance is not supported yet." - # ) + if method not in { None, "ffill", @@ -1220,28 +1218,27 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): ) if method is None: result_series = result_series.fillna(-1) + elif method in {"ffill", "bfill", "pad", "backfill"}: + result_series = _get_indexer_basic( + index=self, + positions=result_series, + method=method, + target_col=needle_table["None"], + tolerance=tolerance, + ) + elif method == "nearest": + result_series = _get_nearest_indexer( + index=self, + positions=result_series, + target_col=needle_table["None"], + tolerance=tolerance, + ) else: - nonexact = result_series.isnull() - result_series[nonexact] = self.searchsorted( - needle_table["None"][nonexact], - side="left" if method in {"pad", "ffill"} else "right", + raise ValueError( + f"{method=} is unsupported, only supported values are: " + f"{['ffill', 'bfill', 'nearest', None]}" ) - if method in {"pad", "ffill"}: - # searchsorted returns "indices into a sorted array such that, - # if the corresponding elements in v were inserted before the - # indices, the order of a would be preserved". - # Thus, we need to subtract 1 to find values to the left. - result_series[nonexact] -= 1 - # This also mapped not found values (values of 0 from - # np.searchsorted) to -1, which conveniently is also our - # sentinel for missing values - else: - # Mark indices to the right of the largest value as not found - result_series[result_series == len(self)] = -1 - if tolerance is not None: - distance = self[result_series] - needle_table["None"] - # return cupy.where(distance <= tolerance, result_series, -1) - return result_series.where(distance <= tolerance, -1).to_cupy() + return result_series.to_cupy() @_cudf_nvtx_annotate @@ -2908,3 +2905,72 @@ def _extended_gcd(a: int, b: int) -> Tuple[int, int, int]: old_s, s = s, old_s - quotient * s old_t, t = t, old_t - quotient * t return old_r, old_s, old_t + + +def _get_indexer_basic(index, positions, method, target_col, tolerance): + nonexact = positions.isnull() + positions[nonexact] = index.searchsorted( + target_col[nonexact], + side="left" if method in {"pad", "ffill"} else "right", + ) + if method in {"pad", "ffill"}: + # searchsorted returns "indices into a sorted array such that, + # if the corresponding elements in v were inserted before the + # indices, the order of a would be preserved". + # Thus, we need to subtract 1 to find values to the left. + positions[nonexact] -= 1 + # This also mapped not found values (values of 0 from + # np.searchsorted) to -1, which conveniently is also our + # sentinel for missing values + else: + # Mark indices to the right of the largest value as not found + positions[positions == len(index)] = -1 + + if tolerance is not None: + distance = abs(index[positions] - target_col) + return positions.where(distance <= tolerance, -1) + return positions + + +def _get_nearest_indexer(index, positions, target_col, tolerance): + """ + Get the indexer for the nearest index labels; requires an index with + values that can be subtracted from each other. + """ + if not len(index): + return _get_indexer_basic( + index=index, + positions=positions.copy(deep=True), + method="pad", + targe_col=target_col, + tolerance=tolerance, + ) + + left_indexer = _get_indexer_basic( + index=index, + positions=positions.copy(deep=True), + method="pad", + target_col=target_col, + tolerance=tolerance, + ) + right_indexer = _get_indexer_basic( + index=index, + positions=positions.copy(deep=True), + method="backfill", + target_col=target_col, + tolerance=tolerance, + ) + + left_distances = abs(index[left_indexer] - target_col) + right_distances = abs(index[right_indexer] - target_col) + + op = operator.lt if index.is_monotonic_increasing else operator.le + indexer = left_indexer.where( + op(left_distances, right_distances) | (right_indexer == -1), + right_indexer, + ) + + if tolerance is not None: + distance = abs(index[indexer] - target_col) + return indexer.where(distance <= tolerance, -1) + return indexer diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 7b5eed2d631..9f41e9db610 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -27,6 +27,7 @@ _index_astype_docstring, _lexsorted_equal_range, as_index, + _get_indexer_basic, ) from cudf.utils.docutils import doc_apply from cudf.utils.utils import NotIterable, _cudf_nvtx_annotate @@ -1696,6 +1697,11 @@ def get_indexer(self, target, method=None, tolerance=None): raise NotImplementedError( "Parameter tolerance is not supported yet." ) + if method == "nearest": + raise NotImplementedError( + f"{method=} is not supported yet for MultiIndex." + ) + target = cudf.MultiIndex.from_tuples(target) needle_table = target.to_frame(index=False) col_names = list(range(0, self.nlevels)) @@ -1712,28 +1718,20 @@ def get_indexer(self, target, method=None, tolerance=None): ) if method is None: result_series = result_series.fillna(-1) + elif method in {"ffill", "bfill", "pad", "backfill"}: + result_series = _get_indexer_basic( + index=self, + positions=result_series, + method=method, + target_col=needle_table[col_names], + tolerance=tolerance, + ) else: - nonexact = result_series.isnull() - result_series[nonexact] = self.searchsorted( - needle_table[col_names][nonexact], - side="left" if method in {"pad", "ffill"} else "right", + raise ValueError( + f"{method=} is unsupported, only supported values are: " + f"{['ffill', 'bfill', None]}" ) - if method in {"pad", "ffill"}: - # searchsorted returns "indices into a sorted array such that, - # if the corresponding elements in v were inserted before the - # indices, the order of a would be preserved". - # Thus, we need to subtract 1 to find values to the left. - result_series[nonexact] -= 1 - # This also mapped not found values (values of 0 from - # np.searchsorted) to -1, which conveniently is also our - # sentinel for missing values - else: - # Mark indices to the right of the largest value as not found - result_series[result_series == len(self)] = -1 - if tolerance is not None: - distance = self[result_series] - needle_table["None"] - # return cupy.where(distance <= tolerance, result_series, -1) - return result_series.where(distance <= tolerance, -1).to_cupy() + return result_series.to_cupy() @_cudf_nvtx_annotate diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index 80707763246..2eb14cb2413 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -1940,7 +1940,7 @@ def test_get_indexer_single_unique_numeric(idx, key, method): list(range(77, 110, 3)), ], ) -@pytest.mark.parametrize("method", [None, "ffill", "bfill"]) +@pytest.mark.parametrize("method", [None, "ffill", "bfill", "nearest"]) @pytest.mark.parametrize("tolerance", [None, 0, 1, 13, 20]) def test_get_indexer_rangeindex(idx, key, method, tolerance): pi = idx @@ -2018,8 +2018,9 @@ def test_get_loc_single_duplicate_numeric(idx, key): ], ) @pytest.mark.parametrize("key", [[0, 3, 1], [6, 7]]) -@pytest.mark.parametrize("method", [None, "ffill", "bfill"]) -def test_get_indexer_single_duplicate_numeric(idx, key, method): +@pytest.mark.parametrize("method", [None, "ffill", "bfill", "nearest"]) +@pytest.mark.parametrize("tolerance", [None, 1, 2]) +def test_get_indexer_single_duplicate_numeric(idx, key, method, tolerance): pi = idx gi = cudf.from_pandas(pi) @@ -2031,8 +2032,12 @@ def test_get_indexer_single_duplicate_numeric(idx, key, method): rfunc_args_and_kwargs=([], {"key": key, "method": method}), ) else: - expected = pi.get_indexer(key, method=method) - got = gi.get_indexer(key, method=method) + expected = pi.get_indexer( + key, method=method, tolerance=None if method is None else tolerance + ) + got = gi.get_indexer( + key, method=method, tolerance=None if method is None else tolerance + ) assert_eq(expected, got) @@ -2179,7 +2184,7 @@ def test_get_loc_multi_numeric(idx, key): ], ) @pytest.mark.parametrize("key", [[(1, 2, 3)], [(9, 9, 9)]]) -@pytest.mark.parametrize("method", [None]) +@pytest.mark.parametrize("method", [None, "ffill", "bfill"]) def test_get_indexer_multi_numeric(idx, key, method): pi = idx.sort_values() gi = cudf.from_pandas(pi) From ac39341437dc12d35e41e00285909829596fbc4c Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 30 May 2023 14:24:09 -0700 Subject: [PATCH 041/384] Update docs --- docs/cudf/source/api_docs/index_objects.rst | 3 + python/cudf/cudf/core/_base_index.py | 82 +++++++++++++- python/cudf/cudf/core/index.py | 77 +------------ python/cudf/cudf/core/multiindex.py | 114 +------------------- 4 files changed, 90 insertions(+), 186 deletions(-) diff --git a/docs/cudf/source/api_docs/index_objects.rst b/docs/cudf/source/api_docs/index_objects.rst index 1b748a8f69f..69b5a5f0631 100644 --- a/docs/cudf/source/api_docs/index_objects.rst +++ b/docs/cudf/source/api_docs/index_objects.rst @@ -135,6 +135,7 @@ Selecting .. autosummary:: :toctree: api/ + Index.get_indexer Index.get_level_values Index.get_loc Index.get_slice_bound @@ -191,6 +192,7 @@ IntervalIndex components IntervalIndex.from_breaks IntervalIndex.values + IntervalIndex.get_indexer IntervalIndex.get_loc .. _api.multiindex: @@ -236,6 +238,7 @@ MultiIndex selecting .. autosummary:: :toctree: api/ + MultiIndex.get_indexer MultiIndex.get_loc MultiIndex.get_level_values diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index 1527232f340..d34405032b5 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -3,6 +3,7 @@ from __future__ import annotations import pickle +import warnings from functools import cached_property from typing import Any, Set @@ -10,7 +11,6 @@ from typing_extensions import Self import cudf -import warnings from cudf._lib.copying import _gather_map_is_valid, gather from cudf._lib.stream_compaction import ( apply_boolean_mask, @@ -91,9 +91,89 @@ def values(self): raise NotImplementedError def get_indexer(self, target, method=None, limit=None, tolerance=None): + """ + Compute indexer and mask for new index given the current index. + + The indexer should be then used as an input to ndarray.take to align + the current data to the new index. + + Parameters + ---------- + target : Index + method : {None, 'pad'/'fill', 'backfill'/'bfill', 'nearest'}, optional + - default: exact matches only. + - pad / ffill: find the PREVIOUS index value if no exact match. + - backfill / bfill: use NEXT index value if no exact match. + - nearest: use the NEAREST index value if no exact match. Tied + distances are broken by preferring the larger index + value. + tolerance : int or float, optional + Maximum distance from index value for inexact matches. The value + of the index at the matching location must satisfy the equation + ``abs(index[loc] - target) <= tolerance``. + + Returns + ------- + cupy.ndarray + Integers from 0 to n - 1 indicating that the index at these + positions matches the corresponding target values. + Missing values in the target are marked by -1. + + Examples + -------- + >>> import cudf + >>> index = cudf.Index(['c', 'a', 'b']) + >>> index + Index(['c', 'a', 'b'], dtype='object') + >>> index.get_indexer(['a', 'b', 'x']) + array([ 1, 2, -1]) + """ raise NotImplementedError def get_loc(self, key): + """ + Get integer location, slice or boolean mask for requested label. + + Parameters + ---------- + key : label + + Returns + ------- + int or slice or boolean mask + - If result is unique, return integer index + - If index is monotonic, loc is returned as a slice object + - Otherwise, a boolean mask is returned + + Examples + -------- + >>> import cudf + >>> unique_index = cudf.Index(list('abc')) + >>> unique_index.get_loc('b') + 1 + >>> monotonic_index = cudf.Index(list('abbc')) + >>> monotonic_index.get_loc('b') + slice(1, 3, None) + >>> non_monotonic_index = cudf.Index(list('abcb')) + >>> non_monotonic_index.get_loc('b') + array([False, True, False, True]) + >>> numeric_unique_index = cudf.Index([1, 2, 3]) + >>> numeric_unique_index.get_loc(3) + 2 + + **MultiIndex** + + >>> multi_index = cudf.MultiIndex.from_tuples([('a', 'd'), ('b', 'e'), ('b', 'f')]) + >>> multi_index + MultiIndex([('a', 'd'), + ('b', 'e'), + ('b', 'f')], + ) + >>> multi_index.get_loc('b') + slice(1, 3, None) + >>> multi_index.get_loc(('b', 'e')) + 1 + """ # noqa: E501 raise NotImplementedError def __getitem__(self, key): diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index a4e716910f1..a7424bf0f04 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -2,6 +2,7 @@ from __future__ import annotations +import operator import pickle import warnings from functools import cached_property @@ -16,7 +17,6 @@ ) import cupy -import operator import numpy as np import pandas as pd from pandas._config import get_option @@ -35,6 +35,7 @@ is_scalar, ) from cudf.core._base_index import BaseIndex, _index_astype_docstring +from cudf.core._compat import PANDAS_GE_200 from cudf.core.column import ( CategoricalColumn, ColumnBase, @@ -61,7 +62,6 @@ numeric_normalize_types, ) from cudf.utils.utils import _cudf_nvtx_annotate, search_range -from cudf.core._compat import PANDAS_GE_200 class IndexMeta(type): @@ -591,7 +591,7 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): ) @_cudf_nvtx_annotate - def get_indexer(self, target, method=None, tolerance=None): + def get_indexer(self, target, limit=None, method=None, tolerance=None): if method is None: if self.step > 0: start, stop, step = self.start, self.stop, self.step @@ -612,15 +612,13 @@ def get_indexer(self, target, method=None, tolerance=None): return locs else: return self._as_int_index().get_indexer( - target=target, method=method, tolerance=tolerance + target=target, limit=limit, method=method, tolerance=tolerance ) @_cudf_nvtx_annotate def get_loc(self, key): - # Given an actual integer, if not is_scalar(key): raise TypeError("Should be a sequence") - # Given an actual integer, idx = (key - self._start) / self._step idx_int_upper_bound = (self._stop - self._start) // self._step if idx > idx_int_upper_bound or idx < 0: @@ -1134,45 +1132,6 @@ def astype(self, dtype, copy: bool = True): @_cudf_nvtx_annotate def get_indexer(self, target, method=None, limit=None, tolerance=None): - """Get integer location, slice or boolean mask for requested label. - - Parameters - ---------- - target : label - method : {None, 'pad'/'fill', 'backfill'/'bfill', 'nearest'}, optional - - default: exact matches only. - - pad / ffill: find the PREVIOUS index value if no exact match. - - backfill / bfill: use NEXT index value if no exact match. - - nearest: use the NEAREST index value if no exact match. Tied - distances are broken by preferring the larger index - value. - tolerance : int or float, optional - Maximum distance from index value for inexact matches. The value - of the index at the matching location must satisfy the equation - ``abs(index[loc] - target) <= tolerance``. - - Returns - ------- - int or slice or boolean mask - - If result is unique, return integer index - - If index is monotonic, loc is returned as a slice object - - Otherwise, a boolean mask is returned - - Examples - -------- - >>> unique_index = cudf.Index(list('abc')) - >>> unique_index.get_loc('b') - 1 - >>> monotonic_index = cudf.Index(list('abbc')) - >>> monotonic_index.get_loc('b') - slice(1, 3, None) - >>> non_monotonic_index = cudf.Index(list('abcb')) - >>> non_monotonic_index.get_loc('b') - array([False, True, False, True]) - >>> numeric_unique_index = cudf.Index([1, 2, 3]) - >>> numeric_unique_index.get_loc(3) - 2 - """ if is_scalar(target): raise TypeError("Should be a sequence") @@ -1243,34 +1202,6 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): @_cudf_nvtx_annotate def get_loc(self, key): - """Get integer location, slice or boolean mask for requested label. - - Parameters - ---------- - key : label - - Returns - ------- - int or slice or boolean mask - - If result is unique, return integer index - - If index is monotonic, loc is returned as a slice object - - Otherwise, a boolean mask is returned - - Examples - -------- - >>> unique_index = cudf.Index(list('abc')) - >>> unique_index.get_loc('b') - 1 - >>> monotonic_index = cudf.Index(list('abbc')) - >>> monotonic_index.get_loc('b') - slice(1, 3, None) - >>> non_monotonic_index = cudf.Index(list('abcb')) - >>> non_monotonic_index.get_loc('b') - array([False, True, False, True]) - >>> numeric_unique_index = cudf.Index([1, 2, 3]) - >>> numeric_unique_index.get_loc(3) - 2 - """ if is_scalar(key): target = [key] else: diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 9f41e9db610..f6da845ba46 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -24,10 +24,10 @@ from cudf.core.frame import Frame from cudf.core.index import ( BaseIndex, + _get_indexer_basic, _index_astype_docstring, _lexsorted_equal_range, as_index, - _get_indexer_basic, ) from cudf.utils.docutils import doc_apply from cudf.utils.utils import NotIterable, _cudf_nvtx_annotate @@ -1637,62 +1637,7 @@ def _level_index_from_level(self, level): return level @_cudf_nvtx_annotate - def get_indexer(self, target, method=None, tolerance=None): - """ - Get location for a label or a tuple of labels. - - The location is returned as an integer/slice or boolean mask. - - Parameters - ---------- - target : label or tuple of labels (one for each level) - method : None - - Returns - ------- - loc : int, slice object or boolean mask - - If index is unique, search result is unique, return a single int. - - If index is monotonic, index is returned as a slice object. - - Otherwise, cudf attempts a best effort to convert the search - result into a slice object, and will return a boolean mask if - failed to do so. Notice this can deviate from Pandas behavior - in some situations. - - Examples - -------- - >>> import cudf - >>> mi = cudf.MultiIndex.from_tuples( - ... [('a', 'd'), ('b', 'e'), ('b', 'f')]) - >>> mi.get_loc('b') - slice(1, 3, None) - >>> mi.get_loc(('b', 'e')) - 1 - >>> non_monotonic_non_unique_idx = cudf.MultiIndex.from_tuples( - ... [('c', 'd'), ('b', 'e'), ('a', 'f'), ('b', 'e')]) - >>> non_monotonic_non_unique_idx.get_loc('b') # differ from pandas - slice(1, 4, 2) - - .. pandas-compat:: - **MultiIndex.get_loc** - - The return types of this function may deviates from the - method provided by Pandas. If the index is neither - lexicographically sorted nor unique, a best effort attempt is made - to coerce the found indices into a slice. For example: - - .. code-block:: - - >>> import pandas as pd - >>> import cudf - >>> x = pd.MultiIndex.from_tuples([ - ... (2, 1, 1), (1, 2, 3), (1, 2, 1), - ... (1, 1, 1), (1, 1, 1), (2, 2, 1), - ... ]) - >>> x.get_loc(1) - array([False, True, True, True, True, False]) - >>> cudf.from_pandas(x).get_loc(1) - slice(1, 5, 1) - """ + def get_indexer(self, target, method=None, limit=None, tolerance=None): if tolerance is not None: raise NotImplementedError( "Parameter tolerance is not supported yet." @@ -1736,61 +1681,6 @@ def get_indexer(self, target, method=None, tolerance=None): @_cudf_nvtx_annotate def get_loc(self, key): - """ - Get location for a label or a tuple of labels. - - The location is returned as an integer/slice or boolean mask. - - Parameters - ---------- - key : label or tuple of labels (one for each level) - method : None - - Returns - ------- - loc : int, slice object or boolean mask - - If index is unique, search result is unique, return a single int. - - If index is monotonic, index is returned as a slice object. - - Otherwise, cudf attempts a best effort to convert the search - result into a slice object, and will return a boolean mask if - failed to do so. Notice this can deviate from Pandas behavior - in some situations. - - Examples - -------- - >>> import cudf - >>> mi = cudf.MultiIndex.from_tuples( - ... [('a', 'd'), ('b', 'e'), ('b', 'f')]) - >>> mi.get_loc('b') - slice(1, 3, None) - >>> mi.get_loc(('b', 'e')) - 1 - >>> non_monotonic_non_unique_idx = cudf.MultiIndex.from_tuples( - ... [('c', 'd'), ('b', 'e'), ('a', 'f'), ('b', 'e')]) - >>> non_monotonic_non_unique_idx.get_loc('b') # differ from pandas - slice(1, 4, 2) - - .. pandas-compat:: - **MultiIndex.get_loc** - - The return types of this function may deviates from the - method provided by Pandas. If the index is neither - lexicographically sorted nor unique, a best effort attempt is made - to coerce the found indices into a slice. For example: - - .. code-block:: - - >>> import pandas as pd - >>> import cudf - >>> x = pd.MultiIndex.from_tuples([ - ... (2, 1, 1), (1, 2, 3), (1, 2, 1), - ... (1, 1, 1), (1, 1, 1), (2, 2, 1), - ... ]) - >>> x.get_loc(1) - array([False, True, True, True, True, False]) - >>> cudf.from_pandas(x).get_loc(1) - slice(1, 5, 1) - """ is_sorted = ( self.is_monotonic_increasing or self.is_monotonic_decreasing ) From a92ad860d15c7357f12be16d395c50d479cc3c9a Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 31 May 2023 08:56:23 -0500 Subject: [PATCH 042/384] Fix parquet paritioning pytest failures (#13474) This PR fixes parquet pytest failures, mostly working around two upstream issues: 1. https://github.com/pandas-dev/pandas/issues/53345 2. https://github.com/apache/arrow/issues/33321 Thus fixes 20 pytest failure: This PR: ``` = 231 failed, 95767 passed, 2045 skipped, 764 xfailed, 300 xpassed in 426.65s (0:07:06) = ``` On `pandas_2.0_feature_branch`: ``` = 251 failed, 95747 passed, 2045 skipped, 764 xfailed, 300 xpassed in 433.50s (0:07:13) = ``` --- python/cudf/cudf/tests/test_parquet.py | 50 ++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index d684e4a19b1..ba49e1fe798 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -1749,6 +1749,15 @@ def test_parquet_partitioned(tmpdir_factory, cols, filename): # Check that cudf and pd return the same read got_cudf = cudf.read_parquet(gdf_dir) + if PANDAS_GE_200 and isinstance(got_pd["c"].dtype, pd.CategoricalDtype): + # Work-around for pandas bug: + # https://github.com/pandas-dev/pandas/issues/53345 + got_pd["c"] = got_pd["c"].astype( + pd.CategoricalDtype( + categories=got_pd["c"].dtype.categories.astype("int64"), + ordered=got_pd["c"].dtype.ordered, + ) + ) assert_eq(got_pd, got_cudf) # If filename is specified, check that it is correct @@ -1796,6 +1805,15 @@ def test_parquet_writer_chunked_partitioned(tmpdir_factory, return_meta): # Check that cudf and pd return the same read got_cudf = cudf.read_parquet(gdf_dir) + if PANDAS_GE_200: + # Work-around for pandas bug: + # https://github.com/pandas-dev/pandas/issues/53345 + got_pd["a"] = got_pd["a"].astype( + pd.CategoricalDtype( + categories=got_pd["a"].dtype.categories.astype("int64"), + ordered=got_pd["a"].dtype.ordered, + ) + ) assert_eq(got_pd, got_cudf) @@ -1836,7 +1854,15 @@ def test_parquet_writer_chunked_max_file_size( # Check that cudf and pd return the same read got_cudf = cudf.read_parquet(gdf_dir) - + if PANDAS_GE_200: + # Work-around for pandas bug: + # https://github.com/pandas-dev/pandas/issues/53345 + got_pd["a"] = got_pd["a"].astype( + pd.CategoricalDtype( + categories=got_pd["a"].dtype.categories.astype("int64"), + ordered=got_pd["a"].dtype.ordered, + ) + ) assert_eq( got_pd.sort_values(["b"]).reset_index(drop=True), got_cudf.sort_values(["b"]).reset_index(drop=True), @@ -1882,6 +1908,15 @@ def test_parquet_writer_chunked_partitioned_context(tmpdir_factory): # Check that cudf and pd return the same read got_cudf = cudf.read_parquet(gdf_dir) + if PANDAS_GE_200: + # Work-around for pandas bug: + # https://github.com/pandas-dev/pandas/issues/53345 + got_pd["a"] = got_pd["a"].astype( + pd.CategoricalDtype( + categories=got_pd["a"].dtype.categories.astype("int64"), + ordered=got_pd["a"].dtype.ordered, + ) + ) assert_eq(got_pd, got_cudf) @@ -1989,6 +2024,15 @@ def test_read_parquet_partitioned_filtered( filters = [[("a", "==", 10)], [("c", "==", 1)]] got = cudf.read_parquet(read_path, filters=filters) expect = pd.read_parquet(read_path, filters=filters) + if PANDAS_GE_200: + # Work-around for pandas bug: + # https://github.com/pandas-dev/pandas/issues/53345 + expect["c"] = expect["c"].astype( + pd.CategoricalDtype( + categories=expect["c"].dtype.categories.astype("int64"), + ordered=expect["c"].dtype.ordered, + ) + ) assert_eq(expect, got) @@ -2803,7 +2847,9 @@ def test_parquet_roundtrip_time_delta(): ) buffer = BytesIO() df.to_parquet(buffer) - assert_eq(df, cudf.read_parquet(buffer)) + # TODO: Remove `check_dtype` once following issue is fixed in arrow: + # https://github.com/apache/arrow/issues/33321 + assert_eq(df, cudf.read_parquet(buffer), check_dtype=not PANDAS_GE_200) def test_parquet_reader_malformed_file(datadir): From 63b8fb1a673bbd92761a8b50ec711edd18dc2618 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 2 Jun 2023 19:00:36 -0500 Subject: [PATCH 043/384] Enforce merge validation deprecation (#13499) This PR raises an error when a merge is being performed between data consisting of different levels. --- python/cudf/cudf/core/join/join.py | 12 +++++------- python/cudf/cudf/tests/test_joining.py | 24 +++++++++++++++--------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/python/cudf/cudf/core/join/join.py b/python/cudf/cudf/core/join/join.py index de4e323a8d7..480a6c64fe6 100644 --- a/python/cudf/cudf/core/join/join.py +++ b/python/cudf/cudf/core/join/join.py @@ -1,7 +1,6 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. from __future__ import annotations -import warnings from typing import Any, ClassVar, List, Optional import cudf @@ -422,11 +421,10 @@ def _validate_merge_params( # modified in the size 0 case. and max(lhs._data.nlevels, 1) != max(rhs._data.nlevels, 1) ): - warnings.warn( - "merging between different levels is deprecated and will be " - f"removed in a future version. ({lhs._data.nlevels} levels on " - f"the left, {rhs._data.nlevels} on the right)", - FutureWarning, + raise ValueError( + "Not allowed to merge between different levels. " + f"({lhs._data.nlevels} levels on " + f"the left, {rhs._data.nlevels} on the right)" ) diff --git a/python/cudf/cudf/tests/test_joining.py b/python/cudf/cudf/tests/test_joining.py index c578266ac22..f8d0bc2ace8 100644 --- a/python/cudf/cudf/tests/test_joining.py +++ b/python/cudf/cudf/tests/test_joining.py @@ -2160,15 +2160,21 @@ def test_join_multiindex_empty(): lhs = pd.DataFrame({"a": [1, 2, 3], "b": [2, 3, 4]}, index=["a", "b", "c"]) lhs.columns = pd.MultiIndex.from_tuples([("a", "x"), ("a", "y")]) rhs = pd.DataFrame(index=["a", "c", "d"]) - with pytest.warns(FutureWarning): - expect = lhs.join(rhs, how="inner") - - lhs = cudf.from_pandas(lhs) - rhs = cudf.from_pandas(rhs) - with pytest.warns(FutureWarning): - got = lhs.join(rhs, how="inner") - - assert_join_results_equal(expect, got, how="inner") + g_lhs = cudf.from_pandas(lhs) + g_rhs = cudf.from_pandas(rhs) + if PANDAS_GE_200: + assert_exceptions_equal( + lfunc=lhs.join, + rfunc=g_lhs.join, + lfunc_args_and_kwargs=([rhs], {"how": "inner"}), + rfunc_args_and_kwargs=([g_rhs], {"how": "inner"}), + check_exception_type=False, + ) + else: + with pytest.warns(FutureWarning): + _ = lhs.join(rhs, how="inner") + with pytest.raises(ValueError): + _ = g_lhs.join(g_rhs, how="inner") def test_join_on_index_with_duplicate_names(): From 139e32d1415c300daff514a5928996b7630ae313 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 2 Jun 2023 19:01:22 -0500 Subject: [PATCH 044/384] Enable `sort=True` for `Index.union`, `Index.difference` and `Index.intersection` (#13497) This PR enables `sort=True` for `union`, `difference`, and `intersection` APIs in `Index`. This also fixes 1 pytest failure and adds 77 pytests: On `Index_sort_2.0`: ``` = 230 failed, 95836 passed, 2045 skipped, 768 xfailed, 308 xpassed in 438.88s (0:07:18) = ``` On `pandas_2.0_feature_branch`: ``` = 231 failed, 95767 passed, 2045 skipped, 764 xfailed, 300 xpassed in 432.59s (0:07:12) = ``` xref: https://github.com/pandas-dev/pandas/issues/25151 --- python/cudf/cudf/core/_base_index.py | 25 ++++++++------ python/cudf/cudf/core/index.py | 6 ++-- python/cudf/cudf/core/multiindex.py | 4 +-- python/cudf/cudf/tests/test_index.py | 50 +++++++++++++++++++++++----- 4 files changed, 61 insertions(+), 24 deletions(-) diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index 46e7cdfac61..49721c23eb9 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -334,6 +334,7 @@ def union(self, other, sort=None): 2. `self` or `other` has length 0. * False : do not sort the result. + * True : Sort the result (which may raise TypeError). Returns ------- @@ -395,10 +396,10 @@ def union(self, other, sort=None): if not isinstance(other, BaseIndex): other = cudf.Index(other, name=self.name) - if sort not in {None, False}: + if sort not in {None, False, True}: raise ValueError( f"The 'sort' keyword only takes the values of " - f"None or False; {sort} was passed." + f"[None, False, True]; {sort} was passed." ) if not len(other) or self.equals(other): @@ -425,6 +426,7 @@ def intersection(self, other, sort=False): * False : do not sort the result. * None : sort the result, except when `self` and `other` are equal or when the values cannot be compared. + * True : Sort the result (which may raise TypeError). Returns ------- @@ -475,10 +477,10 @@ def intersection(self, other, sort=False): if not isinstance(other, BaseIndex): other = cudf.Index(other, name=self.name) - if sort not in {None, False}: + if sort not in {None, False, True}: raise ValueError( f"The 'sort' keyword only takes the values of " - f"None or False; {sort} was passed." + f"[None, False, True]; {sort} was passed." ) if self.equals(other): @@ -768,6 +770,7 @@ def difference(self, other, sort=None): * None : Attempt to sort the result, but catch any TypeErrors from comparing incomparable elements. * False : Do not sort the result. + * True : Sort the result (which may raise TypeError). Returns ------- @@ -787,16 +790,18 @@ def difference(self, other, sort=None): >>> idx1.difference(idx2, sort=False) Index([2, 1], dtype='int64') """ - if sort not in {None, False}: + if sort not in {None, False, True}: raise ValueError( f"The 'sort' keyword only takes the values " - f"of None or False; {sort} was passed." + f"of [None, False, True]; {sort} was passed." ) other = cudf.Index(other) - if is_mixed_with_object_dtype(self, other): + if is_mixed_with_object_dtype(self, other) or len(other) == 0: difference = self.copy() + if sort is True: + return difference.sort_values() else: other = other.copy(deep=False) other.names = self.names @@ -813,7 +818,7 @@ def difference(self, other, sort=None): if self.dtype != other.dtype: difference = difference.astype(self.dtype) - if sort is None and len(other): + if sort in {None, True} and len(other): return difference.sort_values() return difference @@ -1170,7 +1175,7 @@ def _union(self, other, sort=None): ) union_result = cudf.core.index._index_from_data({0: res._data[0]}) - if sort is None and len(other): + if sort in {None, True} and len(other): return union_result.sort_values() return union_result @@ -1187,7 +1192,7 @@ def _intersection(self, other, sort=None): ._data ) - if sort is None and len(other): + if sort is {None, True} and len(other): return intersection_result.sort_values() return intersection_result diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index c0664d3ca4d..a71c285b737 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -699,7 +699,7 @@ def _union(self, other, sort=None): ): result = type(self)(start_r, end_r + step_s / 2, step_s / 2) if result is not None: - if sort is None and not result.is_monotonic_increasing: + if sort in {None, True} and not result.is_monotonic_increasing: return result.sort_values() else: return result @@ -710,7 +710,7 @@ def _union(self, other, sort=None): return self._as_int_index()._union(other, sort=sort) @_cudf_nvtx_annotate - def _intersection(self, other, sort=False): + def _intersection(self, other, sort=None): if not isinstance(other, RangeIndex): return super()._intersection(other, sort=sort) @@ -750,7 +750,7 @@ def _intersection(self, other, sort=False): if (self.step < 0 and other.step < 0) is not (new_index.step < 0): new_index = new_index[::-1] - if sort is None: + if sort in {None, True}: new_index = new_index.sort_values() return new_index diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index cdc120935ee..4803e2b8e4b 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1796,7 +1796,7 @@ def _union(self, other, sort=None): midx = MultiIndex.from_frame(result_df.iloc[:, : self.nlevels]) midx.names = self.names if self.names == other.names else None - if sort is None and len(other): + if sort in {None, True} and len(other): return midx.sort_values() return midx @@ -1819,7 +1819,7 @@ def _intersection(self, other, sort=None): result_df = cudf.merge(self_df, other_df, how="inner") midx = self.__class__.from_frame(result_df, names=res_name) - if sort is None and len(other): + if sort in {None, True} and len(other): return midx.sort_values() return midx diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index de4c72389cf..81369cd2c6e 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -781,7 +781,7 @@ def test_index_to_series(data): [], ], ) -@pytest.mark.parametrize("sort", [None, False]) +@pytest.mark.parametrize("sort", [None, False, True]) def test_index_difference(data, other, sort): pd_data = pd.Index(data) pd_other = pd.Index(other) @@ -801,8 +801,8 @@ def test_index_difference_sort_error(): assert_exceptions_equal( pdi.difference, gdi.difference, - ([pdi], {"sort": True}), - ([gdi], {"sort": True}), + ([pdi], {"sort": "A"}), + ([gdi], {"sort": "A"}), ) @@ -2236,13 +2236,45 @@ def test_range_index_concat(objs): [ (pd.RangeIndex(0, 10), pd.RangeIndex(3, 7)), (pd.RangeIndex(0, 10), pd.RangeIndex(10, 20)), - (pd.RangeIndex(0, 10, 2), pd.RangeIndex(1, 5, 3)), - (pd.RangeIndex(1, 5, 3), pd.RangeIndex(0, 10, 2)), - (pd.RangeIndex(1, 10, 3), pd.RangeIndex(1, 5, 2)), + pytest.param( + pd.RangeIndex(0, 10, 2), + pd.RangeIndex(1, 5, 3), + marks=pytest.mark.xfail( + condition=PANDAS_GE_200, + reason="https://github.com/pandas-dev/pandas/issues/53490", + strict=False, + ), + ), + pytest.param( + pd.RangeIndex(1, 5, 3), + pd.RangeIndex(0, 10, 2), + marks=pytest.mark.xfail( + condition=PANDAS_GE_200, + reason="https://github.com/pandas-dev/pandas/issues/53490", + strict=False, + ), + ), + pytest.param( + pd.RangeIndex(1, 10, 3), + pd.RangeIndex(1, 5, 2), + marks=pytest.mark.xfail( + condition=PANDAS_GE_200, + reason="https://github.com/pandas-dev/pandas/issues/53490", + strict=False, + ), + ), (pd.RangeIndex(1, 5, 2), pd.RangeIndex(1, 10, 3)), (pd.RangeIndex(1, 100, 3), pd.RangeIndex(1, 50, 3)), (pd.RangeIndex(1, 100, 3), pd.RangeIndex(1, 50, 6)), - (pd.RangeIndex(1, 100, 6), pd.RangeIndex(1, 50, 3)), + pytest.param( + pd.RangeIndex(1, 100, 6), + pd.RangeIndex(1, 50, 3), + marks=pytest.mark.xfail( + condition=PANDAS_GE_200, + reason="https://github.com/pandas-dev/pandas/issues/53490", + strict=False, + ), + ), (pd.RangeIndex(0, 10, name="a"), pd.RangeIndex(90, 100, name="b")), (pd.Index([0, 1, 2, 30], name="a"), pd.Index([90, 100])), (pd.Index([0, 1, 2, 30], name="a"), [90, 100]), @@ -2250,7 +2282,7 @@ def test_range_index_concat(objs): (pd.Index(["a", "b", "c", "d", "c"]), pd.Index(["a", "c", "z"])), ], ) -@pytest.mark.parametrize("sort", [None, False]) +@pytest.mark.parametrize("sort", [None, False, True]) def test_union_index(idx1, idx2, sort): expected = idx1.union(idx2, sort=sort) @@ -2280,7 +2312,7 @@ def test_union_index(idx1, idx2, sort): (pd.Index([True, False, True, True]), pd.Index([True, True])), ], ) -@pytest.mark.parametrize("sort", [None, False]) +@pytest.mark.parametrize("sort", [None, False, True]) def test_intersection_index(idx1, idx2, sort): expected = idx1.intersection(idx2, sort=sort) From a6869e8a16928b0e07f17d0992eb8b18ea433715 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 2 Jun 2023 19:01:51 -0500 Subject: [PATCH 045/384] Fix a groupby pytest related to numeric_only (#13496) This PR fixes a groupby pytest by performing a special version based handling, we will need this handling because of no support for numeric_only in groupby.agg yet. --- python/cudf/cudf/tests/test_groupby.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index 7c610eca88c..6d326252d92 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -1876,14 +1876,19 @@ def test_groupby_list_columns_excluded(): ) gdf = cudf.from_pandas(pdf) - # cudf does not yet support numeric_only, so our default is False, but - # pandas defaults to inferring and throws a warning about it, so we need to - # catch that. pandas future behavior will match ours by default (at which - # point supporting numeric_only=True will be the open feature request). - with pytest.warns(FutureWarning): - pandas_result = pdf.groupby("a").mean() - with pytest.warns(FutureWarning): - pandas_agg_result = pdf.groupby("a").agg("mean") + if PANDAS_GE_200: + pandas_result = pdf.groupby("a").mean(numeric_only=True) + pandas_agg_result = pdf.groupby("a").agg("mean", numeric_only=True) + else: + # cudf does not yet support numeric_only, so our default is False, but + # pandas defaults to inferring and throws a warning about it, so + # we need to catch that. pandas future behavior will match ours + # by default (at which point supporting numeric_only=True will + # be the open feature request). + with pytest.warns(FutureWarning): + pandas_result = pdf.groupby("a").mean() + with pytest.warns(FutureWarning): + pandas_agg_result = pdf.groupby("a").agg("mean") assert_groupby_results_equal( pandas_result, gdf.groupby("a").mean(), check_dtype=False From 6001bbfce87bc4032f2b15d38a85504b407c007b Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 2 Jun 2023 19:02:31 -0500 Subject: [PATCH 046/384] Drop special handling of `min_periods` for `Rolling.count` (#13483) This PR drops a special handling for `Rolling.count` where we always default to `min_periods=0`, this is an inconsistency that pandas-2.0 resolves in: https://github.com/pandas-dev/pandas/pull/48839 This PR fixes 2 pytest failures: ``` = 229 failed, 95769 passed, 2045 skipped, 764 xfailed, 300 xpassed in 458.04s (0:07:38) = ``` On `pandas_2.0_feature_branch`: ``` = 231 failed, 95767 passed, 2045 skipped, 764 xfailed, 300 xpassed in 457.58s (0:07:37) = ``` --- python/cudf/cudf/core/window/rolling.py | 2 -- python/cudf/cudf/tests/test_rolling.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/python/cudf/cudf/core/window/rolling.py b/python/cudf/cudf/core/window/rolling.py index 8a92ea86d57..f1500408eb4 100644 --- a/python/cudf/cudf/core/window/rolling.py +++ b/python/cudf/cudf/core/window/rolling.py @@ -543,8 +543,6 @@ def _window_to_window_sizes(self, window): ) def _apply_agg(self, agg_name): - if agg_name == "count" and not self._time_window: - self.min_periods = 0 index = cudf.MultiIndex.from_frame( cudf.DataFrame( { diff --git a/python/cudf/cudf/tests/test_rolling.py b/python/cudf/cudf/tests/test_rolling.py index b4e0983a9e3..a4c41136b1b 100644 --- a/python/cudf/cudf/tests/test_rolling.py +++ b/python/cudf/cudf/tests/test_rolling.py @@ -8,14 +8,14 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_150, PANDAS_LT_140 +from cudf.core._compat import PANDAS_GE_150, PANDAS_LT_140, PANDAS_GE_200 from cudf.testing._utils import _create_pandas_series, assert_eq from cudf.testing.dataset_generator import rand_dataframe @contextmanager def _hide_pandas_rolling_min_periods_warning(agg): - if agg == "count": + if not PANDAS_GE_200 and agg == "count": with pytest.warns( FutureWarning, match="min_periods=None will default to the size of window " From 4416a24cb8784485c869125f3a674cadb4151db1 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 2 Jun 2023 19:02:54 -0500 Subject: [PATCH 047/384] Fix JSON pytests (#13476) This PR fixes 3 json reader pytests: This PR: ``` = 228 failed, 95770 passed, 2045 skipped, 764 xfailed, 300 xpassed in 473.29s (0:07:53) = ``` On `pandas_2.0_feature_branch`: ``` = 231 failed, 95767 passed, 2045 skipped, 764 xfailed, 300 xpassed in 445.90s (0:07:25) = ``` --- python/cudf/cudf/tests/test_json.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/python/cudf/cudf/tests/test_json.py b/python/cudf/cudf/tests/test_json.py index 43b0ca0119a..2f062e3a738 100644 --- a/python/cudf/cudf/tests/test_json.py +++ b/python/cudf/cudf/tests/test_json.py @@ -13,6 +13,7 @@ import pytest import cudf +from cudf.core._compat import PANDAS_GE_200 from cudf.testing._utils import ( DATETIME_TYPES, NUMERIC_TYPES, @@ -212,6 +213,18 @@ def test_cudf_json_writer_read(gdf_writer_types): if pdf2.empty: pdf2.reset_index(drop=True, inplace=True) pdf2.columns = pdf2.columns.astype("object") + if PANDAS_GE_200: + # Pandas moved to consistent datetimes parsing format: + # https://pandas.pydata.org/docs/dev/whatsnew/v2.0.0.html#datetimes-are-now-parsed-with-a-consistent-format + for unit in ["s", "ms"]: + if f"col_datetime64[{unit}]" in pdf2.columns: + pdf2[f"col_datetime64[{unit}]"] = ( + pd.to_datetime( + pdf2[f"col_datetime64[{unit}]"], format="mixed" + ) + .dt.tz_localize(None) + .astype(f"datetime64[{unit}]") + ) assert_eq(pdf2, gdf2) From d6324d144fdb02f0e4e762d5070bcb1a543ad0bd Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Mon, 5 Jun 2023 21:46:36 -0700 Subject: [PATCH 048/384] Fixed strings --- python/cudf/cudf/core/index.py | 2 +- python/cudf/cudf/core/multiindex.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index a7424bf0f04..6d24ccef410 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -1195,7 +1195,7 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): else: raise ValueError( f"{method=} is unsupported, only supported values are: " - f"{['ffill', 'bfill', 'nearest', None]}" + "{['ffill'/'pad', 'bfill'/'backfill', 'nearest', None]}" ) return result_series.to_cupy() diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index f6da845ba46..e8435bee380 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1674,7 +1674,7 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): else: raise ValueError( f"{method=} is unsupported, only supported values are: " - f"{['ffill', 'bfill', None]}" + "{['ffill'/'pad', 'bfill'/'backfill', None]}" ) return result_series.to_cupy() From 361e96e55b92d9c08dd7be8783b3071e28b1b9fe Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 6 Jun 2023 09:28:56 -0500 Subject: [PATCH 049/384] Fix `DataFrame.mode` pytest (#13500) This PR xfails a condition that is failing due to a pandas bug: pandas-dev/pandas#53497 --- python/cudf/cudf/tests/test_dataframe.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index e6f2f9ec448..4e1e07d2bfd 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -8394,8 +8394,17 @@ def test_describe_misc_exclude(df, exclude): ) @pytest.mark.parametrize("numeric_only", [True, False]) @pytest.mark.parametrize("dropna", [True, False]) -def test_dataframe_mode(df, numeric_only, dropna): +def test_dataframe_mode(request, df, numeric_only, dropna): pdf = df.to_pandas() + request.applymarker( + pytest.mark.xfail( + condition=PANDAS_GE_200 + and numeric_only is False + and "b" in df.columns + and df["b"].dtype == np.dtype("timedelta64[s]"), + reason="https://github.com/pandas-dev/pandas/issues/53497", + ) + ) expected = pdf.mode(numeric_only=numeric_only, dropna=dropna) actual = df.mode(numeric_only=numeric_only, dropna=dropna) From 8bf7b04a8b5f36df50be90e2606bf1c711d39ee8 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 6 Jun 2023 11:47:06 -0700 Subject: [PATCH 050/384] Address first round of reviews --- python/cudf/cudf/core/index.py | 52 ++++++++++++++++------------ python/cudf/cudf/core/multiindex.py | 15 +++++--- python/cudf/cudf/tests/test_index.py | 36 +++++++++++++++++++ 3 files changed, 77 insertions(+), 26 deletions(-) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 83c4c17babe..6e59222fc67 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -592,29 +592,34 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): @_cudf_nvtx_annotate def get_indexer(self, target, limit=None, method=None, tolerance=None): - if method is None: - if self.step > 0: - start, stop, step = self.start, self.stop, self.step - else: - # Reversed - reverse = self._range[::-1] - start, stop, step = reverse.start, reverse.stop, reverse.step - - target_array = cupy.asarray(target) - locs = target_array - start - valid = (locs % step == 0) & (locs >= 0) & (target_array < stop) - locs[~valid] = -1 - locs[valid] = locs[valid] / step - - if step != self.step: - # Reversed - locs[valid] = len(self) - 1 - locs[valid] - return locs - else: + target_col = cudf.core.column.as_column(target) + if method is not None or not isinstance( + target_col, cudf.core.column.NumericalColumn + ): + # TODO: See if we can implement this without converting to + # Integer index. return self._as_int_index().get_indexer( target=target, limit=limit, method=method, tolerance=tolerance ) + if self.step > 0: + start, stop, step = self.start, self.stop, self.step + else: + # Reversed + reverse = self._range[::-1] + start, stop, step = reverse.start, reverse.stop, reverse.step + + target_array = target_col.values + locs = target_array - start + valid = (locs % step == 0) & (locs >= 0) & (target_array < stop) + locs[~valid] = -1 + locs[valid] = locs[valid] / step + + if step != self.step: + # Reversed + locs[valid] = len(self) - 1 - locs[valid] + return locs + @_cudf_nvtx_annotate def get_loc(self, key): if not is_scalar(key): @@ -1167,9 +1172,12 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): haystack_table = cudf.DataFrame( {"None": self._column, "order": arange(0, len(self))} ) - merged_table = haystack_table.merge( - needle_table, on="None", how="outer" - ) + try: + merged_table = haystack_table.merge( + needle_table, on="None", how="outer" + ) + except ValueError: + return cupy.full(len(needle_table), -1, dtype="int64") result_series = ( merged_table.sort_values(by="order_y") .head(len(target))["order_x"] diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 485644ee2f5..4afd27873e3 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1647,15 +1647,22 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): f"{method=} is not supported yet for MultiIndex." ) - target = cudf.MultiIndex.from_tuples(target) + try: + target = cudf.MultiIndex.from_tuples(target) + except TypeError: + return cp.full(len(target), -1, dtype="int64") needle_table = target.to_frame(index=False) col_names = list(range(0, self.nlevels)) needle_table["order"] = needle_table.index haystack_table = self.copy(deep=True).to_frame(index=False) haystack_table["order"] = haystack_table.index - merged_table = haystack_table.merge( - needle_table, on=col_names, how="outer" - ) + try: + merged_table = haystack_table.merge( + needle_table, on=col_names, how="outer" + ) + except ValueError: + return cp.full(len(needle_table), -1, dtype="int64") + result_series = ( merged_table.sort_values(by="order_y") .head(len(target))["order_x"] diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index ef674430e1c..db94dadbd2c 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -2396,6 +2396,42 @@ def test_get_indexer_multi_string(idx, key, method): assert_eq(expected, got) +@pytest.mark.parametrize( + "idx1", + [ + lambda: cudf.Index(["a", "b", "c"]), + lambda: cudf.RangeIndex(0, 10), + lambda: cudf.Index([1, 2, 3], dtype="category"), + lambda: cudf.Index(["a", "b", "c", "d"], dtype="category"), + lambda: cudf.MultiIndex.from_tuples( + [ + ("a", "a", "a"), + ("a", "b", "c"), + ("b", "a", "a"), + ("a", "a", "b"), + ("a", "b", "a"), + ("b", "c", "a"), + ] + ), + ], +) +@pytest.mark.parametrize( + "idx2", + [ + lambda: cudf.Index(["a", "b", "c"]), + lambda: cudf.RangeIndex(0, 10), + lambda: cudf.Index([1, 2, 3], dtype="category"), + lambda: cudf.Index(["a", "b", "c", "d"], dtype="category"), + ], +) +def test_get_indexer_invalid(idx1, idx2): + idx1 = idx1() + idx2 = idx2() + assert_eq( + idx1.get_indexer(idx2), idx1.to_pandas().get_indexer(idx2.to_pandas()) + ) + + @pytest.mark.parametrize( "objs", [ From 0dc0a3da5e3d33bee5e90a766bb8602a0b56fd13 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 6 Jun 2023 17:48:20 -0700 Subject: [PATCH 051/384] annotate --- python/cudf/cudf/core/index.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 6e59222fc67..e6eb9c8c54b 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -1172,6 +1172,8 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): haystack_table = cudf.DataFrame( {"None": self._column, "order": arange(0, len(self))} ) + if not len(self): + return cupy.full(len(needle_table), -1, dtype="int64") try: merged_table = haystack_table.merge( needle_table, on="None", how="outer" @@ -2871,20 +2873,16 @@ def _get_indexer_basic(index, positions, method, target_col, tolerance): return positions -def _get_nearest_indexer(index, positions, target_col, tolerance): +def _get_nearest_indexer( + index: Index, + positions: cudf.Series, + target_col: cudf.core.column.ColumnBase, + tolerance: Union[int, float], +): """ Get the indexer for the nearest index labels; requires an index with values that can be subtracted from each other. """ - if not len(index): - return _get_indexer_basic( - index=index, - positions=positions.copy(deep=True), - method="pad", - targe_col=target_col, - tolerance=tolerance, - ) - left_indexer = _get_indexer_basic( index=index, positions=positions.copy(deep=True), From 261f594075d55d2bcb01c8b41f611006fb88c988 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 6 Jun 2023 18:21:36 -0700 Subject: [PATCH 052/384] Fix issues --- python/cudf/cudf/core/index.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index e6eb9c8c54b..2e0d09b9d5f 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -623,7 +623,7 @@ def get_indexer(self, target, limit=None, method=None, tolerance=None): @_cudf_nvtx_annotate def get_loc(self, key): if not is_scalar(key): - raise TypeError("Should be a sequence") + raise TypeError("Should be a scalar-like") idx = (key - self._start) / self._step idx_int_upper_bound = (self._stop - self._start) // self._step if idx > idx_int_upper_bound or idx < 0: @@ -1212,22 +1212,20 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): @_cudf_nvtx_annotate def get_loc(self, key): - if is_scalar(key): - target = [key] - else: - target = key + if not is_scalar(key): + raise TypeError("Should be a scalar-like") is_sorted = ( self.is_monotonic_increasing or self.is_monotonic_decreasing ) - target_as_table = cudf.core.frame.Frame({"None": as_column(target)}) + target_as_table = cudf.core.frame.Frame({"None": as_column([key])}) lower_bound, upper_bound, sort_inds = _lexsorted_equal_range( self, target_as_table, is_sorted ) if lower_bound == upper_bound: - raise KeyError(target) + raise KeyError(key) if lower_bound + 1 == upper_bound: # Search result is unique, return int. From dc08ef05a42c8e163351e2ac02860179d9144621 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 7 Jun 2023 09:57:59 -0700 Subject: [PATCH 053/384] Switch to outer inner --- python/cudf/cudf/core/_base_index.py | 2 +- python/cudf/cudf/core/index.py | 46 +++++++++++++++------------- python/cudf/cudf/core/multiindex.py | 38 +++++++++++++---------- 3 files changed, 46 insertions(+), 40 deletions(-) diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index 0635e970d10..6d20c49cf2e 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -126,7 +126,7 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): >>> index Index(['c', 'a', 'b'], dtype='object') >>> index.get_indexer(['a', 'b', 'x']) - array([ 1, 2, -1]) + array([ 1, 2, -1], dtype=int32) """ raise NotImplementedError diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 2e0d09b9d5f..a251d67e689 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -23,6 +23,7 @@ from typing_extensions import Self import cudf +from cudf import _lib as libcudf from cudf._lib.datetime import extract_quarter, is_leap_year from cudf._lib.filling import sequence from cudf._lib.search import search_sorted @@ -52,6 +53,7 @@ from cudf.core.column.string import StringMethods as StringMethods from cudf.core.dtypes import IntervalDtype from cudf.core.frame import Frame +from cudf.core.join._join_helpers import _match_join_keys from cudf.core.mixins import BinaryOperand from cudf.core.single_column_frame import SingleColumnFrame from cudf.utils.docutils import copy_docstring, doc_apply @@ -1166,43 +1168,43 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): "is specified." ) - needle_table = cudf.DataFrame( - {"None": as_column(target), "order": arange(0, len(target))} + needle = as_column(target) + haystack = self._column + result = cudf.core.column.full( + len(needle), + fill_value=-1 if method is None else None, + dtype=libcudf.types.size_type_dtype, ) - haystack_table = cudf.DataFrame( - {"None": self._column, "order": arange(0, len(self))} - ) - if not len(self): - return cupy.full(len(needle_table), -1, dtype="int64") try: - merged_table = haystack_table.merge( - needle_table, on="None", how="outer" - ) + lcol, rcol = _match_join_keys(needle, haystack, "inner") except ValueError: - return cupy.full(len(needle_table), -1, dtype="int64") - result_series = ( - merged_table.sort_values(by="order_y") - .head(len(target))["order_x"] - .reset_index(drop=True) - ) - if method is None: - result_series = result_series.fillna(-1) - elif method in {"ffill", "bfill", "pad", "backfill"}: + return cupy.full( + len(needle), -1, dtype=libcudf.types.size_type_dtype + ) + scatter_map, indices = libcudf.join.join([lcol], [rcol], how="inner") + (result,) = libcudf.copying.scatter([indices], scatter_map, [result]) + result_series = cudf.Series(result) + if not len(self): + return cupy.full( + len(needle), -1, dtype=libcudf.types.size_type_dtype + ) + + if method in {"ffill", "bfill", "pad", "backfill"}: result_series = _get_indexer_basic( index=self, positions=result_series, method=method, - target_col=needle_table["None"], + target_col=cudf.Series(needle), tolerance=tolerance, ) elif method == "nearest": result_series = _get_nearest_indexer( index=self, positions=result_series, - target_col=needle_table["None"], + target_col=cudf.Series(needle), tolerance=tolerance, ) - else: + elif method is not None: raise ValueError( f"{method=} is unsupported, only supported values are: " "{['ffill'/'pad', 'bfill'/'backfill', 'nearest', None]}" diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 4afd27873e3..9e79779c6f6 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1650,27 +1650,31 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): try: target = cudf.MultiIndex.from_tuples(target) except TypeError: - return cp.full(len(target), -1, dtype="int64") + return cp.full( + len(target), -1, dtype=libcudf.types.size_type_dtype + ) + needle_table = target.to_frame(index=False) col_names = list(range(0, self.nlevels)) - needle_table["order"] = needle_table.index haystack_table = self.copy(deep=True).to_frame(index=False) - haystack_table["order"] = haystack_table.index - try: - merged_table = haystack_table.merge( - needle_table, on=col_names, how="outer" + result = cudf.core.column.full( + len(needle_table), + fill_value=-1 if method is None else None, + dtype=libcudf.types.size_type_dtype, + ) + scatter_map, indices = libcudf.join.join( + list(needle_table._data.columns), + list(haystack_table._data.columns), + how="inner", + ) + (result,) = libcudf.copying.scatter([indices], scatter_map, [result]) + result_series = cudf.Series(result) + if not len(self): + return cp.full( + len(needle_table), -1, dtype=libcudf.types.size_type_dtype ) - except ValueError: - return cp.full(len(needle_table), -1, dtype="int64") - result_series = ( - merged_table.sort_values(by="order_y") - .head(len(target))["order_x"] - .reset_index(drop=True) - ) - if method is None: - result_series = result_series.fillna(-1) - elif method in {"ffill", "bfill", "pad", "backfill"}: + if method in {"ffill", "bfill", "pad", "backfill"}: result_series = _get_indexer_basic( index=self, positions=result_series, @@ -1678,7 +1682,7 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): target_col=needle_table[col_names], tolerance=tolerance, ) - else: + elif method is not None: raise ValueError( f"{method=} is unsupported, only supported values are: " "{['ffill'/'pad', 'bfill'/'backfill', None]}" From 4289ef43272772be819fab15af152ce7c38776c1 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 13 Jun 2023 12:27:15 -0500 Subject: [PATCH 054/384] Fix `dask_cudf` pytest failures for `pandas-2.0` upgrade (#13548) This PR fixes all `dask_cudf` side failures that happen due to `pandas-2.0` upgrade. The fixes are trivial to be broken down into separate PRs. - [x] `check_less_precise` is removed in `pandas-2.0`, since it is a parameter that we never supported and just had it for the sake of matching signature I removed it from all the methods. - [x] Due to the removal of `StringIndex`, we had to perform some re-ordering of `if/elif` logic in `_nonempty_index`. - [x] `dask_cudf.DataFrame.var` got `numeric_only` support. - [x] `Series.count` doesn't have `skipna` support. Hence removed it from the call. This PR fixes 56 pytest failures: ``` == 1100 passed, 13 skipped, 8 xfailed, 5 xpassed, 114 warnings in 57.10s == ``` On `pandas_2.0_feature_branch`: ``` == 56 failed, 1044 passed, 13 skipped, 8 xfailed, 5 xpassed, 114 warnings in 73.73s (0:01:13) == ``` --- python/cudf/cudf/testing/testing.py | 14 -------------- python/dask_cudf/dask_cudf/backends.py | 12 ++++++------ python/dask_cudf/dask_cudf/core.py | 8 ++++++-- .../dask_cudf/io/tests/test_parquet.py | 5 ++++- .../dask_cudf/tests/test_accessor.py | 6 +++--- python/dask_cudf/dask_cudf/tests/test_core.py | 6 +++--- .../dask_cudf/tests/test_reductions.py | 19 ++++++++++++------- 7 files changed, 34 insertions(+), 36 deletions(-) diff --git a/python/cudf/cudf/testing/testing.py b/python/cudf/cudf/testing/testing.py index 0f54391b426..70a96411a7c 100644 --- a/python/cudf/cudf/testing/testing.py +++ b/python/cudf/cudf/testing/testing.py @@ -2,8 +2,6 @@ from __future__ import annotations -from typing import Union - import cupy as cp import numpy as np import pandas as pd @@ -101,7 +99,6 @@ def assert_column_equal( right, check_dtype=True, check_column_type="equiv", - check_less_precise=False, check_exact=False, check_datetimelike_compat=False, check_categorical=True, @@ -129,8 +126,6 @@ def assert_column_equal( Whether to check the columns class, dtype and inferred_type are identical. Currently it is idle, and similar to pandas. - check_less_precise : bool or int, default False - Not yet supported check_exact : bool, default False Whether to compare number exactly. check_datetime_like_compat : bool, default False @@ -292,7 +287,6 @@ def assert_index_equal( right, exact="equiv", check_names: bool = True, - check_less_precise: Union[bool, int] = False, check_exact: bool = True, check_categorical: bool = True, check_order: bool = True, @@ -319,8 +313,6 @@ def assert_index_equal( for Index with an int8/int32/int64 dtype as well. check_names : bool, default True Whether to check the names attribute. - check_less_precise : bool or int, default False - Not yet supported check_exact : bool, default False Whether to compare number exactly. check_categorical : bool, default True @@ -404,7 +396,6 @@ def assert_index_equal( exact=check_exact, check_names=check_names, check_exact=check_exact, - check_less_precise=check_less_precise, check_order=check_order, rtol=rtol, atol=atol, @@ -433,7 +424,6 @@ def assert_series_equal( check_dtype=True, check_index_type="equiv", check_series_type=True, - check_less_precise=False, check_names=True, check_exact=False, check_datetimelike_compat=False, @@ -465,8 +455,6 @@ def assert_series_equal( Whether to check the series class, dtype and inferred_type are identical. Currently it is idle, and similar to pandas. - check_less_precise : bool or int, default False - Not yet supported check_names : bool, default True Whether to check that the names attribute for both the index and column attributes of the Series is identical. @@ -530,7 +518,6 @@ def assert_series_equal( right.index, exact=check_index_type, check_names=check_names, - check_less_precise=check_less_precise, check_exact=check_exact, check_categorical=check_categorical, rtol=rtol, @@ -543,7 +530,6 @@ def assert_series_equal( right._column, check_dtype=check_dtype, check_column_type=check_series_type, - check_less_precise=check_less_precise, check_exact=check_exact, check_datetimelike_compat=check_datetimelike_compat, check_categorical=check_categorical, diff --git a/python/dask_cudf/dask_cudf/backends.py b/python/dask_cudf/dask_cudf/backends.py index e44775e56df..3c7c5c99695 100644 --- a/python/dask_cudf/dask_cudf/backends.py +++ b/python/dask_cudf/dask_cudf/backends.py @@ -57,8 +57,6 @@ def _nonempty_index(idx): data = np.array([start, "1970-01-02"], dtype=idx.dtype) values = cudf.core.column.as_column(data) return cudf.core.index.DatetimeIndex(values, name=idx.name) - elif isinstance(idx._column, cudf.core.column.StringColumn): - return cudf.Index(["cat", "dog"], name=idx.name) elif isinstance(idx, cudf.core.index.CategoricalIndex): key = tuple(idx._data.keys()) assert len(key) == 1 @@ -69,16 +67,18 @@ def _nonempty_index(idx): categories=categories, codes=codes, ordered=ordered ) return cudf.core.index.CategoricalIndex(values, name=idx.name) - elif isinstance(idx, cudf.core.index.Index): - return cudf.core.index.Index( - np.arange(2, dtype=idx.dtype), name=idx.name - ) elif isinstance(idx, cudf.core.multiindex.MultiIndex): levels = [meta_nonempty(lev) for lev in idx.levels] codes = [[0, 0] for i in idx.levels] return cudf.core.multiindex.MultiIndex( levels=levels, codes=codes, names=idx.names ) + elif isinstance(idx._column, cudf.core.column.StringColumn): + return cudf.Index(["cat", "dog"], name=idx.name) + elif isinstance(idx, cudf.core.index.Index): + return cudf.core.index.Index( + np.arange(2, dtype=idx.dtype), name=idx.name + ) raise TypeError(f"Don't know how to handle index of type {type(idx)}") diff --git a/python/dask_cudf/dask_cudf/core.py b/python/dask_cudf/dask_cudf/core.py index d2858876fcd..3e5a40f5554 100644 --- a/python/dask_cudf/dask_cudf/core.py +++ b/python/dask_cudf/dask_cudf/core.py @@ -269,9 +269,12 @@ def var( dtype=None, out=None, naive=False, + numeric_only=False, ): axis = self._validate_axis(axis) - meta = self._meta_nonempty.var(axis=axis, skipna=skipna) + meta = self._meta_nonempty.var( + axis=axis, skipna=skipna, numeric_only=numeric_only + ) if axis == 1: result = map_partitions( M.var, @@ -281,6 +284,7 @@ def var( axis=axis, skipna=skipna, ddof=ddof, + numeric_only=numeric_only, ) return handle_out(out, result) elif naive: @@ -421,7 +425,7 @@ def _naive_var(ddf, meta, skipna, ddof, split_every, out): def _parallel_var(ddf, meta, skipna, split_every, out): def _local_var(x, skipna): if skipna: - n = x.count(skipna=skipna) + n = x.count() avg = x.mean(skipna=skipna) else: # Not skipping nulls, so might as well diff --git a/python/dask_cudf/dask_cudf/io/tests/test_parquet.py b/python/dask_cudf/dask_cudf/io/tests/test_parquet.py index 8e80aad67d1..489608cef1c 100644 --- a/python/dask_cudf/dask_cudf/io/tests/test_parquet.py +++ b/python/dask_cudf/dask_cudf/io/tests/test_parquet.py @@ -13,6 +13,7 @@ from dask.utils import natural_sort_key import cudf +from cudf.core._compat import PANDAS_GE_200 import dask_cudf @@ -173,7 +174,9 @@ def test_dask_timeseries_from_pandas(tmpdir): pdf = ddf2.compute() pdf.to_parquet(fn, engine="pyarrow") read_df = dask_cudf.read_parquet(fn) - dd.assert_eq(ddf2, read_df.compute()) + # Workaround until following issue is fixed: + # https://github.com/apache/arrow/issues/33321 + dd.assert_eq(ddf2, read_df.compute(), check_index_type=not PANDAS_GE_200) @pytest.mark.parametrize("index", [False, None]) diff --git a/python/dask_cudf/dask_cudf/tests/test_accessor.py b/python/dask_cudf/dask_cudf/tests/test_accessor.py index 6b1627c91e8..09d02893c26 100644 --- a/python/dask_cudf/dask_cudf/tests/test_accessor.py +++ b/python/dask_cudf/dask_cudf/tests/test_accessor.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2022, NVIDIA CORPORATION. +# Copyright (c) 2019-2023, NVIDIA CORPORATION. import numpy as np import pandas as pd @@ -53,8 +53,8 @@ def test_dt_series(data, field): sr = Series(pdsr) dsr = dgd.from_cudf(sr, npartitions=5) base = getattr(pdsr.dt, field) - test = getattr(dsr.dt, field).compute().to_pandas().astype("int64") - assert_series_equal(base, test) + test = getattr(dsr.dt, field).compute() + assert_eq(base, test, check_dtype=False) @pytest.mark.parametrize("data", [data_dt_1()]) diff --git a/python/dask_cudf/dask_cudf/tests/test_core.py b/python/dask_cudf/dask_cudf/tests/test_core.py index 7f8876c8564..afd1d91e29c 100644 --- a/python/dask_cudf/dask_cudf/tests/test_core.py +++ b/python/dask_cudf/dask_cudf/tests/test_core.py @@ -809,7 +809,7 @@ def test_series_describe(): dd.assert_eq( dsr.describe(), pdsr.describe(), - check_less_precise=3, + rtol=1e-3, ) @@ -838,7 +838,7 @@ def test_zero_std_describe(): ddf = dgd.from_cudf(df, npartitions=4) pddf = dd.from_pandas(pdf, npartitions=4) - dd.assert_eq(ddf.describe(), pddf.describe(), check_less_precise=3) + dd.assert_eq(ddf.describe(), pddf.describe(), rtol=1e-3) def test_large_numbers_var(): @@ -853,7 +853,7 @@ def test_large_numbers_var(): ddf = dgd.from_cudf(df, npartitions=4) pddf = dd.from_pandas(pdf, npartitions=4) - dd.assert_eq(ddf.var(), pddf.var(), check_less_precise=3) + dd.assert_eq(ddf.var(), pddf.var(), rtol=1e-3) def test_index_map_partitions(): diff --git a/python/dask_cudf/dask_cudf/tests/test_reductions.py b/python/dask_cudf/dask_cudf/tests/test_reductions.py index c34fbc3b0e7..ae1bfa02357 100644 --- a/python/dask_cudf/dask_cudf/tests/test_reductions.py +++ b/python/dask_cudf/dask_cudf/tests/test_reductions.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. import numpy as np import pandas as pd @@ -71,10 +71,15 @@ def test_rowwise_reductions(data, op): pddf = gddf.to_dask_dataframe() if op in ("var", "std"): - expected = getattr(pddf, op)(axis=1, ddof=0) - got = getattr(gddf, op)(axis=1, ddof=0) + expected = getattr(pddf, op)(axis=1, numeric_only=True, ddof=0) + got = getattr(gddf, op)(axis=1, numeric_only=True, ddof=0) else: - expected = getattr(pddf, op)(axis=1) - got = getattr(pddf, op)(axis=1) - - dd.assert_eq(expected.compute(), got.compute(), check_exact=False) + expected = getattr(pddf, op)(numeric_only=True, axis=1) + got = getattr(pddf, op)(numeric_only=True, axis=1) + + dd.assert_eq( + expected, + got, + check_exact=False, + check_dtype=op not in ("var", "std"), + ) From e7eb1d3918779bdf110db2f6cea9251d74131664 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 13 Jun 2023 12:48:10 -0700 Subject: [PATCH 055/384] simplify --- python/cudf/cudf/core/index.py | 19 ++++++------- python/cudf/cudf/core/multiindex.py | 43 +++++++++++++++++------------ 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index a251d67e689..ab551a43bae 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -1169,25 +1169,22 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): ) needle = as_column(target) - haystack = self._column result = cudf.core.column.full( len(needle), - fill_value=-1 if method is None else None, + fill_value=-1, dtype=libcudf.types.size_type_dtype, ) + + if not len(self): + return result.values try: - lcol, rcol = _match_join_keys(needle, haystack, "inner") + lcol, rcol = _match_join_keys(needle, self._column, "inner") except ValueError: - return cupy.full( - len(needle), -1, dtype=libcudf.types.size_type_dtype - ) + return result.values + scatter_map, indices = libcudf.join.join([lcol], [rcol], how="inner") (result,) = libcudf.copying.scatter([indices], scatter_map, [result]) result_series = cudf.Series(result) - if not len(self): - return cupy.full( - len(needle), -1, dtype=libcudf.types.size_type_dtype - ) if method in {"ffill", "bfill", "pad", "backfill"}: result_series = _get_indexer_basic( @@ -2849,7 +2846,7 @@ def _extended_gcd(a: int, b: int) -> Tuple[int, int, int]: def _get_indexer_basic(index, positions, method, target_col, tolerance): - nonexact = positions.isnull() + nonexact = positions == -1 positions[nonexact] = index.searchsorted( target_col[nonexact], side="left" if method in {"pad", "ffill"} else "right", diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 9e79779c6f6..b8390e4e678 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -29,6 +29,7 @@ _lexsorted_equal_range, as_index, ) +from cudf.core.join._join_helpers import _match_join_keys from cudf.utils.docutils import doc_apply from cudf.utils.utils import NotIterable, _cudf_nvtx_annotate @@ -1647,39 +1648,45 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): f"{method=} is not supported yet for MultiIndex." ) + result = cudf.core.column.full( + len(target), + fill_value=-1, + dtype=libcudf.types.size_type_dtype, + ) + if not len(self): + return result.values try: target = cudf.MultiIndex.from_tuples(target) except TypeError: - return cp.full( - len(target), -1, dtype=libcudf.types.size_type_dtype - ) - - needle_table = target.to_frame(index=False) - col_names = list(range(0, self.nlevels)) - haystack_table = self.copy(deep=True).to_frame(index=False) - result = cudf.core.column.full( - len(needle_table), - fill_value=-1 if method is None else None, - dtype=libcudf.types.size_type_dtype, + return result.values + + lcols, rcols = map( + list, + zip( + *[ + _match_join_keys(lcol, rcol, "inner") + for lcol, rcol in zip( + target._data.columns, self._data.columns + ) + ] + ), ) scatter_map, indices = libcudf.join.join( - list(needle_table._data.columns), - list(haystack_table._data.columns), + lcols, + rcols, how="inner", ) (result,) = libcudf.copying.scatter([indices], scatter_map, [result]) result_series = cudf.Series(result) - if not len(self): - return cp.full( - len(needle_table), -1, dtype=libcudf.types.size_type_dtype - ) if method in {"ffill", "bfill", "pad", "backfill"}: result_series = _get_indexer_basic( index=self, positions=result_series, method=method, - target_col=needle_table[col_names], + target_col=target.to_frame(index=False)[ + list(range(0, self.nlevels)) + ], tolerance=tolerance, ) elif method is not None: From fb99b0afe0c2c77799324d5de8601138501769d0 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 14 Jun 2023 09:24:16 -0500 Subject: [PATCH 056/384] Enable writing column names with mixed dtype in parquet writer when `mode.pandas_compatible=True` (#13505) This PR enables writing a dataframe that has column names that are of mixed types to a parquet file when pandas compatibility mode is enabled(`mode.pandas_compatible=True`). --------- Co-authored-by: Bradley Dice --- .../source/user_guide/pandas-comparison.md | 21 ++++++++++++++ python/cudf/cudf/_lib/parquet.pyx | 9 ++++-- python/cudf/cudf/_lib/utils.pyx | 2 +- python/cudf/cudf/tests/test_parquet.py | 29 ++++++++++++++----- 4 files changed, 49 insertions(+), 12 deletions(-) diff --git a/docs/cudf/source/user_guide/pandas-comparison.md b/docs/cudf/source/user_guide/pandas-comparison.md index ba04a231f41..441bc72205a 100644 --- a/docs/cudf/source/user_guide/pandas-comparison.md +++ b/docs/cudf/source/user_guide/pandas-comparison.md @@ -139,6 +139,27 @@ module, which allow you to compare values up to a desired precision. Unlike Pandas, cuDF does not support duplicate column names. It is best to use unique strings for column names. +## Writing a DataFrame to Parquet with non-string column names + +When there is a DataFrame with non-string column names, pandas casts each +column name to `str` before writing to a Parquet file. `cudf` raises an +error by default if this is attempted. However, to achieve similar behavior +as pandas you can enable the `mode.pandas_compatible` option, which will +enable `cudf` to cast the column names to `str` just like pandas. + +```python +>>> import cudf +>>> df = cudf.DataFrame({1: [1, 2, 3], "1": ["a", "b", "c"]}) +>>> df.to_parquet("df.parquet") + +Traceback (most recent call last): +ValueError: Writing a Parquet file requires string column names +>>> cudf.set_option("mode.pandas_compatible", True) +>>> df.to_parquet("df.parquet") + +UserWarning: The DataFrame has column names of non-string type. They will be converted to strings on write. +``` + ## No true `"object"` data type In Pandas and NumPy, the `"object"` data type is used for diff --git a/python/cudf/cudf/_lib/parquet.pyx b/python/cudf/cudf/_lib/parquet.pyx index 923f5c4089f..5519bbd4cd5 100644 --- a/python/cudf/cudf/_lib/parquet.pyx +++ b/python/cudf/cudf/_lib/parquet.pyx @@ -361,9 +361,12 @@ def write_parquet( for i, name in enumerate(table._column_names, num_index_cols_meta): if not isinstance(name, str): - raise ValueError("parquet must have string column names") - - tbl_meta.get().column_metadata[i].set_name(name.encode()) + if cudf.get_option("mode.pandas_compatible"): + tbl_meta.get().column_metadata[i].set_name(str(name).encode()) + else: + raise ValueError("Writing a Parquet file requires string column names") + else: + tbl_meta.get().column_metadata[i].set_name(name.encode()) _set_col_metadata( table[name]._column, tbl_meta.get().column_metadata[i], diff --git a/python/cudf/cudf/_lib/utils.pyx b/python/cudf/cudf/_lib/utils.pyx index 56918799cca..f5a5571a72f 100644 --- a/python/cudf/cudf/_lib/utils.pyx +++ b/python/cudf/cudf/_lib/utils.pyx @@ -174,7 +174,7 @@ cpdef generate_pandas_metadata(table, index): for col in table._columns ], df=table, - column_names=col_names, + column_names=map(str, col_names), index_levels=index_levels, index_descriptors=index_descriptors, preserve_index=index, diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index cd7075e1851..74ed6baead6 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -30,7 +30,6 @@ from cudf.testing._utils import ( TIMEDELTA_TYPES, assert_eq, - assert_exceptions_equal, expect_warning_if, set_random_null_mask_inplace, ) @@ -2528,15 +2527,29 @@ def test_parquet_writer_decimal(decimal_type, data): def test_parquet_writer_column_validation(): - df = cudf.DataFrame({1: [1, 2, 3], "1": ["a", "b", "c"]}) + df = cudf.DataFrame({1: [1, 2, 3], "a": ["a", "b", "c"]}) pdf = df.to_pandas() - assert_exceptions_equal( - lfunc=df.to_parquet, - rfunc=pdf.to_parquet, - lfunc_args_and_kwargs=(["cudf.parquet"],), - rfunc_args_and_kwargs=(["pandas.parquet"],), - ) + with cudf.option_context("mode.pandas_compatible", True): + with pytest.warns(UserWarning): + df.to_parquet("cudf.parquet") + + if PANDAS_GE_200: + with pytest.warns(UserWarning): + pdf.to_parquet("pandas.parquet") + + assert_eq( + pd.read_parquet("cudf.parquet"), + cudf.read_parquet("pandas.parquet"), + ) + assert_eq( + cudf.read_parquet("cudf.parquet"), + pd.read_parquet("pandas.parquet"), + ) + + with cudf.option_context("mode.pandas_compatible", False): + with pytest.raises(ValueError): + df.to_parquet("cudf.parquet") def test_parquet_writer_nulls_pandas_read(tmpdir, pdf): From 2488d9143a5d38765c2294ebc40f1f212a77f2c6 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 20 Jun 2023 15:47:16 -0700 Subject: [PATCH 057/384] address reviews --- python/cudf/cudf/core/index.py | 3 +++ python/cudf/cudf/core/multiindex.py | 22 ++++++++++------------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index ab551a43bae..ccde34c2654 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -2846,6 +2846,9 @@ def _extended_gcd(a: int, b: int) -> Tuple[int, int, int]: def _get_indexer_basic(index, positions, method, target_col, tolerance): + # `positions` will be modified in-place, so it is the + # responsibility of the caller to decide whether or not + # to make a copy of it before passing it to this method. nonexact = positions == -1 positions[nonexact] = index.searchsorted( target_col[nonexact], diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index b8390e4e678..5bb379b94c3 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1660,20 +1660,18 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): except TypeError: return result.values - lcols, rcols = map( - list, - zip( - *[ - _match_join_keys(lcol, rcol, "inner") - for lcol, rcol in zip( - target._data.columns, self._data.columns + scatter_map, indices = libcudf.join.join( + *map( + list, + zip( + *( + _match_join_keys(lcol, rcol, "inner") + for lcol, rcol in zip( + target._data.columns, self._data.columns + ) ) - ] + ), ), - ) - scatter_map, indices = libcudf.join.join( - lcols, - rcols, how="inner", ) (result,) = libcudf.copying.scatter([indices], scatter_map, [result]) From 7f216cf9f7ef463d47aeed248a9588e4b258ace3 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 20 Jun 2023 15:49:03 -0700 Subject: [PATCH 058/384] fix --- python/cudf/cudf/core/multiindex.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 5bb379b94c3..10929dbb804 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1664,12 +1664,12 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): *map( list, zip( - *( + *[ _match_join_keys(lcol, rcol, "inner") for lcol, rcol in zip( target._data.columns, self._data.columns ) - ) + ] ), ), how="inner", From 13d62c5a2fb6b1325627ba1ce6b6e946ed92e85a Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 20 Jun 2023 15:55:55 -0700 Subject: [PATCH 059/384] simplify --- python/cudf/cudf/core/multiindex.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 10929dbb804..649f6d15c76 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1660,18 +1660,13 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): except TypeError: return result.values + join_keys = [ + _match_join_keys(lcol, rcol, "inner") + for lcol, rcol in zip(target._data.columns, self._data.columns) + ] + join_keys = map(list, zip(*join_keys)) scatter_map, indices = libcudf.join.join( - *map( - list, - zip( - *[ - _match_join_keys(lcol, rcol, "inner") - for lcol, rcol in zip( - target._data.columns, self._data.columns - ) - ] - ), - ), + *join_keys, how="inner", ) (result,) = libcudf.copying.scatter([indices], scatter_map, [result]) From 273945b831fc5cb6847677e3391886336d788fb8 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 31 Jul 2023 10:50:11 -0700 Subject: [PATCH 060/384] Fix default behavior of index metaclass instance and subclass checks #13787 The current implementation of IndexMeta gives the wrong result for subclasses of Index: for instance, DatetimeIndex.__instancecheck__(DatetimeIndex(), DatetimeIndex) or DatetimeIndex.__subclasscheck__(DatetimeIndex, DatetimeIndex) would return False. In the case of isinstance, however, we have been saved by https://bugs.python.org/issue35083, wherein Python is silently injecting a if type(instance) is self: return True short-circuit. issubclass currently does have the wrong behavior, though. The fix is to fall back to the original behavior of isinstance/issubclass rather than hardcoding a boolean result. That will also ensure that we get the correct behavior if e.g. a subclass of Index is itself subclassed, e.g. if we introduced a class SpecialDatetimeIndex(DatetimeIndex). --- python/cudf/cudf/core/index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 7235ddc5e50..297ac21fb7e 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -87,13 +87,13 @@ def __instancecheck__(self, instance): if self is cudf.Index: return isinstance(instance, BaseIndex) else: - return False + return type.__instancecheck__(self, instance) def __subclasscheck__(self, subclass): if self is cudf.Index: return issubclass(subclass, BaseIndex) else: - return False + return type.__subclasscheck__(self, subclass) def _lexsorted_equal_range( From db92536c86ea215cec705e80c59344f2d64de709 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 11 Oct 2023 14:16:53 -0700 Subject: [PATCH 061/384] merge fix --- python/cudf/cudf/core/algorithms.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/core/algorithms.py b/python/cudf/cudf/core/algorithms.py index 591816e4161..06c67e831a6 100644 --- a/python/cudf/cudf/core/algorithms.py +++ b/python/cudf/cudf/core/algorithms.py @@ -91,10 +91,10 @@ def factorize(values, sort=False, use_na_sentinel=True, size_hint=None): if use_na_sentinel: na_sentinel = Scalar(-1) - cats = values._column.dropna() + cats = values.dropna() else: na_sentinel = Scalar(None, dtype=values.dtype) - cats = values._column + cats = values cats = cats.unique().astype(values.dtype) From fc6a30f514633176d860cb1eeaf7308a66cdecae Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 11 Oct 2023 17:41:48 -0700 Subject: [PATCH 062/384] Handle PandasArray renaming --- python/cudf/cudf/core/column/column.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index ed23c9574b0..3289f99d237 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -67,7 +67,7 @@ is_string_dtype, is_struct_dtype, ) -from cudf.core._compat import PANDAS_GE_150 +from cudf.core._compat import PANDAS_GE_150, PANDAS_GE_210 from cudf.core.abc import Serializable from cudf.core.buffer import ( Buffer, @@ -101,6 +101,11 @@ else: from pandas.core.arrays._arrow_utils import ArrowIntervalType +if PANDAS_GE_210: + NumpyExtensionArray = pd.arrays.NumpyExtensionArray +else: + NumpyExtensionArray = pd.arrays.PandasArray + class ColumnBase(Column, Serializable, BinaryOperand, Reducible): _VALID_REDUCTIONS = { @@ -2213,7 +2218,7 @@ def as_column( if delayed_cast: data = data.astype(cudf.dtype(dtype)) - elif isinstance(arbitrary, pd.arrays.PandasArray): + elif isinstance(arbitrary, NumpyExtensionArray): if is_categorical_dtype(arbitrary.dtype): arb_dtype = arbitrary.dtype else: From ad3ae65d7bdaa67269e00e48c6eba36c3227bac8 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 12 Oct 2023 17:51:53 -0500 Subject: [PATCH 063/384] Deprecate `is_categorical_dtype` (#14274) This PR deprecates `is_categorical_dtype` to match https://github.com/pandas-dev/pandas/pull/52527 which was introduced in `pandas-2.x`. This PR internalizes the public API since this is a needed utility in our code base. This PR: ``` = 23835 failed, 5698 passed, 1613 skipped, 288 xfailed, 423 errors in 1976.84s (0:32:56) = ``` On `pandas_2.0_feature_branch`: ``` = 24297 failed, 5115 passed, 1613 skipped, 288 xfailed, 480 errors in 1980.46s (0:33:00) = ``` --- python/cudf/cudf/_fuzz_testing/csv.py | 4 +- python/cudf/cudf/_fuzz_testing/json.py | 4 +- python/cudf/cudf/_lib/column.pyx | 10 ++--- python/cudf/cudf/_lib/csv.pyx | 8 ++-- python/cudf/cudf/_lib/groupby.pyx | 6 +-- python/cudf/cudf/_lib/json.pyx | 4 +- python/cudf/cudf/_lib/utils.pyx | 6 +-- python/cudf/cudf/api/types.py | 5 ++- python/cudf/cudf/core/_internals/where.py | 4 +- python/cudf/cudf/core/column/categorical.py | 4 +- python/cudf/cudf/core/column/column.py | 24 ++++++----- python/cudf/cudf/core/column/interval.py | 4 +- python/cudf/cudf/core/dataframe.py | 22 +++++----- python/cudf/cudf/core/dtypes.py | 46 ++++++++++++++------- python/cudf/cudf/core/index.py | 4 +- python/cudf/cudf/core/indexed_frame.py | 16 +++---- python/cudf/cudf/core/reshape.py | 2 +- python/cudf/cudf/core/tools/numeric.py | 6 +-- python/cudf/cudf/testing/testing.py | 16 +++---- python/cudf/cudf/tests/test_api_types.py | 9 ++-- python/cudf/cudf/tests/test_column.py | 4 +- python/cudf/cudf/tests/test_concat.py | 12 +++--- python/cudf/cudf/tests/test_dataframe.py | 4 +- python/cudf/cudf/tests/test_index.py | 2 +- python/cudf/cudf/utils/dtypes.py | 12 +++--- python/dask_cudf/dask_cudf/backends.py | 4 +- python/dask_cudf/dask_cudf/sorting.py | 4 +- 27 files changed, 134 insertions(+), 112 deletions(-) diff --git a/python/cudf/cudf/_fuzz_testing/csv.py b/python/cudf/cudf/_fuzz_testing/csv.py index 8ab7048cff0..13ea07372d0 100644 --- a/python/cudf/cudf/_fuzz_testing/csv.py +++ b/python/cudf/cudf/_fuzz_testing/csv.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. import logging import random @@ -99,7 +99,7 @@ def set_rand_params(self, params): if dtype_val is not None: dtype_val = { col_name: "category" - if cudf.utils.dtypes.is_categorical_dtype(dtype) + if cudf.utils.dtypes._is_categorical_dtype(dtype) else pandas_dtypes_to_np_dtypes[dtype] for col_name, dtype in dtype_val.items() } diff --git a/python/cudf/cudf/_fuzz_testing/json.py b/python/cudf/cudf/_fuzz_testing/json.py index 29e0aeb7050..c6e74798cd7 100644 --- a/python/cudf/cudf/_fuzz_testing/json.py +++ b/python/cudf/cudf/_fuzz_testing/json.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. import logging import random @@ -27,7 +27,7 @@ def _get_dtype_param_value(dtype_val): if dtype_val is not None and isinstance(dtype_val, abc.Mapping): processed_dtypes = {} for col_name, dtype in dtype_val.items(): - if cudf.utils.dtypes.is_categorical_dtype(dtype): + if cudf.utils.dtypes._is_categorical_dtype(dtype): processed_dtypes[col_name] = "category" else: processed_dtypes[col_name] = str( diff --git a/python/cudf/cudf/_lib/column.pyx b/python/cudf/cudf/_lib/column.pyx index 4db3761b1b8..fbd70de9905 100644 --- a/python/cudf/cudf/_lib/column.pyx +++ b/python/cudf/cudf/_lib/column.pyx @@ -11,7 +11,7 @@ import rmm import cudf import cudf._lib as libcudf from cudf._lib import pylibcudf -from cudf.api.types import is_categorical_dtype, is_datetime64tz_dtype +from cudf.api.types import _is_categorical_dtype, is_datetime64tz_dtype from cudf.core.buffer import ( Buffer, ExposureTrackedBuffer, @@ -331,7 +331,7 @@ cdef class Column: ) cdef mutable_column_view mutable_view(self) except *: - if is_categorical_dtype(self.dtype): + if _is_categorical_dtype(self.dtype): col = self.base_children[0] data_dtype = col.dtype elif is_datetime64tz_dtype(self.dtype): @@ -394,7 +394,7 @@ cdef class Column: return self._view(c_null_count) cdef column_view _view(self, libcudf_types.size_type null_count) except *: - if is_categorical_dtype(self.dtype): + if _is_categorical_dtype(self.dtype): col = self.base_children[0] data_dtype = col.dtype elif is_datetime64tz_dtype(self.dtype): @@ -469,7 +469,7 @@ cdef class Column: # categoricals because cudf supports ordered and unordered categoricals # while libcudf supports only unordered categoricals (see # https://github.com/rapidsai/cudf/pull/8567). - if is_categorical_dtype(self.dtype): + if _is_categorical_dtype(self.dtype): col = self.base_children[0] else: col = self @@ -635,7 +635,7 @@ cdef class Column: """ column_owner = isinstance(owner, Column) mask_owner = owner - if column_owner and is_categorical_dtype(owner.dtype): + if column_owner and _is_categorical_dtype(owner.dtype): owner = owner.base_children[0] size = cv.size() diff --git a/python/cudf/cudf/_lib/csv.pyx b/python/cudf/cudf/_lib/csv.pyx index 630dcf73545..399a53c09b5 100644 --- a/python/cudf/cudf/_lib/csv.pyx +++ b/python/cudf/cudf/_lib/csv.pyx @@ -434,7 +434,7 @@ def read_csv( if dtype is not None: if isinstance(dtype, abc.Mapping): for k, v in dtype.items(): - if cudf.api.types.is_categorical_dtype(v): + if cudf.api.types._is_categorical_dtype(v): df._data[str(k)] = df._data[str(k)].astype(v) elif ( cudf.api.types.is_scalar(dtype) or @@ -442,11 +442,11 @@ def read_csv( np.dtype, pd.api.extensions.ExtensionDtype, type )) ): - if cudf.api.types.is_categorical_dtype(dtype): + if cudf.api.types._is_categorical_dtype(dtype): df = df.astype(dtype) elif isinstance(dtype, abc.Collection): for index, col_dtype in enumerate(dtype): - if cudf.api.types.is_categorical_dtype(col_dtype): + if cudf.api.types._is_categorical_dtype(col_dtype): col_name = df._data.names[index] df._data[col_name] = df._data[col_name].astype(col_dtype) @@ -547,7 +547,7 @@ cdef data_type _get_cudf_data_type_from_dtype(object dtype) except *: # TODO: Remove this work-around Dictionary types # in libcudf are fully mapped to categorical columns: # https://github.com/rapidsai/cudf/issues/3960 - if cudf.api.types.is_categorical_dtype(dtype): + if cudf.api.types._is_categorical_dtype(dtype): if isinstance(dtype, str): dtype = "str" else: diff --git a/python/cudf/cudf/_lib/groupby.pyx b/python/cudf/cudf/_lib/groupby.pyx index b9447c96ee6..0067981169c 100644 --- a/python/cudf/cudf/_lib/groupby.pyx +++ b/python/cudf/cudf/_lib/groupby.pyx @@ -3,7 +3,7 @@ from pandas.core.groupby.groupby import DataError from cudf.api.types import ( - is_categorical_dtype, + _is_categorical_dtype, is_decimal_dtype, is_interval_dtype, is_list_dtype, @@ -189,7 +189,7 @@ cdef class GroupBy: valid_aggregations = ( _LIST_AGGS if is_list_dtype(dtype) else _STRING_AGGS if is_string_dtype(dtype) - else _CATEGORICAL_AGGS if is_categorical_dtype(dtype) + else _CATEGORICAL_AGGS if _is_categorical_dtype(dtype) else _STRUCT_AGGS if is_struct_dtype(dtype) else _INTERVAL_AGGS if is_interval_dtype(dtype) else _DECIMAL_AGGS if is_decimal_dtype(dtype) @@ -260,7 +260,7 @@ cdef class GroupBy: valid_aggregations = ( _LIST_AGGS if is_list_dtype(dtype) else _STRING_AGGS if is_string_dtype(dtype) - else _CATEGORICAL_AGGS if is_categorical_dtype(dtype) + else _CATEGORICAL_AGGS if _is_categorical_dtype(dtype) else _STRUCT_AGGS if is_struct_dtype(dtype) else _INTERVAL_AGGS if is_interval_dtype(dtype) else _DECIMAL_AGGS if is_decimal_dtype(dtype) diff --git a/python/cudf/cudf/_lib/json.pyx b/python/cudf/cudf/_lib/json.pyx index 437c3ef6ec4..f66109bccbd 100644 --- a/python/cudf/cudf/_lib/json.pyx +++ b/python/cudf/cudf/_lib/json.pyx @@ -214,7 +214,7 @@ def write_json( cdef schema_element _get_cudf_schema_element_from_dtype(object dtype) except *: cdef schema_element s_element cdef data_type lib_type - if cudf.api.types.is_categorical_dtype(dtype): + if cudf.api.types._is_categorical_dtype(dtype): raise NotImplementedError( "CategoricalDtype as dtype is not yet " "supported in JSON reader" @@ -237,7 +237,7 @@ cdef schema_element _get_cudf_schema_element_from_dtype(object dtype) except *: cdef data_type _get_cudf_data_type_from_dtype(object dtype) except *: - if cudf.api.types.is_categorical_dtype(dtype): + if cudf.api.types._is_categorical_dtype(dtype): raise NotImplementedError( "CategoricalDtype as dtype is not yet " "supported in JSON reader" diff --git a/python/cudf/cudf/_lib/utils.pyx b/python/cudf/cudf/_lib/utils.pyx index 2259d90468f..69b0fe5d8f2 100644 --- a/python/cudf/cudf/_lib/utils.pyx +++ b/python/cudf/cudf/_lib/utils.pyx @@ -23,7 +23,7 @@ except ImportError: import json from cudf.api.types import ( - is_categorical_dtype, + _is_categorical_dtype, is_decimal_dtype, is_list_dtype, is_struct_dtype, @@ -92,7 +92,7 @@ cpdef generate_pandas_metadata(table, index): # Columns for name, col in table._data.items(): col_names.append(name) - if is_categorical_dtype(col): + if _is_categorical_dtype(col): raise ValueError( "'category' column dtypes are currently not " + "supported by the gpu accelerated parquet writer" @@ -147,7 +147,7 @@ cpdef generate_pandas_metadata(table, index): level=level, column_names=col_names ) - if is_categorical_dtype(idx): + if _is_categorical_dtype(idx): raise ValueError( "'category' column dtypes are currently not " + "supported by the gpu accelerated parquet writer" diff --git a/python/cudf/cudf/api/types.py b/python/cudf/cudf/api/types.py index 7ed5bc31420..007b9f3ee02 100644 --- a/python/cudf/cudf/api/types.py +++ b/python/cudf/cudf/api/types.py @@ -18,7 +18,7 @@ from cudf.core.dtypes import ( # noqa: F401 _BaseDtype, dtype, - is_categorical_dtype, + _is_categorical_dtype, is_decimal32_dtype, is_decimal64_dtype, is_decimal128_dtype, @@ -112,7 +112,7 @@ def is_string_dtype(obj): or ( pd.api.types.is_string_dtype(obj) # Reject all cudf extension types. - and not is_categorical_dtype(obj) + and not _is_categorical_dtype(obj) and not is_decimal_dtype(obj) and not is_list_dtype(obj) and not is_struct_dtype(obj) @@ -486,6 +486,7 @@ def is_any_real_numeric_dtype(arr_or_dtype) -> bool: is_iterator = pd_types.is_iterator is_bool = pd_types.is_bool is_categorical = pd_types.is_categorical_dtype +# TODO is_complex = pd_types.is_complex is_float = pd_types.is_float is_hashable = pd_types.is_hashable diff --git a/python/cudf/cudf/core/_internals/where.py b/python/cudf/cudf/core/_internals/where.py index 0f65861dc72..ce22d4c8860 100644 --- a/python/cudf/cudf/core/_internals/where.py +++ b/python/cudf/cudf/core/_internals/where.py @@ -8,8 +8,8 @@ import cudf from cudf._typing import ScalarLike from cudf.api.types import ( + _is_categorical_dtype, _is_non_decimal_numeric_dtype, - is_categorical_dtype, is_scalar, ) from cudf.core.column import ColumnBase @@ -45,7 +45,7 @@ def _check_and_cast_columns_with_other( ) -> Tuple[ColumnBase, Union[ScalarLike, ColumnBase]]: # Returns type-casted `source_col` & `other` based on `inplace`. source_dtype = source_col.dtype - if is_categorical_dtype(source_dtype): + if _is_categorical_dtype(source_dtype): return _normalize_categorical(source_col, other) other_is_scalar = is_scalar(other) diff --git a/python/cudf/cudf/core/column/categorical.py b/python/cudf/cudf/core/column/categorical.py index 40abd5b7db8..af0e3257d4e 100644 --- a/python/cudf/cudf/core/column/categorical.py +++ b/python/cudf/cudf/core/column/categorical.py @@ -16,7 +16,7 @@ from cudf import _lib as libcudf from cudf._lib.transform import bools_to_mask from cudf._typing import ColumnBinaryOperand, ColumnLike, Dtype, ScalarLike -from cudf.api.types import is_categorical_dtype, is_interval_dtype +from cudf.api.types import _is_categorical_dtype, is_interval_dtype from cudf.core.buffer import Buffer from cudf.core.column import column from cudf.core.column.methods import ColumnMethods @@ -98,7 +98,7 @@ class CategoricalAccessor(ColumnMethods): _column: CategoricalColumn def __init__(self, parent: SeriesOrSingleColumnIndex): - if not is_categorical_dtype(parent.dtype): + if not _is_categorical_dtype(parent.dtype): raise AttributeError( "Can only use .cat accessor with a 'category' dtype" ) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 3289f99d237..048ce620a8d 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -49,10 +49,10 @@ from cudf._lib.types import size_type_dtype from cudf._typing import ColumnLike, Dtype, ScalarLike from cudf.api.types import ( + _is_categorical_dtype, _is_non_decimal_numeric_dtype, infer_dtype, is_bool_dtype, - is_categorical_dtype, is_datetime64_dtype, is_datetime64tz_dtype, is_decimal32_dtype, @@ -977,7 +977,7 @@ def can_cast_safely(self, to_dtype: Dtype) -> bool: def astype(self, dtype: Dtype, **kwargs) -> ColumnBase: if self.dtype == dtype: return self - if is_categorical_dtype(dtype): + if _is_categorical_dtype(dtype): return self.as_categorical_column(dtype, **kwargs) dtype = ( @@ -987,7 +987,7 @@ def astype(self, dtype: Dtype, **kwargs) -> ColumnBase: ) if _is_non_decimal_numeric_dtype(dtype): return self.as_numerical_column(dtype, **kwargs) - elif is_categorical_dtype(dtype): + elif _is_categorical_dtype(dtype): return self.as_categorical_column(dtype, **kwargs) elif cudf.dtype(dtype).type in { np.str_, @@ -1423,7 +1423,7 @@ def column_empty_like( if ( hasattr(column, "dtype") - and is_categorical_dtype(column.dtype) + and _is_categorical_dtype(column.dtype) and dtype == column.dtype ): catcolumn = cast("cudf.core.column.CategoricalColumn", column) @@ -1476,7 +1476,7 @@ def column_empty( full(row_count + 1, 0, dtype=libcudf.types.size_type_dtype), column_empty(row_count, dtype=dtype.element_type), ) - elif is_categorical_dtype(dtype): + elif _is_categorical_dtype(dtype): data = None children = ( build_column( @@ -1553,7 +1553,7 @@ def build_column( offset=offset, null_count=null_count, ) - if is_categorical_dtype(dtype): + if _is_categorical_dtype(dtype): if not len(children) == 1: raise ValueError( "Must specify exactly one child column for CategoricalColumn" @@ -2037,7 +2037,7 @@ def as_column( f"{arbitrary.dtype} is not supported. Convert first to " f"{arbitrary.dtype.subtype}." ) - if is_categorical_dtype(arbitrary.dtype): + if _is_categorical_dtype(arbitrary.dtype): if isinstance( arbitrary.dtype.categories.dtype, pd.DatetimeTZDtype ): @@ -2219,7 +2219,7 @@ def as_column( data = data.astype(cudf.dtype(dtype)) elif isinstance(arbitrary, NumpyExtensionArray): - if is_categorical_dtype(arbitrary.dtype): + if _is_categorical_dtype(arbitrary.dtype): arb_dtype = arbitrary.dtype else: if arbitrary.dtype == pd.StringDtype(): @@ -2347,7 +2347,9 @@ def as_column( np_type = None try: if dtype is not None: - if is_categorical_dtype(dtype) or is_interval_dtype(dtype): + if _is_categorical_dtype(dtype) or is_interval_dtype( + dtype + ): raise TypeError if is_datetime64tz_dtype(dtype): raise NotImplementedError( @@ -2491,7 +2493,7 @@ def as_column( except (pa.ArrowInvalid, pa.ArrowTypeError, TypeError) as e: if isinstance(e, MixedTypeError): raise TypeError(str(e)) - if is_categorical_dtype(dtype): + if _is_categorical_dtype(dtype): sr = pd.Series(arbitrary, dtype="category") data = as_column(sr, nan_as_null=nan_as_null, dtype=dtype) elif np_type == np.str_: @@ -2774,7 +2776,7 @@ def concat_columns(objs: "MutableSequence[ColumnBase]") -> ColumnBase: # ColumnBase._concat so that all subclasses can override necessary # behavior. However, at the moment it's not clear what that API should look # like, so CategoricalColumn simply implements a minimal working API. - if all(is_categorical_dtype(o.dtype) for o in objs): + if all(_is_categorical_dtype(o.dtype) for o in objs): return cudf.core.column.categorical.CategoricalColumn._concat( cast( MutableSequence[ diff --git a/python/cudf/cudf/core/column/interval.py b/python/cudf/cudf/core/column/interval.py index 38384d09126..7b87552f1a0 100644 --- a/python/cudf/cudf/core/column/interval.py +++ b/python/cudf/cudf/core/column/interval.py @@ -5,7 +5,7 @@ import pyarrow as pa import cudf -from cudf.api.types import is_categorical_dtype, is_interval_dtype +from cudf.api.types import _is_categorical_dtype, is_interval_dtype from cudf.core.column import StructColumn from cudf.core.dtypes import IntervalDtype @@ -102,7 +102,7 @@ def copy(self, deep=True): def as_interval_column(self, dtype, **kwargs): if is_interval_dtype(dtype): - if is_categorical_dtype(self): + if _is_categorical_dtype(self): new_struct = self._get_decategorized_column() return IntervalColumn.from_struct_column(new_struct) if is_interval_dtype(dtype): diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 23c2405b58e..0cfa37c224d 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -45,7 +45,7 @@ from cudf.api.types import ( _is_scalar_or_zero_d_array, is_bool_dtype, - is_categorical_dtype, + _is_categorical_dtype, is_datetime_dtype, is_dict_like, is_dtype_equal, @@ -1403,7 +1403,7 @@ def _get_numeric_data(self): columns = [ c for c, dt in self.dtypes.items() - if dt != object and not is_categorical_dtype(dt) + if dt != object and not _is_categorical_dtype(dt) ] return self[columns] @@ -1646,9 +1646,9 @@ def _concat( out._index._data, indices[:first_data_column_position], ) - if not isinstance(out._index, MultiIndex) and is_categorical_dtype( - out._index._values.dtype - ): + if not isinstance( + out._index, MultiIndex + ) and _is_categorical_dtype(out._index._values.dtype): out = out.set_index( cudf.core.index.as_index(out.index._values) ) @@ -3807,8 +3807,8 @@ def transpose(self): # No column from index is transposed with libcudf. source_columns = [*self._columns] source_dtype = source_columns[0].dtype - if is_categorical_dtype(source_dtype): - if any(not is_categorical_dtype(c.dtype) for c in source_columns): + if _is_categorical_dtype(source_dtype): + if any(not _is_categorical_dtype(c.dtype) for c in source_columns): raise ValueError("Columns must all have the same dtype") cats = list(c.categories for c in source_columns) cats = cudf.core.column.concat_columns(cats).unique() @@ -3822,7 +3822,7 @@ def transpose(self): result_columns = libcudf.transpose.transpose(source_columns) - if is_categorical_dtype(source_dtype): + if _is_categorical_dtype(source_dtype): result_columns = [ codes._with_type_metadata( cudf.core.dtypes.CategoricalDtype(categories=cats) @@ -4524,7 +4524,7 @@ def apply_rows( """ for col in incols: current_col_dtype = self._data[col].dtype - if is_string_dtype(current_col_dtype) or is_categorical_dtype( + if is_string_dtype(current_col_dtype) or _is_categorical_dtype( current_col_dtype ): raise TypeError( @@ -6308,7 +6308,7 @@ def select_dtypes(self, include=None, exclude=None): for dtype in self.dtypes: for i_dtype in include: # category handling - if is_categorical_dtype(i_dtype): + if _is_categorical_dtype(i_dtype): include_subtypes.add(i_dtype) elif inspect.isclass(dtype.type): if issubclass(dtype.type, i_dtype): @@ -6319,7 +6319,7 @@ def select_dtypes(self, include=None, exclude=None): for dtype in self.dtypes: for e_dtype in exclude: # category handling - if is_categorical_dtype(e_dtype): + if _is_categorical_dtype(e_dtype): exclude_subtypes.add(e_dtype) elif inspect.isclass(dtype.type): if issubclass(dtype.type, e_dtype): diff --git a/python/cudf/cudf/core/dtypes.py b/python/cudf/cudf/core/dtypes.py index 15fbaa04418..6fae552c6ed 100644 --- a/python/cudf/cudf/core/dtypes.py +++ b/python/cudf/cudf/core/dtypes.py @@ -4,6 +4,7 @@ import operator import pickle import textwrap +import warnings from functools import cached_property from typing import Any, Callable, Dict, List, Tuple, Type, Union @@ -957,19 +958,7 @@ def deserialize(cls, header: dict, frames: list): return klass(subtype, closed=closed) -def is_categorical_dtype(obj): - """Check whether an array-like or dtype is of the Categorical dtype. - - Parameters - ---------- - obj : array-like or dtype - The array-like or dtype to check. - - Returns - ------- - bool - Whether or not the array-like or dtype is of a categorical dtype. - """ +def _is_categorical_dtype(obj): if obj is None: return False @@ -1013,13 +1002,40 @@ def is_categorical_dtype(obj): pd.Series, ), ): - return is_categorical_dtype(obj.dtype) + return _is_categorical_dtype(obj.dtype) if hasattr(obj, "type"): if obj.type is pd_CategoricalDtypeType: return True # TODO: A lot of the above checks are probably redundant and should be # farmed out to this function here instead. - return pd_types.is_categorical_dtype(obj) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return pd_types.is_categorical_dtype(obj) + + +def is_categorical_dtype(obj): + """Check whether an array-like or dtype is of the Categorical dtype. + + .. deprecated:: 23.12 + Use isinstance(dtype, cudf.CategoricalDtype) instead + + Parameters + ---------- + obj : array-like or dtype + The array-like or dtype to check. + + Returns + ------- + bool + Whether or not the array-like or dtype is of a categorical dtype. + """ + # Do not remove until pandas 3.0 support is added. + warnings.warn( + "is_categorical_dtype is deprecated and will be removed in a future " + "version. Use isinstance(dtype, cudf.CategoricalDtype) instead", + FutureWarning, + ) + return _is_categorical_dtype(obj) def is_list_dtype(obj): diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 30e24409dbe..92b244d1999 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -31,7 +31,7 @@ from cudf.api.extensions import no_default from cudf.api.types import ( _is_non_decimal_numeric_dtype, - is_categorical_dtype, + _is_categorical_dtype, is_dtype_equal, is_interval_dtype, is_list_like, @@ -2496,7 +2496,7 @@ def __init__( if isinstance(data, CategoricalColumn): data = data elif isinstance(data, pd.Series) and ( - is_categorical_dtype(data.dtype) + _is_categorical_dtype(data.dtype) ): codes_data = column.as_column(data.cat.codes.values) data = column.build_categorical_column( diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index a3f919c6091..f9435eebe96 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -41,8 +41,8 @@ from cudf.api.extensions import no_default from cudf.api.types import ( _is_non_decimal_numeric_dtype, + _is_categorical_dtype, is_bool_dtype, - is_categorical_dtype, is_decimal_dtype, is_dict_like, is_list_dtype, @@ -170,7 +170,7 @@ def _indices_from_labels(obj, labels): if not isinstance(labels, cudf.MultiIndex): labels = cudf.core.column.as_column(labels) - if is_categorical_dtype(obj.index): + if _is_categorical_dtype(obj.index): labels = labels.astype("category") codes = labels.codes.astype(obj.index._values.codes.dtype) labels = cudf.core.column.build_categorical_column( @@ -5418,21 +5418,21 @@ def _is_same_dtype(lhs_dtype, rhs_dtype): if lhs_dtype == rhs_dtype: return True elif ( - is_categorical_dtype(lhs_dtype) - and is_categorical_dtype(rhs_dtype) + _is_categorical_dtype(lhs_dtype) + and _is_categorical_dtype(rhs_dtype) and lhs_dtype.categories.dtype == rhs_dtype.categories.dtype ): # OK if categories are not all the same return True elif ( - is_categorical_dtype(lhs_dtype) - and not is_categorical_dtype(rhs_dtype) + _is_categorical_dtype(lhs_dtype) + and not _is_categorical_dtype(rhs_dtype) and lhs_dtype.categories.dtype == rhs_dtype ): return True elif ( - is_categorical_dtype(rhs_dtype) - and not is_categorical_dtype(lhs_dtype) + _is_categorical_dtype(rhs_dtype) + and not _is_categorical_dtype(lhs_dtype) and rhs_dtype.categories.dtype == lhs_dtype ): return True diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index 85973ee194b..7a80d70acb3 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -540,7 +540,7 @@ def melt( # Error for unimplemented support for datatype dtypes = [frame[col].dtype for col in id_vars + value_vars] - if any(cudf.api.types.is_categorical_dtype(t) for t in dtypes): + if any(cudf.api.types._is_categorical_dtype(t) for t in dtypes): raise NotImplementedError( "Categorical columns are not yet supported for function" ) diff --git a/python/cudf/cudf/core/tools/numeric.py b/python/cudf/cudf/core/tools/numeric.py index 0273227010b..35ddffb0f01 100644 --- a/python/cudf/cudf/core/tools/numeric.py +++ b/python/cudf/cudf/core/tools/numeric.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2022, NVIDIA CORPORATION. +# Copyright (c) 2018-2023, NVIDIA CORPORATION. import warnings @@ -9,8 +9,8 @@ from cudf import _lib as libcudf from cudf._lib import strings as libstrings from cudf.api.types import ( + _is_categorical_dtype, _is_non_decimal_numeric_dtype, - is_categorical_dtype, is_datetime_dtype, is_list_dtype, is_string_dtype, @@ -110,7 +110,7 @@ def to_numeric(arg, errors="raise", downcast=None): if is_datetime_dtype(dtype) or is_timedelta_dtype(dtype): col = col.as_numerical_column(cudf.dtype("int64")) - elif is_categorical_dtype(dtype): + elif _is_categorical_dtype(dtype): cat_dtype = col.dtype.type if _is_non_decimal_numeric_dtype(cat_dtype): col = col.as_numerical_column(cat_dtype) diff --git a/python/cudf/cudf/testing/testing.py b/python/cudf/cudf/testing/testing.py index 0ab3a244ebe..9c2ee637584 100644 --- a/python/cudf/cudf/testing/testing.py +++ b/python/cudf/cudf/testing/testing.py @@ -9,7 +9,7 @@ import cudf from cudf._lib.unary import is_nan from cudf.api.types import ( - is_categorical_dtype, + _is_categorical_dtype, is_decimal_dtype, is_interval_dtype, is_list_dtype, @@ -86,7 +86,7 @@ def _check_types( if ( exact and not isinstance(left, cudf.MultiIndex) - and is_categorical_dtype(left) + and _is_categorical_dtype(left) ): if left.dtype != right.dtype: raise_assert_detail( @@ -144,8 +144,8 @@ def assert_column_equal( """ if check_dtype is True: if ( - is_categorical_dtype(left) - and is_categorical_dtype(right) + _is_categorical_dtype(left) + and _is_categorical_dtype(right) and not check_categorical ): pass @@ -173,7 +173,7 @@ def assert_column_equal( return if check_exact and check_categorical: - if is_categorical_dtype(left) and is_categorical_dtype(right): + if _is_categorical_dtype(left) and _is_categorical_dtype(right): left_cat = left.categories right_cat = right.categories @@ -207,8 +207,8 @@ def assert_column_equal( if ( not check_dtype - and is_categorical_dtype(left) - and is_categorical_dtype(right) + and _is_categorical_dtype(left) + and _is_categorical_dtype(right) ): left = left.astype(left.categories.dtype) right = right.astype(right.categories.dtype) @@ -254,7 +254,7 @@ def assert_column_equal( raise e else: columns_equal = False - if is_categorical_dtype(left) and is_categorical_dtype(right): + if _is_categorical_dtype(left) and _is_categorical_dtype(right): left = left.astype(left.categories.dtype) right = right.astype(right.categories.dtype) if not columns_equal: diff --git a/python/cudf/cudf/tests/test_api_types.py b/python/cudf/cudf/tests/test_api_types.py index ae3d232e542..da29972ea82 100644 --- a/python/cudf/cudf/tests/test_api_types.py +++ b/python/cudf/cudf/tests/test_api_types.py @@ -116,7 +116,7 @@ ), ) def test_is_categorical_dtype(obj, expect): - assert types.is_categorical_dtype(obj) == expect + assert types._is_categorical_dtype(obj) == expect @pytest.mark.parametrize( @@ -1036,9 +1036,10 @@ def test_is_decimal_dtype(obj, expect): ), ) def test_pandas_agreement(obj): - assert types.is_categorical_dtype(obj) == pd_types.is_categorical_dtype( - obj - ) + with pytest.warns(FutureWarning): + assert types.is_categorical_dtype( + obj + ) == pd_types.is_categorical_dtype(obj) assert types.is_numeric_dtype(obj) == pd_types.is_numeric_dtype(obj) assert types.is_integer_dtype(obj) == pd_types.is_integer_dtype(obj) assert types.is_integer(obj) == pd_types.is_integer(obj) diff --git a/python/cudf/cudf/tests/test_column.py b/python/cudf/cudf/tests/test_column.py index db0446d506c..ad585518b83 100644 --- a/python/cudf/cudf/tests/test_column.py +++ b/python/cudf/cudf/tests/test_column.py @@ -81,7 +81,7 @@ def test_column_offset_and_size(pandas_input, offset, size): children=col.base_children, ) - if cudf.api.types.is_categorical_dtype(col.dtype): + if cudf.api.types._is_categorical_dtype(col.dtype): assert col.size == col.codes.size assert col.size == (col.codes.data.size / col.codes.dtype.itemsize) elif cudf.api.types.is_string_dtype(col.dtype): @@ -120,7 +120,7 @@ def column_slicing_test(col, offset, size, cast_to_float=False): else: pd_series = series.to_pandas() - if cudf.api.types.is_categorical_dtype(col.dtype): + if cudf.api.types._is_categorical_dtype(col.dtype): # The cudf.Series is constructed from an already sliced column, whereas # the pandas.Series is constructed from the unsliced series and then # sliced, so the indexes should be different and we must ignore it. diff --git a/python/cudf/cudf/tests/test_concat.py b/python/cudf/cudf/tests/test_concat.py index 925a522399d..32d22c3e2f5 100644 --- a/python/cudf/cudf/tests/test_concat.py +++ b/python/cudf/cudf/tests/test_concat.py @@ -7,7 +7,7 @@ import pytest import cudf as gd -from cudf.api.types import is_categorical_dtype +from cudf.api.types import _is_categorical_dtype from cudf.core._compat import PANDAS_GE_150, PANDAS_LT_140, PANDAS_GE_200 from cudf.core.dtypes import Decimal32Dtype, Decimal64Dtype, Decimal128Dtype from cudf.testing._utils import ( @@ -582,8 +582,8 @@ def test_concat_empty_dataframes(df, other, ignore_index): actual = gd.concat(other_gd, ignore_index=ignore_index) if expected.shape != df.shape: for key, col in actual[actual.columns].items(): - if is_categorical_dtype(col.dtype): - if not is_categorical_dtype(expected[key].dtype): + if _is_categorical_dtype(col.dtype): + if not _is_categorical_dtype(expected[key].dtype): # TODO: Pandas bug: # https://github.com/pandas-dev/pandas/issues/42840 expected[key] = expected[key].fillna("-1").astype("str") @@ -1213,8 +1213,8 @@ def test_concat_join_empty_dataframes( if expected.shape != df.shape: if axis == 0: for key, col in actual[actual.columns].items(): - if is_categorical_dtype(col.dtype): - if not is_categorical_dtype(expected[key].dtype): + if _is_categorical_dtype(col.dtype): + if not _is_categorical_dtype(expected[key].dtype): # TODO: Pandas bug: # https://github.com/pandas-dev/pandas/issues/42840 expected[key] = ( @@ -1336,7 +1336,7 @@ def test_concat_join_empty_dataframes_axis_1( if expected.shape != df.shape: if axis == 0: for key, col in actual[actual.columns].items(): - if is_categorical_dtype(col.dtype): + if _is_categorical_dtype(col.dtype): expected[key] = expected[key].fillna("-1") actual[key] = col.astype("str").fillna("-1") # if not expected.empty: diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index d873597f849..9ab5b835049 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -5943,7 +5943,9 @@ def test_df_sr_mask_where(data, condition, other, error, inplace): expect_mask = ps_mask got_mask = gs_mask - if pd.api.types.is_categorical_dtype(expect_where): + if hasattr(expect_where, "dtype") and isinstance( + expect_where, pd.CategoricalDtype + ): np.testing.assert_array_equal( expect_where.cat.codes, got_where.cat.codes.astype(expect_where.cat.codes.dtype) diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index 01715e3be52..e82f3d581e5 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -659,7 +659,7 @@ def test_index_where(data, condition, other, error): gs_other = other if error is None: - if pd.api.types.is_categorical_dtype(ps): + if hasattr(ps, "dtype") and isinstance(ps, pd.CategoricalDtype): expect = ps.where(ps_condition, other=ps_other) got = gs.where(gs_condition, other=gs_other) np.testing.assert_array_equal( diff --git a/python/cudf/cudf/utils/dtypes.py b/python/cudf/cudf/utils/dtypes.py index f3e245f8769..a5e3d1230fa 100644 --- a/python/cudf/cudf/utils/dtypes.py +++ b/python/cudf/cudf/utils/dtypes.py @@ -184,7 +184,7 @@ def cudf_dtype_from_pydata_dtype(dtype): Python dtype. """ - if cudf.api.types.is_categorical_dtype(dtype): + if cudf.api.types._is_categorical_dtype(dtype): return cudf.core.dtypes.CategoricalDtype elif cudf.api.types.is_decimal32_dtype(dtype): return cudf.core.dtypes.Decimal32Dtype @@ -202,7 +202,7 @@ def cudf_dtype_to_pa_type(dtype): """Given a cudf pandas dtype, converts it into the equivalent cuDF Python dtype. """ - if cudf.api.types.is_categorical_dtype(dtype): + if cudf.api.types._is_categorical_dtype(dtype): raise NotImplementedError() elif ( cudf.api.types.is_list_dtype(dtype) @@ -427,9 +427,9 @@ def get_min_float_dtype(col): def is_mixed_with_object_dtype(lhs, rhs): - if cudf.api.types.is_categorical_dtype(lhs.dtype): + if cudf.api.types._is_categorical_dtype(lhs.dtype): return is_mixed_with_object_dtype(lhs.dtype.categories, rhs) - elif cudf.api.types.is_categorical_dtype(rhs.dtype): + elif cudf.api.types._is_categorical_dtype(rhs.dtype): return is_mixed_with_object_dtype(lhs, rhs.dtype.categories) return (lhs.dtype == "object" and rhs.dtype != "object") or ( @@ -529,10 +529,10 @@ def find_common_type(dtypes): # Early exit for categoricals since they're not hashable and therefore # can't be put in a set. - if any(cudf.api.types.is_categorical_dtype(dtype) for dtype in dtypes): + if any(cudf.api.types._is_categorical_dtype(dtype) for dtype in dtypes): if all( ( - cudf.api.types.is_categorical_dtype(dtype) + cudf.api.types._is_categorical_dtype(dtype) and (not dtype.ordered if hasattr(dtype, "ordered") else True) ) for dtype in dtypes diff --git a/python/dask_cudf/dask_cudf/backends.py b/python/dask_cudf/dask_cudf/backends.py index c98e724a72c..155f2d81c23 100644 --- a/python/dask_cudf/dask_cudf/backends.py +++ b/python/dask_cudf/dask_cudf/backends.py @@ -312,7 +312,7 @@ def tolist_cudf(obj): ) @_dask_cudf_nvtx_annotate def is_categorical_dtype_cudf(obj): - return cudf.api.types.is_categorical_dtype(obj) + return cudf.api.types._is_categorical_dtype(obj) @grouper_dispatch.register((cudf.Series, cudf.DataFrame)) @@ -341,7 +341,7 @@ def percentile_cudf(a, q, interpolation="linear"): if isinstance(q, Iterator): q = list(q) - if cudf.api.types.is_categorical_dtype(a.dtype): + if cudf.api.types._is_categorical_dtype(a.dtype): result = cp.percentile(a.cat.codes, q, interpolation=interpolation) return ( diff --git a/python/dask_cudf/dask_cudf/sorting.py b/python/dask_cudf/dask_cudf/sorting.py index d6c9c1be73c..c8ddef54e2b 100644 --- a/python/dask_cudf/dask_cudf/sorting.py +++ b/python/dask_cudf/dask_cudf/sorting.py @@ -15,7 +15,7 @@ from dask.utils import M import cudf as gd -from cudf.api.types import is_categorical_dtype +from cudf.api.types import _is_categorical_dtype from cudf.utils.utils import _dask_cudf_nvtx_annotate _SHUFFLE_SUPPORT = ("tasks", "p2p") # "disk" not supported @@ -203,7 +203,7 @@ def quantile_divisions(df, by, npartitions): if ( len(columns) == 1 and df[columns[0]].dtype != "object" - and not is_categorical_dtype(df[columns[0]].dtype) + and not _is_categorical_dtype(df[columns[0]].dtype) ): dtype = df[columns[0]].dtype divisions = divisions[columns[0]].astype("int64") From 7c6d8f2a6faf92c76ba379656133b4dda8358fa6 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 12 Oct 2023 18:24:00 -0500 Subject: [PATCH 064/384] Deprecate is_interval_dtype and is_datetime64tz_dtype (#14275) This PR deprecates `is_datetime64tz_dtype` and `is_interval_dtype` to have parity with pandas-2.x: https://github.com/pandas-dev/pandas/pull/52607, alternatively this PR internalizes these utilities. This PR: ``` = 1584 failed, 98570 passed, 2069 skipped, 776 xfailed, 312 xpassed, 20 errors in 737.24s (0:12:17) = ``` On `pandas_2.0_feature_branch`: ``` = 23835 failed, 5698 passed, 1613 skipped, 288 xfailed, 423 errors in 1976.84s (0:32:56) = ``` --- python/cudf/cudf/_lib/column.pyx | 6 +-- python/cudf/cudf/_lib/groupby.pyx | 9 ++-- python/cudf/cudf/api/types.py | 29 ++++++++--- python/cudf/cudf/core/column/categorical.py | 6 +-- python/cudf/cudf/core/column/column.py | 24 ++++----- python/cudf/cudf/core/column/datetime.py | 4 +- python/cudf/cudf/core/column/interval.py | 8 +-- python/cudf/cudf/core/dtypes.py | 55 ++++++++++++--------- python/cudf/cudf/core/index.py | 19 +++---- python/cudf/cudf/testing/testing.py | 6 +-- 10 files changed, 90 insertions(+), 76 deletions(-) diff --git a/python/cudf/cudf/_lib/column.pyx b/python/cudf/cudf/_lib/column.pyx index fbd70de9905..da9ef1f1697 100644 --- a/python/cudf/cudf/_lib/column.pyx +++ b/python/cudf/cudf/_lib/column.pyx @@ -11,7 +11,7 @@ import rmm import cudf import cudf._lib as libcudf from cudf._lib import pylibcudf -from cudf.api.types import _is_categorical_dtype, is_datetime64tz_dtype +from cudf.api.types import _is_categorical_dtype, _is_datetime64tz_dtype from cudf.core.buffer import ( Buffer, ExposureTrackedBuffer, @@ -334,7 +334,7 @@ cdef class Column: if _is_categorical_dtype(self.dtype): col = self.base_children[0] data_dtype = col.dtype - elif is_datetime64tz_dtype(self.dtype): + elif _is_datetime64tz_dtype(self.dtype): col = self data_dtype = _get_base_dtype(col.dtype) else: @@ -397,7 +397,7 @@ cdef class Column: if _is_categorical_dtype(self.dtype): col = self.base_children[0] data_dtype = col.dtype - elif is_datetime64tz_dtype(self.dtype): + elif _is_datetime64tz_dtype(self.dtype): col = self data_dtype = _get_base_dtype(col.dtype) else: diff --git a/python/cudf/cudf/_lib/groupby.pyx b/python/cudf/cudf/_lib/groupby.pyx index 0067981169c..5b882bf9d3c 100644 --- a/python/cudf/cudf/_lib/groupby.pyx +++ b/python/cudf/cudf/_lib/groupby.pyx @@ -1,16 +1,15 @@ # Copyright (c) 2020-2023, NVIDIA CORPORATION. -from pandas.core.groupby.groupby import DataError - from cudf.api.types import ( _is_categorical_dtype, + _is_interval_dtype, is_decimal_dtype, - is_interval_dtype, is_list_dtype, is_string_dtype, is_struct_dtype, ) from cudf.core.buffer import acquire_spill_lock +from pandas.core.groupby.groupby import DataError from libcpp cimport bool from libcpp.memory cimport unique_ptr @@ -191,7 +190,7 @@ cdef class GroupBy: else _STRING_AGGS if is_string_dtype(dtype) else _CATEGORICAL_AGGS if _is_categorical_dtype(dtype) else _STRUCT_AGGS if is_struct_dtype(dtype) - else _INTERVAL_AGGS if is_interval_dtype(dtype) + else _INTERVAL_AGGS if _is_interval_dtype(dtype) else _DECIMAL_AGGS if is_decimal_dtype(dtype) else "ALL" ) @@ -262,7 +261,7 @@ cdef class GroupBy: else _STRING_AGGS if is_string_dtype(dtype) else _CATEGORICAL_AGGS if _is_categorical_dtype(dtype) else _STRUCT_AGGS if is_struct_dtype(dtype) - else _INTERVAL_AGGS if is_interval_dtype(dtype) + else _INTERVAL_AGGS if _is_interval_dtype(dtype) else _DECIMAL_AGGS if is_decimal_dtype(dtype) else "ALL" ) diff --git a/python/cudf/cudf/api/types.py b/python/cudf/cudf/api/types.py index 007b9f3ee02..4f948fddab7 100644 --- a/python/cudf/cudf/api/types.py +++ b/python/cudf/cudf/api/types.py @@ -4,6 +4,7 @@ from __future__ import annotations +import warnings from collections import abc from functools import wraps from inspect import isclass @@ -11,14 +12,14 @@ import cupy as cp import numpy as np -import pandas as pd -from pandas.api import types as pd_types import cudf +import pandas as pd from cudf.core.dtypes import ( # noqa: F401 _BaseDtype, - dtype, _is_categorical_dtype, + _is_interval_dtype, + dtype, is_decimal32_dtype, is_decimal64_dtype, is_decimal128_dtype, @@ -27,6 +28,7 @@ is_list_dtype, is_struct_dtype, ) +from pandas.api import types as pd_types def is_numeric_dtype(obj): @@ -116,7 +118,7 @@ def is_string_dtype(obj): and not is_decimal_dtype(obj) and not is_list_dtype(obj) and not is_struct_dtype(obj) - and not is_interval_dtype(obj) + and not _is_interval_dtype(obj) ) ) @@ -451,6 +453,22 @@ def is_any_real_numeric_dtype(arr_or_dtype) -> bool: ) +def _is_datetime64tz_dtype(obj): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return _wrap_pandas_is_dtype_api(pd_types.is_datetime64tz_dtype)(obj) + + +def is_datetime64tz_dtype(obj): + # Do not remove until pandas 3.0 support is added. + warnings.warn( + "is_datetime64tz_dtype is deprecated and will be removed in a future " + "version.", + FutureWarning, + ) + return _is_datetime64tz_dtype(obj) + + # TODO: The below alias is removed for now since improving cudf categorical # support is ongoing and we don't want to introduce any ambiguities. The above # method _union_categoricals will take its place once exposed. @@ -465,9 +483,6 @@ def is_any_real_numeric_dtype(arr_or_dtype) -> bool: is_datetime64_ns_dtype = _wrap_pandas_is_dtype_api( pd_types.is_datetime64_ns_dtype ) -is_datetime64tz_dtype = _wrap_pandas_is_dtype_api( - pd_types.is_datetime64tz_dtype -) is_extension_array_dtype = pd_types.is_extension_array_dtype is_int64_dtype = pd_types.is_int64_dtype is_period_dtype = pd_types.is_period_dtype diff --git a/python/cudf/cudf/core/column/categorical.py b/python/cudf/cudf/core/column/categorical.py index af0e3257d4e..73ca529b248 100644 --- a/python/cudf/cudf/core/column/categorical.py +++ b/python/cudf/cudf/core/column/categorical.py @@ -7,16 +7,16 @@ from typing import TYPE_CHECKING, Any, Mapping, Optional, Sequence, Tuple, cast import numpy as np -import pandas as pd import pyarrow as pa from numba import cuda from typing_extensions import Self import cudf +import pandas as pd from cudf import _lib as libcudf from cudf._lib.transform import bools_to_mask from cudf._typing import ColumnBinaryOperand, ColumnLike, Dtype, ScalarLike -from cudf.api.types import _is_categorical_dtype, is_interval_dtype +from cudf.api.types import _is_categorical_dtype, _is_interval_dtype from cudf.core.buffer import Buffer from cudf.core.column import column from cudf.core.column.methods import ColumnMethods @@ -802,7 +802,7 @@ def to_pandas( .fillna(_DEFAULT_CATEGORICAL_VALUE) .values_host ) - if is_interval_dtype(col.categories.dtype): + if _is_interval_dtype(col.categories.dtype): # leaving out dropna because it temporarily changes an interval # index into a struct and throws off results. # TODO: work on interval index dropna diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 048ce620a8d..c1b74a01fc2 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -23,7 +23,6 @@ import cupy import numpy as np -import pandas as pd import pyarrow as pa from numba import cuda from typing_extensions import Self @@ -31,6 +30,7 @@ import rmm import cudf +import pandas as pd from cudf import _lib as libcudf from cudf._lib.column import Column from cudf._lib.null_mask import ( @@ -50,18 +50,18 @@ from cudf._typing import ColumnLike, Dtype, ScalarLike from cudf.api.types import ( _is_categorical_dtype, + _is_datetime64tz_dtype, + _is_interval_dtype, _is_non_decimal_numeric_dtype, infer_dtype, is_bool_dtype, is_datetime64_dtype, - is_datetime64tz_dtype, is_decimal32_dtype, is_decimal64_dtype, is_decimal128_dtype, is_decimal_dtype, is_dtype_equal, is_integer_dtype, - is_interval_dtype, is_list_dtype, is_scalar, is_string_dtype, @@ -1014,7 +1014,7 @@ def astype(self, dtype: Dtype, **kwargs) -> ColumnBase: "Casting struct columns not currently supported" ) return self - elif is_interval_dtype(self.dtype): + elif _is_interval_dtype(self.dtype): return self.as_interval_column(dtype, **kwargs) elif is_decimal_dtype(dtype): return self.as_decimal_column(dtype, **kwargs) @@ -1579,7 +1579,7 @@ def build_column( offset=offset, null_count=null_count, ) - elif is_datetime64tz_dtype(dtype): + elif _is_datetime64tz_dtype(dtype): if data is None: raise TypeError("Must specify data buffer") return cudf.core.column.datetime.DatetimeTZColumn( @@ -1618,7 +1618,7 @@ def build_column( null_count=null_count, children=children, ) - elif is_interval_dtype(dtype): + elif _is_interval_dtype(dtype): return cudf.core.column.IntervalColumn( dtype=dtype, mask=mask, @@ -1675,7 +1675,7 @@ def build_column( null_count=null_count, children=children, ) - elif is_interval_dtype(dtype): + elif _is_interval_dtype(dtype): return cudf.core.column.IntervalColumn( dtype=dtype, mask=mask, @@ -2045,7 +2045,7 @@ def as_column( "cuDF does not yet support timezone-aware datetimes" ) data = as_column(pa.array(arbitrary, from_pandas=True)) - elif is_interval_dtype(arbitrary.dtype): + elif _is_interval_dtype(arbitrary.dtype): if isinstance(arbitrary.dtype.subtype, pd.DatetimeTZDtype): raise NotImplementedError( "cuDF does not yet support timezone-aware datetimes" @@ -2287,7 +2287,7 @@ def as_column( ) or ( isinstance(arbitrary, pd.IntervalIndex) - and is_datetime64tz_dtype(arbitrary.dtype.subtype) + and _is_datetime64tz_dtype(arbitrary.dtype.subtype) ) or ( isinstance(arbitrary, pd.CategoricalIndex) @@ -2347,11 +2347,11 @@ def as_column( np_type = None try: if dtype is not None: - if _is_categorical_dtype(dtype) or is_interval_dtype( + if _is_categorical_dtype(dtype) or _is_interval_dtype( dtype ): raise TypeError - if is_datetime64tz_dtype(dtype): + if _is_datetime64tz_dtype(dtype): raise NotImplementedError( "Use `tz_localize()` to construct " "timezone aware data." @@ -2499,7 +2499,7 @@ def as_column( elif np_type == np.str_: sr = pd.Series(arbitrary, dtype="str") data = as_column(sr, nan_as_null=nan_as_null) - elif is_interval_dtype(dtype): + elif _is_interval_dtype(dtype): sr = pd.Series(arbitrary, dtype="interval") data = as_column(sr, nan_as_null=nan_as_null, dtype=dtype) elif ( diff --git a/python/cudf/cudf/core/column/datetime.py b/python/cudf/cudf/core/column/datetime.py index 75548daf310..3998ca99dba 100644 --- a/python/cudf/cudf/core/column/datetime.py +++ b/python/cudf/cudf/core/column/datetime.py @@ -22,8 +22,8 @@ ScalarLike, ) from cudf.api.types import ( + _is_datetime64tz_dtype, is_datetime64_dtype, - is_datetime64tz_dtype, is_scalar, is_timedelta64_dtype, ) @@ -566,7 +566,7 @@ def can_cast_safely(self, to_dtype: Dtype) -> bool: return False def _with_type_metadata(self, dtype): - if is_datetime64tz_dtype(dtype): + if _is_datetime64tz_dtype(dtype): return DatetimeTZColumn( data=self.base_data, dtype=dtype, diff --git a/python/cudf/cudf/core/column/interval.py b/python/cudf/cudf/core/column/interval.py index 7b87552f1a0..bcbe777ee66 100644 --- a/python/cudf/cudf/core/column/interval.py +++ b/python/cudf/cudf/core/column/interval.py @@ -1,11 +1,11 @@ # Copyright (c) 2018-2023, NVIDIA CORPORATION. from typing import Optional -import pandas as pd import pyarrow as pa import cudf -from cudf.api.types import _is_categorical_dtype, is_interval_dtype +import pandas as pd +from cudf.api.types import _is_categorical_dtype, _is_interval_dtype from cudf.core.column import StructColumn from cudf.core.dtypes import IntervalDtype @@ -101,11 +101,11 @@ def copy(self, deep=True): ) def as_interval_column(self, dtype, **kwargs): - if is_interval_dtype(dtype): + if _is_interval_dtype(dtype): if _is_categorical_dtype(self): new_struct = self._get_decategorized_column() return IntervalColumn.from_struct_column(new_struct) - if is_interval_dtype(dtype): + if _is_interval_dtype(dtype): # a user can directly input the string `interval` as the dtype # when creating an interval series or interval dataframe if dtype == "interval": diff --git a/python/cudf/cudf/core/dtypes.py b/python/cudf/cudf/core/dtypes.py index 6fae552c6ed..e293b8a61f5 100644 --- a/python/cudf/cudf/core/dtypes.py +++ b/python/cudf/cudf/core/dtypes.py @@ -9,21 +9,21 @@ from typing import Any, Callable, Dict, List, Tuple, Type, Union import numpy as np -import pandas as pd import pyarrow as pa -from pandas.api import types as pd_types -from pandas.api.extensions import ExtensionDtype -from pandas.core.dtypes.dtypes import ( - CategoricalDtype as pd_CategoricalDtype, - CategoricalDtypeType as pd_CategoricalDtypeType, -) import cudf +import pandas as pd from cudf._typing import Dtype from cudf.core._compat import PANDAS_GE_150 from cudf.core.abc import Serializable from cudf.core.buffer import Buffer from cudf.utils.docutils import doc_apply +from pandas.api import types as pd_types +from pandas.api.extensions import ExtensionDtype +from pandas.core.dtypes.dtypes import ( + CategoricalDtype as pd_CategoricalDtype, + CategoricalDtypeType as pd_CategoricalDtypeType, +) if PANDAS_GE_150: from pandas.core.arrays.arrow.extension_types import ArrowIntervalType @@ -261,7 +261,7 @@ def to_pandas(self) -> pd.CategoricalDtype: def _init_categories(self, categories: Any): if categories is None: return categories - if len(categories) == 0 and not is_interval_dtype(categories): + if len(categories) == 0 and not _is_interval_dtype(categories): dtype = "object" # type: Any else: dtype = None @@ -1107,21 +1107,7 @@ def is_decimal_dtype(obj): ) -def is_interval_dtype(obj): - """Check whether an array-like or dtype is of the interval dtype. - - Parameters - ---------- - obj : array-like or dtype - The array-like or dtype to check. - - Returns - ------- - bool - Whether or not the array-like or dtype is of the interval dtype. - """ - # TODO: Should there be any branch in this function that calls - # pd.api.types.is_interval_dtype? +def _is_interval_dtype(obj): return ( isinstance( obj, @@ -1135,8 +1121,29 @@ def is_interval_dtype(obj): or ( isinstance(obj, str) and obj == cudf.core.dtypes.IntervalDtype.name ) - or (hasattr(obj, "dtype") and is_interval_dtype(obj.dtype)) + or (hasattr(obj, "dtype") and _is_interval_dtype(obj.dtype)) + ) + + +def is_interval_dtype(obj): + """Check whether an array-like or dtype is of the interval dtype. + + Parameters + ---------- + obj : array-like or dtype + The array-like or dtype to check. + + Returns + ------- + bool + Whether or not the array-like or dtype is of the interval dtype. + """ + warnings.warn( + "is_interval_dtype is deprecated and will be removed in a " + "future version. Use `isinstance(dtype, cudf.IntervalDtype)` instead", + FutureWarning, ) + return _is_interval_dtype(obj) def is_decimal32_dtype(obj): diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 92b244d1999..6d144b36a65 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -7,22 +7,14 @@ import warnings from functools import cache, cached_property from numbers import Number -from typing import ( - Any, - List, - MutableMapping, - Optional, - Tuple, - Union, -) +from typing import Any, List, MutableMapping, Optional, Tuple, Union import cupy import numpy as np -import pandas as pd -from pandas._config import get_option from typing_extensions import Self import cudf +import pandas as pd from cudf import _lib as libcudf from cudf._lib.datetime import extract_quarter, is_leap_year from cudf._lib.filling import sequence @@ -30,10 +22,10 @@ from cudf._lib.types import size_type_dtype from cudf.api.extensions import no_default from cudf.api.types import ( - _is_non_decimal_numeric_dtype, _is_categorical_dtype, + _is_interval_dtype, + _is_non_decimal_numeric_dtype, is_dtype_equal, - is_interval_dtype, is_list_like, is_scalar, ) @@ -70,6 +62,7 @@ _warn_no_dask_cudf, search_range, ) +from pandas._config import get_option class IndexMeta(type): @@ -2713,7 +2706,7 @@ def __init__( if isinstance(data, IntervalColumn): data = data - elif isinstance(data, pd.Series) and (is_interval_dtype(data.dtype)): + elif isinstance(data, pd.Series) and (_is_interval_dtype(data.dtype)): data = column.as_column(data, data.dtype) elif isinstance(data, (pd._libs.interval.Interval, pd.IntervalIndex)): data = column.as_column( diff --git a/python/cudf/cudf/testing/testing.py b/python/cudf/cudf/testing/testing.py index 9c2ee637584..3e8c986ab95 100644 --- a/python/cudf/cudf/testing/testing.py +++ b/python/cudf/cudf/testing/testing.py @@ -4,14 +4,14 @@ import cupy as cp import numpy as np -import pandas as pd import cudf +import pandas as pd from cudf._lib.unary import is_nan from cudf.api.types import ( _is_categorical_dtype, + _is_interval_dtype, is_decimal_dtype, - is_interval_dtype, is_list_dtype, is_numeric_dtype, is_string_dtype, @@ -28,7 +28,7 @@ def dtype_can_compare_equal_to_other(dtype): or is_list_dtype(dtype) or is_struct_dtype(dtype) or is_decimal_dtype(dtype) - or is_interval_dtype(dtype) + or _is_interval_dtype(dtype) ) From 2461315ed223d214dcb38414344f3207eafd6630 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 13 Oct 2023 11:50:56 -0500 Subject: [PATCH 065/384] Deprecate `method` in `fillna` API (#14278) This PR deprecates `method` parameter in all public `fillna` APIs to match pandas: https://github.com/pandas-dev/pandas/pull/53496/ This PR: ``` = 1056 failed, 99098 passed, 2069 skipped, 776 xfailed, 312 xpassed, 20 errors in 670.87s (0:11:10) = ``` On `pandas_2.0_feature_branch`: ``` = 1584 failed, 98570 passed, 2069 skipped, 776 xfailed, 312 xpassed, 20 errors in 737.24s (0:12:17) = ``` --- python/cudf/cudf/core/dataframe.py | 4 +++- python/cudf/cudf/core/frame.py | 4 +++- python/cudf/cudf/core/groupby/groupby.py | 13 ++++++++++++- python/cudf/cudf/core/indexed_frame.py | 8 ++++++++ python/cudf/cudf/core/resample.py | 5 ++++- python/cudf/cudf/core/series.py | 4 +++- python/cudf/cudf/tests/test_groupby.py | 12 ++++++++---- python/cudf/cudf/tests/test_replace.py | 20 +++++++++++++++----- 8 files changed, 56 insertions(+), 14 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 0cfa37c224d..793742604a2 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -7031,7 +7031,9 @@ def pct_change( "'bfill', or 'backfill'." ) - data = self.fillna(method=fill_method, limit=limit) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + data = self.fillna(method=fill_method, limit=limit) return data.diff(periods=periods) / data.shift( periods=periods, freq=freq diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index 43a713e273d..0b627c12d97 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -735,13 +735,15 @@ def fillna( are filled with values in corresponding indices. A dict can be used to provide different values to fill nulls in different columns. Cannot be used with ``method``. - method : {'ffill', 'bfill'}, default None Method to use for filling null values in the dataframe or series. `ffill` propagates the last non-null values forward to the next non-null value. `bfill` propagates backward with the next non-null value. Cannot be used with ``value``. + .. deprecated:: 23.12 + `method` is deprecated. + Returns ------- result : DataFrame, Series, or Index diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 0fc61713eeb..fcff8e805bf 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -2183,6 +2183,14 @@ def fillna( if method is not None: if method not in {"ffill", "bfill"}: raise ValueError("Method can only be of 'ffill', 'bfill'.") + # Do not remove until pandas 3.0 support is added. + warnings.warn( + f"{type(self).__name__}.fillna with 'method' is " + "deprecated and will raise in a future version. " + "Use obj.ffill() or obj.bfill() instead.", + FutureWarning, + ) + return getattr(self, method, limit)() values = self.obj.__class__._from_data( @@ -2295,7 +2303,10 @@ def pct_change( FutureWarning, ) - filled = self.fillna(method=fill_method, limit=limit) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + filled = self.fillna(method=fill_method, limit=limit) + fill_grp = filled.groupby(self.grouping) shifted = fill_grp.shift(periods=periods, freq=freq) return (filled / shifted) - 1 diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index f9435eebe96..028c6cf208b 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -2148,6 +2148,14 @@ def _split(self, splits, keep_index=True): def fillna( self, value=None, method=None, axis=None, inplace=False, limit=None ): # noqa: D102 + if method is not None: + # Do not remove until pandas 3.0 support is added. + warnings.warn( + f"{type(self).__name__}.fillna with 'method' is " + "deprecated and will raise in a future version. " + "Use obj.ffill() or obj.bfill() instead.", + FutureWarning, + ) old_index = self._index ret = super().fillna(value, method, axis, inplace, limit) if inplace: diff --git a/python/cudf/cudf/core/resample.py b/python/cudf/cudf/core/resample.py index eb59cf83926..83a003cb949 100644 --- a/python/cudf/cudf/core/resample.py +++ b/python/cudf/cudf/core/resample.py @@ -15,6 +15,7 @@ # limitations under the License. import pickle +import warnings import numpy as np import pandas as pd @@ -73,7 +74,9 @@ def _scan_fill(self, method: str, limit: int) -> DataFrameOrSeries: ) # fill the gaps: - filled = upsampled.fillna(method=method) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + filled = upsampled.fillna(method=method) # filter the result to only include the values corresponding # to the bin labels: diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 47808259f14..9e82a353282 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -3506,7 +3506,9 @@ def pct_change( "'bfill', or 'backfill'." ) - data = self.fillna(method=fill_method, limit=limit) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + data = self.fillna(method=fill_method, limit=limit) diff = data.diff(periods=periods) change = diff / data.shift(periods=periods, freq=freq) return change diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index 50a749a25b8..2f83348bcff 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -2646,10 +2646,12 @@ def test_groupby_various_by_fillna(by, data, args): ps = pd.Series(data) gs = cudf.from_pandas(ps) - expect = ps.groupby(by).fillna(**args) + with expect_warning_if(PANDAS_GE_210 and "method" in args): + expect = ps.groupby(by).fillna(**args) if isinstance(by, pd.Grouper): by = cudf.Grouper(level=by.level) - got = gs.groupby(by).fillna(**args) + with expect_warning_if("method" in args): + got = gs.groupby(by).fillna(**args) assert_groupby_results_equal(expect, got, check_dtype=False) @@ -2693,8 +2695,10 @@ def test_groupby_fillna_method(nelem, method): pdf = t.to_pandas() gdf = cudf.from_pandas(pdf) - expect = pdf.groupby(key_col).fillna(method=method) - got = gdf.groupby(key_col).fillna(method=method) + with expect_warning_if(PANDAS_GE_210): + expect = pdf.groupby(key_col).fillna(method=method) + with pytest.warns(FutureWarning): + got = gdf.groupby(key_col).fillna(method=method) assert_groupby_results_equal( expect[value_cols], got[value_cols], sort=False diff --git a/python/cudf/cudf/tests/test_replace.py b/python/cudf/cudf/tests/test_replace.py index 364afacd261..3ab7064e2d0 100644 --- a/python/cudf/cudf/tests/test_replace.py +++ b/python/cudf/cudf/tests/test_replace.py @@ -8,13 +8,19 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_134, PANDAS_GE_150, PANDAS_GE_200 +from cudf.core._compat import ( + PANDAS_GE_134, + PANDAS_GE_150, + PANDAS_GE_200, + PANDAS_GE_210, +) from cudf.core.dtypes import Decimal32Dtype, Decimal64Dtype, Decimal128Dtype from cudf.testing._utils import ( INTEGER_TYPES, NUMERIC_TYPES, assert_eq, assert_exceptions_equal, + expect_warning_if, ) @@ -348,8 +354,10 @@ def test_fillna_method_numerical(data, container, data_dtype, method, inplace): # Explicitly using nans_as_nulls=True gdata = cudf.from_pandas(pdata, nan_as_null=True) - expected = pdata.fillna(method=method, inplace=inplace) - actual = gdata.fillna(method=method, inplace=inplace) + with expect_warning_if(PANDAS_GE_210): + expected = pdata.fillna(method=method, inplace=inplace) + with pytest.warns(FutureWarning): + actual = gdata.fillna(method=method, inplace=inplace) if inplace: expected = pdata @@ -665,8 +673,10 @@ def test_fillna_method_fixed_width_non_num(data, container, method, inplace): # Explicitly using nans_as_nulls=True gdata = cudf.from_pandas(pdata, nan_as_null=True) - expected = pdata.fillna(method=method, inplace=inplace) - actual = gdata.fillna(method=method, inplace=inplace) + with expect_warning_if(PANDAS_GE_210): + expected = pdata.fillna(method=method, inplace=inplace) + with pytest.warns(FutureWarning): + actual = gdata.fillna(method=method, inplace=inplace) if inplace: expected = pdata From 90788f27953f23b75d88ba53dce99c93b8990292 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 29 Nov 2023 11:46:35 -0600 Subject: [PATCH 066/384] Deprecate `fill_method` and `limit` in `pct_change` APIs (#14277) This PR deprecated `fill_method` and `limit` in `Series.pct_change`, `DataFrame.pct_change` and `Groupby.pct_change` This PR: ``` = 1263 failed, 98996 passed, 2069 skipped, 776 xfailed, 312 xpassed, 20 errors in 701.08s (0:11:41) = ``` On `pandas_2.0_feature_branch`: ``` = 1584 failed, 98570 passed, 2069 skipped, 776 xfailed, 312 xpassed, 20 errors in 737.24s (0:12:17) = ``` --- python/cudf/cudf/core/dataframe.py | 36 +++++++++++++++++++--- python/cudf/cudf/core/groupby/groupby.py | 38 +++++++++++++++++------- python/cudf/cudf/core/series.py | 36 +++++++++++++++++++--- python/cudf/cudf/tests/test_dataframe.py | 15 ++++++++-- python/cudf/cudf/tests/test_groupby.py | 23 +++++++++----- python/cudf/cudf/tests/test_stats.py | 18 +++++++++-- 6 files changed, 135 insertions(+), 31 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index b4f8669ba9d..63139231d75 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -7137,7 +7137,7 @@ def explode(self, column, ignore_index=False): return super()._explode(column, ignore_index) def pct_change( - self, periods=1, fill_method="ffill", limit=None, freq=None + self, periods=1, fill_method=no_default, limit=no_default, freq=None ): """ Calculates the percent change between sequential elements @@ -7149,9 +7149,15 @@ def pct_change( Periods to shift for forming percent change. fill_method : str, default 'ffill' How to handle NAs before computing percent changes. + + .. deprecated:: 23.12 + All options of `fill_method` are deprecated except `fill_method=None`. limit : int, optional The number of consecutive NAs to fill before stopping. Not yet implemented. + + .. deprecated:: 23.12 + `limit` is deprecated. freq : str, optional Increment to use from time series API. Not yet implemented. @@ -7160,16 +7166,38 @@ def pct_change( ------- DataFrame """ - if limit is not None: + if limit is not no_default: raise NotImplementedError("limit parameter not supported yet.") if freq is not None: raise NotImplementedError("freq parameter not supported yet.") - elif fill_method not in {"ffill", "pad", "bfill", "backfill"}: + elif fill_method not in { + no_default, + None, + "ffill", + "pad", + "bfill", + "backfill", + }: raise ValueError( - "fill_method must be one of 'ffill', 'pad', " + "fill_method must be one of None, 'ffill', 'pad', " "'bfill', or 'backfill'." ) + if fill_method not in (no_default, None) or limit is not no_default: + # Do not remove until pandas 3.0 support is added. + warnings.warn( + "The 'fill_method' and 'limit' keywords in " + f"{type(self).__name__}.pct_change are deprecated and will be " + "removed in a future version. Either fill in any non-leading NA values prior " + "to calling pct_change or specify 'fill_method=None' to not fill NA " + "values.", + FutureWarning, + ) + if fill_method is no_default: + fill_method = "ffill" + if limit is no_default: + limit = None + with warnings.catch_warnings(): warnings.simplefilter("ignore") data = self.fillna(method=fill_method, limit=limit) diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index bad5106970e..414a86470f0 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -21,6 +21,7 @@ from cudf._lib.sort import segmented_sort_by_key from cudf._lib.types import size_type_dtype from cudf._typing import AggType, DataFrameOrSeries, MultiColumnAggType +from cudf.api.extensions import no_default from cudf.api.types import is_bool_dtype, is_float_dtype, is_list_like from cudf.core.abc import Serializable from cudf.core.column.column import ColumnBase, arange, as_column @@ -2286,7 +2287,12 @@ def shift(self, periods=1, freq=None, axis=0, fill_value=None): @_cudf_nvtx_annotate def pct_change( - self, periods=1, fill_method="ffill", axis=0, limit=None, freq=None + self, + periods=1, + fill_method=no_default, + axis=0, + limit=no_default, + freq=None, ): """ Calculates the percent change between sequential elements @@ -2298,9 +2304,15 @@ def pct_change( Periods to shift for forming percent change. fill_method : str, default 'ffill' How to handle NAs before computing percent changes. + + .. deprecated:: 23.12 + All options of `fill_method` are deprecated except `fill_method=None`. limit : int, optional The number of consecutive NAs to fill before stopping. Not yet implemented. + + .. deprecated:: 23.12 + `limit` is deprecated. freq : str, optional Increment to use from time series API. Not yet implemented. @@ -2312,25 +2324,31 @@ def pct_change( """ if not axis == 0: raise NotImplementedError("Only axis=0 is supported.") - if limit is not None: + if limit is not no_default: raise NotImplementedError("limit parameter not supported yet.") if freq is not None: raise NotImplementedError("freq parameter not supported yet.") - elif fill_method not in {"ffill", "pad", "bfill", "backfill"}: + elif fill_method not in {no_default, None, "ffill", "bfill"}: raise ValueError( - "fill_method must be one of 'ffill', 'pad', " - "'bfill', or 'backfill'." + "fill_method must be one of 'ffill', or" "'bfill'." ) - if fill_method in ("pad", "backfill"): - alternative = "ffill" if fill_method == "pad" else "bfill" - # Do not remove until pandas 2.0 support is added. + if fill_method not in (no_default, None) or limit is not no_default: + # Do not remove until pandas 3.0 support is added. warnings.warn( - f"{fill_method} is deprecated and will be removed in a future " - f"version. Use f{alternative} instead.", + "The 'fill_method' keyword being not None and the 'limit' keywords in " + f"{type(self).__name__}.pct_change are deprecated and will be " + "removed in a future version. Either fill in any non-leading NA values prior " + "to calling pct_change or specify 'fill_method=None' to not fill NA " + "values.", FutureWarning, ) + if fill_method in (no_default, None): + fill_method = "ffill" + if limit is no_default: + limit = None + with warnings.catch_warnings(): warnings.simplefilter("ignore") filled = self.fillna(method=fill_method, limit=limit) diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index f0323d6f55b..f9987569070 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -3569,7 +3569,7 @@ def explode(self, ignore_index=False): @_cudf_nvtx_annotate def pct_change( - self, periods=1, fill_method="ffill", limit=None, freq=None + self, periods=1, fill_method=no_default, limit=no_default, freq=None ): """ Calculates the percent change between sequential elements @@ -3581,9 +3581,15 @@ def pct_change( Periods to shift for forming percent change. fill_method : str, default 'ffill' How to handle NAs before computing percent changes. + + .. deprecated:: 23.12 + All options of `fill_method` are deprecated except `fill_method=None`. limit : int, optional The number of consecutive NAs to fill before stopping. Not yet implemented. + + .. deprecated:: 23.12 + `limit` is deprecated. freq : str, optional Increment to use from time series API. Not yet implemented. @@ -3592,15 +3598,37 @@ def pct_change( ------- Series """ - if limit is not None: + if limit is not no_default: raise NotImplementedError("limit parameter not supported yet.") if freq is not None: raise NotImplementedError("freq parameter not supported yet.") - elif fill_method not in {"ffill", "pad", "bfill", "backfill"}: + elif fill_method not in { + no_default, + None, + "ffill", + "pad", + "bfill", + "backfill", + }: raise ValueError( - "fill_method must be one of 'ffill', 'pad', " + "fill_method must be one of None, 'ffill', 'pad', " "'bfill', or 'backfill'." ) + if fill_method not in (no_default, None) or limit is not no_default: + # Do not remove until pandas 3.0 support is added. + warnings.warn( + "The 'fill_method' and 'limit' keywords in " + f"{type(self).__name__}.pct_change are deprecated and will be " + "removed in a future version. Either fill in any non-leading NA values prior " + "to calling pct_change or specify 'fill_method=None' to not fill NA " + "values.", + FutureWarning, + ) + + if fill_method is no_default: + fill_method = "ffill" + if limit is no_default: + limit = None with warnings.catch_warnings(): warnings.simplefilter("ignore") diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 9192e5e7ca0..9a51ef5ed57 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -25,8 +25,10 @@ PANDAS_GE_134, PANDAS_GE_150, PANDAS_GE_200, + PANDAS_GE_210, PANDAS_LT_140, ) +from cudf.api.extensions import no_default from cudf.core.buffer.spill_manager import get_global_manager from cudf.core.column import column from cudf.testing import _utils as utils @@ -9896,13 +9898,20 @@ def test_dataframe_rename_duplicate_column(): ], ) @pytest.mark.parametrize("periods", [-5, -2, 0, 2, 5]) -@pytest.mark.parametrize("fill_method", ["ffill", "bfill", "pad", "backfill"]) +@pytest.mark.parametrize( + "fill_method", ["ffill", "bfill", "pad", "backfill", no_default] +) def test_dataframe_pct_change(data, periods, fill_method): gdf = cudf.DataFrame(data) pdf = gdf.to_pandas() - actual = gdf.pct_change(periods=periods, fill_method=fill_method) - expected = pdf.pct_change(periods=periods, fill_method=fill_method) + with expect_warning_if(fill_method is not no_default): + actual = gdf.pct_change(periods=periods, fill_method=fill_method) + with expect_warning_if( + PANDAS_GE_210 + and (fill_method is not no_default or pdf.isna().any().any()) + ): + expected = pdf.pct_change(periods=periods, fill_method=fill_method) assert_eq(expected, actual) diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index 65c48c1b12d..fd0f7863d2b 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -19,6 +19,7 @@ import cudf from cudf import DataFrame, Series +from cudf.api.extensions import no_default from cudf.core._compat import ( PANDAS_GE_150, PANDAS_LT_140, @@ -3062,17 +3063,25 @@ def test_groupby_transform_maintain_index(by): ], ) @pytest.mark.parametrize("periods", [-5, -2, 0, 2, 5]) -@pytest.mark.parametrize("fill_method", ["ffill", "bfill"]) +@pytest.mark.parametrize("fill_method", ["ffill", "bfill", no_default, None]) def test_groupby_pct_change(data, gkey, periods, fill_method): gdf = cudf.DataFrame(data) pdf = gdf.to_pandas() - actual = gdf.groupby(gkey).pct_change( - periods=periods, fill_method=fill_method - ) - expected = pdf.groupby(gkey).pct_change( - periods=periods, fill_method=fill_method - ) + with expect_warning_if(fill_method not in (no_default, None)): + actual = gdf.groupby(gkey).pct_change( + periods=periods, fill_method=fill_method + ) + with expect_warning_if( + PANDAS_GE_210 + and ( + fill_method not in (no_default, None) + or (fill_method is not None and pdf.isna().any().any()) + ) + ): + expected = pdf.groupby(gkey).pct_change( + periods=periods, fill_method=fill_method + ) assert_eq(expected, actual) diff --git a/python/cudf/cudf/tests/test_stats.py b/python/cudf/cudf/tests/test_stats.py index 8eae74a34f7..41fac49ea83 100644 --- a/python/cudf/cudf/tests/test_stats.py +++ b/python/cudf/cudf/tests/test_stats.py @@ -8,6 +8,7 @@ import pytest import cudf +from cudf.api.extensions import no_default from cudf.datasets import randomdata from cudf.testing._utils import ( _create_cudf_series_float64_default, @@ -16,6 +17,7 @@ assert_exceptions_equal, expect_warning_if, ) +from cudf.core._compat import PANDAS_GE_210 params_dtypes = [np.int32, np.uint32, np.float32, np.float64] methods = ["min", "max", "sum", "mean", "var", "std"] @@ -356,14 +358,24 @@ def test_series_median(dtype, num_na): ], ) @pytest.mark.parametrize("periods", range(-5, 5)) -@pytest.mark.parametrize("fill_method", ["ffill", "bfill", "pad", "backfill"]) +@pytest.mark.parametrize( + "fill_method", ["ffill", "bfill", "pad", "backfill", no_default, None] +) def test_series_pct_change(data, periods, fill_method): cs = cudf.Series(data) ps = cs.to_pandas() if np.abs(periods) <= len(cs): - got = cs.pct_change(periods=periods, fill_method=fill_method) - expected = ps.pct_change(periods=periods, fill_method=fill_method) + with expect_warning_if(fill_method not in (no_default, None)): + got = cs.pct_change(periods=periods, fill_method=fill_method) + with expect_warning_if( + PANDAS_GE_210 + and ( + fill_method not in (no_default, None) + or (fill_method is not None and ps.isna().any()) + ) + ): + expected = ps.pct_change(periods=periods, fill_method=fill_method) np.testing.assert_array_almost_equal( got.to_numpy(na_value=np.nan), expected ) From c51444fef24bd6ab812808f614b024c31f8bbe22 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 5 Dec 2023 02:10:31 -0500 Subject: [PATCH 067/384] Replace PandasArray with NumpyExtensionArray (#14549) This PR replaces usages of `PandasArray` with `NumpyExtensionArray` to not have warnings during runtime. On `pandas_2.0_feature_branch`: ``` = 15895 failed, 61649 passed, 1840 skipped, 735 xfailed, 312 xpassed, 371 errors in 4361.35s (1:12:41) = ``` On this PR: ``` = 923 failed, 100684 passed, 2071 skipped, 786 xfailed, 312 xpassed, 20 errors in 1216.98s (0:20:16) = ``` --- python/cudf/cudf/core/column/column.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 05517da24b1..b79d0644696 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -2109,13 +2109,13 @@ def as_column( ) elif isinstance( arbitrary.dtype, pd.api.extensions.ExtensionDtype - ) and not isinstance(arbitrary, pd.arrays.PandasArray): + ) and not isinstance(arbitrary, NumpyExtensionArray): raise NotImplementedError( "Custom pandas ExtensionDtypes are not supported" ) elif arbitrary.dtype.kind in "fiubmM": # numpy dtype like - if isinstance(arbitrary, pd.arrays.PandasArray): + if isinstance(arbitrary, NumpyExtensionArray): arbitrary = np.array(arbitrary) arb_dtype = np.dtype(arbitrary.dtype) if arb_dtype.kind == "f" and arb_dtype.itemsize == 2: @@ -2129,8 +2129,8 @@ def as_column( arbitrary, nan_as_null=nan_as_null, dtype=dtype, length=length ) elif arbitrary.dtype.kind == "O": - if isinstance(arbitrary, pd.arrays.PandasArray): - # infer_dtype does not handle PandasArray + if isinstance(arbitrary, NumpyExtensionArray): + # infer_dtype does not handle NumpyExtensionArray arbitrary = np.array(arbitrary, dtype=object) inferred_dtype = infer_dtype(arbitrary) if inferred_dtype in ("mixed-integer", "mixed-integer-float"): From e04b88b5dc86f696843b896dd0f2dc3cfbec09a7 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 5 Dec 2023 02:13:58 -0500 Subject: [PATCH 068/384] Fix copy creation of a columnAccessor (#14551) This PR fixes a copy creation in ColumnAccessor by properly passing the rangeindex and label_dtype to it's newly constructed object. --- python/cudf/cudf/core/column_accessor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/cudf/cudf/core/column_accessor.py b/python/cudf/cudf/core/column_accessor.py index f6f3fe7d8fd..679b3e340f7 100644 --- a/python/cudf/cudf/core/column_accessor.py +++ b/python/cudf/cudf/core/column_accessor.py @@ -366,11 +366,15 @@ def copy(self, deep=False) -> ColumnAccessor: {k: v.copy(deep=deep) for k, v in self._data.items()}, multiindex=self.multiindex, level_names=self.level_names, + rangeindex=self.rangeindex, + label_dtype=self.label_dtype, ) return self.__class__( self._data.copy(), multiindex=self.multiindex, level_names=self.level_names, + rangeindex=self.rangeindex, + label_dtype=self.label_dtype, ) def select_by_label(self, key: Any) -> ColumnAccessor: From 29b3ac80d05a1a36c2868b4e15d79ffd7185fce6 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 5 Dec 2023 02:14:39 -0500 Subject: [PATCH 069/384] Fix to_pandas calls (#14552) This PR removes nullable=True in two pytests as we error when nullable is passed when there is decimal / list / struct data. --- python/cudf/cudf/tests/test_parquet.py | 1 - python/cudf/cudf/tests/test_udf_masked_ops.py | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index 5390ca72c0a..4d16bb4857e 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -2661,7 +2661,6 @@ def test_parquet_writer_decimal(decimal_type, data): gdf.to_parquet(buff) got = pd.read_parquet(buff, dtype_backend="numpy_nullable") - assert_eq(gdf.to_pandas(nullable=True), got) assert_eq(gdf["val"].to_pandas(nullable=True), got["val"]) assert_eq(gdf["dec_val"].to_pandas(), got["dec_val"]) diff --git a/python/cudf/cudf/tests/test_udf_masked_ops.py b/python/cudf/cudf/tests/test_udf_masked_ops.py index 04f4d12b78e..bd31fbd7f51 100644 --- a/python/cudf/cudf/tests/test_udf_masked_ops.py +++ b/python/cudf/cudf/tests/test_udf_masked_ops.py @@ -185,14 +185,13 @@ def func(row): gdf["a"] = gdf["a"].astype(dtype_l) gdf["b"] = gdf["b"].astype(dtype_r) - pdf = gdf.to_pandas(nullable=True) - + pdf = gdf.to_pandas() expect = op(pdf["a"], pdf["b"]) obtain = gdf.apply(func, axis=1) assert_eq(expect, obtain, check_dtype=False) # TODO: After the following pandas issue is # fixed, uncomment the following line and delete - # through `to_pandas(nullable=True)` statement. + # through `to_pandas()` statement. # https://github.com/pandas-dev/pandas/issues/52411 # run_masked_udf_test(func, gdf, nullable=False, check_dtype=False) From 19952eb92cbc1d118d2613ac452d3771bae4e458 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 5 Dec 2023 02:15:34 -0500 Subject: [PATCH 070/384] Add missing `is_categorical_dtype` to `cudf.api.types` namespace (#14555) This PR adds back cudf.api.types.is_categorical that was missing due to a bad merge. --- python/cudf/cudf/api/types.py | 1 + python/cudf/cudf/tests/test_api_types.py | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/python/cudf/cudf/api/types.py b/python/cudf/cudf/api/types.py index 4ad7e4b1db2..c921a48a599 100644 --- a/python/cudf/cudf/api/types.py +++ b/python/cudf/cudf/api/types.py @@ -21,6 +21,7 @@ _is_categorical_dtype, _is_interval_dtype, dtype, + is_categorical_dtype, is_decimal32_dtype, is_decimal64_dtype, is_decimal128_dtype, diff --git a/python/cudf/cudf/tests/test_api_types.py b/python/cudf/cudf/tests/test_api_types.py index da29972ea82..d640e8e1376 100644 --- a/python/cudf/cudf/tests/test_api_types.py +++ b/python/cudf/cudf/tests/test_api_types.py @@ -6,9 +6,11 @@ from pandas.api import types as pd_types import cudf -from cudf.core._compat import PANDAS_GE_200 +from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210 from cudf.api import types +from cudf.testing._utils import expect_warning_if + @pytest.mark.parametrize( "obj, expect", @@ -1036,10 +1038,11 @@ def test_is_decimal_dtype(obj, expect): ), ) def test_pandas_agreement(obj): + with expect_warning_if(PANDAS_GE_210): + expected = pd_types.is_categorical_dtype(obj) with pytest.warns(FutureWarning): - assert types.is_categorical_dtype( - obj - ) == pd_types.is_categorical_dtype(obj) + actual = types.is_categorical_dtype(obj) + assert expected == actual assert types.is_numeric_dtype(obj) == pd_types.is_numeric_dtype(obj) assert types.is_integer_dtype(obj) == pd_types.is_integer_dtype(obj) assert types.is_integer(obj) == pd_types.is_integer(obj) From ac07b3d21d522d8298f51a88236d5203a418e109 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 5 Dec 2023 02:25:55 -0500 Subject: [PATCH 071/384] Fix name in Index.difference (#14556) This PR fixes result names for Index.difference in some early exit scenarios. --- python/cudf/cudf/core/_base_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index 0f7e85f1cc2..61a5a4a5d68 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -1109,6 +1109,7 @@ def difference(self, other, sort=None): if is_mixed_with_object_dtype(self, other) or len(other) == 0: difference = self.copy() + difference.name = res_name if sort is True: return difference.sort_values() else: @@ -1122,12 +1123,11 @@ def difference(self, other, sort=None): ) ._data ) + difference.name = res_name if self.dtype != other.dtype: difference = difference.astype(self.dtype) - difference.name = res_name - if sort in {None, True} and len(other): return difference.sort_values() From 2bdd8b8acd0e365931ef418bb815a52cdf237772 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 5 Dec 2023 10:13:29 -0500 Subject: [PATCH 072/384] Filter deprecation warning in `ffill` and `bfill` APIs (#14554) This PR doesn't let the fillna warnings propagate to the user when ffill and bfill APIs are invoked. --- python/cudf/cudf/core/indexed_frame.py | 32 +++++++++++++++----------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 79d3bdf4fc3..246d5b934a5 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -2174,13 +2174,15 @@ def bfill(self, value=None, axis=None, inplace=None, limit=None): ------- Object with missing values filled or None if ``inplace=True``. """ - return self.fillna( - method="bfill", - value=value, - axis=axis, - inplace=inplace, - limit=limit, - ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + return self.fillna( + method="bfill", + value=value, + axis=axis, + inplace=inplace, + limit=limit, + ) @_cudf_nvtx_annotate def backfill(self, value=None, axis=None, inplace=None, limit=None): @@ -2211,13 +2213,15 @@ def ffill(self, value=None, axis=None, inplace=None, limit=None): ------- Object with missing values filled or None if ``inplace=True``. """ - return self.fillna( - method="ffill", - value=value, - axis=axis, - inplace=inplace, - limit=limit, - ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + return self.fillna( + method="ffill", + value=value, + axis=axis, + inplace=inplace, + limit=limit, + ) @_cudf_nvtx_annotate def pad(self, value=None, axis=None, inplace=None, limit=None): From a068b10fad7f6ca6a0cbe5cdb4be299e77e494f8 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 5 Dec 2023 10:15:28 -0500 Subject: [PATCH 073/384] Fix typo in value_counts (#14550) This PR fixes the return type of Series.value_counts to return int64, correcting a typo that was int34. --- python/cudf/cudf/core/series.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index c66b893f757..61d7c8d5437 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -3120,7 +3120,7 @@ def value_counts( if dropna and self.null_count == len(self): return Series( [], - dtype=np.int34, + dtype=np.int64, name=result_name, index=cudf.Index([], dtype=self.dtype, name=self.name), ) From ccfbe7161e729e3a6d1f7b232973cf827f55e113 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 5 Dec 2023 18:33:03 -0500 Subject: [PATCH 074/384] Enforce `Index.to_frame` deprecations (#14553) This PR enforces deprecations of Index.to_frame and updates pytests related to this API. --- python/cudf/cudf/core/_base_index.py | 20 +++++--------------- python/cudf/cudf/core/multiindex.py | 21 +++++++-------------- python/cudf/cudf/tests/test_index.py | 5 ++--- python/cudf/cudf/tests/test_multiindex.py | 22 ++++++++++------------ 4 files changed, 24 insertions(+), 44 deletions(-) diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index 61a5a4a5d68..0a70f3050eb 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -844,22 +844,12 @@ def to_frame(self, index=True, name=no_default): 1 Bear 2 Cow """ - if name is None: - warnings.warn( - "Explicitly passing `name=None` currently preserves " - "the Index's name or uses a default name of 0. This " - "behaviour is deprecated, and in the future `None` " - "will be used as the name of the " - "resulting DataFrame column.", - FutureWarning, - ) - name = no_default - if name is not no_default: - col_name = name - elif self.name is None: - col_name = 0 + + if name is no_default: + col_name = 0 if self.name is None else self.name else: - col_name = self.name + col_name = name + return cudf.DataFrame( {col_name: self._values}, index=self if index else None ) diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 3218d8a735f..5c2b4e6c7b0 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -6,7 +6,6 @@ import numbers import operator import pickle -import warnings from collections import abc from functools import cached_property from numbers import Integral @@ -1023,25 +1022,19 @@ def to_frame(self, index=True, name=no_default, allow_duplicates=False): # TODO: Currently this function makes a shallow copy, which is # incorrect. We want to make a deep copy, otherwise further # modifications of the resulting DataFrame will affect the MultiIndex. - if name is None: - warnings.warn( - "Explicitly passing `name=None` currently preserves the " - "Index's name or uses a default name of 0. This behaviour " - "is deprecated, and in the future `None` will be used " - "as the name of the resulting DataFrame column.", - FutureWarning, - ) - name = no_default - - if name is not no_default: + if name is no_default: + column_names = [ + level if name is None else name + for level, name in enumerate(self.names) + ] + else: if len(name) != len(self.levels): raise ValueError( "'name' should have the same length as " "number of levels on index." ) column_names = name - else: - column_names = self.names + all_none_names = None if not ( all_none_names := all(x is None for x in column_names) diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index 031143fc9f5..445fc84981b 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -3098,10 +3098,9 @@ def test_index_to_frame(data, data_name, index, name): pidx = pd.Index(data, name=data_name) gidx = cudf.from_pandas(pidx) - with expect_warning_if(name is None): + with expect_warning_if(not PANDAS_GE_200 and name is None): expected = pidx.to_frame(index=index, name=name) - with expect_warning_if(name is None): - actual = gidx.to_frame(index=index, name=name) + actual = gidx.to_frame(index=index, name=name) assert_eq(expected, actual) diff --git a/python/cudf/cudf/tests/test_multiindex.py b/python/cudf/cudf/tests/test_multiindex.py index ddaf83a4c9b..5fdeacc346f 100644 --- a/python/cudf/cudf/tests/test_multiindex.py +++ b/python/cudf/cudf/tests/test_multiindex.py @@ -1989,22 +1989,20 @@ def test_multiindex_to_frame_allow_duplicates( ) or (isinstance(name, list) and len(name) != len(set(name))): # cudf doesn't have the ability to construct dataframes # with duplicate column names - with expect_warning_if(name is None): - with pytest.raises(ValueError): - gidx.to_frame( - index=index, - name=name, - allow_duplicates=allow_duplicates, - ) + with pytest.raises(ValueError): + gidx.to_frame( + index=index, + name=name, + allow_duplicates=allow_duplicates, + ) else: - with expect_warning_if(name is None): + with expect_warning_if(not PANDAS_GE_200 and name is None): expected = pidx.to_frame( index=index, name=name, allow_duplicates=allow_duplicates ) - with expect_warning_if(name is None): - actual = gidx.to_frame( - index=index, name=name, allow_duplicates=allow_duplicates - ) + actual = gidx.to_frame( + index=index, name=name, allow_duplicates=allow_duplicates + ) assert_eq(expected, actual) From 9b478b002aa036c1b8252214b3911e7e10902db9 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 5 Dec 2023 19:59:22 -0500 Subject: [PATCH 075/384] Deprecate DataFrame.applymap and use map instead (#14579) Pandas 2.1.0 deprecated DataFrame.applymap, This PR deprecated applymap and introduces map to be used as the new alternative API. --- .../source/user_guide/api_docs/dataframe.rst | 5 +-- python/cudf/cudf/core/dataframe.py | 32 +++++++++++++++++++ python/cudf/cudf/tests/test_applymap.py | 19 +++++++---- python/cudf/cudf/tests/test_parquet.py | 2 +- 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/docs/cudf/source/user_guide/api_docs/dataframe.rst b/docs/cudf/source/user_guide/api_docs/dataframe.rst index dd685447025..90227541e4a 100644 --- a/docs/cudf/source/user_guide/api_docs/dataframe.rst +++ b/docs/cudf/source/user_guide/api_docs/dataframe.rst @@ -105,13 +105,14 @@ Function application, GroupBy & window .. autosummary:: :toctree: api/ + DataFrame.agg DataFrame.apply DataFrame.applymap DataFrame.apply_chunks DataFrame.apply_rows - DataFrame.pipe - DataFrame.agg DataFrame.groupby + DataFrame.map + DataFrame.pipe DataFrame.rolling .. _api.dataframe.stats: diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index a8e5aecfb30..3118dfa4490 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -4532,6 +4532,38 @@ def applymap( This method applies a function that accepts and returns a scalar to every element of a DataFrame. + Parameters + ---------- + func : callable + Python function, returns a single value from a single value. + na_action : {None, 'ignore'}, default None + If 'ignore', propagate NaN values, without passing them to func. + + Returns + ------- + DataFrame + Transformed DataFrame. + """ + # Do not remove until pandas 3.0 support is added. + warnings.warn( + "DataFrame.applymap has been deprecated. Use DataFrame.map " + "instead.", + FutureWarning, + ) + return self.map(func=func, na_action=na_action, **kwargs) + + def map( + self, + func: Callable[[Any], Any], + na_action: Union[str, None] = None, + **kwargs, + ) -> DataFrame: + """ + Apply a function to a Dataframe elementwise. + + This method applies a function that accepts and returns a scalar + to every element of a DataFrame. + Parameters ---------- func : callable diff --git a/python/cudf/cudf/tests/test_applymap.py b/python/cudf/cudf/tests/test_applymap.py index 32f3e39dd7c..9c0115fbc29 100644 --- a/python/cudf/cudf/tests/test_applymap.py +++ b/python/cudf/cudf/tests/test_applymap.py @@ -1,9 +1,10 @@ -# Copyright (c) 2018-2022, NVIDIA CORPORATION. +# Copyright (c) 2018-2023, NVIDIA CORPORATION. import pytest from cudf import NA, DataFrame from cudf.testing import _utils as utils +from cudf.core._compat import PANDAS_GE_210 @pytest.mark.parametrize( @@ -29,8 +30,10 @@ def test_applymap_dataframe(data, func, na_action): gdf = DataFrame(data) pdf = gdf.to_pandas(nullable=True) - expect = pdf.applymap(func, na_action=na_action) - got = gdf.applymap(func, na_action=na_action) + with utils.expect_warning_if(PANDAS_GE_210): + expect = pdf.applymap(func, na_action=na_action) + with pytest.warns(FutureWarning): + got = gdf.applymap(func, na_action=na_action) utils.assert_eq(expect, got, check_dtype=False) @@ -41,8 +44,10 @@ def test_applymap_raise_cases(): def f(x, some_kwarg=0): return x + some_kwarg - with pytest.raises(NotImplementedError): - df.applymap(f, some_kwarg=1) + with pytest.warns(FutureWarning): + with pytest.raises(NotImplementedError): + df.applymap(f, some_kwarg=1) - with pytest.raises(ValueError): - df.applymap(f, na_action="some_invalid_option") + with pytest.warns(FutureWarning): + with pytest.raises(ValueError): + df.applymap(f, na_action="some_invalid_option") diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index 4d16bb4857e..adadf147503 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -2823,7 +2823,7 @@ def postprocess(val): fname = datadir / "one_level_list2.parquet" expect = pd.read_parquet(fname) - expect = expect.applymap(postprocess) + expect = expect.map(postprocess) got = cudf.read_parquet(fname) assert_eq(expect, got, check_dtype=False) From 0e83e2094a3f2c9a1bc9f1f796eb174d715d70f6 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 7 Dec 2023 08:48:31 -0600 Subject: [PATCH 076/384] Deprecate first and last (#14583) This PR deprecates first and last APIs to bring parity with pandas, where these APIs were deprecated starting 2.1.0 --- python/cudf/cudf/core/indexed_frame.py | 12 ++++++++++++ python/cudf/cudf/tests/test_datetime.py | 25 ++++++++++++++++++------- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 246d5b934a5..ff626c12e0e 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -3318,6 +3318,12 @@ def first(self, offset): 2018-04-09 1 2018-04-11 2 """ + # Do not remove until pandas 3.0 support is added. + warnings.warn( + "first is deprecated and will be removed in a future version. " + "Please create a mask and filter using `.loc` instead", + FutureWarning, + ) return self._first_or_last( offset, idx=0, @@ -3364,6 +3370,12 @@ def last(self, offset): 2018-04-13 3 2018-04-15 4 """ + # Do not remove until pandas 3.0 support is added. + warnings.warn( + "last is deprecated and will be removed in a future version. " + "Please create a mask and filter using `.loc` instead", + FutureWarning, + ) return self._first_or_last( offset, idx=-1, diff --git a/python/cudf/cudf/tests/test_datetime.py b/python/cudf/cudf/tests/test_datetime.py index 521c1303a52..2368b3e539c 100644 --- a/python/cudf/cudf/tests/test_datetime.py +++ b/python/cudf/cudf/tests/test_datetime.py @@ -13,7 +13,12 @@ import warnings import cudf.testing.dataset_generator as dataset_generator from cudf import DataFrame, Series -from cudf.core._compat import PANDAS_GE_150, PANDAS_LT_140, PANDAS_EQ_200 +from cudf.core._compat import ( + PANDAS_GE_150, + PANDAS_LT_140, + PANDAS_EQ_200, + PANDAS_GE_210, +) from cudf.core.index import DatetimeIndex from cudf.testing._utils import ( DATETIME_TYPES, @@ -2070,8 +2075,10 @@ def test_first(idx, offset): p = pd.Series(range(len(idx)), dtype="int64", index=idx) g = cudf.from_pandas(p) - expect = p.first(offset=offset) - got = g.first(offset=offset) + with expect_warning_if(PANDAS_GE_210): + expect = p.first(offset=offset) + with pytest.warns(FutureWarning): + got = g.first(offset=offset) assert_eq(expect, got) @@ -2100,8 +2107,10 @@ def test_first_start_at_end_of_month(idx, offset): p = pd.Series(range(len(idx)), index=idx) g = cudf.from_pandas(p) - expect = p.first(offset=offset) - got = g.first(offset=offset) + with expect_warning_if(PANDAS_GE_210): + expect = p.first(offset=offset) + with pytest.warns(FutureWarning): + got = g.first(offset=offset) assert_eq(expect, got) @@ -2137,8 +2146,10 @@ def test_last(idx, offset): p = pd.Series(range(len(idx)), dtype="int64", index=idx) g = cudf.from_pandas(p) - expect = p.last(offset=offset) - got = g.last(offset=offset) + with expect_warning_if(PANDAS_GE_210): + expect = p.last(offset=offset) + with pytest.warns(FutureWarning): + got = g.last(offset=offset) assert_eq(expect, got) From 5f3ecd6a7909dc46d9d85dc2b2a162cff4a2c377 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 14 Dec 2023 00:37:49 +0530 Subject: [PATCH 077/384] Fix CategoricalDtype docstring (#14622) This PR fixes the docstring in CategoricalDtype where the repr has added a new field (categories_dtype). This PR fixes 2 doctest failures. --- python/cudf/cudf/core/dtypes.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/python/cudf/cudf/core/dtypes.py b/python/cudf/cudf/core/dtypes.py index 070aacd49c8..834b384d892 100644 --- a/python/cudf/cudf/core/dtypes.py +++ b/python/cudf/cudf/core/dtypes.py @@ -228,11 +228,11 @@ def from_pandas(cls, dtype: pd.CategoricalDtype) -> "CategoricalDtype": >>> import pandas as pd >>> pd_dtype = pd.CategoricalDtype(categories=['b', 'a'], ordered=True) >>> pd_dtype - CategoricalDtype(categories=['b', 'a'], ordered=True) + CategoricalDtype(categories=['b', 'a'], ordered=True, categories_dtype=object) >>> cudf_dtype = cudf.CategoricalDtype.from_pandas(pd_dtype) >>> cudf_dtype - CategoricalDtype(categories=['b', 'a'], ordered=True) - """ + CategoricalDtype(categories=['b', 'a'], ordered=True, categories_dtype=object) + """ # noqa: E501 return CategoricalDtype( categories=dtype.categories, ordered=dtype.ordered ) @@ -246,10 +246,10 @@ def to_pandas(self) -> pd.CategoricalDtype: >>> import cudf >>> dtype = cudf.CategoricalDtype(categories=['b', 'a'], ordered=True) >>> dtype - CategoricalDtype(categories=['b', 'a'], ordered=True) + CategoricalDtype(categories=['b', 'a'], ordered=True, categories_dtype=object) >>> dtype.to_pandas() - CategoricalDtype(categories=['b', 'a'], ordered=True) - """ + CategoricalDtype(categories=['b', 'a'], ordered=True, categories_dtype=object) + """ # noqa: E501 if self._categories is None: categories = None else: From 72221b3c72efbe521e08c350594a312c246024d9 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 14 Dec 2023 00:39:03 +0530 Subject: [PATCH 078/384] Fix `DataFrame.sort_index` when a index is a `MultiIndex` (#14621) This PR fixes sorting of a MultiIndex by removing an existing hard-coded na_position value that was based on ascending flag, essentially ignoring the user-passed parameter. On pandas_2.0_feature_branch: = 501 failed, 101106 passed, 2071 skipped, 786 xfailed, 312 xpassed, 20 errors in 1234.91s (0:20:34) = This PR: = 405 failed, 101034 passed, 2071 skipped, 954 xfailed, 312 xpassed, 20 errors in 1124.69s (0:18:44) = --- python/cudf/cudf/core/indexed_frame.py | 2 -- python/cudf/cudf/tests/test_dataframe.py | 19 ++++++++++--------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index ff626c12e0e..b4fba1eef07 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -1611,8 +1611,6 @@ def sort_index( idx = self.index if isinstance(idx, MultiIndex): if level is not None: - # Pandas doesn't handle na_position in case of MultiIndex. - na_position = "first" if ascending is True else "last" if not is_list_like(level): level = [level] by = list(map(idx._get_level_label, level)) diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index c89b5b507f5..e18c1809fd4 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -3492,8 +3492,16 @@ def test_dataframe_sort_index( @pytest.mark.parametrize("inplace", [True, False]) @pytest.mark.parametrize("na_position", ["first", "last"]) def test_dataframe_mulitindex_sort_index( - axis, level, ascending, inplace, ignore_index, na_position + request, axis, level, ascending, inplace, ignore_index, na_position ): + request.applymarker( + pytest.mark.xfail( + condition=axis in (1, "columns") + and ignore_index + and not (level is None and not ascending), + reason="https://github.com/pandas-dev/pandas/issues/56478", + ) + ) pdf = pd.DataFrame( { "b": [1.0, 3.0, np.nan], @@ -3505,17 +3513,14 @@ def test_dataframe_mulitindex_sort_index( ).set_index(["b", "a", 1]) gdf = cudf.DataFrame.from_pandas(pdf) - # ignore_index is supported in v.1.0 - expected = pdf.sort_index( axis=axis, level=level, ascending=ascending, inplace=inplace, na_position=na_position, + ignore_index=ignore_index, ) - if ignore_index is True: - expected = expected got = gdf.sort_index( axis=axis, level=level, @@ -3526,12 +3531,8 @@ def test_dataframe_mulitindex_sort_index( ) if inplace is True: - if ignore_index is True: - pdf = pdf.reset_index(drop=True) assert_eq(pdf, gdf) else: - if ignore_index is True: - expected = expected.reset_index(drop=True) assert_eq(expected, got) From d7dc16e85e2ab9bb38bb12c916e65ff8dd24e852 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 14 Dec 2023 00:40:04 +0530 Subject: [PATCH 079/384] Deprecate reading literal string in cudf.read_json (#14619) This PR deprecates reading literal strings in read_json, instead users will need to pass StringIO for these cases to silence the warning. This change is to match: pandas-dev/pandas#53409 On pandas_2.0_feature_branch: = 501 failed, 101106 passed, 2071 skipped, 786 xfailed, 312 xpassed, 20 errors in 1234.91s (0:20:34) = This PR: = 426 failed, 101181 passed, 2091 skipped, 786 xfailed, 312 xpassed in 1126.93s (0:18:46) = --- python/cudf/cudf/io/json.py | 2 + python/cudf/cudf/tests/test_json.py | 95 +++++++++++++++++++---------- python/cudf/cudf/tests/test_s3.py | 4 +- python/cudf/cudf/utils/ioutils.py | 29 +++++++++ 4 files changed, 96 insertions(+), 34 deletions(-) diff --git a/python/cudf/cudf/io/json.py b/python/cudf/cudf/io/json.py index ae2f0203642..b499fa23ede 100644 --- a/python/cudf/cudf/io/json.py +++ b/python/cudf/cudf/io/json.py @@ -102,6 +102,8 @@ def read_json( iotypes=(BytesIO, StringIO), allow_raw_text_input=True, storage_options=storage_options, + warn_on_raw_text_input=True, + warn_meta=("json", "read_json"), ) if isinstance(tmp_source, list): filepaths_or_buffers.extend(tmp_source) diff --git a/python/cudf/cudf/tests/test_json.py b/python/cudf/cudf/tests/test_json.py index f44b7495aab..5bc9a33fd8d 100644 --- a/python/cudf/cudf/tests/test_json.py +++ b/python/cudf/cudf/tests/test_json.py @@ -13,12 +13,13 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_200 +from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210 from cudf.testing._utils import ( DATETIME_TYPES, NUMERIC_TYPES, TIMEDELTA_TYPES, assert_eq, + expect_warning_if, ) @@ -95,6 +96,8 @@ def json_files(request, tmp_path_factory, pdf): ) if index is False and orient == "table": pytest.skip("'index=False' isn't valid when 'orient' is 'table'") + if index is True and orient not in ("split", "table", "index", "columns"): + pytest.skip("'index=False' isn't valid when 'orient' is 'table'") fname_df = tmp_path_factory.mktemp("json") / "test_df.json" fname_series = tmp_path_factory.mktemp("json") / "test_series.json" pdf.to_json(fname_df, index=index, compression=compression, orient=orient) @@ -338,8 +341,16 @@ def json_input(request, tmp_path_factory): @pytest.mark.filterwarnings("ignore:Using CPU") @pytest.mark.parametrize("engine", ["auto", "cudf", "pandas"]) def test_json_lines_basic(json_input, engine): - cu_df = cudf.read_json(json_input, engine=engine, lines=True) - pd_df = pd.read_json(json_input, lines=True) + with expect_warning_if( + isinstance(json_input, str) and not json_input.endswith(".json") + ): + cu_df = cudf.read_json(json_input, engine=engine, lines=True) + with expect_warning_if( + isinstance(json_input, str) + and PANDAS_GE_210 + and not json_input.endswith(".json") + ): + pd_df = pd.read_json(json_input, lines=True) assert all(cu_df.dtypes == ["int64", "int64", "int64"]) for cu_col, pd_col in zip(cu_df.columns, pd_df.columns): @@ -353,7 +364,12 @@ def test_json_lines_multiple(tmpdir, json_input, engine): tmp_file1 = tmpdir.join("MultiInputs1.json") tmp_file2 = tmpdir.join("MultiInputs2.json") - pdf = pd.read_json(json_input, lines=True) + with expect_warning_if( + isinstance(json_input, str) + and PANDAS_GE_210 + and not json_input.endswith(".json") + ): + pdf = pd.read_json(json_input, lines=True) pdf.to_json(tmp_file1, compression="infer", lines=True, orient="records") pdf.to_json(tmp_file2, compression="infer", lines=True, orient="records") @@ -368,7 +384,12 @@ def test_json_lines_multiple(tmpdir, json_input, engine): @pytest.mark.parametrize("engine", ["auto", "cudf"]) def test_json_read_directory(tmpdir, json_input, engine): - pdf = pd.read_json(json_input, lines=True) + with expect_warning_if( + isinstance(json_input, str) + and PANDAS_GE_210 + and not json_input.endswith(".json") + ): + pdf = pd.read_json(json_input, lines=True) pdf.to_json( tmpdir.join("MultiInputs1.json"), compression="infer", @@ -400,37 +421,47 @@ def test_json_read_directory(tmpdir, json_input, engine): def test_json_lines_byte_range(json_input): # include the first row and half of the second row # should parse the first two rows - df = cudf.read_json( - copy.deepcopy(json_input), lines=True, byte_range=(0, 15) + will_warn = isinstance(json_input, str) and not json_input.endswith( + ".json" ) + with expect_warning_if(will_warn): + df = cudf.read_json( + copy.deepcopy(json_input), lines=True, byte_range=(0, 15) + ) assert df.shape == (2, 3) # include half of the second row and half of the third row # should parse only the third row - df = cudf.read_json( - copy.deepcopy(json_input), lines=True, byte_range=(15, 10) - ) + with expect_warning_if(will_warn): + df = cudf.read_json( + copy.deepcopy(json_input), lines=True, byte_range=(15, 10) + ) assert df.shape == (1, 3) # include half of the second row and entire third row # should parse only the third row - df = cudf.read_json( - copy.deepcopy(json_input), lines=True, byte_range=(15, 0) - ) + with expect_warning_if(will_warn): + df = cudf.read_json( + copy.deepcopy(json_input), lines=True, byte_range=(15, 0) + ) assert df.shape == (1, 3) # include half of the second row till past the end of the file # should parse only the third row - df = cudf.read_json( - copy.deepcopy(json_input), lines=True, byte_range=(10, 50) - ) + with expect_warning_if(will_warn): + df = cudf.read_json( + copy.deepcopy(json_input), lines=True, byte_range=(10, 50) + ) assert df.shape == (1, 3) def test_json_lines_dtypes(json_input): - df = cudf.read_json( - json_input, lines=True, dtype={1: "int", 2: "short", 0: "float"} - ) + with expect_warning_if( + isinstance(json_input, str) and not json_input.endswith(".json") + ): + df = cudf.read_json( + json_input, lines=True, dtype={1: "int", 2: "short", 0: "float"} + ) assert all(df.dtypes == ["float64", "int64", "int16"]) @@ -470,32 +501,32 @@ def test_json_engine_selection(): json = "[1, 2, 3]" # should use the cudf engine - df = cudf.read_json(json, lines=True) + df = cudf.read_json(StringIO(json), lines=True) # column names are strings when parsing with cudf for col_name in df.columns: assert isinstance(col_name, str) # should use the pandas engine - df = cudf.read_json(json, lines=False, engine="pandas") + df = cudf.read_json(StringIO(json), lines=False, engine="pandas") # column names are ints when parsing with pandas for col_name in df.columns: assert isinstance(col_name, int) # should use the pandas engine - df = cudf.read_json(json, lines=True, engine="pandas") + df = cudf.read_json(StringIO(json), lines=True, engine="pandas") # column names are ints when parsing with pandas for col_name in df.columns: assert isinstance(col_name, int) # should raise an exception with pytest.raises(ValueError): - cudf.read_json(json, lines=False, engine="cudf_legacy") + cudf.read_json(StringIO(json), lines=False, engine="cudf_legacy") def test_json_bool_values(): buffer = "[true,1]\n[false,false]\n[true,true]" - cu_df = cudf.read_json(buffer, lines=True) - pd_df = pd.read_json(buffer, lines=True) + cu_df = cudf.read_json(StringIO(buffer), lines=True) + pd_df = pd.read_json(StringIO(buffer), lines=True) # types should be ['bool', 'int64'] np.testing.assert_array_equal(pd_df.dtypes, cu_df.dtypes) @@ -504,7 +535,7 @@ def test_json_bool_values(): np.testing.assert_array_equal(pd_df[1], cu_df["1"].to_numpy()) cu_df = cudf.read_json( - buffer, lines=True, dtype={"0": "bool", "1": "long"} + StringIO(buffer), lines=True, dtype={"0": "bool", "1": "long"} ) np.testing.assert_array_equal(pd_df.dtypes, cu_df.dtypes) @@ -522,7 +553,7 @@ def test_json_bool_values(): ], ) def test_json_null_literal(buffer): - df = cudf.read_json(buffer, lines=True, engine="cudf_legacy") + df = cudf.read_json(StringIO(buffer), lines=True, engine="cudf_legacy") # first column contains a null field, type should be set to float # second column contains only empty fields, type should be set to int8 @@ -534,7 +565,7 @@ def test_json_null_literal(buffer): def test_json_bad_protocol_string(): - test_string = '{"field": "s3://path"}' + test_string = StringIO('{"field": "s3://path"}') expect = pd.DataFrame([{"field": "s3://path"}]) got = cudf.read_json(test_string, lines=True) @@ -748,7 +779,7 @@ def test_default_integer_bitwidth_extremes(default_integer_bitwidth, engine): def test_default_float_bitwidth(default_float_bitwidth): # Test that float columns in json are _inferred_ as 32 bit columns. df = cudf.read_json( - '{"a": 1.0, "b": 2.5}\n{"a": 3.5, "b": 4.0}', + StringIO('{"a": 1.0, "b": 2.5}\n{"a": 3.5, "b": 4.0}'), engine="cudf", lines=True, orient="records", @@ -1231,7 +1262,7 @@ def test_json_round_trip_gzip(): @pytest.mark.parametrize("lines", [True, False]) def test_json_array_of_arrays(data, lines): data = data if lines else "[" + data.replace("\n", ",") + "]" - pdf = pd.read_json(data, orient="values", lines=lines) + pdf = pd.read_json(StringIO(data), orient="values", lines=lines) df = cudf.read_json( StringIO(data), engine="cudf", @@ -1325,8 +1356,8 @@ def _replace_with_nulls(df, replace_items): # both json lines and json string tested. json_string = "[" + jsonl_string.replace("\n", ",") + "]" - pdf = pd.read_json(jsonl_string, orient="records", lines=True) - pdf2 = pd.read_json(json_string, orient="records", lines=False) + pdf = pd.read_json(StringIO(jsonl_string), orient="records", lines=True) + pdf2 = pd.read_json(StringIO(json_string), orient="records", lines=False) assert_eq(pdf, pdf2) # replace list elements with None if it has dict and non-dict # in above test cases, these items are mixed with dict/list items diff --git a/python/cudf/cudf/tests/test_s3.py b/python/cudf/cudf/tests/test_s3.py index b92f84b677c..8db46f87d65 100644 --- a/python/cudf/cudf/tests/test_s3.py +++ b/python/cudf/cudf/tests/test_s3.py @@ -3,7 +3,7 @@ import os import socket from contextlib import contextmanager -from io import BytesIO +from io import BytesIO, StringIO import numpy as np import pandas as pd @@ -433,7 +433,7 @@ def test_read_json(s3_base, s3so): storage_options=s3so, ) - expect = pd.read_json(buffer, lines=True) + expect = pd.read_json(StringIO(buffer), lines=True) assert_eq(expect, got) diff --git a/python/cudf/cudf/utils/ioutils.py b/python/cudf/cudf/utils/ioutils.py index 6641bd8290a..c3b89d64435 100644 --- a/python/cudf/cudf/utils/ioutils.py +++ b/python/cudf/cudf/utils/ioutils.py @@ -1666,6 +1666,8 @@ def get_reader_filepath_or_buffer( allow_raw_text_input=False, storage_options=None, bytes_per_thread=_BYTES_PER_THREAD_DEFAULT, + warn_on_raw_text_input=None, + warn_meta=None, ): """{docstring}""" @@ -1679,6 +1681,15 @@ def get_reader_filepath_or_buffer( path_or_data, storage_options ) if fs is None: + if warn_on_raw_text_input: + # Do not remove until pandas 3.0 support is added. + warnings.warn( + f"Passing literal {warn_meta[0]} to {warn_meta[1]} is " + "deprecated and will be removed in a future version. " + "To read from a literal string, wrap it in a " + "'StringIO' object.", + FutureWarning, + ) return path_or_data, compression if _is_local_filesystem(fs): @@ -1691,6 +1702,24 @@ def get_reader_filepath_or_buffer( raise FileNotFoundError( f"{path_or_data} could not be resolved to any files" ) + elif warn_on_raw_text_input: + # Do not remove until pandas 3.0 support is added. + warnings.warn( + f"Passing literal {warn_meta[0]} to {warn_meta[1]} is " + "deprecated and will be removed in a future version. " + "To read from a literal string, wrap it in a " + "'StringIO' object.", + FutureWarning, + ) + elif warn_on_raw_text_input: + # Do not remove until pandas 3.0 support is added. + warnings.warn( + f"Passing literal {warn_meta[0]} to {warn_meta[1]} is " + "deprecated and will be removed in a future version. " + "To read from a literal string, wrap it in a " + "'StringIO' object.", + FutureWarning, + ) else: if len(paths) == 0: From eea5f107cbb062cc47c935728bb1ae234729de09 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Sun, 17 Dec 2023 01:08:23 +0530 Subject: [PATCH 080/384] Preserve column ordering in DataFrame.stack (#14626) This PR preserves original column ordering in DataFrame.stack On pandas_2.0_feature_branch: = 328 failed, 101111 passed, 2091 skipped, 954 xfailed, 312 xpassed in 1113.40s (0:18:33) = This PR: = 316 failed, 101123 passed, 2091 skipped, 954 xfailed, 312 xpassed in 1123.65s (0:18:43) = --- python/cudf/cudf/core/dataframe.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 3118dfa4490..50fe5adebf8 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -6749,11 +6749,11 @@ def stack(self, level=-1, dropna=True): cat 1.0 2.0 dog 3.0 4.0 >>> df_multi_level_cols2.stack() - height weight - cat kg 1.0 - m 2.0 - dog kg 3.0 - m 4.0 + weight height + cat kg 1.0 + m 2.0 + dog kg 3.0 + m 4.0 **Prescribing the level(s) to be stacked** @@ -6925,10 +6925,18 @@ def unnamed_group_generator(): else: if unnamed_level_values.nlevels == 1: unnamed_level_values = unnamed_level_values.get_level_values(0) - unnamed_level_values = unnamed_level_values.unique().sort_values() + unnamed_level_values = unnamed_level_values.unique() data = ColumnAccessor( - dict(zip(unnamed_level_values, stacked)), + dict( + zip( + unnamed_level_values, + [ + stacked[i] + for i in unnamed_level_values.argsort().argsort() + ], + ) + ), isinstance(unnamed_level_values, pd.MultiIndex), unnamed_level_values.names, ) From bc5584b159671f9e92281e22a44d8da9610d8748 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 18 Dec 2023 23:01:19 +0530 Subject: [PATCH 081/384] Change `is_.._dtype` deprecations to `DeprecationWarning` instead of `FutureWarning` (#14617) This PR changes all FutureWarning's to DeprecationWarning's to match with pandas: pandas-dev/pandas#55703 On pandas_2.0_feature_branch: = 501 failed, 101106 passed, 2071 skipped, 786 xfailed, 312 xpassed, 20 errors in 1234.91s (0:20:34) = This PR: = 445 failed, 101162 passed, 2071 skipped, 786 xfailed, 312 xpassed, 20 errors in 1216.79s (0:20:16) = --- .pre-commit-config.yaml | 4 ++++ python/cudf/cudf/core/_compat.py | 1 + python/cudf/cudf/core/dtypes.py | 4 ++-- python/cudf/cudf/tests/test_api_types.py | 8 +++++--- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7db8d9ab52f..d14a34ad1a3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -91,6 +91,10 @@ repos: entry: '(category=|\s)DeprecationWarning[,)]' language: pygrep types_or: [python, cython] + exclude: | + (?x)^( + ^python/cudf/cudf/core/dtypes.py + ) - id: no-programmatic-xfail name: no-programmatic-xfail description: 'Enforce that pytest.xfail is not introduced (see dev docs for details)' diff --git a/python/cudf/cudf/core/_compat.py b/python/cudf/cudf/core/_compat.py index d0b6dcf2e6d..b57fa4e83ed 100644 --- a/python/cudf/cudf/core/_compat.py +++ b/python/cudf/cudf/core/_compat.py @@ -12,4 +12,5 @@ PANDAS_EQ_200 = PANDAS_VERSION == version.parse("2.0.0") PANDAS_GE_200 = PANDAS_VERSION >= version.parse("2.0.0") PANDAS_GE_210 = PANDAS_VERSION >= version.parse("2.1.0") +PANDAS_GE_214 = PANDAS_VERSION >= version.parse("2.1.4") PANDAS_GE_220 = PANDAS_VERSION >= version.parse("2.2.0") diff --git a/python/cudf/cudf/core/dtypes.py b/python/cudf/cudf/core/dtypes.py index 834b384d892..c32969a401b 100644 --- a/python/cudf/cudf/core/dtypes.py +++ b/python/cudf/cudf/core/dtypes.py @@ -1043,7 +1043,7 @@ def is_categorical_dtype(obj): warnings.warn( "is_categorical_dtype is deprecated and will be removed in a future " "version. Use isinstance(dtype, cudf.CategoricalDtype) instead", - FutureWarning, + DeprecationWarning, ) return _is_categorical_dtype(obj) @@ -1151,7 +1151,7 @@ def is_interval_dtype(obj): warnings.warn( "is_interval_dtype is deprecated and will be removed in a " "future version. Use `isinstance(dtype, cudf.IntervalDtype)` instead", - FutureWarning, + DeprecationWarning, ) return _is_interval_dtype(obj) diff --git a/python/cudf/cudf/tests/test_api_types.py b/python/cudf/cudf/tests/test_api_types.py index d640e8e1376..4d617056c10 100644 --- a/python/cudf/cudf/tests/test_api_types.py +++ b/python/cudf/cudf/tests/test_api_types.py @@ -6,7 +6,7 @@ from pandas.api import types as pd_types import cudf -from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210 +from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210, PANDAS_GE_214 from cudf.api import types from cudf.testing._utils import expect_warning_if @@ -1038,9 +1038,11 @@ def test_is_decimal_dtype(obj, expect): ), ) def test_pandas_agreement(obj): - with expect_warning_if(PANDAS_GE_210): + with expect_warning_if( + PANDAS_GE_210, DeprecationWarning if PANDAS_GE_214 else FutureWarning + ): expected = pd_types.is_categorical_dtype(obj) - with pytest.warns(FutureWarning): + with pytest.warns(DeprecationWarning): actual = types.is_categorical_dtype(obj) assert expected == actual assert types.is_numeric_dtype(obj) == pd_types.is_numeric_dtype(obj) From 194e487edf838054e937091472955daa343dd286 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 19 Dec 2023 05:32:31 +0530 Subject: [PATCH 082/384] Version dataframe.mode pytest (#14650) This PR versions the xfail properly to not fail in version of pandas where this bug is fixed. --- python/cudf/cudf/core/_compat.py | 1 + python/cudf/cudf/tests/test_dataframe.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/python/cudf/cudf/core/_compat.py b/python/cudf/cudf/core/_compat.py index b57fa4e83ed..c326b19307d 100644 --- a/python/cudf/cudf/core/_compat.py +++ b/python/cudf/cudf/core/_compat.py @@ -14,3 +14,4 @@ PANDAS_GE_210 = PANDAS_VERSION >= version.parse("2.1.0") PANDAS_GE_214 = PANDAS_VERSION >= version.parse("2.1.4") PANDAS_GE_220 = PANDAS_VERSION >= version.parse("2.2.0") +PANDAS_LT_203 = PANDAS_VERSION < version.parse("2.0.3") diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index e18c1809fd4..f3cd65a72a1 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -27,6 +27,7 @@ PANDAS_GE_200, PANDAS_GE_210, PANDAS_LT_140, + PANDAS_LT_203, ) from cudf.api.extensions import no_default from cudf.core.buffer.spill_manager import get_global_manager @@ -8593,6 +8594,7 @@ def test_dataframe_mode(request, df, numeric_only, dropna): request.applymarker( pytest.mark.xfail( condition=PANDAS_GE_200 + and PANDAS_LT_203 and numeric_only is False and "b" in df.columns and df["b"].dtype == np.dtype("timedelta64[s]"), From f736d72c5e1e8d400d9335e39e8ca7c42ef33263 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 19 Dec 2023 07:10:38 +0530 Subject: [PATCH 083/384] Filter ufunc related warnings in pytests (#14652) This PR ignores ufunc runtime warnings that show up in eval API and setitem deprecation warnings. On pandas_2.0_feature_branch: = 260 failed, 101179 passed, 2091 skipped, 954 xfailed, 312 xpassed in 1104.58s (0:18:24) = This PR: = 211 failed, 101228 passed, 2091 skipped, 954 xfailed, 312 xpassed in 1095.49s (0:18:15) = --- python/cudf/cudf/tests/test_array_ufunc.py | 67 ++++++++++++++++++++-- python/cudf/cudf/tests/test_dataframe.py | 34 ++++++++++- 2 files changed, 95 insertions(+), 6 deletions(-) diff --git a/python/cudf/cudf/tests/test_array_ufunc.py b/python/cudf/cudf/tests/test_array_ufunc.py index 305f935bcb4..40966f6b6c9 100644 --- a/python/cudf/cudf/tests/test_array_ufunc.py +++ b/python/cudf/cudf/tests/test_array_ufunc.py @@ -10,8 +10,12 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_150, PANDAS_GE_200 -from cudf.testing._utils import assert_eq, set_random_null_mask_inplace +from cudf.core._compat import PANDAS_GE_150, PANDAS_GE_200, PANDAS_GE_210 +from cudf.testing._utils import ( + assert_eq, + set_random_null_mask_inplace, + expect_warning_if, +) _UFUNCS = [ obj @@ -47,6 +51,21 @@ def _hide_ufunc_warnings(ufunc): category=RuntimeWarning, ) yield + elif name in { + "bitwise_and", + "bitwise_or", + "bitwise_xor", + }: + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + "Operation between non boolean Series with different " + "indexes will no longer return a boolean result in " + "a future version. Cast both Series to object type " + "to maintain the prior behavior.", + category=FutureWarning, + ) + yield else: yield @@ -217,7 +236,27 @@ def test_ufunc_series(request, ufunc, has_nulls, indexed): assert_eq(g, e, check_exact=False) else: if has_nulls: - expect[mask] = np.nan + with expect_warning_if( + PANDAS_GE_210 + and fname + in ( + "isfinite", + "isinf", + "isnan", + "logical_and", + "logical_not", + "logical_or", + "logical_xor", + "signbit", + "equal", + "greater", + "greater_equal", + "less", + "less_equal", + "not_equal", + ) + ): + expect[mask] = np.nan assert_eq(got, expect, check_exact=False) @@ -443,5 +482,25 @@ def test_ufunc_dataframe(request, ufunc, has_nulls, indexed): assert_eq(g, e, check_exact=False) else: if has_nulls: - expect[mask] = np.nan + with expect_warning_if( + PANDAS_GE_210 + and fname + in ( + "isfinite", + "isinf", + "isnan", + "logical_and", + "logical_not", + "logical_or", + "logical_xor", + "signbit", + "equal", + "greater", + "greater_equal", + "less", + "less_equal", + "not_equal", + ) + ): + expect[mask] = np.nan assert_eq(got, expect, check_exact=False) diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index f3cd65a72a1..6e9b9a37ac0 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -9,6 +9,8 @@ import re import string import textwrap +import warnings +from contextlib import contextmanager from collections import OrderedDict, defaultdict, namedtuple from copy import copy @@ -65,6 +67,32 @@ pytest_xfail = pytest.mark.skipif +@contextmanager +def _hide_ufunc_warnings(eval_str): + # pandas raises warnings for some inputs to the following ufuncs: + if any( + x in eval_str + for x in { + "arctanh", + "log", + } + ): + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + "invalid value encountered in", + category=RuntimeWarning, + ) + warnings.filterwarnings( + "ignore", + "divide by zero encountered in", + category=RuntimeWarning, + ) + yield + else: + yield + + def test_init_via_list_of_tuples(): data = [ (5, "cats", "jump", np.nan), @@ -10071,7 +10099,8 @@ def df_eval(request): ) def test_dataframe_eval(df_eval, expr, dtype): df_eval = df_eval.astype(dtype) - expect = df_eval.to_pandas().eval(expr) + with _hide_ufunc_warnings(expr): + expect = df_eval.to_pandas().eval(expr) got = df_eval.eval(expr) # In the specific case where the evaluated expression is a unary function # of a single column with no nesting, pandas will retain the name. This @@ -10081,7 +10110,8 @@ def test_dataframe_eval(df_eval, expr, dtype): # Test inplace if re.search("[^=><]=[^=]", expr) is not None: pdf_eval = df_eval.to_pandas() - pdf_eval.eval(expr, inplace=True) + with _hide_ufunc_warnings(expr): + pdf_eval.eval(expr, inplace=True) df_eval.eval(expr, inplace=True) assert_eq(pdf_eval, df_eval) From 4539f4f83b4297039c56c87b6fab741994b61334 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 19 Dec 2023 07:11:53 +0530 Subject: [PATCH 084/384] Deprecate positional access for label based indexes in Series.__getitem__ (#14654) This PR deprecates positional access in `Series.__getitem__` when a label-based index is present. xref: https://github.com/pandas-dev/pandas/pull/53201 On `pandas_2.0_feature_branch`: ``` = 260 failed, 101179 passed, 2091 skipped, 954 xfailed, 312 xpassed in 1104.58s (0:18:24) = ``` This PR: ``` = 248 failed, 101190 passed, 2091 skipped, 954 xfailed, 312 xpassed in 1105.78s (0:18:25) = ``` --- python/cudf/cudf/core/series.py | 16 ++++++++++++---- python/cudf/cudf/tests/test_csv.py | 4 ++-- python/cudf/cudf/tests/test_indexing.py | 10 +++++++--- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 61d7c8d5437..5876a577b87 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -371,6 +371,12 @@ def _loc_to_iloc(self, arg): arg = arg[0] if _is_scalar_or_zero_d_array(arg): index_dtype = self._frame.index.dtype + warn_msg = ( + "Series.__getitem__ treating keys as positions is deprecated. " + "In a future version, integer keys will always be treated " + "as labels (consistent with DataFrame behavior). To access " + "a value by position, use `ser.iloc[pos]`" + ) if not _is_non_decimal_numeric_dtype(index_dtype) and not ( isinstance(index_dtype, cudf.CategoricalDtype) and is_integer_dtype(index_dtype.categories.dtype) @@ -379,11 +385,13 @@ def _loc_to_iloc(self, arg): if isinstance(arg, cudf.Scalar) and is_integer_dtype( arg.dtype ): - found_index = arg.value - return found_index + # Do not remove until pandas 3.0 support is added. + warnings.warn(warn_msg, FutureWarning) + return arg.value elif is_integer(arg): - found_index = arg - return found_index + # Do not remove until pandas 3.0 support is added. + warnings.warn(warn_msg, FutureWarning) + return arg try: indices = self._frame.index._indices_of(arg) if (n := len(indices)) == 0: diff --git a/python/cudf/cudf/tests/test_csv.py b/python/cudf/cudf/tests/test_csv.py index 1ccf91fe63e..cbb262cd649 100644 --- a/python/cudf/cudf/tests/test_csv.py +++ b/python/cudf/cudf/tests/test_csv.py @@ -595,12 +595,12 @@ def test_csv_reader_NaN_values(): header=None, na_values=custom_na_values, ) - assert gdf.dtypes[0] == "int8" + assert gdf.dtypes.iloc[0] == "int8" assert all(gdf["0"][idx] is cudf.NA for idx in range(len(gdf["0"]))) # data type detection should evaluate the column to object if some nulls gdf = read_csv(StringIO(all_cells), header=None) - assert gdf.dtypes[0] == np.dtype("object") + assert gdf.dtypes.iloc[0] == np.dtype("object") def test_csv_reader_thousands(tmpdir): diff --git a/python/cudf/cudf/tests/test_indexing.py b/python/cudf/cudf/tests/test_indexing.py index 27e84f179b6..e921a6ccf3f 100644 --- a/python/cudf/cudf/tests/test_indexing.py +++ b/python/cudf/cudf/tests/test_indexing.py @@ -9,11 +9,13 @@ import pytest import cudf +from cudf.core._compat import PANDAS_GE_210 from cudf.testing import _utils as utils from cudf.testing._utils import ( INTEGER_TYPES, assert_eq, assert_exceptions_equal, + expect_warning_if, ) index_dtypes = INTEGER_TYPES @@ -151,8 +153,10 @@ def test_series_get_item_iloc_defer(arg): ps = pd.Series([1, 2, 3], index=pd.Index(["a", "b", "c"])) gs = cudf.from_pandas(ps) - expect = ps[arg] - got = gs[arg] + with expect_warning_if(PANDAS_GE_210 and not isinstance(arg, str)): + expect = ps[arg] + with expect_warning_if(not isinstance(arg, str)): + got = gs[arg] assert_eq(expect, got) @@ -163,7 +167,7 @@ def test_series_iloc_defer_cudf_scalar(): for t in index_dtypes: arg = cudf.Scalar(1, dtype=t) - got = gs[arg] + got = gs.iloc[arg] expect = 2 assert_eq(expect, got) From c1411b6c1df40d98d1c3172175f73fc56e1b7a82 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 28 Dec 2023 00:17:57 +0530 Subject: [PATCH 085/384] Deprecate `method` in `interpolate` and calculation on `object` dtype (#14667) This PR: - [x] Deprecates `method` in `interpolate`. - [x] Deprecates performing `interpolate` on string columns. On `pandas_2.0_feature_branch`: ``` = 198 failed, 101241 passed, 2091 skipped, 954 xfailed, 312 xpassed in 1098.81s (0:18:18) = ``` This PR: ``` = 187 failed, 101252 passed, 2091 skipped, 954 xfailed, 312 xpassed in 1090.48s (0:18:10) = ``` --- python/cudf/cudf/core/indexed_frame.py | 16 +++++++++++- python/cudf/cudf/tests/test_interpolate.py | 29 ++++++++++++++-------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index b4fba1eef07..d2223ff004a 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -40,8 +40,8 @@ ) from cudf.api.extensions import no_default from cudf.api.types import ( - _is_non_decimal_numeric_dtype, _is_categorical_dtype, + _is_non_decimal_numeric_dtype, is_bool_dtype, is_decimal_dtype, is_dict_like, @@ -1067,6 +1067,14 @@ def interpolate( f"`limit_direction` must be 'backward' for method `{method}`" ) + if method.lower() in {"ffill", "bfill", "pad", "backfill"}: + warnings.warn( + f"{type(self).__name__}.interpolate with method={method} is " + "deprecated and will raise in a future version. " + "Use obj.ffill() or obj.bfill() instead.", + FutureWarning, + ) + data = self if not isinstance(data._index, cudf.RangeIndex): @@ -1082,6 +1090,12 @@ def interpolate( interpolator = cudf.core.algorithms.get_column_interpolator(method) columns = {} for colname, col in data._data.items(): + if isinstance(col, cudf.core.column.StringColumn): + warnings.warn( + f"{type(self).__name__}.interpolate with object dtype is " + "deprecated and will raise in a future version.", + FutureWarning, + ) if col.nullable: col = col.astype("float64").fillna(np.nan) diff --git a/python/cudf/cudf/tests/test_interpolate.py b/python/cudf/cudf/tests/test_interpolate.py index c0b085a5097..3acda9165fd 100644 --- a/python/cudf/cudf/tests/test_interpolate.py +++ b/python/cudf/cudf/tests/test_interpolate.py @@ -1,9 +1,14 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. import pytest import cudf -from cudf.testing._utils import assert_eq, assert_exceptions_equal +from cudf.core._compat import PANDAS_GE_210 +from cudf.testing._utils import ( + assert_eq, + assert_exceptions_equal, + expect_warning_if, +) @pytest.mark.parametrize( @@ -49,8 +54,10 @@ def test_interpolate_series(data, method, axis): gsr = cudf.Series(data) psr = gsr.to_pandas() - expect = psr.interpolate(method=method, axis=axis) - got = gsr.interpolate(method=method, axis=axis) + with expect_warning_if(PANDAS_GE_210 and psr.dtype == "object"): + expect = psr.interpolate(method=method, axis=axis) + with expect_warning_if(gsr.dtype == "object"): + got = gsr.interpolate(method=method, axis=axis) assert_eq(expect, got, check_dtype=psr.dtype != "object") @@ -87,8 +94,10 @@ def test_interpolate_series_values_or_index(data, index, method): gsr = cudf.Series(data, index=index) psr = gsr.to_pandas() - expect = psr.interpolate(method=method) - got = gsr.interpolate(method=method) + with expect_warning_if(PANDAS_GE_210 and gsr.dtype == "object"): + expect = psr.interpolate(method=method) + with expect_warning_if(gsr.dtype == "object"): + got = gsr.interpolate(method=method) assert_eq(expect, got, check_dtype=psr.dtype != "object") @@ -100,12 +109,12 @@ def test_interpolate_series_values_or_index(data, index, method): {"A": ["a", "b", "c"], "B": ["d", "e", "f"]}, {"axis": 0, "method": "linear"}, ), - ({"A": [1, 2, 3]}, {"method": "pad", "limit_direction": "backward"}), - ({"A": [1, 2, 3]}, {"method": "ffill", "limit_direction": "backward"}), - ({"A": [1, 2, 3]}, {"method": "bfill", "limit_direction": "forward"}), + ({"A": [1, 2, 3]}, {"method": "pad", "limit_direction": "forward"}), + ({"A": [1, 2, 3]}, {"method": "ffill", "limit_direction": "forward"}), + ({"A": [1, 2, 3]}, {"method": "bfill", "limit_direction": "backward"}), ( {"A": [1, 2, 3]}, - {"method": "backfill", "limit_direction": "forward"}, + {"method": "backfill", "limit_direction": "backward"}, ), ], ) From 2b9ab53599511b636ed5067ff7d18c617c1172b5 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 28 Dec 2023 00:19:56 +0530 Subject: [PATCH 086/384] Add more validation to MultiIndex.to_frame (#14671) This PR adds validation to `name` inputs in `MultiIndex.to_frame` API. On `pandas_2.0_feature_branch`: ``` = 198 failed, 101241 passed, 2091 skipped, 954 xfailed, 312 xpassed in 1098.81s (0:18:18) = ``` This PR: ``` = 180 failed, 101247 passed, 2091 skipped, 954 xfailed, 312 xpassed in 1025.07s (0:17:05) = ``` --- python/cudf/cudf/core/multiindex.py | 4 ++++ python/cudf/cudf/tests/test_multiindex.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 5c2b4e6c7b0..a2cc5450ca4 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1028,6 +1028,10 @@ def to_frame(self, index=True, name=no_default, allow_duplicates=False): for level, name in enumerate(self.names) ] else: + if not is_list_like(name): + raise TypeError( + "'name' must be a list / sequence of column names." + ) if len(name) != len(self.levels): raise ValueError( "'name' should have the same length as " diff --git a/python/cudf/cudf/tests/test_multiindex.py b/python/cudf/cudf/tests/test_multiindex.py index 5fdeacc346f..0cdc0e42cc1 100644 --- a/python/cudf/cudf/tests/test_multiindex.py +++ b/python/cudf/cudf/tests/test_multiindex.py @@ -1953,13 +1953,13 @@ def test_multiindex_to_frame_allow_duplicates( ): gidx = cudf.from_pandas(pidx) - if ( + if name is None or ( ( len(pidx.names) != len(set(pidx.names)) and not all(x is None for x in pidx.names) ) and not allow_duplicates - and (name is None or name is no_default) + and name is no_default ): assert_exceptions_equal( pidx.to_frame, From 46ef14838e6aa97e1400ae704bd47e9b97c86324 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 28 Dec 2023 00:20:40 +0530 Subject: [PATCH 087/384] Deprecate ignoring empty objects in concat (#14672) This PR deprecates ignoring `empty` objects for dtype calculation in `concat`. On `pandas_2.0_feature_branch`: ``` = 198 failed, 101241 passed, 2091 skipped, 954 xfailed, 312 xpassed in 1098.81s (0:18:18) = ``` This PR: ``` = 179 failed, 101260 passed, 2091 skipped, 954 xfailed, 312 xpassed in 1225.23s (0:20:25) = ``` ## Checklist - [x] I am familiar with the [Contributing Guidelines](https://github.com/rapidsai/cudf/blob/HEAD/CONTRIBUTING.md). - [x] New or existing tests cover these changes. - [x] The documentation is up to date with these changes. --- python/cudf/cudf/core/dataframe.py | 36 ++-- python/cudf/cudf/core/groupby/groupby.py | 12 +- python/cudf/cudf/core/index.py | 14 +- python/cudf/cudf/core/join/_join_helpers.py | 9 +- python/cudf/cudf/core/multiindex.py | 21 ++- python/cudf/cudf/core/reshape.py | 14 +- python/cudf/cudf/core/series.py | 4 +- python/cudf/cudf/io/parquet.py | 16 +- python/cudf/cudf/tests/test_concat.py | 187 ++++++++++++-------- python/cudf/cudf/tests/test_dataframe.py | 72 +++++--- python/cudf/cudf/tests/test_index.py | 22 ++- 11 files changed, 263 insertions(+), 144 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 50fe5adebf8..bfb5fbe4d48 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -910,7 +910,9 @@ def _init_from_series_list(self, data, columns, index): transpose = self.T else: - concat_df = cudf.concat(data, axis=1) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + concat_df = cudf.concat(data, axis=1) cols = concat_df._data.to_pandas_index() if cols.dtype == "object": @@ -1920,9 +1922,11 @@ def _get_renderable_dataframe(self): lower_left = self.tail(lower_rows).iloc[:, :left_cols] lower_right = self.tail(lower_rows).iloc[:, right_cols:] - upper = cudf.concat([upper_left, upper_right], axis=1) - lower = cudf.concat([lower_left, lower_right], axis=1) - output = cudf.concat([upper, lower]) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + upper = cudf.concat([upper_left, upper_right], axis=1) + lower = cudf.concat([lower_left, lower_right], axis=1) + output = cudf.concat([upper, lower]) output = self._clean_nulls_from_dataframe(output) output._index = output._index._clean_nulls_from_index() @@ -5154,14 +5158,17 @@ def describe( None, ) - return cudf.concat( - [ - series.reindex(names, copy=False) - for series in describe_series_list - ], - axis=1, - sort=False, - ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + res = cudf.concat( + [ + series.reindex(names, copy=False) + for series in describe_series_list + ], + axis=1, + sort=False, + ) + return res @_cudf_nvtx_annotate def to_pandas(self, *, nullable: bool = False) -> pd.DataFrame: @@ -6258,7 +6265,10 @@ def mode(self, axis=0, numeric_only=False, dropna=True): if len(mode_results) == 0: return DataFrame() - df = cudf.concat(mode_results, axis=1) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + df = cudf.concat(mode_results, axis=1) + if isinstance(df, Series): df = df.to_frame() diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 0262e586807..849ec46f74d 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -1319,13 +1319,17 @@ def _post_process_chunk_results( # group is a row-like "Series" where the index labels # are the same as the original calling DataFrame if _is_row_of(chunk_results[0], self.obj): - result = cudf.concat(chunk_results, axis=1).T + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + result = cudf.concat(chunk_results, axis=1).T result.index = group_names result.index.names = self.grouping.names # When the UDF is like df.x + df.y, the result for each # group is the same length as the original group elif len(self.obj) == sum(len(chk) for chk in chunk_results): - result = cudf.concat(chunk_results) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + result = cudf.concat(chunk_results) index_data = group_keys._data.copy(deep=True) index_data[None] = grouped_values.index._column result.index = cudf.MultiIndex._from_data(index_data) @@ -1336,7 +1340,9 @@ def _post_process_chunk_results( f"type {type(chunk_results[0])}" ) else: - result = cudf.concat(chunk_results) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + result = cudf.concat(chunk_results) if self._group_keys: index_data = group_keys._data.copy(deep=True) index_data[None] = grouped_values.index._column diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 3cce1ab515e..25a58d77830 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -1103,6 +1103,16 @@ def _values(self): @_cudf_nvtx_annotate def _concat(cls, objs): non_empties = [index for index in objs if len(index)] + if len(objs) != len(non_empties): + # Do not remove until pandas-3.0 support is added. + warnings.warn( + "The behavior of array concatenation with empty entries is " + "deprecated. In a future version, this will no longer exclude " + "empty items when determining the result dtype. " + "To retain the old behavior, exclude the empty entries before " + "the concat operation.", + FutureWarning, + ) if all(isinstance(obj, RangeIndex) for obj in non_empties): result = _concat_range_index(non_empties) else: @@ -1300,7 +1310,9 @@ def __repr__(self): top = self[0:mr] bottom = self[-1 * mr :] - preprocess = cudf.concat([top, bottom]) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + preprocess = cudf.concat([top, bottom]) else: preprocess = self diff --git a/python/cudf/cudf/core/join/_join_helpers.py b/python/cudf/cudf/core/join/_join_helpers.py index 1071261044f..822c1848d58 100644 --- a/python/cudf/cudf/core/join/_join_helpers.py +++ b/python/cudf/cudf/core/join/_join_helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +import warnings from collections import abc from typing import TYPE_CHECKING, Any, Tuple, cast @@ -170,9 +171,11 @@ def _match_categorical_dtypes_both( return lcol, rcol.astype(ltype) else: # merge categories - merged_categories = cudf.concat( - [ltype.categories, rtype.categories] - ).unique() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + merged_categories = cudf.concat( + [ltype.categories, rtype.categories] + ).unique() common_type = cudf.CategoricalDtype( categories=merged_categories, ordered=False ) diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index a2cc5450ca4..1bca738590f 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -6,6 +6,7 @@ import numbers import operator import pickle +import warnings from collections import abc from functools import cached_property from numbers import Integral @@ -717,15 +718,17 @@ def _compute_validity_mask(self, index, row_tuple, max_length): continue lookup[i] = cudf.Series(row) frame = cudf.DataFrame(dict(enumerate(index._data.columns))) - data_table = cudf.concat( - [ - frame, - cudf.DataFrame( - {"idx": cudf.Series(column.arange(len(frame)))} - ), - ], - axis=1, - ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + data_table = cudf.concat( + [ + frame, + cudf.DataFrame( + {"idx": cudf.Series(column.arange(len(frame)))} + ), + ], + axis=1, + ) # Sort indices in pandas compatible mode # because we want the indices to be fetched # in a deterministic order. diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index 7a80d70acb3..465186d81d2 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -5,6 +5,7 @@ from typing import Dict, Optional import cupy +import warnings import numpy as np import pandas as pd @@ -320,9 +321,20 @@ def concat(objs, axis=0, join="outer", ignore_index=False, sort=None): df = cudf.DataFrame() _normalize_series_and_dataframe(objs, axis=axis) + any_empty = any(obj.empty for obj in objs) + if any_empty: + # Do not remove until pandas-3.0 support is added. + warnings.warn( + "The behavior of array concatenation with empty entries is " + "deprecated. In a future version, this will no longer exclude " + "empty items when determining the result dtype. " + "To retain the old behavior, exclude the empty entries before " + "the concat operation.", + FutureWarning, + ) # Inner joins involving empty data frames always return empty dfs, but # We must delay returning until we have set the column names. - empty_inner = any(obj.empty for obj in objs) and join == "inner" + empty_inner = any_empty and join == "inner" objs = [obj for obj in objs if obj.shape != (0, 0)] diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 5876a577b87..959b91afd32 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -1429,7 +1429,9 @@ def __repr__(self): if max_rows not in (0, None) and len(self) > max_rows: top = self.head(int(max_rows / 2 + 1)) bottom = self.tail(int(max_rows / 2 + 1)) - preprocess = cudf.concat([top, bottom]) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + preprocess = cudf.concat([top, bottom]) else: preprocess = self.copy() preprocess.index = preprocess.index._clean_nulls_from_index() diff --git a/python/cudf/cudf/io/parquet.py b/python/cudf/cudf/io/parquet.py index bcc24a85cf9..a6da55c1a7f 100644 --- a/python/cudf/cudf/io/parquet.py +++ b/python/cudf/cudf/io/parquet.py @@ -794,13 +794,15 @@ def _parquet_to_frame( dtype=_dtype, ) - # Concatenate dfs and return. - # Assume we can ignore the index if it has no name. - return ( - cudf.concat(dfs, ignore_index=dfs[-1].index.name is None) - if len(dfs) > 1 - else dfs[0] - ) + if len(dfs) > 1: + # Concatenate dfs and return. + # Assume we can ignore the index if it has no name. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + res = cudf.concat(dfs, ignore_index=dfs[-1].index.name is None) + return res + else: + return dfs[0] @_cudf_nvtx_annotate diff --git a/python/cudf/cudf/tests/test_concat.py b/python/cudf/cudf/tests/test_concat.py index a265618e4ba..7fa1b634185 100644 --- a/python/cudf/cudf/tests/test_concat.py +++ b/python/cudf/cudf/tests/test_concat.py @@ -2,10 +2,13 @@ from decimal import Decimal +import warnings import numpy as np import pandas as pd import pytest +from contextlib import contextmanager + import cudf as gd from cudf.api.types import _is_categorical_dtype from cudf.core._compat import PANDAS_GE_150, PANDAS_LT_140, PANDAS_GE_200 @@ -17,6 +20,20 @@ ) +@contextmanager +def _hide_concat_empty_dtype_warning(): + with warnings.catch_warnings(): + # Ignoring warnings in this test as warnings are + # being caught and validated in other tests. + warnings.filterwarnings( + "ignore", + "The behavior of array concatenation with empty entries " + "is deprecated.", + category=FutureWarning, + ) + yield + + def make_frames(index=None, nulls="none"): df = pd.DataFrame( { @@ -66,8 +83,9 @@ def test_concat_dataframe(index, nulls, axis): df_empty1 = gdf_empty1.to_pandas() # DataFrame - res = gd.concat([gdf, gdf2, gdf, gdf_empty1], axis=axis).to_pandas() - sol = pd.concat([df, df2, df, df_empty1], axis=axis) + with _hide_concat_empty_dtype_warning(): + res = gd.concat([gdf, gdf2, gdf, gdf_empty1], axis=axis).to_pandas() + sol = pd.concat([df, df2, df, df_empty1], axis=axis) assert_eq( res, sol, @@ -476,8 +494,9 @@ def test_concat_series_dataframe_input(objs): pd_objs = objs gd_objs = [gd.from_pandas(obj) for obj in objs] - expected = pd.concat(pd_objs) - actual = gd.concat(gd_objs) + with _hide_concat_empty_dtype_warning(): + expected = pd.concat(pd_objs) + actual = gd.concat(gd_objs) assert_eq( expected.fillna(-1), @@ -843,23 +862,24 @@ def test_concat_join_many_df_and_empty_df(ignore_index, sort, join, axis): gdf3 = gd.from_pandas(pdf3) gdf_empty1 = gd.from_pandas(pdf_empty1) - assert_eq( - pd.concat( - [pdf1, pdf2, pdf3, pdf_empty1], - sort=sort, - join=join, - ignore_index=ignore_index, - axis=axis, - ), - gd.concat( - [gdf1, gdf2, gdf3, gdf_empty1], - sort=sort, - join=join, - ignore_index=ignore_index, - axis=axis, - ), - check_index_type=False, - ) + with _hide_concat_empty_dtype_warning(): + assert_eq( + pd.concat( + [pdf1, pdf2, pdf3, pdf_empty1], + sort=sort, + join=join, + ignore_index=ignore_index, + axis=axis, + ), + gd.concat( + [gdf1, gdf2, gdf3, gdf_empty1], + sort=sort, + join=join, + ignore_index=ignore_index, + axis=axis, + ), + check_index_type=False, + ) @pytest.mark.parametrize("ignore_index", [True, False]) @@ -970,20 +990,21 @@ def test_concat_join_no_overlapping_columns_many_and_empty( gdf6 = gd.from_pandas(pdf6) gdf_empty = gd.from_pandas(pdf_empty) - expected = pd.concat( - [pdf4, pdf5, pdf6, pdf_empty], - sort=sort, - join=join, - ignore_index=ignore_index, - axis=axis, - ) - actual = gd.concat( - [gdf4, gdf5, gdf6, gdf_empty], - sort=sort, - join=join, - ignore_index=ignore_index, - axis=axis, - ) + with _hide_concat_empty_dtype_warning(): + expected = pd.concat( + [pdf4, pdf5, pdf6, pdf_empty], + sort=sort, + join=join, + ignore_index=ignore_index, + axis=axis, + ) + actual = gd.concat( + [gdf4, gdf5, gdf6, gdf_empty], + sort=sort, + join=join, + ignore_index=ignore_index, + axis=axis, + ) assert_eq( expected, actual, @@ -1042,20 +1063,21 @@ def test_concat_join_no_overlapping_columns_many_and_empty2( ): objs_gd = [gd.from_pandas(o) if o is not None else o for o in objs] - expected = pd.concat( - objs, - sort=sort, - join=join, - ignore_index=ignore_index, - axis=axis, - ) - actual = gd.concat( - objs_gd, - sort=sort, - join=join, - ignore_index=ignore_index, - axis=axis, - ) + with _hide_concat_empty_dtype_warning(): + expected = pd.concat( + objs, + sort=sort, + join=join, + ignore_index=ignore_index, + axis=axis, + ) + actual = gd.concat( + objs_gd, + sort=sort, + join=join, + ignore_index=ignore_index, + axis=axis, + ) assert_eq(expected, actual, check_index_type=False) @@ -1079,20 +1101,21 @@ def test_concat_join_no_overlapping_columns_empty_df_basic( gdf6 = gd.from_pandas(pdf6) gdf_empty = gd.from_pandas(pdf_empty) - expected = pd.concat( - [pdf6, pdf_empty], - sort=sort, - join=join, - ignore_index=ignore_index, - axis=axis, - ) - actual = gd.concat( - [gdf6, gdf_empty], - sort=sort, - join=join, - ignore_index=ignore_index, - axis=axis, - ) + with _hide_concat_empty_dtype_warning(): + expected = pd.concat( + [pdf6, pdf_empty], + sort=sort, + join=join, + ignore_index=ignore_index, + axis=axis, + ) + actual = gd.concat( + [gdf6, gdf_empty], + sort=sort, + join=join, + ignore_index=ignore_index, + axis=axis, + ) assert_eq( expected, actual, @@ -1109,7 +1132,7 @@ def test_concat_join_series(ignore_index, sort, join, axis): s1 = gd.Series(["a", "b", "c"]) s2 = gd.Series(["a", "b"]) s3 = gd.Series(["a", "b", "c", "d"]) - s4 = gd.Series() + s4 = gd.Series(dtype="str") ps1 = s1.to_pandas() ps2 = s2.to_pandas() @@ -1123,13 +1146,14 @@ def test_concat_join_series(ignore_index, sort, join, axis): ignore_index=ignore_index, axis=axis, ) - actual = gd.concat( - [s1, s2, s3, s4], - sort=sort, - join=join, - ignore_index=ignore_index, - axis=axis, - ) + with expect_warning_if(axis == 1): + actual = gd.concat( + [s1, s2, s3, s4], + sort=sort, + join=join, + ignore_index=ignore_index, + axis=axis, + ) if PANDAS_GE_150: assert_eq( @@ -1327,12 +1351,21 @@ def test_concat_join_empty_dataframes_axis_1( gdf = gd.from_pandas(df) other_gd = [gdf] + [gd.from_pandas(o) for o in other] - expected = pd.concat( - other_pd, ignore_index=ignore_index, axis=axis, join=join, sort=sort - ) - actual = gd.concat( - other_gd, ignore_index=ignore_index, axis=axis, join=join, sort=sort - ) + with _hide_concat_empty_dtype_warning(): + expected = pd.concat( + other_pd, + ignore_index=ignore_index, + axis=axis, + join=join, + sort=sort, + ) + actual = gd.concat( + other_gd, + ignore_index=ignore_index, + axis=axis, + join=join, + sort=sort, + ) if expected.shape != df.shape: if axis == 0: for key, col in actual[actual.columns].items(): diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 6e9b9a37ac0..94aff555c7f 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -93,6 +93,20 @@ def _hide_ufunc_warnings(eval_str): yield +@contextmanager +def _hide_concat_empty_dtype_warning(): + with warnings.catch_warnings(): + # Ignoring warnings in this test as warnings are + # being caught and validated in other tests. + warnings.filterwarnings( + "ignore", + "The behavior of array concatenation with empty " + "entries is deprecated.", + category=FutureWarning, + ) + yield + + def test_init_via_list_of_tuples(): data = [ (5, "cats", "jump", np.nan), @@ -1601,8 +1615,9 @@ def test_dataframe_concat_different_column_types(): "df_2", [cudf.DataFrame({"a": [], "b": []}), cudf.DataFrame({})] ) def test_concat_empty_dataframe(df_1, df_2): - got = cudf.concat([df_1, df_2]) - expect = pd.concat([df_1.to_pandas(), df_2.to_pandas()], sort=False) + with _hide_concat_empty_dtype_warning(): + got = cudf.concat([df_1, df_2]) + expect = pd.concat([df_1.to_pandas(), df_2.to_pandas()], sort=False) # ignoring dtypes as pandas upcasts int to float # on concatenation with empty dataframes @@ -1628,10 +1643,15 @@ def test_concat_empty_dataframe(df_1, df_2): ], ) def test_concat_different_column_dataframe(df1_d, df2_d): - got = cudf.concat( - [cudf.DataFrame(df1_d), cudf.DataFrame(df2_d), cudf.DataFrame(df1_d)], - sort=False, - ) + with _hide_concat_empty_dtype_warning(): + got = cudf.concat( + [ + cudf.DataFrame(df1_d), + cudf.DataFrame(df2_d), + cudf.DataFrame(df1_d), + ], + sort=False, + ) pdf1 = pd.DataFrame(df1_d) pdf2 = pd.DataFrame(df2_d) @@ -1670,8 +1690,9 @@ def is_invalid_concat(left, right): ) @pytest.mark.parametrize("ser_2", [pd.Series([], dtype="float64")]) def test_concat_empty_series(ser_1, ser_2): - got = cudf.concat([cudf.Series(ser_1), cudf.Series(ser_2)]) - expect = pd.concat([ser_1, ser_2]) + with _hide_concat_empty_dtype_warning(): + got = cudf.concat([cudf.Series(ser_1), cudf.Series(ser_2)]) + expect = pd.concat([ser_1, ser_2]) assert_eq(got, expect, check_index_type=True) @@ -7500,8 +7521,13 @@ def test_dataframe_concat_dataframe(df, other, sort, ignore_index): gdf = cudf.from_pandas(df) other_gd = cudf.from_pandas(other) - expected = pd.concat([pdf, other_pd], sort=sort, ignore_index=ignore_index) - actual = cudf.concat([gdf, other_gd], sort=sort, ignore_index=ignore_index) + with _hide_concat_empty_dtype_warning(): + expected = pd.concat( + [pdf, other_pd], sort=sort, ignore_index=ignore_index + ) + actual = cudf.concat( + [gdf, other_gd], sort=sort, ignore_index=ignore_index + ) # In empty dataframe cases, Pandas & cudf differ in columns # creation, pandas creates RangeIndex(0, 0) @@ -7739,12 +7765,13 @@ def test_dataframe_concat_dataframe_lists(df, other, sort, ignore_index): gdf = cudf.from_pandas(df) other_gd = [cudf.from_pandas(o) for o in other] - expected = pd.concat( - [pdf] + other_pd, sort=sort, ignore_index=ignore_index - ) - actual = cudf.concat( - [gdf] + other_gd, sort=sort, ignore_index=ignore_index - ) + with _hide_concat_empty_dtype_warning(): + expected = pd.concat( + [pdf] + other_pd, sort=sort, ignore_index=ignore_index + ) + actual = cudf.concat( + [gdf] + other_gd, sort=sort, ignore_index=ignore_index + ) # In some cases, Pandas creates an empty Index([], dtype="object") for # columns whereas cudf creates a RangeIndex(0, 0). @@ -7854,12 +7881,13 @@ def test_dataframe_concat_lists(df, other, sort, ignore_index): gdf = cudf.from_pandas(df) other_gd = [cudf.from_pandas(o) for o in other_pd] - expected = pd.concat( - [pdf] + other_pd, sort=sort, ignore_index=ignore_index - ) - actual = cudf.concat( - [gdf] + other_gd, sort=sort, ignore_index=ignore_index - ) + with _hide_concat_empty_dtype_warning(): + expected = pd.concat( + [pdf] + other_pd, sort=sort, ignore_index=ignore_index + ) + actual = cudf.concat( + [gdf] + other_gd, sort=sort, ignore_index=ignore_index + ) if expected.shape != df.shape: assert_eq( diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index 445fc84981b..d06041301b9 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -1034,16 +1034,19 @@ def test_index_append(data, other): pd_data = pd.Index(data) pd_other = pd.Index(other) - gd_data = cudf.core.index.as_index(data) - gd_other = cudf.core.index.as_index(other) + gd_data = cudf.Index(data) + gd_other = cudf.Index(other) if cudf.utils.dtypes.is_mixed_with_object_dtype(gd_data, gd_other): gd_data = gd_data.astype("str") gd_other = gd_other.astype("str") - expected = pd_data.append(pd_other) - - actual = gd_data.append(gd_other) + with expect_warning_if( + (len(data) == 0 or len(other) == 0) and pd_data.dtype != pd_other.dtype + ): + expected = pd_data.append(pd_other) + with expect_warning_if(len(data) == 0 or len(other) == 0): + actual = gd_data.append(gd_other) if len(data) == 0 and len(other) == 0: # Pandas default dtype to "object" for empty list # cudf default dtype to "float" for empty list @@ -1233,8 +1236,13 @@ def test_index_append_list(data, other): gd_data = cudf.from_pandas(data) gd_other = [cudf.from_pandas(i) for i in other] - expected = pd_data.append(pd_other) - actual = gd_data.append(gd_other) + with expect_warning_if( + (len(data) == 0 or any(len(d) == 0 for d in other)) + and (any(d.dtype != data.dtype for d in other)) + ): + expected = pd_data.append(pd_other) + with expect_warning_if(len(data) == 0 or any(len(d) == 0 for d in other)): + actual = gd_data.append(gd_other) assert_eq(expected, actual) From e218f5c384b64a93597273265435f19d087206ee Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 28 Dec 2023 00:56:56 +0530 Subject: [PATCH 088/384] Deprecate setting of incompatible dtypes to an existing column (#14668) This PR deprecates the setting of a value that is not of same dtype as that of a column. On `pandas_2.0_feature_branch`: ``` = 198 failed, 101241 passed, 2091 skipped, 954 xfailed, 312 xpassed in 1098.81s (0:18:18) = ``` This PR: ``` = 176 failed, 101263 passed, 2091 skipped, 954 xfailed, 312 xpassed in 1096.08s (0:18:16) = ``` --- python/cudf/cudf/core/index.py | 2 +- python/cudf/cudf/core/series.py | 9 +++++++ python/cudf/cudf/tests/test_indexing.py | 34 ++++++++++++++++++++----- python/cudf/cudf/tests/test_setitem.py | 14 +++++++--- 4 files changed, 48 insertions(+), 11 deletions(-) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 25a58d77830..0b0b25281ce 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -3016,7 +3016,7 @@ def _get_indexer_basic(index, positions, method, target_col, tolerance): # sentinel for missing values else: # Mark indices to the right of the largest value as not found - positions[positions == len(index)] = -1 + positions[positions == len(index)] = np.int32(-1) if tolerance is not None: distance = abs(index[positions] - target_col) diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 959b91afd32..6080a37f0a2 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -284,6 +284,15 @@ def __setitem__(self, key, value): to_dtype = np.result_type(value.dtype, self._frame._column.dtype) value = value.astype(to_dtype) if to_dtype != self._frame._column.dtype: + # Do not remove until pandas-3.0 support is added. + warnings.warn( + f"Setting an item of incompatible dtype is deprecated " + "and will raise in a future error of pandas. " + f"Value '{value}' has dtype incompatible with " + f"{self._frame._column.dtype}, " + "please explicitly cast to a compatible dtype first.", + FutureWarning, + ) self._frame._column._mimic_inplace( self._frame._column.astype(to_dtype), inplace=True ) diff --git a/python/cudf/cudf/tests/test_indexing.py b/python/cudf/cudf/tests/test_indexing.py index e921a6ccf3f..f2b58a80362 100644 --- a/python/cudf/cudf/tests/test_indexing.py +++ b/python/cudf/cudf/tests/test_indexing.py @@ -930,8 +930,17 @@ def test_series_setitem_basics(key, value, nulls): elif nulls == "all": psr[:] = None gsr = cudf.from_pandas(psr) - psr[key] = value - gsr[key] = value + with expect_warning_if( + PANDAS_GE_210 + and isinstance(value, list) + and len(value) == 0 + and nulls == "none" + ): + psr[key] = value + with expect_warning_if( + isinstance(value, list) and len(value) == 0 and nulls == "none" + ): + gsr[key] = value assert_eq(psr, gsr, check_dtype=False) @@ -974,8 +983,17 @@ def test_series_setitem_iloc(key, value, nulls): elif nulls == "all": psr[:] = None gsr = cudf.from_pandas(psr) - psr.iloc[key] = value - gsr.iloc[key] = value + with expect_warning_if( + PANDAS_GE_210 + and isinstance(value, list) + and len(value) == 0 + and nulls == "none" + ): + psr.iloc[key] = value + with expect_warning_if( + isinstance(value, list) and len(value) == 0 and nulls == "none" + ): + gsr.iloc[key] = value assert_eq(psr, gsr, check_dtype=False) @@ -994,8 +1012,12 @@ def test_series_setitem_iloc(key, value, nulls): def test_series_setitem_dtype(key, value): psr = pd.Series([1, 2, 3], dtype="int32") gsr = cudf.from_pandas(psr) - psr[key] = value - gsr[key] = value + + with expect_warning_if(isinstance(value, (float, list))): + psr[key] = value + with expect_warning_if(isinstance(value, (float, list))): + gsr[key] = value + assert_eq(psr, gsr) diff --git a/python/cudf/cudf/tests/test_setitem.py b/python/cudf/cudf/tests/test_setitem.py index 6e1e53fc869..2d663a6c329 100644 --- a/python/cudf/cudf/tests/test_setitem.py +++ b/python/cudf/cudf/tests/test_setitem.py @@ -5,8 +5,12 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_150 -from cudf.testing._utils import assert_eq, assert_exceptions_equal +from cudf.core._compat import PANDAS_GE_150, PANDAS_GE_210 +from cudf.testing._utils import ( + assert_eq, + assert_exceptions_equal, + expect_warning_if, +) @pytest.mark.parametrize("df", [pd.DataFrame({"a": [1, 2, 3]})]) @@ -310,8 +314,10 @@ def test_series_setitem_upcasting(dtype, indices): # column dtype. new_value = np.float64(np.pi) col_ref = cr._column - sr[indices] = new_value - cr[indices] = new_value + with expect_warning_if(PANDAS_GE_210 and dtype != np.float64): + sr[indices] = new_value + with expect_warning_if(dtype != np.float64): + cr[indices] = new_value if PANDAS_GE_150: assert_eq(sr, cr) else: From fd1f98641fe5c705e24ea645d12b27cd8ee4cea2 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 28 Dec 2023 01:05:34 +0530 Subject: [PATCH 089/384] Fix datetime related assertions and warnings in pytests (#14673) This PR fixes all `datetime` related pytests by properly handling their assertions with bug-fixes made in pandas-2.x and filtering newly introduced warnings where not necessary to propagate to the end-user. On `pandas_2.0_feature_branch`: ``` = 198 failed, 101241 passed, 2091 skipped, 954 xfailed, 312 xpassed in 1098.81s (0:18:18) = ``` This PR: ``` = 161 failed, 101280 passed, 2091 skipped, 952 xfailed, 312 xpassed in 1106.29s (0:18:26) = ``` --- python/cudf/cudf/core/column/column.py | 8 ++- python/cudf/cudf/core/tools/datetimes.py | 12 +++-- python/cudf/cudf/tests/test_datetime.py | 65 +++++++++--------------- python/cudf/cudf/tests/test_joining.py | 8 +-- python/cudf/cudf/tests/test_orc.py | 29 +++-------- python/cudf/cudf/tests/test_parquet.py | 7 --- 6 files changed, 45 insertions(+), 84 deletions(-) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index b79d0644696..e83d82307e5 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -4,6 +4,8 @@ import builtins import pickle +import warnings + from collections import abc from functools import cached_property from itertools import chain @@ -2596,7 +2598,11 @@ def _construct_array( ): # We may have date-like strings with timezones try: - pd_arbitrary = pd.to_datetime(arbitrary) + with warnings.catch_warnings(): + # Need to ignore userwarnings when + # datetime format cannot be inferred. + warnings.simplefilter("ignore", UserWarning) + pd_arbitrary = pd.to_datetime(arbitrary) if isinstance(pd_arbitrary.dtype, pd.DatetimeTZDtype): raise NotImplementedError( "cuDF does not yet support timezone-aware datetimes" diff --git a/python/cudf/cudf/core/tools/datetimes.py b/python/cudf/cudf/core/tools/datetimes.py index 3d06e82d4cb..1525dd1da22 100644 --- a/python/cudf/cudf/core/tools/datetimes.py +++ b/python/cudf/cudf/core/tools/datetimes.py @@ -923,10 +923,14 @@ def date_range( # FIXME: when `end_estim` is out of bound, but the actual `end` is not, # we shouldn't raise but compute the sequence as is. The trailing overflow # part should get trimmed at the end. - end_estim = ( - pd.Timestamp(start.value) - + periods * offset._maybe_as_fast_pandas_offset() - ).to_datetime64() + with warnings.catch_warnings(): + # Need to ignore userwarnings where nonzero nanoseconds + # are dropped in conversion during the binops + warnings.simplefilter("ignore", UserWarning) + end_estim = ( + pd.Timestamp(start.value) + + periods * offset._maybe_as_fast_pandas_offset() + ).to_datetime64() if "months" in offset.kwds or "years" in offset.kwds: # If `offset` is non-fixed frequency, resort to libcudf. diff --git a/python/cudf/cudf/tests/test_datetime.py b/python/cudf/cudf/tests/test_datetime.py index 2368b3e539c..88a50c7936e 100644 --- a/python/cudf/cudf/tests/test_datetime.py +++ b/python/cudf/cudf/tests/test_datetime.py @@ -642,21 +642,10 @@ def test_cudf_to_datetime(data, dayfirst): expected = pd.to_datetime(pd_data, dayfirst=dayfirst) actual = cudf.to_datetime(gd_data, dayfirst=dayfirst) - # TODO: Remove typecast to `ns` and following if/else - # workaround after following issue is fixed: - # https://github.com/pandas-dev/pandas/issues/52449 - - if actual is not None and expected is not None: - assert_eq( - actual.astype(pd_data.dtype) - if pd_data is not None - and hasattr(pd_data, "dtype") - and cudf.api.types.is_datetime_dtype(pd_data.dtype) - else actual.astype("datetime64[ns]"), - expected, - ) + if isinstance(expected, pd.Series): + assert_eq(actual, expected, check_dtype=False) else: - assert_eq(actual, expected) + assert_eq(actual, expected, check_exact=False) @pytest.mark.parametrize( @@ -748,11 +737,10 @@ def test_to_datetime_units(data, unit): expected = pd.to_datetime(pd_data, unit=unit) actual = cudf.to_datetime(gd_data, unit=unit) - # TODO: Remove typecast to `ns` after following - # issue is fixed: - # https://github.com/pandas-dev/pandas/issues/52449 - - assert_eq(actual.astype("datetime64[ns]"), expected) + if isinstance(expected, pd.Series): + assert_eq(actual, expected, check_dtype=False) + else: + assert_eq(actual, expected, exact=False, check_exact=False) @pytest.mark.parametrize( @@ -810,11 +798,11 @@ def test_to_datetime_format(data, format, infer_datetime_format): actual = cudf.to_datetime( gd_data, format=format, infer_datetime_format=infer_datetime_format ) - # TODO: Remove typecast to `ns` after following - # issue is fixed: - # https://github.com/pandas-dev/pandas/issues/52449 - assert_eq(actual.astype("datetime64[ns]"), expected) + if isinstance(expected, pd.Series): + assert_eq(actual, expected, check_dtype=False) + else: + assert_eq(actual, expected, check_exact=False) def test_to_datetime_data_out_of_range_for_format(): @@ -879,11 +867,8 @@ def test_datetime_scalar_timeunit_cast(timeunit): gs = Series(testscalar) ps = pd.Series(testscalar) - # TODO: Remove typecast to `ns` after following - # issue is fixed: - # https://github.com/pandas-dev/pandas/issues/52449 - assert_eq(ps, gs.astype("datetime64[ns]")) + assert_eq(ps, gs, check_dtype=False) gdf = DataFrame() gdf["a"] = np.arange(5) @@ -894,11 +879,7 @@ def test_datetime_scalar_timeunit_cast(timeunit): pdf["b"] = testscalar assert gdf["b"].dtype == cudf.dtype("datetime64[s]") - # TODO: Remove typecast to `ns` after following - # issue is fixed: - # https://github.com/pandas-dev/pandas/issues/52449 - gdf["b"] = gdf["b"].astype("datetime64[ns]") - assert_eq(pdf, gdf) + assert_eq(pdf, gdf, check_dtype=True) @pytest.mark.parametrize( @@ -1328,14 +1309,13 @@ def test_datetime_infer_format(data, timezone, dtype): assert_eq(expected, actual) else: - with pytest.raises(NotImplementedError): - assert_exceptions_equal( - lfunc=psr.astype, - rfunc=sr.astype, - lfunc_args_and_kwargs=([], {"dtype": dtype}), - rfunc_args_and_kwargs=([], {"dtype": dtype}), - check_exception_type=False, - ) + assert_exceptions_equal( + lfunc=psr.astype, + rfunc=sr.astype, + lfunc_args_and_kwargs=([], {"dtype": dtype}), + rfunc_args_and_kwargs=([], {"dtype": dtype}), + check_exception_type=False, + ) def test_dateoffset_instance_subclass_check(): @@ -1634,7 +1614,8 @@ def test_date_range_end_freq_periods(request, end, freq, periods): request.applymarker( pytest.mark.xfail( condition=( - "nanoseconds" in freq + not PANDAS_GE_210 + and "nanoseconds" in freq and periods != 1 and end == "1970-01-01 00:00:00" ), @@ -2268,7 +2249,7 @@ def test_format_timezone_not_implemented(code): @pytest.mark.parametrize("tz", ["Z", "UTC-3", "+01:00"]) def test_no_format_timezone_not_implemented(tz): - with pytest.raises(NotImplementedError): + with pytest.raises((NotImplementedError, ValueError)): cudf.to_datetime([f"2020-01-01 00:00:00{tz}"]) diff --git a/python/cudf/cudf/tests/test_joining.py b/python/cudf/cudf/tests/test_joining.py index 0544406924f..b273e554158 100644 --- a/python/cudf/cudf/tests/test_joining.py +++ b/python/cudf/cudf/tests/test_joining.py @@ -787,13 +787,7 @@ def test_join_datetimes_index(dtype): assert gdf["d"].dtype == cudf.dtype(dtype) - if PANDAS_GE_200: - # TODO: Remove typecast to `ns` after following - # issue is fixed: - # https://github.com/pandas-dev/pandas/issues/52449 - gdf = gdf.astype("datetime64[ns]") - - assert_join_results_equal(pdf, gdf, how="inner") + assert_join_results_equal(pdf, gdf, how="inner", check_dtype=False) def test_join_with_different_names(): diff --git a/python/cudf/cudf/tests/test_orc.py b/python/cudf/cudf/tests/test_orc.py index e3f1c8eec4d..e53fa1fb4bf 100644 --- a/python/cudf/cudf/tests/test_orc.py +++ b/python/cudf/cudf/tests/test_orc.py @@ -21,7 +21,6 @@ gen_rand_series, supported_numpy_dtypes, ) -from cudf.core._compat import PANDAS_GE_200 # Removal of these deprecated features is no longer imminent. They will not be # removed until a suitable alternative has been implemented. As a result, we @@ -160,13 +159,7 @@ def test_orc_reader_datetimestamp(datadir, inputfile, use_index): pdf = orcfile.read().to_pandas(date_as_object=False) gdf = cudf.read_orc(path, use_index=use_index) - if PANDAS_GE_200: - # TODO: Remove typecast to `ns` after following - # issue is fixed: - # https://github.com/pandas-dev/pandas/issues/52449 - gdf = gdf.astype("datetime64[ns]") - - assert_eq(pdf, gdf, check_categorical=False) + assert_eq(pdf, gdf, check_categorical=False, check_exact=False) def test_orc_reader_strings(datadir): @@ -1832,13 +1825,7 @@ def test_orc_reader_negative_timestamp(negative_timestamp_df, engine): with expect_warning_if(engine == "pyarrow", UserWarning): got = cudf.read_orc(buffer, engine=engine) - if PANDAS_GE_200: - # TODO: Remove typecast to `ns` after following - # issue is fixed: - # https://github.com/pandas-dev/pandas/issues/52449 - negative_timestamp_df = negative_timestamp_df.astype("datetime64[ns]") - - assert_eq(negative_timestamp_df, got) + assert_eq(negative_timestamp_df, got, check_dtype=False) def test_orc_writer_negative_timestamp(negative_timestamp_df): @@ -1847,14 +1834,10 @@ def test_orc_writer_negative_timestamp(negative_timestamp_df): buffer = BytesIO() negative_timestamp_df.to_orc(buffer) - if PANDAS_GE_200: - # TODO: Remove typecast to `ns` after following - # issue is fixed: - # https://github.com/pandas-dev/pandas/issues/52449 - negative_timestamp_df = negative_timestamp_df.astype("datetime64[ns]") - - assert_eq(negative_timestamp_df, pd.read_orc(buffer)) - assert_eq(negative_timestamp_df, orc.ORCFile(buffer).read()) + assert_eq(negative_timestamp_df, pd.read_orc(buffer), check_dtype=False) + assert_eq( + negative_timestamp_df, orc.ORCFile(buffer).read(), check_dtype=False + ) def test_orc_reader_apache_negative_timestamp(datadir): diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index adadf147503..971bfe74185 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -632,13 +632,6 @@ def test_parquet_reader_microsecond_timestamps(datadir): expect = pd.read_parquet(fname) got = cudf.read_parquet(fname) - if PANDAS_GE_200: - # TODO: Remove typecast to `ns` after following - # issue is fixed: - # https://github.com/pandas-dev/pandas/issues/52449 - assert got["a"].dtype == cudf.dtype("datetime64[us]") - got = got.astype("datetime64[ns]") - assert_eq(expect, got) From cb09a3911ff75f4a9557912a0a426827b52e2ed3 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 29 Dec 2023 06:13:09 +0530 Subject: [PATCH 090/384] Fix pytest condition to include more warning scenarios (#14680) This PR fixes calculation of cond variable in test_corr1d which will include more cases for warnings. This change fixes, 9 pytest failures. --- python/cudf/cudf/tests/test_stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cudf/cudf/tests/test_stats.py b/python/cudf/cudf/tests/test_stats.py index 41fac49ea83..f24a5ea7b41 100644 --- a/python/cudf/cudf/tests/test_stats.py +++ b/python/cudf/cudf/tests/test_stats.py @@ -480,7 +480,7 @@ def test_corr1d(data1, data2, method): # Spearman allows for size 1 samples, but will error if all data in a # sample is identical since the covariance is zero and so the correlation # coefficient is not defined. - cond = (is_singular and method == "pearson") or ( + cond = ((is_singular or is_identical) and method == "pearson") or ( is_identical and not is_singular and method == "spearman" ) if method == "spearman": From 1c54354bcdde1c2728213ff3fac8e5be0e613242 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 29 Dec 2023 14:38:41 +0530 Subject: [PATCH 091/384] Sort `Index.difference` & `union` results for early exit scenarios (#14681) This PR sorts results in `Index.difference` & `union` in the early exit scenarios similar to: https://github.com/pandas-dev/pandas/pull/51346/ On `pandas_2.0_feature_branch`: ``` = 110 failed, 101331 passed, 2091 skipped, 952 xfailed, 312 xpassed in 1064.30s (0:17:44) = ``` This PR: ``` = 87 failed, 101354 passed, 2091 skipped, 952 xfailed, 312 xpassed in 1004.34s (0:16:44) = ``` --- python/cudf/cudf/core/_base_index.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index 0a70f3050eb..82d496a5c78 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -629,12 +629,18 @@ def union(self, other, sort=None): common_dtype = cudf.utils.dtypes.find_common_type( [self.dtype, other.dtype] ) - return self._get_reconciled_name_object(other).astype(common_dtype) + res = self._get_reconciled_name_object(other).astype(common_dtype) + if sort: + return res.sort_values() + return res elif not len(self): common_dtype = cudf.utils.dtypes.find_common_type( [self.dtype, other.dtype] ) - return other._get_reconciled_name_object(self).astype(common_dtype) + res = other._get_reconciled_name_object(self).astype(common_dtype) + if sort: + return res.sort_values() + return res result = self._union(other, sort=sort) result.name = _get_result_name(self.name, other.name) @@ -1091,9 +1097,15 @@ def difference(self, other, sort=None): other = cudf.Index(other, name=getattr(other, "name", self.name)) if not len(other): - return self._get_reconciled_name_object(other) + res = self._get_reconciled_name_object(other) + if sort: + return res.sort_values() + return res elif self.equals(other): - return self[:0]._get_reconciled_name_object(other) + res = self[:0]._get_reconciled_name_object(other) + if sort: + return res.sort_values() + return res res_name = _get_result_name(self.name, other.name) From 8a8b627076b42f44aab2eaac9f166a3e3e6ff2fc Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Sat, 30 Dec 2023 14:51:27 +0530 Subject: [PATCH 092/384] Fix column parameter handling in `read_orc` (#14666) When `columns=[]` for `read_orc`, pandas actually only drops the column and preserves the `index` while reading an orc file. Fixing the `cudf` behavior to match the same. On `pandas_2.0_feature_branch`: ``` = 198 failed, 101241 passed, 2091 skipped, 954 xfailed, 312 xpassed in 1098.81s (0:18:18) = ``` This PR: ``` = 185 failed, 101254 passed, 2091 skipped, 954 xfailed, 312 xpassed in 1088.47s (0:18:08) = ``` --- python/cudf/cudf/_lib/orc.pyx | 11 +++++++++-- python/cudf/cudf/core/_compat.py | 1 + python/cudf/cudf/tests/test_orc.py | 13 ++----------- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/python/cudf/cudf/_lib/orc.pyx b/python/cudf/cudf/_lib/orc.pyx index 0ae039b14d2..62a9a2886b6 100644 --- a/python/cudf/cudf/_lib/orc.pyx +++ b/python/cudf/cudf/_lib/orc.pyx @@ -116,6 +116,7 @@ cpdef read_orc(object filepaths_or_buffers, ) cdef table_with_metadata c_result + cdef size_type nrows with nogil: c_result = move(libcudf_read_orc(c_orc_reader_options)) @@ -127,6 +128,12 @@ cpdef read_orc(object filepaths_or_buffers, skip_rows, num_rows) + if columns is not None and (isinstance(columns, list) and len(columns) == 0): + # When `columns=[]`, index needs to be + # established, but not the columns. + nrows = c_result.tbl.get()[0].view().num_rows() + return {}, cudf.RangeIndex(nrows) + data, index = data_from_unique_ptr( move(c_result.tbl), col_names if columns is None else names, @@ -173,7 +180,6 @@ cdef tuple _get_index_from_metadata( range_idx = None if json_str != "": meta = json.loads(json_str) - if 'index_columns' in meta and len(meta['index_columns']) > 0: index_col = meta['index_columns'] if isinstance(index_col[0], dict) and \ @@ -353,7 +359,8 @@ cdef orc_reader_options make_orc_reader_options( c_column_names.reserve(len(column_names)) for col in column_names: c_column_names.push_back(str(col).encode()) - opts.set_columns(c_column_names) + if len(column_names) > 0: + opts.set_columns(c_column_names) return opts diff --git a/python/cudf/cudf/core/_compat.py b/python/cudf/cudf/core/_compat.py index c326b19307d..3783e9ded6d 100644 --- a/python/cudf/cudf/core/_compat.py +++ b/python/cudf/cudf/core/_compat.py @@ -11,6 +11,7 @@ PANDAS_LT_153 = PANDAS_VERSION < version.parse("1.5.3") PANDAS_EQ_200 = PANDAS_VERSION == version.parse("2.0.0") PANDAS_GE_200 = PANDAS_VERSION >= version.parse("2.0.0") +PANDAS_GE_201 = PANDAS_VERSION >= version.parse("2.0.1") PANDAS_GE_210 = PANDAS_VERSION >= version.parse("2.1.0") PANDAS_GE_214 = PANDAS_VERSION >= version.parse("2.1.4") PANDAS_GE_220 = PANDAS_VERSION >= version.parse("2.2.0") diff --git a/python/cudf/cudf/tests/test_orc.py b/python/cudf/cudf/tests/test_orc.py index e53fa1fb4bf..fc3e0ce56e1 100644 --- a/python/cudf/cudf/tests/test_orc.py +++ b/python/cudf/cudf/tests/test_orc.py @@ -576,7 +576,7 @@ def test_int_overflow(tmpdir): # The number of rows and the large element trigger delta encoding num_rows = 513 - df = cudf.DataFrame({"a": [None] * num_rows}, dtype="int32") + df = cudf.DataFrame({"a": [None] * num_rows}, dtype="int64") df["a"][0] = 1024 * 1024 * 1024 df["a"][num_rows - 1] = 1 df.to_orc(file_path) @@ -1669,16 +1669,7 @@ def run_orc_columns_and_index_param(index_obj, index, columns): expected = pd.read_orc(buffer, columns=columns) got = cudf.read_orc(buffer, columns=columns) - if columns: - # TODO: Remove workaround after this issue is fixed: - # https://github.com/pandas-dev/pandas/issues/47944 - assert_eq( - expected.sort_index(axis=1), - got.sort_index(axis=1), - check_index_type=True, - ) - else: - assert_eq(expected, got, check_index_type=True) + assert_eq(expected, got, check_index_type=True) @pytest.mark.parametrize("index_obj", [None, [10, 11, 12], ["x", "y", "z"]]) From 3344377b118ae63c71d79405c694f89c885dbdf8 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Sat, 30 Dec 2023 14:52:11 +0530 Subject: [PATCH 093/384] Handle missing warning assertions for concat pytests (#14682) This PR adds warning assertions that were missed in https://github.com/rapidsai/cudf/pull/14672 On `pandas_2.0_feature_branch`: ``` = 110 failed, 101331 passed, 2091 skipped, 952 xfailed, 312 xpassed in 1064.30s (0:17:44) = ``` This PR: ``` = 105 failed, 101336 passed, 2091 skipped, 952 xfailed, 312 xpassed in 1068.90s (0:17:48) = ``` --- python/cudf/cudf/tests/test_index.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index d06041301b9..9a927e65eb1 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -1062,10 +1062,12 @@ def test_index_empty_append_name_conflict(): non_empty = cudf.Index([1], name="bar") expected = cudf.Index([1]) - result = non_empty.append(empty) + with pytest.warns(FutureWarning): + result = non_empty.append(empty) assert_eq(result, expected) - result = empty.append(non_empty) + with pytest.warns(FutureWarning): + result = empty.append(non_empty) assert_eq(result, expected) @@ -2861,7 +2863,8 @@ def test_index_methods(index, func): if func == "append": expected = pidx.append(other=pidx) - actual = gidx.append(other=gidx) + with expect_warning_if(len(gidx) == 0): + actual = gidx.append(other=gidx) else: expected = getattr(pidx, func)() actual = getattr(gidx, func)() From eabba98c593b4603240b187e9207f01d5459e248 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Sat, 30 Dec 2023 14:52:41 +0530 Subject: [PATCH 094/384] Fix a typo error in where pytest (#14683) This PR fixes a typo in isinstance check, thus fixing 6 pytest failures. --- python/cudf/cudf/tests/test_dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 94aff555c7f..72a232f3d41 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -6108,7 +6108,7 @@ def test_df_sr_mask_where(data, condition, other, error, inplace): got_mask = gs_mask if hasattr(expect_where, "dtype") and isinstance( - expect_where, pd.CategoricalDtype + expect_where.dtype, pd.CategoricalDtype ): np.testing.assert_array_equal( expect_where.cat.codes, From bcdeb19ed66ce5dbcdb711ebb8cf803c23adb38a Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 8 Jan 2024 20:34:40 +0530 Subject: [PATCH 095/384] Change empty column dtype to `string` from `float64` (#14691) This PR enforces deprecation where an empty column now defaults to `str` dtype rather than `float64` dtype. On `pandas_2.0_feature_branch`: ``` = 68 failed, 101373 passed, 2091 skipped, 952 xfailed, 312 xpassed in 1000.21s (0:16:40) = ``` This PR: ``` = 65 failed, 101364 passed, 2091 skipped, 964 xfailed, 312 xpassed in 1054.55s (0:17:34) = ``` --- python/cudf/cudf/core/column/column.py | 4 +-- python/cudf/cudf/core/column/numerical.py | 7 ++++- python/cudf/cudf/core/reshape.py | 8 ++--- python/cudf/cudf/core/series.py | 19 +++--------- python/cudf/cudf/testing/_utils.py | 27 ----------------- python/cudf/cudf/tests/test_dataframe.py | 24 ++++++++++----- python/cudf/cudf/tests/test_dropna.py | 7 ++--- python/cudf/cudf/tests/test_duplicates.py | 3 +- python/cudf/cudf/tests/test_index.py | 8 ++--- python/cudf/cudf/tests/test_indexing.py | 13 +++++--- python/cudf/cudf/tests/test_joining.py | 11 ++++++- python/cudf/cudf/tests/test_onehot.py | 2 +- python/cudf/cudf/tests/test_rolling.py | 11 +++---- python/cudf/cudf/tests/test_series.py | 37 ++++++++++------------- python/cudf/cudf/tests/test_sorting.py | 2 +- python/cudf/cudf/tests/test_stats.py | 14 ++++----- 16 files changed, 86 insertions(+), 111 deletions(-) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index e83d82307e5..5dbbbe5ac10 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -2039,8 +2039,8 @@ def as_column( new_dtype = dtype elif len(arbitrary) == 0: # If the column is empty, it has to be - # a `float64` dtype. - new_dtype = cudf.dtype("float64") + # a `str` dtype. + new_dtype = cudf.dtype("str") else: # If the null column is not empty, it has to # be of `object` dtype. diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index cdecf44cc8f..fb9fa954c68 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -173,7 +173,12 @@ def __setitem__(self, key: Any, value: Any): if isinstance(key, slice): out = self._scatter_by_slice(key, device_value) else: - key = as_column(key) + key = as_column( + key, + dtype="float64" + if isinstance(key, list) and len(key) == 0 + else None, + ) if not isinstance(key, cudf.core.column.NumericalColumn): raise ValueError(f"Invalid scatter map type {key.dtype}.") out = self._scatter_by_column(key, device_value) diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index 465186d81d2..5f9d333811a 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -429,11 +429,9 @@ def concat(objs, axis=0, join="outer", ignore_index=False, sort=None): return result elif typ is cudf.Series: - objs = [obj for obj in objs if len(obj)] - if len(objs) == 0: - return cudf.Series() - elif len(objs) == 1 and not ignore_index: - return objs[0] + new_objs = [obj for obj in objs if len(obj)] + if len(new_objs) == 1 and not ignore_index: + return new_objs[0] else: return cudf.Series._concat( objs, axis=axis, index=None if ignore_index else True diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 6080a37f0a2..7b40e172da3 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -14,7 +14,6 @@ Dict, MutableMapping, Optional, - Sequence, Set, Tuple, Union, @@ -601,18 +600,6 @@ def __init__( copy=False, nan_as_null=True, ): - if ( - isinstance(data, Sequence) - and len(data) == 0 - and dtype is None - and getattr(data, "dtype", None) is None - ): - warnings.warn( - "The default dtype for empty Series will be 'object' instead " - "of 'float64' in a future version. Specify a dtype explicitly " - "to silence this warning.", - FutureWarning, - ) if isinstance(data, pd.Series): if name is None: name = data.name @@ -1621,7 +1608,11 @@ def _concat(cls, objs, axis=0, index=True): if isinstance(objs[0].index, cudf.MultiIndex): index = cudf.MultiIndex._concat([o.index for o in objs]) else: - index = cudf.core.index.Index._concat([o.index for o in objs]) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + index = cudf.core.index.Index._concat( + [o.index for o in objs] + ) names = {obj.name for obj in objs} if len(names) == 1: diff --git a/python/cudf/cudf/testing/_utils.py b/python/cudf/cudf/testing/_utils.py index 9182246826f..af8a38b8f01 100644 --- a/python/cudf/cudf/testing/_utils.py +++ b/python/cudf/cudf/testing/_utils.py @@ -19,7 +19,6 @@ import cudf from cudf._lib.null_mask import bitmask_allocation_size_bytes -from cudf.api.types import is_scalar from cudf.core.column.timedelta import _unit_to_nanoseconds_conversion from cudf.core.udf.strings_lowering import cast_string_view_to_udf_string from cudf.core.udf.strings_typing import StringView, string_view, udf_string @@ -397,32 +396,6 @@ def assert_column_memory_ne( raise AssertionError("lhs and rhs holds the same memory.") -def _create_pandas_series_float64_default( - data=None, index=None, dtype=None, *args, **kwargs -): - # Wrapper around pd.Series using a float64 - # default dtype for empty data to silence warnings. - # TODO: Remove this in pandas-2.0 upgrade - if dtype is None and ( - data is None or (not is_scalar(data) and len(data) == 0) - ): - dtype = "float64" - return pd.Series(data=data, index=index, dtype=dtype, *args, **kwargs) - - -def _create_cudf_series_float64_default( - data=None, index=None, dtype=None, *args, **kwargs -): - # Wrapper around cudf.Series using a float64 - # default dtype for empty data to silence warnings. - # TODO: Remove this in pandas-2.0 upgrade - if dtype is None and ( - data is None or (not is_scalar(data) and len(data) == 0) - ): - dtype = "float64" - return cudf.Series(data=data, index=index, dtype=dtype, *args, **kwargs) - - parametrize_numeric_dtypes_pairwise = pytest.mark.parametrize( "left_dtype,right_dtype", list(itertools.combinations_with_replacement(NUMERIC_TYPES, 2)), diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 72a232f3d41..e4007ec9cf9 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -40,7 +40,6 @@ ALL_TYPES, DATETIME_TYPES, NUMERIC_TYPES, - _create_cudf_series_float64_default, assert_eq, assert_exceptions_equal, assert_neq, @@ -1376,6 +1375,11 @@ def test_dataframe_setitem_from_masked_object(): def test_dataframe_append_to_empty(): pdf = pd.DataFrame() pdf["a"] = [] + if PANDAS_GE_200: + # TODO: Remove this workaround after + # the following bug is fixed: + # https://github.com/pandas-dev/pandas/issues/56679 + pdf["a"] = pdf["a"].astype("str") pdf["b"] = [1, 2, 3] gdf = cudf.DataFrame() @@ -2713,8 +2717,8 @@ def test_decimal_quantile(q, interpolation, decimal_type): def test_empty_quantile(): - pdf = pd.DataFrame({"x": []}) - df = cudf.DataFrame({"x": []}) + pdf = pd.DataFrame({"x": []}, dtype="float64") + df = cudf.DataFrame({"x": []}, dtype="float64") actual = df.quantile() expected = pdf.quantile() @@ -2972,7 +2976,7 @@ def test_series_all_null(num_elements, null_type): @pytest.mark.parametrize("num_elements", [0, 2, 10, 100]) def test_series_all_valid_nan(num_elements): data = [np.nan] * num_elements - sr = _create_cudf_series_float64_default(data, nan_as_null=False) + sr = cudf.Series(data, nan_as_null=False) np.testing.assert_equal(sr.null_count, 0) @@ -4653,7 +4657,7 @@ def test_dataframe_columns_empty_data_preserves_dtype(dtype, idx_data, data): ) def test_series_values_host_property(data): pds = pd.Series(data=data, dtype=None if data else float) - gds = _create_cudf_series_float64_default(data) + gds = cudf.Series(data=data, dtype=None if data else float) np.testing.assert_array_equal(pds.values, gds.values_host) @@ -4676,7 +4680,7 @@ def test_series_values_host_property(data): ) def test_series_values_property(data): pds = pd.Series(data=data, dtype=None if data else float) - gds = _create_cudf_series_float64_default(data) + gds = cudf.from_pandas(pds) gds_vals = gds.values assert isinstance(gds_vals, cupy.ndarray) np.testing.assert_array_equal(gds_vals.get(), pds.values) @@ -6663,7 +6667,13 @@ def test_dataframe_init_from_arrays_cols(data, cols, index): None, ], ) -def test_dataframe_assign_scalar(col_data, assign_val): +def test_dataframe_assign_scalar(request, col_data, assign_val): + request.applymarker( + pytest.mark.xfail( + condition=PANDAS_GE_200 and len(col_data) == 0, + reason="https://github.com/pandas-dev/pandas/issues/56679", + ) + ) pdf = pd.DataFrame({"a": col_data}) gdf = cudf.DataFrame({"a": col_data}) diff --git a/python/cudf/cudf/tests/test_dropna.py b/python/cudf/cudf/tests/test_dropna.py index d53d24cd6c6..b9dbfd330a4 100644 --- a/python/cudf/cudf/tests/test_dropna.py +++ b/python/cudf/cudf/tests/test_dropna.py @@ -5,10 +5,7 @@ import pytest import cudf -from cudf.testing._utils import ( - _create_pandas_series_float64_default, - assert_eq, -) +from cudf.testing._utils import assert_eq @pytest.mark.parametrize( @@ -25,7 +22,7 @@ @pytest.mark.parametrize("inplace", [True, False]) def test_dropna_series(data, nulls, inplace): - psr = _create_pandas_series_float64_default(data) + psr = pd.Series(data) if len(data) > 0: if nulls == "one": diff --git a/python/cudf/cudf/tests/test_duplicates.py b/python/cudf/cudf/tests/test_duplicates.py index ddbfdf5eee2..c6f025aa956 100644 --- a/python/cudf/cudf/tests/test_duplicates.py +++ b/python/cudf/cudf/tests/test_duplicates.py @@ -10,7 +10,6 @@ import cudf from cudf import concat from cudf.testing._utils import ( - _create_pandas_series_float64_default, assert_eq, assert_exceptions_equal, ) @@ -62,7 +61,7 @@ def test_duplicated_with_misspelled_column_name(subset): ], ) def test_drop_duplicates_series(data, keep): - pds = _create_pandas_series_float64_default(data) + pds = pd.Series(data) gds = cudf.from_pandas(pds) assert_df(pds.drop_duplicates(keep=keep), gds.drop_duplicates(keep=keep)) diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index 9a927e65eb1..38ac1a844b8 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -31,8 +31,6 @@ SERIES_OR_INDEX_NAMES, SIGNED_INTEGER_TYPES, UNSIGNED_TYPES, - _create_cudf_series_float64_default, - _create_pandas_series_float64_default, assert_column_memory_eq, assert_column_memory_ne, assert_eq, @@ -987,8 +985,8 @@ def test_index_equal_misc(data, other): actual = gd_data.equals(np.array(gd_other)) assert_eq(expected, actual) - expected = pd_data.equals(_create_pandas_series_float64_default(pd_other)) - actual = gd_data.equals(_create_cudf_series_float64_default(gd_other)) + expected = pd_data.equals(pd.Series(pd_other)) + actual = gd_data.equals(cudf.Series(gd_other)) assert_eq(expected, actual) expected = pd_data.astype("category").equals(pd_other) @@ -2559,7 +2557,7 @@ def test_index_nan_as_null(data, nan_idx, NA_idx, nan_as_null): ], ) def test_isin_index(data, values): - psr = _create_pandas_series_float64_default(data) + psr = pd.Series(data) gsr = cudf.Series.from_pandas(psr) got = gsr.index.isin(values) diff --git a/python/cudf/cudf/tests/test_indexing.py b/python/cudf/cudf/tests/test_indexing.py index f2b58a80362..3e09a11ad35 100644 --- a/python/cudf/cudf/tests/test_indexing.py +++ b/python/cudf/cudf/tests/test_indexing.py @@ -938,7 +938,7 @@ def test_series_setitem_basics(key, value, nulls): ): psr[key] = value with expect_warning_if( - isinstance(value, list) and len(value) == 0 and nulls == "none" + isinstance(value, list) and len(value) == 0 and not len(key) == 0 ): gsr[key] = value assert_eq(psr, gsr, check_dtype=False) @@ -991,7 +991,7 @@ def test_series_setitem_iloc(key, value, nulls): ): psr.iloc[key] = value with expect_warning_if( - isinstance(value, list) and len(value) == 0 and nulls == "none" + isinstance(value, list) and len(value) == 0 and not len(key) == 0 ): gsr.iloc[key] = value assert_eq(psr, gsr, check_dtype=False) @@ -1610,9 +1610,12 @@ def test_dataframe_loc_inplace_update_with_invalid_RHS_df_columns(): actual = gdf.loc[[0, 2], ["x", "y"]] = cudf.DataFrame( {"b": [10, 20], "y": [30, 40]}, index=cudf.Index([0, 2]) ) - expected = pdf.loc[[0, 2], ["x", "y"]] = pd.DataFrame( - {"b": [10, 20], "y": [30, 40]}, index=pd.Index([0, 2]) - ) + with pytest.warns(FutureWarning): + # Seems to be a false warning from pandas, + # but nevertheless catching it. + expected = pdf.loc[[0, 2], ["x", "y"]] = pd.DataFrame( + {"b": [10, 20], "y": [30, 40]}, index=pd.Index([0, 2]) + ) assert_eq(expected, actual) diff --git a/python/cudf/cudf/tests/test_joining.py b/python/cudf/cudf/tests/test_joining.py index b273e554158..670536ac32e 100644 --- a/python/cudf/cudf/tests/test_joining.py +++ b/python/cudf/cudf/tests/test_joining.py @@ -1941,7 +1941,11 @@ def test_string_join_key(str_data, num_keys, how): gdf[i] = cudf.Series(str_data, dtype="str") pdf["a"] = other_data gdf["a"] = other_data - + if PANDAS_GE_200 and len(other_data) == 0: + # TODO: Remove this workaround after + # the following bug is fixed: + # https://github.com/pandas-dev/pandas/issues/56679 + pdf["a"] = pdf["a"].astype("str") pdf2 = pdf.copy() gdf2 = gdf.copy() @@ -2017,6 +2021,11 @@ def test_string_join_non_key(str_data, num_cols, how): gdf[i] = cudf.Series(str_data, dtype="str") pdf["a"] = other_data gdf["a"] = other_data + if PANDAS_GE_200 and len(other_data) == 0: + # TODO: Remove this workaround after + # the following bug is fixed: + # https://github.com/pandas-dev/pandas/issues/56679 + pdf["a"] = pdf["a"].astype("str") pdf2 = pdf.copy() gdf2 = gdf.copy() diff --git a/python/cudf/cudf/tests/test_onehot.py b/python/cudf/cudf/tests/test_onehot.py index 9a4e71b6c9d..f60f80fcec7 100644 --- a/python/cudf/cudf/tests/test_onehot.py +++ b/python/cudf/cudf/tests/test_onehot.py @@ -24,8 +24,8 @@ ) @pytest.mark.parametrize("dtype", ["bool", "uint8"]) def test_get_dummies(data, index, dtype): - gdf = cudf.DataFrame({"x": data}, index=index) pdf = pd.DataFrame({"x": data}, index=index) + gdf = cudf.from_pandas(pdf) encoded_expected = pd.get_dummies(pdf, prefix="test", dtype=dtype) encoded_actual = cudf.get_dummies(gdf, prefix="test", dtype=dtype) diff --git a/python/cudf/cudf/tests/test_rolling.py b/python/cudf/cudf/tests/test_rolling.py index 8aa5050671a..b4e69f47d2a 100644 --- a/python/cudf/cudf/tests/test_rolling.py +++ b/python/cudf/cudf/tests/test_rolling.py @@ -9,10 +9,7 @@ import cudf from cudf.core._compat import PANDAS_GE_150, PANDAS_GE_200 -from cudf.testing._utils import ( - _create_pandas_series_float64_default, - assert_eq, -) +from cudf.testing._utils import assert_eq from cudf.testing.dataset_generator import rand_dataframe @@ -58,8 +55,8 @@ def test_rolling_series_basic(data, index, agg, nulls, center): elif nulls == "all": data = [np.nan] * len(data) - psr = _create_pandas_series_float64_default(data, index=index) - gsr = cudf.Series(psr) + psr = pd.Series(data, index=index) + gsr = cudf.from_pandas(psr) for window_size in range(1, len(data) + 1): for min_periods in range(1, window_size + 1): expect = getattr( @@ -316,7 +313,7 @@ def test_rolling_getitem_window(): @pytest.mark.parametrize("center", [True, False]) def test_rollling_series_numba_udf_basic(data, index, center): - psr = _create_pandas_series_float64_default(data, index=index) + psr = pd.Series(data, index=index) gsr = cudf.from_pandas(psr) def some_func(A): diff --git a/python/cudf/cudf/tests/test_series.py b/python/cudf/cudf/tests/test_series.py index 1e5d41888de..1aae58f47d1 100644 --- a/python/cudf/cudf/tests/test_series.py +++ b/python/cudf/cudf/tests/test_series.py @@ -20,8 +20,6 @@ NUMERIC_TYPES, SERIES_OR_INDEX_NAMES, TIMEDELTA_TYPES, - _create_cudf_series_float64_default, - _create_pandas_series_float64_default, assert_eq, assert_exceptions_equal, expect_warning_if, @@ -392,8 +390,8 @@ def test_series_tolist(data): [[], [None, None], ["a"], ["a", "b", "c"] * 500, [1.0, 2.0, 0.3] * 57], ) def test_series_size(data): - psr = _create_pandas_series_float64_default(data) - gsr = _create_cudf_series_float64_default(data) + psr = pd.Series(data) + gsr = cudf.Series(data) assert_eq(psr.size, gsr.size) @@ -475,7 +473,7 @@ def test_series_describe_other_types(ps): ) @pytest.mark.parametrize("use_na_sentinel", [True, False]) def test_series_factorize_use_na_sentinel(data, use_na_sentinel): - gsr = _create_cudf_series_float64_default(data) + gsr = cudf.Series(data) psr = gsr.to_pandas(nullable=True) expected_labels, expected_cats = psr.factorize( @@ -499,7 +497,7 @@ def test_series_factorize_use_na_sentinel(data, use_na_sentinel): ) @pytest.mark.parametrize("sort", [True, False]) def test_series_factorize_sort(data, sort): - gsr = _create_cudf_series_float64_default(data) + gsr = cudf.Series(data) psr = gsr.to_pandas(nullable=True) expected_labels, expected_cats = psr.factorize(sort=sort) @@ -1665,7 +1663,7 @@ def test_series_nunique_index(data): ], ) def test_axes(data): - csr = _create_cudf_series_float64_default(data) + csr = cudf.Series(data) psr = csr.to_pandas() expected = psr.axes @@ -1743,7 +1741,7 @@ def test_series_truncate_datetimeindex(): ) def test_isin_numeric(data, values): index = np.random.randint(0, 100, len(data)) - psr = _create_pandas_series_float64_default(data, index=index) + psr = pd.Series(data, index=index) gsr = cudf.Series.from_pandas(psr, nan_as_null=False) expected = psr.isin(values) @@ -1803,7 +1801,7 @@ def test_fill_new_category(): ], ) def test_isin_datetime(data, values): - psr = _create_pandas_series_float64_default(data) + psr = pd.Series(data) gsr = cudf.Series.from_pandas(psr) got = gsr.isin(values) @@ -1832,7 +1830,7 @@ def test_isin_datetime(data, values): ], ) def test_isin_string(data, values): - psr = _create_pandas_series_float64_default(data) + psr = pd.Series(data) gsr = cudf.Series.from_pandas(psr) got = gsr.isin(values) @@ -1861,7 +1859,7 @@ def test_isin_string(data, values): ], ) def test_isin_categorical(data, values): - psr = _create_pandas_series_float64_default(data) + psr = pd.Series(data) gsr = cudf.Series.from_pandas(psr) got = gsr.isin(values) @@ -2082,7 +2080,7 @@ def test_series_to_dict(into): ], ) def test_series_hasnans(data): - gs = _create_cudf_series_float64_default(data, nan_as_null=False) + gs = cudf.Series(data, nan_as_null=False) ps = gs.to_pandas(nullable=True) # Check type to avoid mixing Python bool and NumPy bool @@ -2155,8 +2153,8 @@ def test_series_init_dict_with_index(data, index): "index", [None, ["b", "c"], ["d", "a", "c", "b"], ["a"]] ) def test_series_init_scalar_with_index(data, index): - pandas_series = _create_pandas_series_float64_default(data, index=index) - cudf_series = _create_cudf_series_float64_default(data, index=index) + pandas_series = pd.Series(data, index=index) + cudf_series = cudf.Series(data, index=index) assert_eq( pandas_series, @@ -2305,15 +2303,12 @@ def test_series_round_builtin(data, digits): assert_eq(expected, actual) -def test_series_empty_warning(): - with pytest.warns(FutureWarning): - expected = pd.Series([]) - with pytest.warns(FutureWarning): - actual = cudf.Series([]) - assert_eq(expected, actual) +def test_series_empty_dtype(): + expected = pd.Series([]) + actual = cudf.Series([]) + assert_eq(expected, actual, check_dtype=True) -@pytest.mark.filterwarnings("ignore::FutureWarning") # tested above @pytest.mark.parametrize("data", [None, {}, []]) def test_series_empty_index_rangeindex(data): expected = cudf.RangeIndex(0) diff --git a/python/cudf/cudf/tests/test_sorting.py b/python/cudf/cudf/tests/test_sorting.py index b3db1310adb..518b7597a12 100644 --- a/python/cudf/cudf/tests/test_sorting.py +++ b/python/cudf/cudf/tests/test_sorting.py @@ -394,6 +394,6 @@ def test_dataframe_scatter_by_map_7513(ids): def test_dataframe_scatter_by_map_empty(): - df = DataFrame({"a": [], "b": []}) + df = DataFrame({"a": [], "b": []}, dtype="float64") scattered = df.scatter_by_map(df["a"]) assert len(scattered) == 0 diff --git a/python/cudf/cudf/tests/test_stats.py b/python/cudf/cudf/tests/test_stats.py index f24a5ea7b41..8e1a91c7a4f 100644 --- a/python/cudf/cudf/tests/test_stats.py +++ b/python/cudf/cudf/tests/test_stats.py @@ -11,8 +11,6 @@ from cudf.api.extensions import no_default from cudf.datasets import randomdata from cudf.testing._utils import ( - _create_cudf_series_float64_default, - _create_pandas_series_float64_default, assert_eq, assert_exceptions_equal, expect_warning_if, @@ -225,8 +223,8 @@ def test_approx_quantiles_int(): ) def test_misc_quantiles(data, q): - pdf_series = _create_pandas_series_float64_default(data) - gdf_series = _create_cudf_series_float64_default(data) + pdf_series = pd.Series(data, dtype="float64" if len(data) == 0 else None) + gdf_series = cudf.from_pandas(pdf_series) expected = pdf_series.quantile(q.get() if isinstance(q, cp.ndarray) else q) actual = gdf_series.quantile(q) @@ -539,14 +537,16 @@ def test_df_corr(method): ) @pytest.mark.parametrize("skipna", [True, False]) def test_nans_stats(data, ops, skipna): - psr = _create_pandas_series_float64_default(data) - gsr = _create_cudf_series_float64_default(data, nan_as_null=False) + psr = pd.Series(data, dtype="float64" if len(data) == 0 else None) + gsr = cudf.from_pandas(psr) assert_eq( getattr(psr, ops)(skipna=skipna), getattr(gsr, ops)(skipna=skipna) ) - gsr = _create_cudf_series_float64_default(data, nan_as_null=False) + gsr = cudf.Series( + data, dtype="float64" if len(data) == 0 else None, nan_as_null=False + ) # Since there is no concept of `nan_as_null` in pandas, # nulls will be returned in the operations. So only # testing for `skipna=True` when `nan_as_null=False` From 6bcaf443d902accc3def2bb15cd6542abef6885b Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 23 Jan 2024 11:39:17 +0530 Subject: [PATCH 096/384] Preserve empty index types in parquet reader (#14818) This PR preserves types of empty column index objects whose metadata is already present in the parquet file. This PR: = 107 failed, 101869 passed, 2091 skipped, 976 xfailed, 312 xpassed in 1265.57s (0:21:05) = On pandas_2.0_feature_branch: = 111 failed, 101865 passed, 2091 skipped, 976 xfailed, 312 xpassed in 1292.26s (0:21:32) = --- python/cudf/cudf/_lib/parquet.pyx | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/_lib/parquet.pyx b/python/cudf/cudf/_lib/parquet.pyx index 78606a45fc1..e12b5be3c71 100644 --- a/python/cudf/cudf/_lib/parquet.pyx +++ b/python/cudf/cudf/_lib/parquet.pyx @@ -110,6 +110,7 @@ cdef class BufferArrayFromVector: def _parse_metadata(meta): file_is_range_index = False file_index_cols = None + file_column_dtype = None if 'index_columns' in meta and len(meta['index_columns']) > 0: file_index_cols = meta['index_columns'] @@ -117,7 +118,9 @@ def _parse_metadata(meta): if isinstance(file_index_cols[0], dict) and \ file_index_cols[0]['kind'] == 'range': file_is_range_index = True - return file_is_range_index, file_index_cols + if 'column_indexes' in meta and len(meta['column_indexes']) == 1: + file_column_dtype = meta['column_indexes'][0]["numpy_type"] + return file_is_range_index, file_index_cols, file_column_dtype cpdef read_parquet(filepaths_or_buffers, columns=None, row_groups=None, @@ -185,6 +188,7 @@ cpdef read_parquet(filepaths_or_buffers, columns=None, row_groups=None, cdef vector[unordered_map[string, string]] per_file_user_data = \ c_result.metadata.per_file_user_data + column_index_type = None index_col_names = None is_range_index = True for single_file in per_file_user_data: @@ -192,7 +196,7 @@ cpdef read_parquet(filepaths_or_buffers, columns=None, row_groups=None, meta = None if json_str != "": meta = json.loads(json_str) - file_is_range_index, index_col = _parse_metadata(meta) + file_is_range_index, index_col, column_index_type = _parse_metadata(meta) is_range_index &= file_is_range_index if not file_is_range_index and index_col is not None \ @@ -302,6 +306,9 @@ cpdef read_parquet(filepaths_or_buffers, columns=None, row_groups=None, if use_pandas_metadata: df.index.names = index_col + # Set column dtype for empty types. + if len(df._data.names) == 0 and column_index_type is not None: + df._data.label_dtype = cudf.dtype(column_index_type) return df From 48367a9d0cec39915e6c4e3ec33336480359260f Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 23 Jan 2024 08:18:26 -0800 Subject: [PATCH 097/384] Migrate binary operations to pylibcudf (#14821) This PR migrates the binary operations in cuDF Python to pylibcudf. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/14821 --- .../api_docs/pylibcudf/binaryop.rst | 6 + .../user_guide/api_docs/pylibcudf/index.rst | 1 + python/cudf/cudf/_lib/binaryop.pxd | 5 - python/cudf/cudf/_lib/binaryop.pyx | 261 +++--------------- python/cudf/cudf/_lib/cpp/CMakeLists.txt | 4 +- python/cudf/cudf/_lib/cpp/binaryop.pxd | 75 ++--- python/cudf/cudf/_lib/cpp/binaryop.pyx | 0 .../cudf/cudf/_lib/pylibcudf/CMakeLists.txt | 6 +- python/cudf/cudf/_lib/pylibcudf/__init__.pxd | 5 +- python/cudf/cudf/_lib/pylibcudf/__init__.py | 5 +- python/cudf/cudf/_lib/pylibcudf/binaryop.pxd | 14 + python/cudf/cudf/_lib/pylibcudf/binaryop.pyx | 86 ++++++ python/cudf/cudf/tests/test_udf_binops.py | 51 ---- 13 files changed, 185 insertions(+), 334 deletions(-) create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/binaryop.rst delete mode 100644 python/cudf/cudf/_lib/binaryop.pxd create mode 100644 python/cudf/cudf/_lib/cpp/binaryop.pyx create mode 100644 python/cudf/cudf/_lib/pylibcudf/binaryop.pxd create mode 100644 python/cudf/cudf/_lib/pylibcudf/binaryop.pyx delete mode 100644 python/cudf/cudf/tests/test_udf_binops.py diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/binaryop.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/binaryop.rst new file mode 100644 index 00000000000..e5bc6aa7cda --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/binaryop.rst @@ -0,0 +1,6 @@ +======== +binaryop +======== + +.. automodule:: cudf._lib.pylibcudf.binaryop + :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst index 435278afeeb..7504295de92 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst @@ -8,6 +8,7 @@ This page provides API documentation for pylibcudf. :maxdepth: 1 :caption: API Documentation + binaryop column copying gpumemoryview diff --git a/python/cudf/cudf/_lib/binaryop.pxd b/python/cudf/cudf/_lib/binaryop.pxd deleted file mode 100644 index 1f6022251b3..00000000000 --- a/python/cudf/cudf/_lib/binaryop.pxd +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. - -from libc.stdint cimport int32_t - -ctypedef int32_t underlying_type_t_binary_operator diff --git a/python/cudf/cudf/_lib/binaryop.pyx b/python/cudf/cudf/_lib/binaryop.pyx index 6212347b5b1..969be426044 100644 --- a/python/cudf/cudf/_lib/binaryop.pyx +++ b/python/cudf/cudf/_lib/binaryop.pyx @@ -1,160 +1,30 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. -from enum import IntEnum - -from libcpp.memory cimport unique_ptr -from libcpp.string cimport string -from libcpp.utility cimport move - -from cudf._lib.binaryop cimport underlying_type_t_binary_operator from cudf._lib.column cimport Column - -from cudf._lib.scalar import as_device_scalar - from cudf._lib.scalar cimport DeviceScalar +from cudf._lib.types cimport dtype_to_pylibcudf_type -from cudf._lib.types import SUPPORTED_NUMPY_TO_LIBCUDF_TYPES - -from cudf._lib.cpp.column.column cimport column -from cudf._lib.cpp.column.column_view cimport column_view -from cudf._lib.cpp.scalar.scalar cimport scalar -from cudf._lib.cpp.types cimport data_type, type_id -from cudf._lib.types cimport dtype_to_data_type, underlying_type_t_type_id - -from cudf.api.types import is_scalar +from cudf._lib import pylibcudf +from cudf._lib.scalar import as_device_scalar from cudf.core.buffer import acquire_spill_lock -cimport cudf._lib.cpp.binaryop as cpp_binaryop -from cudf._lib.cpp.binaryop cimport binary_operator -import cudf - - -class BinaryOperation(IntEnum): - ADD = ( - binary_operator.ADD - ) - SUB = ( - binary_operator.SUB - ) - MUL = ( - binary_operator.MUL - ) - DIV = ( - binary_operator.DIV - ) - TRUEDIV = ( - binary_operator.TRUE_DIV - ) - FLOORDIV = ( - binary_operator.FLOOR_DIV - ) - MOD = ( - binary_operator.PYMOD - ) - POW = ( - binary_operator.POW - ) - INT_POW = ( - binary_operator.INT_POW - ) - EQ = ( - binary_operator.EQUAL - ) - NE = ( - binary_operator.NOT_EQUAL - ) - LT = ( - binary_operator.LESS - ) - GT = ( - binary_operator.GREATER - ) - LE = ( - binary_operator.LESS_EQUAL - ) - GE = ( - binary_operator.GREATER_EQUAL - ) - AND = ( - binary_operator.BITWISE_AND - ) - OR = ( - binary_operator.BITWISE_OR - ) - XOR = ( - binary_operator.BITWISE_XOR - ) - L_AND = ( - binary_operator.LOGICAL_AND - ) - L_OR = ( - binary_operator.LOGICAL_OR - ) - GENERIC_BINARY = ( - binary_operator.GENERIC_BINARY - ) - NULL_EQUALS = ( - binary_operator.NULL_EQUALS - ) - - -cdef binaryop_v_v(Column lhs, Column rhs, - binary_operator c_op, data_type c_dtype): - cdef column_view c_lhs = lhs.view() - cdef column_view c_rhs = rhs.view() - - cdef unique_ptr[column] c_result - - with nogil: - c_result = move( - cpp_binaryop.binary_operation( - c_lhs, - c_rhs, - c_op, - c_dtype - ) - ) - - return Column.from_unique_ptr(move(c_result)) - - -cdef binaryop_v_s(Column lhs, DeviceScalar rhs, - binary_operator c_op, data_type c_dtype): - cdef column_view c_lhs = lhs.view() - cdef const scalar* c_rhs = rhs.get_raw_ptr() - - cdef unique_ptr[column] c_result - - with nogil: - c_result = move( - cpp_binaryop.binary_operation( - c_lhs, - c_rhs[0], - c_op, - c_dtype - ) - ) - - return Column.from_unique_ptr(move(c_result)) - -cdef binaryop_s_v(DeviceScalar lhs, Column rhs, - binary_operator c_op, data_type c_dtype): - cdef const scalar* c_lhs = lhs.get_raw_ptr() - cdef column_view c_rhs = rhs.view() - - cdef unique_ptr[column] c_result - - with nogil: - c_result = move( - cpp_binaryop.binary_operation( - c_lhs[0], - c_rhs, - c_op, - c_dtype - ) - ) - - return Column.from_unique_ptr(move(c_result)) +# Map pandas operation names to pylibcudf operation names. +_op_map = { + "TRUEDIV": "TRUE_DIV", + "FLOORDIV": "FLOOR_DIV", + "MOD": "PYMOD", + "EQ": "EQUAL", + "NE": "NOT_EQUAL", + "LT": "LESS", + "GT": "GREATER", + "LE": "LESS_EQUAL", + "GE": "GREATER_EQUAL", + "AND": "BITWISE_AND", + "OR": "BITWISE_OR", + "XOR": "BITWISE_XOR", + "L_AND": "LOGICAL_AND", + "L_OR": "LOGICAL_OR", +} @acquire_spill_lock() @@ -166,74 +36,25 @@ def binaryop(lhs, rhs, op, dtype): # pipeline for libcudf binops that don't map to Python binops. if op not in {"INT_POW", "NULL_EQUALS"}: op = op[2:-2] - - op = BinaryOperation[op.upper()] - cdef binary_operator c_op = ( - op - ) - - cdef data_type c_dtype = dtype_to_data_type(dtype) - - if is_scalar(lhs) or lhs is None: - s_lhs = as_device_scalar(lhs, dtype=rhs.dtype if lhs is None else None) - result = binaryop_s_v( - s_lhs, - rhs, - c_op, - c_dtype - ) - - elif is_scalar(rhs) or rhs is None: - s_rhs = as_device_scalar(rhs, dtype=lhs.dtype if rhs is None else None) - result = binaryop_v_s( - lhs, - s_rhs, - c_op, - c_dtype - ) - - else: - result = binaryop_v_v( - lhs, - rhs, - c_op, - c_dtype - ) - return result - - -@acquire_spill_lock() -def binaryop_udf(Column lhs, Column rhs, udf_ptx, dtype): - """ - Apply a user-defined binary operator (a UDF) defined in `udf_ptx` on - the two input columns `lhs` and `rhs`. The output type of the UDF - has to be specified in `dtype`, a numpy data type. - Currently ONLY int32, int64, float32 and float64 are supported. - """ - cdef column_view c_lhs = lhs.view() - cdef column_view c_rhs = rhs.view() - - cdef type_id tid = ( - ( - ( - SUPPORTED_NUMPY_TO_LIBCUDF_TYPES[cudf.dtype(dtype)] - ) + op = op.upper() + op = _op_map.get(op, op) + + return Column.from_pylibcudf( + # Check if the dtype args are desirable here. + pylibcudf.binaryop.binary_operation( + lhs.to_pylibcudf(mode="read") if isinstance(lhs, Column) + else ( + as_device_scalar( + lhs, dtype=rhs.dtype if lhs is None else None + ) + ).c_value, + rhs.to_pylibcudf(mode="read") if isinstance(rhs, Column) + else ( + as_device_scalar( + rhs, dtype=lhs.dtype if rhs is None else None + ) + ).c_value, + pylibcudf.binaryop.BinaryOperator[op], + dtype_to_pylibcudf_type(dtype), ) ) - cdef data_type c_dtype = data_type(tid) - - cdef string cpp_str = udf_ptx.encode("UTF-8") - - cdef unique_ptr[column] c_result - - with nogil: - c_result = move( - cpp_binaryop.binary_operation( - c_lhs, - c_rhs, - cpp_str, - c_dtype - ) - ) - - return Column.from_unique_ptr(move(c_result)) diff --git a/python/cudf/cudf/_lib/cpp/CMakeLists.txt b/python/cudf/cudf/_lib/cpp/CMakeLists.txt index a99aa58dfe8..764f28add0e 100644 --- a/python/cudf/cudf/_lib/cpp/CMakeLists.txt +++ b/python/cudf/cudf/_lib/cpp/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -12,7 +12,7 @@ # the License. # ============================================================================= -set(cython_sources copying.pyx types.pyx) +set(cython_sources binaryop.pyx copying.pyx types.pyx) set(linked_libraries cudf::cudf) diff --git a/python/cudf/cudf/_lib/cpp/binaryop.pxd b/python/cudf/cudf/_lib/cpp/binaryop.pxd index f73a9502cd1..735216e656a 100644 --- a/python/cudf/cudf/_lib/cpp/binaryop.pxd +++ b/python/cudf/cudf/_lib/cpp/binaryop.pxd @@ -1,5 +1,6 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. +from libc.stdint cimport int32_t from libcpp.memory cimport unique_ptr from libcpp.string cimport string @@ -10,30 +11,30 @@ from cudf._lib.cpp.types cimport data_type cdef extern from "cudf/binaryop.hpp" namespace "cudf" nogil: - ctypedef enum binary_operator: - ADD "cudf::binary_operator::ADD" - SUB "cudf::binary_operator::SUB" - MUL "cudf::binary_operator::MUL" - DIV "cudf::binary_operator::DIV" - TRUE_DIV "cudf::binary_operator::TRUE_DIV" - FLOOR_DIV "cudf::binary_operator::FLOOR_DIV" - MOD "cudf::binary_operator::MOD" - PYMOD "cudf::binary_operator::PYMOD" - POW "cudf::binary_operator::POW" - INT_POW "cudf::binary_operator::INT_POW" - EQUAL "cudf::binary_operator::EQUAL" - NOT_EQUAL "cudf::binary_operator::NOT_EQUAL" - LESS "cudf::binary_operator::LESS" - GREATER "cudf::binary_operator::GREATER" - LESS_EQUAL "cudf::binary_operator::LESS_EQUAL" - GREATER_EQUAL "cudf::binary_operator::GREATER_EQUAL" - NULL_EQUALS "cudf::binary_operator::NULL_EQUALS" - BITWISE_AND "cudf::binary_operator::BITWISE_AND" - BITWISE_OR "cudf::binary_operator::BITWISE_OR" - BITWISE_XOR "cudf::binary_operator::BITWISE_XOR" - LOGICAL_AND "cudf::binary_operator::LOGICAL_AND" - LOGICAL_OR "cudf::binary_operator::LOGICAL_OR" - GENERIC_BINARY "cudf::binary_operator::GENERIC_BINARY" + cpdef enum class binary_operator(int32_t): + ADD + SUB + MUL + DIV + TRUE_DIV + FLOOR_DIV + MOD + PYMOD + POW + INT_POW + EQUAL + NOT_EQUAL + LESS + GREATER + LESS_EQUAL + GREATER_EQUAL + NULL_EQUALS + BITWISE_AND + BITWISE_OR + BITWISE_XOR + LOGICAL_AND + LOGICAL_OR + GENERIC_BINARY cdef unique_ptr[column] binary_operation ( const scalar& lhs, @@ -62,27 +63,3 @@ cdef extern from "cudf/binaryop.hpp" namespace "cudf" nogil: const string& op, data_type output_type ) except + - - unique_ptr[column] jit_binary_operation \ - "cudf::jit::binary_operation" ( - const column_view& lhs, - const column_view& rhs, - binary_operator op, - data_type output_type - ) except + - - unique_ptr[column] jit_binary_operation \ - "cudf::jit::binary_operation" ( - const column_view& lhs, - const scalar& rhs, - binary_operator op, - data_type output_type - ) except + - - unique_ptr[column] jit_binary_operation \ - "cudf::jit::binary_operation" ( - const scalar& lhs, - const column_view& rhs, - binary_operator op, - data_type output_type - ) except + diff --git a/python/cudf/cudf/_lib/cpp/binaryop.pyx b/python/cudf/cudf/_lib/cpp/binaryop.pyx new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt index 870a00f99a9..acb013c8b8c 100644 --- a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt +++ b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -12,8 +12,8 @@ # the License. # ============================================================================= -set(cython_sources column.pyx copying.pyx gpumemoryview.pyx interop.pyx scalar.pyx table.pyx - types.pyx utils.pyx +set(cython_sources binaryop.pyx column.pyx copying.pyx gpumemoryview.pyx interop.pyx scalar.pyx + table.pyx types.pyx utils.pyx ) set(linked_libraries cudf::cudf) rapids_cython_create_modules( diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd index 7a35854392c..f4b8c50eecc 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd @@ -1,7 +1,7 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # TODO: Verify consistent usage of relative/absolute imports in pylibcudf. -from . cimport copying, interop +from . cimport binaryop, copying, interop from .column cimport Column from .gpumemoryview cimport gpumemoryview from .scalar cimport Scalar @@ -15,6 +15,7 @@ __all__ = [ "DataType", "Scalar", "Table", + "binaryop", "copying", "gpumemoryview", "interop", diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.py b/python/cudf/cudf/_lib/pylibcudf/__init__.py index 72b74a57b87..a27d80fc5a2 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.py +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.py @@ -1,6 +1,6 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. -from . import copying, interop +from . import binaryop, copying, interop from .column import Column from .gpumemoryview import gpumemoryview from .scalar import Scalar @@ -13,6 +13,7 @@ "Scalar", "Table", "TypeId", + "binaryop", "copying", "gpumemoryview", "interop", diff --git a/python/cudf/cudf/_lib/pylibcudf/binaryop.pxd b/python/cudf/cudf/_lib/pylibcudf/binaryop.pxd new file mode 100644 index 00000000000..56b98333757 --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/binaryop.pxd @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from cudf._lib.cpp.binaryop cimport binary_operator + +from .column cimport Column +from .types cimport DataType + + +cpdef Column binary_operation( + object lhs, + object rhs, + binary_operator op, + DataType data_type +) diff --git a/python/cudf/cudf/_lib/pylibcudf/binaryop.pyx b/python/cudf/cudf/_lib/pylibcudf/binaryop.pyx new file mode 100644 index 00000000000..af248ba2071 --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/binaryop.pyx @@ -0,0 +1,86 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from cython.operator import dereference + +from libcpp.memory cimport unique_ptr +from libcpp.utility cimport move + +from cudf._lib.cpp cimport binaryop as cpp_binaryop +from cudf._lib.cpp.binaryop cimport binary_operator +from cudf._lib.cpp.column.column cimport column + +from cudf._lib.cpp.binaryop import \ + binary_operator as BinaryOperator # no-cython-lint + +from .column cimport Column +from .scalar cimport Scalar +from .types cimport DataType + + +cpdef Column binary_operation( + object lhs, + object rhs, + binary_operator op, + DataType data_type +): + """Perform a binary operation between a column and another column or scalar. + + Either ``lhs`` or ``rhs`` must be a + :py:class:`~cudf._lib.pylibcudf.column.Column`. The other may be a + :py:class:`~cudf._lib.pylibcudf.column.Column` or a + :py:class:`~cudf._lib.pylibcudf.scalar.Scalar`. + + For details, see :cpp:func:`binary_operation`. + + Parameters + ---------- + lhs : Column or Scalar + The left hand side argument. + rhs : Column or Scalar + The right hand side argument. + op : BinaryOperator + The operation to perform. + data_type : DataType + The output to use for the output. + + Returns + ------- + pylibcudf.Column + The result of the binary operation + """ + cdef unique_ptr[column] result + + if isinstance(lhs, Column) and isinstance(rhs, Column): + with nogil: + result = move( + cpp_binaryop.binary_operation( + ( lhs).view(), + ( rhs).view(), + op, + data_type.c_obj + ) + ) + elif isinstance(lhs, Column) and isinstance(rhs, Scalar): + with nogil: + result = move( + cpp_binaryop.binary_operation( + ( lhs).view(), + dereference(( rhs).c_obj), + op, + data_type.c_obj + ) + ) + elif isinstance(lhs, Scalar) and isinstance(rhs, Column): + with nogil: + result = move( + cpp_binaryop.binary_operation( + dereference(( lhs).c_obj), + ( rhs).view(), + op, + data_type.c_obj + ) + ) + else: + raise ValueError(f"Invalid arguments {lhs} and {rhs}") + + return Column.from_libcudf(move(result)) diff --git a/python/cudf/cudf/tests/test_udf_binops.py b/python/cudf/cudf/tests/test_udf_binops.py deleted file mode 100644 index 1ad45e721a3..00000000000 --- a/python/cudf/cudf/tests/test_udf_binops.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (c) 2018-2022, NVIDIA CORPORATION. - -import numpy as np -import pytest -from numba.cuda import compile_ptx -from numba.np import numpy_support - -import rmm - -import cudf -from cudf import Series, _lib as libcudf -from cudf.utils import dtypes as dtypeutils - -_driver_version = rmm._cuda.gpu.driverGetVersion() -_runtime_version = rmm._cuda.gpu.runtimeGetVersion() -_CUDA_JIT128INT_SUPPORTED = (_driver_version >= 11050) and ( - _runtime_version >= 11050 -) - - -@pytest.mark.skipif(not _CUDA_JIT128INT_SUPPORTED, reason="requires CUDA 11.5") -@pytest.mark.parametrize( - "dtype", sorted(list(dtypeutils.NUMERIC_TYPES - {"int8"})) -) -def test_generic_ptx(dtype): - - size = 500 - - lhs_arr = np.random.random(size).astype(dtype) - lhs_col = Series(lhs_arr)._column - - rhs_arr = np.random.random(size).astype(dtype) - rhs_col = Series(rhs_arr)._column - - def generic_function(a, b): - return a**3 + b - - nb_type = numpy_support.from_dtype(cudf.dtype(dtype)) - type_signature = (nb_type, nb_type) - - ptx_code, output_type = compile_ptx( - generic_function, type_signature, device=True - ) - - dtype = numpy_support.as_dtype(output_type).type - - out_col = libcudf.binaryop.binaryop_udf(lhs_col, rhs_col, ptx_code, dtype) - - result = lhs_arr**3 + rhs_arr - - np.testing.assert_almost_equal(result, out_col.values_host) From c83b9fdcf45aa0b7204ef0313dc0a778dc15e017 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 23 Jan 2024 06:19:34 -1000 Subject: [PATCH 098/384] Refactor and add validation to IntervalIndex.__init__ (#14778) * Adding validation to `closed`, `dtype` arguments in `ItervalIndex.__init__` * Ensure `closed` attribute always maps to `IntervalDtype.closed` * `build_interval_column` was no longer necessary by using `IntervalColumn` directly Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/14778 --- python/cudf/cudf/core/column/column.py | 50 +------- python/cudf/cudf/core/column/interval.py | 25 +--- python/cudf/cudf/core/index.py | 120 ++++++++++++------ .../cudf/cudf/tests/indexes/test_interval.py | 29 ++++- python/cudf/cudf/tests/test_udf_masked_ops.py | 4 +- 5 files changed, 114 insertions(+), 114 deletions(-) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 7a99ef9f470..dc060a7117e 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -999,14 +999,14 @@ def astype(self, dtype: Dtype, copy: bool = False) -> ColumnBase: "`.astype('str')` instead." ) return col.as_string_column(dtype) + elif isinstance(dtype, IntervalDtype): + return col.as_interval_column(dtype) elif isinstance(dtype, (ListDtype, StructDtype)): if not col.dtype == dtype: raise NotImplementedError( f"Casting {self.dtype} columns not currently supported" ) return col - elif isinstance(dtype, IntervalDtype): - return col.as_interval_column(dtype) elif isinstance(dtype, cudf.core.dtypes.DecimalDtype): return col.as_decimal_column(dtype) elif np.issubdtype(cast(Any, dtype), np.datetime64): @@ -1689,52 +1689,6 @@ def build_categorical_column( return cast("cudf.core.column.CategoricalColumn", result) -def build_interval_column( - left_col, - right_col, - mask=None, - size=None, - offset=0, - null_count=None, - closed="right", -): - """ - Build an IntervalColumn - - Parameters - ---------- - left_col : Column - Column of values representing the left of the interval - right_col : Column - Column of representing the right of the interval - mask : Buffer - Null mask - size : int, optional - offset : int, optional - closed : {"left", "right", "both", "neither"}, default "right" - Whether the intervals are closed on the left-side, right-side, - both or neither. - """ - left = as_column(left_col) - right = as_column(right_col) - if closed not in {"left", "right", "both", "neither"}: - closed = "right" - if type(left_col) is not list: - dtype = IntervalDtype(left_col.dtype, closed) - else: - dtype = IntervalDtype("int64", closed) - size = len(left) - return build_column( - data=None, - dtype=dtype, - mask=mask, - size=size, - offset=offset, - null_count=null_count, - children=(left, right), - ) - - def build_list_column( indices: ColumnBase, elements: ColumnBase, diff --git a/python/cudf/cudf/core/column/interval.py b/python/cudf/cudf/core/column/interval.py index 6a7e7729123..7227ef8ba3a 100644 --- a/python/cudf/cudf/core/column/interval.py +++ b/python/cudf/cudf/core/column/interval.py @@ -18,7 +18,6 @@ def __init__( offset=0, null_count=None, children=(), - closed="right", ): super().__init__( data=None, @@ -29,14 +28,6 @@ def __init__( null_count=null_count, children=children, ) - if closed in ["left", "right", "neither", "both"]: - self._closed = closed - else: - raise ValueError("closed value is not valid") - - @property - def closed(self): - return self._closed @classmethod def from_arrow(cls, data): @@ -50,7 +41,6 @@ def from_arrow(cls, data): offset = data.offset null_count = data.null_count children = new_col.children - closed = dtype.closed return IntervalColumn( size=size, @@ -59,7 +49,6 @@ def from_arrow(cls, data): offset=offset, null_count=null_count, children=children, - closed=closed, ) def to_arrow(self): @@ -73,7 +62,7 @@ def to_arrow(self): @classmethod def from_struct_column(cls, struct_column: StructColumn, closed="right"): - first_field_name = list(struct_column.dtype.fields.keys())[0] + first_field_name = next(iter(struct_column.dtype.fields.keys())) return IntervalColumn( size=struct_column.size, dtype=IntervalDtype( @@ -83,20 +72,19 @@ def from_struct_column(cls, struct_column: StructColumn, closed="right"): offset=struct_column.offset, null_count=struct_column.null_count, children=struct_column.base_children, - closed=closed, ) def copy(self, deep=True): - closed = self.closed struct_copy = super().copy(deep=deep) return IntervalColumn( size=struct_copy.size, - dtype=IntervalDtype(struct_copy.dtype.fields["left"], closed), + dtype=IntervalDtype( + struct_copy.dtype.fields["left"], self.dtype.closed + ), mask=struct_copy.base_mask, offset=struct_copy.offset, null_count=struct_copy.null_count, children=struct_copy.base_children, - closed=closed, ) def as_interval_column(self, dtype): @@ -109,7 +97,7 @@ def as_interval_column(self, dtype): # when creating an interval series or interval dataframe if dtype == "interval": dtype = IntervalDtype( - self.dtype.fields["left"], self.closed + self.dtype.subtype, self.dtype.closed ) children = self.children return IntervalColumn( @@ -119,7 +107,6 @@ def as_interval_column(self, dtype): offset=self.offset, null_count=self.null_count, children=children, - closed=dtype.closed, ) else: raise ValueError("dtype must be IntervalDtype") @@ -141,5 +128,5 @@ def to_pandas( def element_indexing(self, index: int): result = super().element_indexing(index) if cudf.get_option("mode.pandas_compatible"): - return pd.Interval(**result, closed=self._closed) + return pd.Interval(**result, closed=self.dtype.closed) return result diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index fa7173f1d0f..c10124f4de6 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -3174,10 +3174,12 @@ def interval_range( data = column.column_empty_like_same_mask(left_col, dtype) return IntervalIndex(data, closed=closed) - interval_col = column.build_interval_column( - left_col, right_col, closed=closed + interval_col = IntervalColumn( + dtype=IntervalDtype(left_col.dtype, closed), + size=len(left_col), + children=(left_col, right_col), ) - return IntervalIndex(interval_col) + return IntervalIndex(interval_col, closed=closed) class IntervalIndex(GenericIndex): @@ -3217,44 +3219,72 @@ class IntervalIndex(GenericIndex): def __init__( self, data, - closed=None, + closed: Optional[Literal["left", "right", "neither", "both"]] = None, dtype=None, - copy=False, + copy: bool = False, name=None, ): - if copy: - data = column.as_column(data, dtype=dtype).copy() - kwargs = _setdefault_name(data, name=name) - - if closed is None: - closed = "right" + name = _setdefault_name(data, name=name)["name"] - if isinstance(data, IntervalColumn): - data = data - elif isinstance(data, pd.Series) and isinstance( - data.dtype, pd.IntervalDtype - ): - data = column.as_column(data, data.dtype) - elif isinstance(data, (pd.Interval, pd.IntervalIndex)): - data = column.as_column( - data, - dtype=dtype, - ) - elif len(data) == 0: - subtype = getattr(data, "dtype", "int64") - dtype = IntervalDtype(subtype, closed) - data = column.column_empty_like_same_mask( - column.as_column(data), dtype + if dtype is not None: + dtype = cudf.dtype(dtype) + if not isinstance(dtype, IntervalDtype): + raise TypeError("dtype must be an IntervalDtype") + if closed is not None and closed != dtype.closed: + raise ValueError("closed keyword does not match dtype.closed") + closed = dtype.closed + + if closed is None and isinstance(dtype, IntervalDtype): + closed = dtype.closed + + closed = closed or "right" + + if len(data) == 0: + if not hasattr(data, "dtype"): + data = np.array([], dtype=np.int64) + elif isinstance(data.dtype, (pd.IntervalDtype, IntervalDtype)): + data = np.array([], dtype=data.dtype.subtype) + interval_col = IntervalColumn( + dtype=IntervalDtype(data.dtype, closed), + size=len(data), + children=(as_column(data), as_column(data)), ) else: - data = column.as_column(data) - data.dtype.closed = closed + col = as_column(data) + if not isinstance(col, IntervalColumn): + raise TypeError("data must be an iterable of Interval data") + if copy: + col = col.copy() + interval_col = IntervalColumn( + dtype=IntervalDtype(col.dtype.subtype, closed), + mask=col.mask, + size=col.size, + offset=col.offset, + null_count=col.null_count, + children=col.children, + ) - self.closed = closed - super().__init__(data, **kwargs) + if dtype: + interval_col = interval_col.astype(dtype) # type: ignore[assignment] + super().__init__(interval_col, name=name) + + @property + def closed(self): + return self._values.dtype.closed + + @classmethod @_cudf_nvtx_annotate - def from_breaks(breaks, closed="right", name=None, copy=False, dtype=None): + def from_breaks( + cls, + breaks, + closed: Optional[ + Literal["left", "right", "neither", "both"] + ] = "right", + name=None, + copy: bool = False, + dtype=None, + ): """ Construct an IntervalIndex from an array of splits. @@ -3283,16 +3313,28 @@ def from_breaks(breaks, closed="right", name=None, copy=False, dtype=None): >>> cudf.IntervalIndex.from_breaks([0, 1, 2, 3]) IntervalIndex([(0, 1], (1, 2], (2, 3]], dtype='interval[int64, right]') """ + breaks = as_column(breaks, dtype=dtype) if copy: - breaks = column.as_column(breaks, dtype=dtype).copy() - left_col = breaks[:-1:] - right_col = breaks[+1::] - - interval_col = column.build_interval_column( - left_col, right_col, closed=closed + breaks = breaks.copy() + left_col = breaks.slice(0, len(breaks) - 1) + right_col = breaks.slice(1, len(breaks)) + # For indexing, children should both have 0 offset + right_col = column.build_column( + data=right_col.data, + dtype=right_col.dtype, + size=right_col.size, + mask=right_col.mask, + offset=0, + null_count=right_col.null_count, + children=right_col.children, ) - return IntervalIndex(interval_col, name=name) + interval_col = IntervalColumn( + dtype=IntervalDtype(left_col.dtype, closed), + size=len(left_col), + children=(left_col, right_col), + ) + return IntervalIndex(interval_col, name=name, closed=closed) def __getitem__(self, index): raise NotImplementedError( diff --git a/python/cudf/cudf/tests/indexes/test_interval.py b/python/cudf/cudf/tests/indexes/test_interval.py index 52c49aebf35..5a6155ece29 100644 --- a/python/cudf/cudf/tests/indexes/test_interval.py +++ b/python/cudf/cudf/tests/indexes/test_interval.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. import numpy as np import pandas as pd import pyarrow as pa @@ -57,11 +57,9 @@ def test_interval_range_dtype_basic(start_t, end_t): @pytest.mark.parametrize("closed", ["left", "right", "both", "neither"]) -@pytest.mark.parametrize("start", [0]) -@pytest.mark.parametrize("end", [0]) -def test_interval_range_empty(start, end, closed): - pindex = pd.interval_range(start=start, end=end, closed=closed) - gindex = cudf.interval_range(start=start, end=end, closed=closed) +def test_interval_range_empty(closed): + pindex = pd.interval_range(start=0, end=0, closed=closed) + gindex = cudf.interval_range(start=0, end=0, closed=closed) assert_eq(pindex, gindex) @@ -315,3 +313,22 @@ def test_intervalindex_empty_typed_non_int(): result = cudf.IntervalIndex(data) expected = pd.IntervalIndex(data) assert_eq(result, expected) + + +def test_intervalindex_invalid_dtype(): + with pytest.raises(TypeError): + cudf.IntervalIndex([pd.Interval(1, 2)], dtype="int64") + + +def test_intervalindex_conflicting_closed(): + with pytest.raises(ValueError): + cudf.IntervalIndex( + [pd.Interval(1, 2)], + dtype=cudf.IntervalDtype("int64", closed="left"), + closed="right", + ) + + +def test_intervalindex_invalid_data(): + with pytest.raises(TypeError): + cudf.IntervalIndex([1, 2]) diff --git a/python/cudf/cudf/tests/test_udf_masked_ops.py b/python/cudf/cudf/tests/test_udf_masked_ops.py index ad0c961a749..11970944a95 100644 --- a/python/cudf/cudf/tests/test_udf_masked_ops.py +++ b/python/cudf/cudf/tests/test_udf_masked_ops.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2023, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. import math import operator @@ -636,7 +636,7 @@ def func(row): ["1.0", "2.0", "3.0"], dtype=cudf.Decimal64Dtype(2, 1) ), cudf.Series([1, 2, 3], dtype="category"), - cudf.interval_range(start=0, end=3, closed=True), + cudf.interval_range(start=0, end=3), [[1, 2], [3, 4], [5, 6]], [{"a": 1}, {"a": 2}, {"a": 3}], ], From c949abeef0a62d94430746995ba1ff68865365db Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Tue, 23 Jan 2024 11:47:04 -0500 Subject: [PATCH 099/384] Add ci check for external kernels (#14768) Adds CI checks so that libcudf doesn't reintroduce weak/external CUDA kernels. Authors: - Robert Maynard (https://github.com/robertmaynard) Approvers: - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/cudf/pull/14768 --- .github/workflows/pr.yaml | 8 ++++++++ .github/workflows/test.yaml | 9 +++++++++ cpp/cmake/thirdparty/patches/cccl_override.json | 5 +++++ 3 files changed, 22 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c94724bcf8c..2fe4bf0b05e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -14,6 +14,7 @@ jobs: needs: - checks - conda-cpp-build + - conda-cpp-checks - conda-cpp-tests - conda-python-build - conda-python-cudf-tests @@ -43,6 +44,13 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.02 with: build_type: pull-request + conda-cpp-checks: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.02 + with: + build_type: pull-request + enable_check_symbols: true conda-cpp-tests: needs: conda-cpp-build secrets: inherit diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index df26a8c5916..7bb2530a7bc 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -14,6 +14,15 @@ on: type: string jobs: + conda-cpp-checks: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.02 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + enable_check_symbols: true conda-cpp-tests: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.02 diff --git a/cpp/cmake/thirdparty/patches/cccl_override.json b/cpp/cmake/thirdparty/patches/cccl_override.json index fa82bfb5421..68fc8979c46 100644 --- a/cpp/cmake/thirdparty/patches/cccl_override.json +++ b/cpp/cmake/thirdparty/patches/cccl_override.json @@ -8,6 +8,11 @@ "issue" : "CCCL installs header-search.cmake files in nondeterministic order and has a typo in checking target creation that leads to duplicates", "fixed_in" : "2.3" }, + { + "file" : "cccl/hide_kernels.diff", + "issue" : "Mark all cub and thrust kernels with hidden visibility [https://github.com/nvidia/cccl/pulls/443]", + "fixed_in" : "2.3" + }, { "file" : "cccl/revert_pr_211.diff", "issue" : "thrust::copy introduced a change in behavior that causes failures with cudaErrorInvalidValue.", From bb025142b40d10125cf3297085f23cfe28e02d20 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 23 Jan 2024 11:01:41 -0600 Subject: [PATCH 100/384] Add developer guideline to use east const. (#14836) This PR documents the libcudf preference for "east const." Follow-up from #13491, #13492, #13493, #13494. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - David Wendt (https://github.com/davidwendt) - Nghia Truong (https://github.com/ttnghia) - Yunsong Wang (https://github.com/PointKernel) URL: https://github.com/rapidsai/cudf/pull/14836 --- cpp/doxygen/developer_guide/DEVELOPER_GUIDE.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cpp/doxygen/developer_guide/DEVELOPER_GUIDE.md b/cpp/doxygen/developer_guide/DEVELOPER_GUIDE.md index c38151d7518..2606b487c07 100644 --- a/cpp/doxygen/developer_guide/DEVELOPER_GUIDE.md +++ b/cpp/doxygen/developer_guide/DEVELOPER_GUIDE.md @@ -129,6 +129,10 @@ and we try to follow his rules: "No raw loops. No raw pointers. No raw synchroni Additional style guidelines for libcudf code include: + * Prefer "east const", placing `const` after the type. This is not + automatically enforced by `clang-format` because the option + `QualifierAlignment: Right` has been observed to produce false negatives and + false positives. * [NL.11: Make Literals Readable](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#nl11-make-literals-readable): Decimal values should use integer separators every thousands place, like From 67a36a9104097cd6a8ae6efee1018e249f2fe441 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 23 Jan 2024 09:06:58 -1000 Subject: [PATCH 101/384] Simplify ColumnAccessor methods; avoid unnecessary validations (#14758) For methods that essentially do ```python def select_by_foo(self, ...): ... return self.__class__(data={subset of self._data}) ``` The `return` would perform validation on the returned subset of column, but I think that's unnecessary since that was done during initialization Additionally * Removed `_create_unsafe` in favor of a `verify=True|False` keyword in the constructor * `_column_length` == `nrows` so removed `_column_length` * Renamed `_compare_keys` to `_keys_equal` * Remove seldom used/unnecessary methods Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/14758 --- python/cudf/cudf/core/column_accessor.py | 141 +++++++++-------------- python/cudf/cudf/core/dataframe.py | 14 ++- python/cudf/cudf/core/frame.py | 6 +- python/cudf/cudf/core/multiindex.py | 3 +- 4 files changed, 70 insertions(+), 94 deletions(-) diff --git a/python/cudf/cudf/core/column_accessor.py b/python/cudf/cudf/core/column_accessor.py index 021d4994613..d87580fcfac 100644 --- a/python/cudf/cudf/core/column_accessor.py +++ b/python/cudf/cudf/core/column_accessor.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2023, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. from __future__ import annotations @@ -21,7 +21,6 @@ import pandas as pd from packaging.version import Version from pandas.api.types import is_bool -from typing_extensions import Self import cudf from cudf.core import column @@ -66,7 +65,7 @@ def __getitem__(self, key): return super().__getitem__(key) -def _to_flat_dict_inner(d, parents=()): +def _to_flat_dict_inner(d: dict, parents: tuple = ()): for k, v in d.items(): if not isinstance(v, d.__class__): if parents: @@ -76,14 +75,6 @@ def _to_flat_dict_inner(d, parents=()): yield from _to_flat_dict_inner(d=v, parents=parents + (k,)) -def _to_flat_dict(d): - """ - Convert the given nested dictionary to a flat dictionary - with tuple keys. - """ - return {k: v for k, v in _to_flat_dict_inner(d)} - - class ColumnAccessor(abc.MutableMapping): """ Parameters @@ -103,6 +94,9 @@ class ColumnAccessor(abc.MutableMapping): label_dtype : Dtype, optional What dtype should be returned in `to_pandas_index` (default=None). + verify : bool, optional + For non ColumnAccessor inputs, whether to verify + column length and type """ _data: "Dict[Any, ColumnBase]" @@ -116,6 +110,7 @@ def __init__( level_names=None, rangeindex: bool = False, label_dtype: Dtype | None = None, + verify: bool = True, ): self.rangeindex = rangeindex self.label_dtype = label_dtype @@ -133,9 +128,9 @@ def __init__( else: # This code path is performance-critical for copies and should be # modified with care. - self._data = {} - if data: - data = dict(data) + data = dict(data) + if data and verify: + result = {} # Faster than next(iter(data.values())) column_length = len(data[next(iter(data))]) for k, v in data.items(): @@ -146,30 +141,14 @@ def __init__( v = column.as_column(v) if len(v) != column_length: raise ValueError("All columns must be of equal length") - self._data[k] = v + result[k] = v + self._data = result + else: + self._data = data self.multiindex = multiindex self._level_names = level_names - @classmethod - def _create_unsafe( - cls, - data: Dict[Any, ColumnBase], - multiindex: bool = False, - level_names=None, - rangeindex: bool = False, - label_dtype: Dtype | None = None, - ) -> ColumnAccessor: - # create a ColumnAccessor without verifying column - # type or size - obj = cls() - obj._data = data - obj.multiindex = multiindex - obj._level_names = level_names - obj.rangeindex = rangeindex - obj.label_dtype = label_dtype - return obj - def __iter__(self): return iter(self._data) @@ -217,7 +196,7 @@ def nlevels(self) -> int: def name(self) -> Any: return self.level_names[-1] - @property + @cached_property def nrows(self) -> int: if len(self._data) == 0: return 0 @@ -243,13 +222,6 @@ def _grouped_data(self) -> abc.MutableMapping: else: return self._data - @cached_property - def _column_length(self): - try: - return len(self._data[next(iter(self._data))]) - except StopIteration: - return 0 - def _clear_cache(self): cached_properties = ("columns", "names", "_grouped_data") for attr in cached_properties: @@ -258,9 +230,9 @@ def _clear_cache(self): except AttributeError: pass - # Column length should only be cleared if no data is present. - if len(self._data) == 0 and hasattr(self, "_column_length"): - del self._column_length + # nrows should only be cleared if no data is present. + if len(self._data) == 0 and hasattr(self, "nrows"): + del self.nrows def to_pandas_index(self) -> pd.Index: """Convert the keys of the ColumnAccessor to a Pandas Index object.""" @@ -345,11 +317,8 @@ def insert( if loc == len(self._data): if validate: value = column.as_column(value) - if len(self._data) > 0: - if len(value) != self._column_length: - raise ValueError("All columns must be of equal length") - else: - self._column_length = len(value) + if len(self._data) > 0 and len(value) != self.nrows: + raise ValueError("All columns must be of equal length") self._data[name] = value else: new_keys = self.names[:loc] + (name,) + self.names[loc:] @@ -362,15 +331,16 @@ def copy(self, deep=False) -> ColumnAccessor: Make a copy of this ColumnAccessor. """ if deep or cudf.get_option("copy_on_write"): - return self.__class__( - {k: v.copy(deep=deep) for k, v in self._data.items()}, - multiindex=self.multiindex, - level_names=self.level_names, - ) + data = {k: v.copy(deep=deep) for k, v in self._data.items()} + else: + data = self._data.copy() return self.__class__( - self._data.copy(), + data=data, multiindex=self.multiindex, level_names=self.level_names, + rangeindex=self.rangeindex, + label_dtype=self.label_dtype, + verify=False, ) def select_by_label(self, key: Any) -> ColumnAccessor: @@ -508,22 +478,12 @@ def set_by_label(self, key: Any, value: Any, validate: bool = True): key = self._pad_key(key) if validate: value = column.as_column(value) - if len(self._data) > 0: - if len(value) != self._column_length: - raise ValueError("All columns must be of equal length") - else: - self._column_length = len(value) + if len(self._data) > 0 and len(value) != self.nrows: + raise ValueError("All columns must be of equal length") self._data[key] = value self._clear_cache() - def _select_by_names(self, names: abc.Sequence) -> Self: - return self.__class__( - {key: self[key] for key in names}, - multiindex=self.multiindex, - level_names=self.level_names, - ) - def _select_by_label_list_like(self, key: Any) -> ColumnAccessor: # Might be a generator key = tuple(key) @@ -541,7 +501,7 @@ def _select_by_label_list_like(self, key: Any) -> ColumnAccessor: else: data = {k: self._grouped_data[k] for k in key} if self.multiindex: - data = _to_flat_dict(data) + data = dict(_to_flat_dict_inner(data)) return self.__class__( data, multiindex=self.multiindex, @@ -550,11 +510,16 @@ def _select_by_label_list_like(self, key: Any) -> ColumnAccessor: def _select_by_label_grouped(self, key: Any) -> ColumnAccessor: result = self._grouped_data[key] - if isinstance(result, cudf.core.column.ColumnBase): - return self.__class__({key: result}, multiindex=self.multiindex) + if isinstance(result, column.ColumnBase): + # self._grouped_data[key] = self._data[key] so skip validation + return self.__class__( + data={key: result}, + multiindex=self.multiindex, + verify=False, + ) else: if self.multiindex: - result = _to_flat_dict(result) + result = dict(_to_flat_dict_inner(result)) if not isinstance(key, tuple): key = (key,) return self.__class__( @@ -575,11 +540,11 @@ def _select_by_label_slice(self, key: slice) -> ColumnAccessor: start = self._pad_key(start, slice(None)) stop = self._pad_key(stop, slice(None)) for idx, name in enumerate(self.names): - if _compare_keys(name, start): + if _keys_equal(name, start): start_idx = idx break for idx, name in enumerate(reversed(self.names)): - if _compare_keys(name, stop): + if _keys_equal(name, stop): stop_idx = len(self.names) - idx break keys = self.names[start_idx:stop_idx] @@ -587,14 +552,16 @@ def _select_by_label_slice(self, key: slice) -> ColumnAccessor: {k: self._data[k] for k in keys}, multiindex=self.multiindex, level_names=self.level_names, + verify=False, ) def _select_by_label_with_wildcard(self, key: Any) -> ColumnAccessor: key = self._pad_key(key, slice(None)) return self.__class__( - {k: self._data[k] for k in self._data if _compare_keys(k, key)}, + {k: self._data[k] for k in self._data if _keys_equal(k, key)}, multiindex=self.multiindex, level_names=self.level_names, + verify=False, ) def _pad_key(self, key: Any, pad_value="") -> Any: @@ -639,6 +606,7 @@ def rename_levels( to the given mapper and level. """ + new_col_names: abc.Iterable if self.multiindex: def rename_column(x): @@ -655,12 +623,7 @@ def rename_column(x): "Renaming columns with a MultiIndex and level=None is" "not supported" ) - new_names = map(rename_column, self.keys()) - ca = ColumnAccessor( - dict(zip(new_names, self.values())), - level_names=self.level_names, - multiindex=self.multiindex, - ) + new_col_names = (rename_column(k) for k in self.keys()) else: if level is None: @@ -680,13 +643,13 @@ def rename_column(x): if len(new_col_names) != len(set(new_col_names)): raise ValueError("Duplicate column names are not allowed") - ca = ColumnAccessor( - dict(zip(new_col_names, self.values())), - level_names=self.level_names, - multiindex=self.multiindex, - ) - - return self.__class__(ca) + data = dict(zip(new_col_names, self.values())) + return self.__class__( + data=data, + level_names=self.level_names, + multiindex=self.multiindex, + verify=False, + ) def droplevel(self, level): # drop the nth level @@ -708,7 +671,7 @@ def droplevel(self, level): self._clear_cache() -def _compare_keys(target: Any, key: Any) -> bool: +def _keys_equal(target: Any, key: Any) -> bool: """ Compare `key` to `target`. diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 2f18c194fde..2acb250ee13 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -481,12 +481,22 @@ def __getitem__(self, arg): index = self._frame.index if col_is_scalar: s = Series._from_data( - ca._select_by_names(column_names), index=index + data=ColumnAccessor( + {key: ca._data[key] for key in column_names}, + multiindex=ca.multiindex, + level_names=ca.level_names, + ), + index=index, ) return s._getitem_preprocessed(row_spec) if column_names != list(self._frame._column_names): frame = self._frame._from_data( - ca._select_by_names(column_names), index=index + data=ColumnAccessor( + {key: ca._data[key] for key in column_names}, + multiindex=ca.multiindex, + level_names=ca.level_names, + ), + index=index, ) else: frame = self._frame diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index fc313a62fd0..eb14a8948af 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -278,12 +278,13 @@ def astype(self, dtype, copy: bool = False): for col_name, col in self._data.items() } - return ColumnAccessor._create_unsafe( + return ColumnAccessor( data=result_data, multiindex=self._data.multiindex, level_names=self._data.level_names, rangeindex=self._data.rangeindex, label_dtype=self._data.label_dtype, + verify=False, ) @_cudf_nvtx_annotate @@ -881,12 +882,13 @@ def fillna( return self._mimic_inplace( self._from_data( - data=ColumnAccessor._create_unsafe( + data=ColumnAccessor( data=filled_data, multiindex=self._data.multiindex, level_names=self._data.level_names, rangeindex=self._data.rangeindex, label_dtype=self._data.label_dtype, + verify=False, ) ), inplace=inplace, diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 8ba47795437..d19fb966194 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -224,9 +224,10 @@ def names(self, value): # to unexpected behavior in some cases. This is # definitely buggy, but we can't disallow non-unique # names either... - self._data = self._data.__class__._create_unsafe( + self._data = self._data.__class__( dict(zip(value, self._data.values())), level_names=self._data.level_names, + verify=False, ) self._names = pd.core.indexes.frozen.FrozenList(value) From 16a942da598ec818b27916e4217a35e31a89d353 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 23 Jan 2024 12:02:07 -0800 Subject: [PATCH 102/384] Use `rapids_cuda_set_runtime` to determine cuda runtime usage by target (#14833) This PR uses rapids-cmake to handle per-target CMake linking to cudart. Replaces #13543 and #11641. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Mark Harris (https://github.com/harrism) - Robert Maynard (https://github.com/robertmaynard) URL: https://github.com/rapidsai/cudf/pull/14833 --- cpp/CMakeLists.txt | 16 +++++----------- cpp/cmake/Modules/JitifyPreprocessKernels.cmake | 5 +++-- cpp/tests/CMakeLists.txt | 1 + 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index df158a64d0b..c9d93f83e5c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -18,6 +18,7 @@ include(../fetch_rapids.cmake) include(rapids-cmake) include(rapids-cpm) include(rapids-cuda) +include(${rapids-cmake-dir}/cuda/set_runtime.cmake) include(rapids-export) include(rapids-find) @@ -780,17 +781,7 @@ if(TARGET conda_env) target_link_libraries(cudf PRIVATE conda_env) endif() -if(CUDA_STATIC_RUNTIME) - # Tell CMake what CUDA language runtime to use - set_target_properties(cudf PROPERTIES CUDA_RUNTIME_LIBRARY Static) - # Make sure to export to consumers what runtime we used - target_link_libraries(cudf PUBLIC CUDA::cudart_static) -else() - # Tell CMake what CUDA language runtime to use - set_target_properties(cudf PROPERTIES CUDA_RUNTIME_LIBRARY Shared) - # Make sure to export to consumers what runtime we used - target_link_libraries(cudf PUBLIC CUDA::cudart) -endif() +rapids_cuda_set_runtime(cudf USE_STATIC ${CUDA_STATIC_RUNTIME}) file( WRITE "${CUDF_BINARY_DIR}/fatbin.ld" @@ -838,6 +829,7 @@ if(CUDF_BUILD_TESTUTIL) PUBLIC cudf PRIVATE $ ) + rapids_cuda_set_runtime(cudftest_default_stream USE_STATIC ${CUDA_STATIC_RUNTIME}) add_library(cudf::cudftest_default_stream ALIAS cudftest_default_stream) @@ -881,6 +873,7 @@ if(CUDF_BUILD_TESTUTIL) cudftestutil PUBLIC "$" "$" ) + rapids_cuda_set_runtime(cudftestutil USE_STATIC ${CUDA_STATIC_RUNTIME}) add_library(cudf::cudftestutil ALIAS cudftestutil) endif() @@ -919,6 +912,7 @@ if(CUDF_BUILD_STREAMS_TEST_UTIL) if(CUDF_BUILD_STACKTRACE_DEBUG) target_link_libraries(${_tgt} PRIVATE cudf_backtrace) endif() + rapids_cuda_set_runtime(${_tgt} USE_STATIC ${CUDA_STATIC_RUNTIME}) add_library(cudf::${_tgt} ALIAS ${_tgt}) if("${_mode}" STREQUAL "testing") diff --git a/cpp/cmake/Modules/JitifyPreprocessKernels.cmake b/cpp/cmake/Modules/JitifyPreprocessKernels.cmake index baabffceeac..8a40be1dc94 100644 --- a/cpp/cmake/Modules/JitifyPreprocessKernels.cmake +++ b/cpp/cmake/Modules/JitifyPreprocessKernels.cmake @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2021-2023, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -16,7 +16,8 @@ add_executable(jitify_preprocess "${JITIFY_INCLUDE_DIR}/jitify2_preprocess.cpp") target_compile_definitions(jitify_preprocess PRIVATE "_FILE_OFFSET_BITS=64") -target_link_libraries(jitify_preprocess CUDA::cudart ${CMAKE_DL_LIBS}) +rapids_cuda_set_runtime(jitify_preprocess USE_STATIC ${CUDA_STATIC_RUNTIME}) +target_link_libraries(jitify_preprocess PUBLIC ${CMAKE_DL_LIBS}) # Take a list of files to JIT-compile and run them through jitify_preprocess. function(jit_preprocess_files) diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index eee736613fe..064d0c49f80 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -58,6 +58,7 @@ function(ConfigureTest CMAKE_TEST_NAME) ${CMAKE_TEST_NAME} PRIVATE cudftestutil GTest::gmock_main GTest::gtest_main $ ) + rapids_cuda_set_runtime(${CMAKE_TEST_NAME} USE_STATIC ${CUDA_STATIC_RUNTIME}) rapids_test_add( NAME ${CMAKE_TEST_NAME} COMMAND ${CMAKE_TEST_NAME} From 45bf274de87ea29d5ba8e6f4aac2fa048141312a Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 23 Jan 2024 18:25:17 -0600 Subject: [PATCH 103/384] Fix calls to deprecated strings factory API in examples. (#14838) Follow-up PR to #14771. I noticed the strings example code still had a deprecated function call: ``` -- Build files have been written to: /opt/conda/conda-bld/work/cpp/examples/strings/build [1/8] Building CXX object CMakeFiles/libcudf_apis.dir/libcudf_apis.cpp.o [2/8] Linking CXX executable libcudf_apis [3/8] Building CUDA object CMakeFiles/custom_prealloc.dir/custom_prealloc.cu.o [4/8] Building CUDA object CMakeFiles/custom_with_malloc.dir/custom_with_malloc.cu.o [5/8] Linking CUDA executable custom_prealloc [6/8] Linking CUDA executable custom_with_malloc [7/8] Building CUDA object CMakeFiles/custom_optimized.dir/custom_optimized.cu.o /opt/conda/conda-bld/work/cpp/examples/strings/custom_optimized.cu: In function 'std::unique_ptr redact_strings(const cudf::column_view&, const cudf::column_view&)': /opt/conda/conda-bld/work/cpp/examples/strings/custom_optimized.cu:158:40: warning: 'std::unique_ptr cudf::make_strings_column(cudf::size_type, rmm::device_uvector&&, rmm::device_uvector&&, rmm::device_buffer&&, cudf::size_type)' is deprecated [-Wdeprecated-declarations] 158 | auto result = | ~ ^ /opt/conda/conda-bld/work/cpp/include/cudf/column/column_factories.hpp:510:42: note: declared here 510 | [[deprecated]] std::unique_ptr make_strings_column(size_type num_strings, | ^~~~~~~~~~~~~~~~~~~ [8/8] Linking CUDA executable custom_optimized ``` Authors: - Bradley Dice (https://github.com/bdice) Approvers: - David Wendt (https://github.com/davidwendt) - Vyas Ramasubramani (https://github.com/vyasr) - Mark Harris (https://github.com/harrism) URL: https://github.com/rapidsai/cudf/pull/14838 --- cpp/examples/strings/custom_optimized.cu | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/examples/strings/custom_optimized.cu b/cpp/examples/strings/custom_optimized.cu index 522093bc647..aa1468ea790 100644 --- a/cpp/examples/strings/custom_optimized.cu +++ b/cpp/examples/strings/custom_optimized.cu @@ -155,8 +155,7 @@ std::unique_ptr redact_strings(cudf::column_view const& names, *d_names, *d_visibilities, offsets.data(), chars.data()); // create column from offsets and chars vectors (no copy is performed) - auto result = - cudf::make_strings_column(names.size(), std::move(offsets), std::move(chars), {}, 0); + auto result = cudf::make_strings_column(names.size(), std::move(offsets), chars.release(), {}, 0); // wait for all of the above to finish stream.synchronize(); From 0a4ce5136685761d3a9d0541f3e2e8ec8867e3cf Mon Sep 17 00:00:00 2001 From: Mark Harris <783069+harrism@users.noreply.github.com> Date: Wed, 24 Jan 2024 12:14:05 +1100 Subject: [PATCH 104/384] Remove get_mem_info functions from custom memory resources (#14832) Part of rapidsai/rmm#1388. This removes now-optional and soon-to-be deprecated functions from cuDF's custom device_memory_resource implementations: * `supports_get_mem_info()` * `do_get_mem_info()` Authors: - Mark Harris (https://github.com/harrism) Approvers: - Jason Lowe (https://github.com/jlowe) - Nghia Truong (https://github.com/ttnghia) - Vukasin Milovanovic (https://github.com/vuule) URL: https://github.com/rapidsai/cudf/pull/14832 --- .../stream_checking_resource_adaptor.hpp | 27 +------------------ java/src/main/native/src/RmmJni.cpp | 14 +--------- 2 files changed, 2 insertions(+), 39 deletions(-) diff --git a/cpp/include/cudf_test/stream_checking_resource_adaptor.hpp b/cpp/include/cudf_test/stream_checking_resource_adaptor.hpp index e18400422aa..90a8c2ccc2f 100644 --- a/cpp/include/cudf_test/stream_checking_resource_adaptor.hpp +++ b/cpp/include/cudf_test/stream_checking_resource_adaptor.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -71,16 +71,6 @@ class stream_checking_resource_adaptor final : public rmm::mr::device_memory_res */ bool supports_streams() const noexcept override { return upstream_->supports_streams(); } - /** - * @brief Query whether the resource supports the get_mem_info API. - * - * @return Whether or not the upstream resource supports get_mem_info - */ - bool supports_get_mem_info() const noexcept override - { - return upstream_->supports_get_mem_info(); - } - private: /** * @brief Allocates memory of size at least `bytes` using the upstream @@ -131,21 +121,6 @@ class stream_checking_resource_adaptor final : public rmm::mr::device_memory_res : upstream_->is_equal(other); } - /** - * @brief Get free and available memory from upstream resource. - * - * @throws `rmm::cuda_error` if unable to retrieve memory info. - * @throws `cudf::logic_error` if attempted on a default stream - * - * @param stream Stream on which to get the mem info. - * @return std::pair with available and free memory for resource - */ - std::pair do_get_mem_info(rmm::cuda_stream_view stream) const override - { - verify_stream(stream); - return upstream_->get_mem_info(stream); - } - /** * @brief Throw an error if the provided stream is invalid. * diff --git a/java/src/main/native/src/RmmJni.cpp b/java/src/main/native/src/RmmJni.cpp index 3c49d153cb6..b92d9e4e891 100644 --- a/java/src/main/native/src/RmmJni.cpp +++ b/java/src/main/native/src/RmmJni.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -96,8 +96,6 @@ class tracking_resource_adaptor final : public base_tracking_resource_adaptor { return scoped_max_total_allocated; } - bool supports_get_mem_info() const noexcept override { return resource->supports_get_mem_info(); } - bool supports_streams() const noexcept override { return resource->supports_streams(); } private: @@ -144,10 +142,6 @@ class tracking_resource_adaptor final : public base_tracking_resource_adaptor { scoped_allocated -= size; } } - - std::pair do_get_mem_info(rmm::cuda_stream_view stream) const override { - return resource->get_mem_info(stream); - } }; template @@ -213,8 +207,6 @@ class java_event_handler_memory_resource : public device_memory_resource { device_memory_resource *get_wrapped_resource() { return resource; } - bool supports_get_mem_info() const noexcept override { return resource->supports_get_mem_info(); } - bool supports_streams() const noexcept override { return resource->supports_streams(); } private: @@ -277,10 +269,6 @@ class java_event_handler_memory_resource : public device_memory_resource { } } - std::pair do_get_mem_info(rmm::cuda_stream_view stream) const override { - return resource->get_mem_info(stream); - } - protected: JavaVM *jvm; jobject handler_obj; From bdbf0bc97cc257d018c1b58433a71c294030da4b Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 24 Jan 2024 12:14:30 +0530 Subject: [PATCH 105/384] Fix `Dataframe.agg` to not return incorrect dtypes (#14851) This PR fixes `DataFrame.agg` API where the actual dataframe was being casted to incorrect dtype (object dtype) and then the operations were being performed. This PR adds strict checks at two places. This PR: ``` = 95 failed, 101829 passed, 2091 skipped, 976 xfailed, 312 xpassed in 1479.83s (0:24:39) = ``` On `pandas_2.0_feature_branch`: ``` = 111 failed, 101865 passed, 2091 skipped, 976 xfailed, 312 xpassed in 1265.57s (0:21:05) = ``` --- python/cudf/cudf/core/dataframe.py | 32 +++++++++------ python/cudf/cudf/tests/test_dataframe.py | 51 ++++++++++++++---------- 2 files changed, 50 insertions(+), 33 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 5ef7c6027a9..1057fd0b716 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -84,6 +84,7 @@ from cudf.core.resample import DataFrameResampler from cudf.core.series import Series from cudf.core.udf.row_function import _get_row_kernel +from cudf.errors import MixedTypeError from cudf.utils import applyutils, docutils, ioutils, queryutils from cudf.utils.docutils import copy_docstring from cudf.utils.dtypes import ( @@ -3609,11 +3610,12 @@ def agg(self, aggs, axis=None): * Not supporting: ``axis``, ``*args``, ``**kwargs`` """ - # TODO: Remove the typecasting below once issue #6846 is fixed - # link dtypes = [self[col].dtype for col in self._column_names] common_dtype = find_common_type(dtypes) - df_normalized = self.astype(common_dtype) + if not is_bool_dtype(common_dtype) and any( + is_bool_dtype(dtype) for dtype in dtypes + ): + raise MixedTypeError("Cannot create a column with mixed types") if any(is_string_dtype(dt) for dt in dtypes): raise NotImplementedError( @@ -3631,17 +3633,17 @@ def agg(self, aggs, axis=None): # TODO : Allow simultaneous pass for multi-aggregation as # a future optimization for agg in aggs: - result[agg] = getattr(df_normalized, agg)() + result[agg] = getattr(self, agg)() return result.T.sort_index(axis=1, ascending=True) elif isinstance(aggs, str): - if not hasattr(df_normalized, aggs): + if not hasattr(self, aggs): raise AttributeError( f"{aggs} is not a valid function for " f"'DataFrame' object" ) result = DataFrame() - result[aggs] = getattr(df_normalized, aggs)() + result[aggs] = getattr(self, aggs)() result = result.iloc[:, 0] result.name = None return result @@ -3653,15 +3655,16 @@ def agg(self, aggs, axis=None): "callable parameter is not implemented yet" ) elif all(isinstance(val, str) for val in aggs.values()): - result = cudf.Series(index=cols) + res = {} for key, value in aggs.items(): - col = df_normalized[key] + col = self[key] if not hasattr(col, value): raise AttributeError( f"{value} is not a valid function for " f"'Series' object" ) - result[key] = getattr(col, value)() + res[key] = getattr(col, value)() + result = cudf.Series(list(res.values()), index=res.keys()) elif all(isinstance(val, abc.Iterable) for val in aggs.values()): idxs = set() for val in aggs.values(): @@ -3677,7 +3680,7 @@ def agg(self, aggs, axis=None): ) result = DataFrame(index=idxs, columns=cols) for key in aggs.keys(): - col = df_normalized[key] + col = self[key] col_empty = column_empty( len(idxs), dtype=col.dtype, masked=True ) @@ -6160,8 +6163,13 @@ def _reduce( else: source_dtypes = [c.dtype for c in source._data.columns] common_dtype = find_common_type(source_dtypes) - if is_object_dtype(common_dtype) and any( - not is_object_dtype(dtype) for dtype in source_dtypes + if ( + is_object_dtype(common_dtype) + and any( + not is_object_dtype(dtype) for dtype in source_dtypes + ) + or not is_bool_dtype(common_dtype) + and any(is_bool_dtype(dtype) for dtype in source_dtypes) ): raise TypeError( "Columns must all have the same dtype to " diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 325097968f7..026f0aa845d 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -9173,17 +9173,8 @@ def test_dataframe_constructor_column_index_only(): @pytest.mark.parametrize( "data", [ - {"a": [1, 2, 3], "b": [3.0, 4.0, 5.0], "c": [True, True, False]}, - {"a": [1.0, 2.0, 3.0], "b": [3.0, 4.0, 5.0], "c": [True, True, False]}, - {"a": [1, 2, 3], "b": [3, 4, 5], "c": [True, True, False]}, - {"a": [1, 2, 3], "b": [True, True, False], "c": [False, True, False]}, - { - "a": [1.0, 2.0, 3.0], - "b": [True, True, False], - "c": [False, True, False], - }, - {"a": [1, 2, 3], "b": [3, 4, 5], "c": [2.0, 3.0, 4.0]}, - {"a": [1, 2, 3], "b": [2.0, 3.0, 4.0], "c": [5.0, 6.0, 4.0]}, + {"a": [1, 2.5, 3], "b": [3, 4.5, 5], "c": [2.0, 3.0, 4.0]}, + {"a": [1, 2.2, 3], "b": [2.0, 3.0, 4.0], "c": [5.0, 6.0, 4.0]}, ], ) @pytest.mark.parametrize( @@ -9208,14 +9199,36 @@ def test_agg_for_dataframes(data, aggs): expect = pdf.agg(aggs).sort_index() got = gdf.agg(aggs).sort_index() - assert_eq(expect, got, check_dtype=False) + + assert_eq(expect, got, check_dtype=True) + + +@pytest_unmark_spilling +@pytest.mark.parametrize( + "data", + [ + {"a": [1, 2, 3], "b": [3.0, 4.0, 5.0], "c": [True, True, False]}, + {"a": [1, 2, 3], "b": [True, True, False], "c": [False, True, False]}, + ], +) +@pytest.mark.parametrize( + "aggs", + [ + ["min", "sum", "max"], + "sum", + {"a": "sum", "b": "min", "c": "max"}, + ], +) +def test_agg_for_dataframes_error(data, aggs): + gdf = cudf.DataFrame(data) + + with pytest.raises(TypeError): + gdf.agg(aggs) @pytest.mark.parametrize("aggs", [{"a": np.sum, "b": np.min, "c": np.max}]) def test_agg_for_unsupported_function(aggs): - gdf = cudf.DataFrame( - {"a": [1, 2, 3], "b": [3.0, 4.0, 5.0], "c": [True, True, False]} - ) + gdf = cudf.DataFrame({"a": [1, 2, 3], "b": [3.0, 4.0, 5.0]}) with pytest.raises(NotImplementedError): gdf.agg(aggs) @@ -9223,9 +9236,7 @@ def test_agg_for_unsupported_function(aggs): @pytest.mark.parametrize("aggs", ["asdf"]) def test_agg_for_dataframe_with_invalid_function(aggs): - gdf = cudf.DataFrame( - {"a": [1, 2, 3], "b": [3.0, 4.0, 5.0], "c": [True, True, False]} - ) + gdf = cudf.DataFrame({"a": [1, 2, 3], "b": [3.0, 4.0, 5.0]}) with pytest.raises( AttributeError, @@ -9236,9 +9247,7 @@ def test_agg_for_dataframe_with_invalid_function(aggs): @pytest.mark.parametrize("aggs", [{"a": "asdf"}]) def test_agg_for_series_with_invalid_function(aggs): - gdf = cudf.DataFrame( - {"a": [1, 2, 3], "b": [3.0, 4.0, 5.0], "c": [True, True, False]} - ) + gdf = cudf.DataFrame({"a": [1, 2, 3], "b": [3.0, 4.0, 5.0]}) with pytest.raises( AttributeError, From 28b1814e9fb509fb2bfe6783613e5a8f792ee34f Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 24 Jan 2024 12:15:01 +0530 Subject: [PATCH 106/384] Catch warnings in reductions (#14852) This PR validates the warnings generated by certain reduction ops. --- python/cudf/cudf/tests/test_reductions.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/python/cudf/cudf/tests/test_reductions.py b/python/cudf/cudf/tests/test_reductions.py index e8bbffcacaa..1a38cb3dd22 100644 --- a/python/cudf/cudf/tests/test_reductions.py +++ b/python/cudf/cudf/tests/test_reductions.py @@ -360,10 +360,9 @@ def test_reductions_axis_none_warning(op): FutureWarning, ): actual = getattr(df, op)(axis=None) - # with expect_warning_if( - # op in {"kurt", "kurtosis", "skew", "min", "max", "mean", "median"}, - # FutureWarning, - # ): - - expected = getattr(pdf, op)(axis=None) + with expect_warning_if( + op in {"sum", "product", "std", "var"}, + FutureWarning, + ): + expected = getattr(pdf, op)(axis=None) assert_eq(expected, actual, check_dtype=False) From df5c78b6fa5e3e3c3c673a2d8e5b8757d903cbd3 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 24 Jan 2024 22:24:31 +0530 Subject: [PATCH 107/384] Catch groupby jit apply warnings (#14858) This PR catches `RuntimeWarning`'s in jit groupby pytests. This PR: ``` = 61 failed, 101866 passed, 2091 skipped, 976 xfailed, 312 xpassed in 1446.19s (0:24:06) = ``` On `pandas_2.0_feature_branch`: ``` = 91 failed, 101836 passed, 2091 skipped, 976 xfailed, 312 xpassed in 1348.36s (0:22:28) = ``` --- python/cudf/cudf/tests/test_groupby.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index 456ce961a79..f594963dcda 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -565,9 +565,10 @@ def test_groupby_apply_jit_reductions_special_vals( func, dtype, dataset, groupby_jit_datasets, special_val ): dataset = groupby_jit_datasets[dataset] - groupby_apply_jit_reductions_special_vals_inner( - func, dataset, dtype, special_val - ) + with expect_warning_if(func in {"var", "std"} and not np.isnan(special_val), RuntimeWarning): + groupby_apply_jit_reductions_special_vals_inner( + func, dataset, dtype, special_val + ) @pytest.mark.parametrize("dtype", ["float64"]) @@ -652,7 +653,8 @@ def func(group): with pytest.raises(UDFError, match=m): run_groupby_apply_jit_test(dataset, func, keys) return - run_groupby_apply_jit_test(dataset, func, keys) + with expect_warning_if(dtype in {"int32", "int64"}, RuntimeWarning): + run_groupby_apply_jit_test(dataset, func, keys) @pytest.mark.parametrize("dtype", ["int32", "int64"]) @@ -667,7 +669,8 @@ def test_groupby_apply_jit_correlation_zero_variance(dtype): def func(group): return group["b"].corr(group["c"]) - run_groupby_apply_jit_test(data, func, ["a"]) + with expect_warning_if(dtype in {"int32", "int64"}, RuntimeWarning): + run_groupby_apply_jit_test(data, func, ["a"]) @pytest.mark.parametrize("op", unary_ops) From 60f04cefaf699daf621125368398cd62635a583d Mon Sep 17 00:00:00 2001 From: Pantakan Kanprawet Date: Thu, 25 Jan 2024 02:52:58 +0700 Subject: [PATCH 108/384] Notes convert to Pandas-compat (#12641) --- python/cudf/cudf/core/column/lists.py | 10 +- python/cudf/cudf/core/column/string.py | 107 +++++++------ python/cudf/cudf/core/dataframe.py | 187 ++++++++++++----------- python/cudf/cudf/core/frame.py | 119 ++++++++++----- python/cudf/cudf/core/groupby/groupby.py | 17 ++- python/cudf/cudf/core/indexed_frame.py | 99 ++++++------ python/cudf/cudf/core/series.py | 62 ++++---- python/cudf/cudf/core/tools/datetimes.py | 1 - python/cudf/cudf/core/tools/numeric.py | 15 +- 9 files changed, 343 insertions(+), 274 deletions(-) diff --git a/python/cudf/cudf/core/column/lists.py b/python/cudf/cudf/core/column/lists.py index 0cccec6f28a..c28489a2f98 100644 --- a/python/cudf/cudf/core/column/lists.py +++ b/python/cudf/cudf/core/column/lists.py @@ -620,11 +620,6 @@ def sort_values( ------- Series or Index with each list sorted - Notes - ----- - Difference from pandas: - * Not supporting: `inplace`, `kind` - Examples -------- >>> s = cudf.Series([[4, 2, None, 9], [8, 8, 2], [2, 1]]) @@ -633,6 +628,11 @@ def sort_values( 1 [2.0, 8.0, 8.0] 2 [1.0, 2.0] dtype: list + + .. pandas-compat:: + **ListMethods.sort_values** + + The ``inplace`` and ``kind`` arguments are currently not supported. """ if inplace: raise NotImplementedError("`inplace` not currently implemented.") diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index c47088caebc..fcb993e1a78 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -594,11 +594,6 @@ def extract( for each group. If `expand=False` and `pat` has only one capture group, then return a Series/Index. - Notes - ----- - The `flags` parameter currently only supports re.DOTALL and - re.MULTILINE. - Examples -------- >>> import cudf @@ -625,6 +620,12 @@ def extract( 1 2 2 dtype: object + + .. pandas-compat:: + **StringMethods.extract** + + The `flags` parameter currently only supports re.DOTALL and + re.MULTILINE. """ # noqa W605 if not _is_supported_regex_flags(flags): raise NotImplementedError( @@ -672,14 +673,6 @@ def contains( pattern is contained within the string of each element of the Series/Index. - Notes - ----- - The parameters `case` and `na` are not yet supported and will - raise a NotImplementedError if anything other than the default - value is set. - The `flags` parameter currently only supports re.DOTALL and - re.MULTILINE. - Examples -------- >>> import cudf @@ -753,6 +746,15 @@ def contains( 3 True 4 dtype: bool + + .. pandas-compat:: + **StringMethods.contains** + + The parameters `case` and `na` are not yet supported and will + raise a NotImplementedError if anything other than the default + value is set. + The `flags` parameter currently only supports re.DOTALL and + re.MULTILINE. """ # noqa W605 if na is not np.nan: raise NotImplementedError("`na` parameter is not yet supported") @@ -951,12 +953,6 @@ def replace( A copy of the object with all matching occurrences of pat replaced by repl. - Notes - ----- - The parameters `case` and `flags` are not yet supported and will raise - a `NotImplementedError` if anything other than the default value - is set. - Examples -------- >>> import cudf @@ -986,6 +982,13 @@ def replace( 1 fuz 2 dtype: object + + .. pandas-compat:: + **StringMethods.replace** + + The parameters `case` and `flags` are not yet supported and will + raise a `NotImplementedError` if anything other than the default + value is set. """ if case is not None: raise NotImplementedError("`case` parameter is not yet supported") @@ -2769,11 +2772,6 @@ def partition(self, sep: str = " ", expand: bool = True) -> SeriesOrIndex: DataFrame or MultiIndex Returns a DataFrame / MultiIndex - Notes - ----- - The parameter `expand` is not yet supported and will raise a - `NotImplementedError` if anything other than the default value is set. - See Also -------- rpartition @@ -2815,6 +2813,14 @@ def partition(self, sep: str = " ", expand: bool = True) -> SeriesOrIndex: MultiIndex([('X', ' ', '123'), ('Y', ' ', '999')], ) + + .. pandas-compat:: + **StringMethods.partition** + + The parameter `expand` is not yet supported and will raise a + `NotImplementedError` if anything other than the default + value is set. + """ if expand is not True: raise NotImplementedError( @@ -3500,14 +3506,6 @@ def count(self, pat: str, flags: int = 0) -> SeriesOrIndex: ------- Series or Index - Notes - ----- - - `flags` parameter currently only supports re.DOTALL - and re.MULTILINE. - - Some characters need to be escaped when passing - in pat. e.g. ``'$'`` has a special meaning in regex - and must be escaped when finding this literal character. - Examples -------- >>> import cudf @@ -3539,6 +3537,15 @@ def count(self, pat: str, flags: int = 0) -> SeriesOrIndex: >>> index = cudf.Index(['A', 'A', 'Aaba', 'cat']) >>> index.str.count('a') Int64Index([0, 0, 2, 1], dtype='int64') + + .. pandas-compat:: + **StringMethods.count** + + - `flags` parameter currently only supports re.DOTALL + and re.MULTILINE. + - Some characters need to be escaped when passing + in pat. e.g. ``'$'`` has a special meaning in regex + and must be escaped when finding this literal character. """ # noqa W605 if isinstance(pat, re.Pattern): flags = pat.flags & ~re.U @@ -3570,11 +3577,6 @@ def findall(self, pat: str, flags: int = 0) -> SeriesOrIndex: All non-overlapping matches of pattern or regular expression in each string of this Series/Index. - Notes - ----- - The `flags` parameter currently only supports re.DOTALL and - re.MULTILINE. - Examples -------- >>> import cudf @@ -3615,6 +3617,12 @@ def findall(self, pat: str, flags: int = 0) -> SeriesOrIndex: 1 [] 2 [b, b] dtype: list + + .. pandas-compat:: + **StringMethods.findall** + + The `flags` parameter currently only supports re.DOTALL and + re.MULTILINE. """ if isinstance(pat, re.Pattern): flags = pat.flags & ~re.U @@ -3797,11 +3805,6 @@ def endswith(self, pat: str) -> SeriesOrIndex: A Series of booleans indicating whether the given pattern matches the end of each string element. - Notes - ----- - `na` parameter is not yet supported, as cudf uses - native strings instead of Python objects. - Examples -------- >>> import cudf @@ -3818,6 +3821,12 @@ def endswith(self, pat: str) -> SeriesOrIndex: 2 False 3 dtype: bool + + .. pandas-compat:: + **StringMethods.endswith** + + `na` parameter is not yet supported, as cudf uses + native strings instead of Python objects. """ if pat is None: raise TypeError( @@ -4245,13 +4254,6 @@ def match( ------- Series or Index of boolean values. - Notes - ----- - Parameters `case` and `na` are currently not supported. - The `flags` parameter currently only supports re.DOTALL and - re.MULTILINE. - - Examples -------- >>> import cudf @@ -4272,6 +4274,13 @@ def match( 1 True 2 True dtype: bool + + .. pandas-compat:: + **StringMethods.match** + + Parameters `case` and `na` are currently not supported. + The `flags` parameter currently only supports re.DOTALL and + re.MULTILINE. """ if case is not True: raise NotImplementedError("`case` parameter is not yet supported") diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 2acb250ee13..7c48352d861 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -3265,10 +3265,6 @@ def diff(self, periods=1, axis=0): DataFrame First differences of the DataFrame. - Notes - ----- - Diff currently only supports numeric dtype columns. - Examples -------- >>> import cudf @@ -3292,6 +3288,10 @@ def diff(self, periods=1, axis=0): 4 2 3 16 5 2 5 20 + .. pandas-compat:: + **DataFrame.diff** + + Diff currently only supports numeric dtype columns. """ if not is_integer(periods): if not (is_float(periods) and periods.is_integer()): @@ -3467,14 +3467,6 @@ def rename( ------- DataFrame - Notes - ----- - Difference from pandas: - * Not supporting: level - - Rename will not overwrite column names. If a list with duplicates is - passed, column names will be postfixed with a number. - Examples -------- >>> import cudf @@ -3500,6 +3492,15 @@ def rename( 10 1 4 20 2 5 30 3 6 + + .. pandas-compat:: + **DataFrame.rename** + + * Not Supporting: level + + Rename will not overwrite column names. If a list with + duplicates is passed, column names will be postfixed + with a number. """ if errors != "ignore": raise NotImplementedError( @@ -3599,10 +3600,10 @@ def agg(self, aggs, axis=None): When ``DataFrame.agg`` is called with several aggs, ``DataFrame`` is returned. - Notes - ----- - Difference from pandas: - * Not supporting: ``axis``, ``*args``, ``**kwargs`` + .. pandas-compat:: + **DataFrame.agg** + + * Not supporting: ``axis``, ``*args``, ``**kwargs`` """ # TODO: Remove the typecasting below once issue #6846 is fixed @@ -3735,11 +3736,6 @@ def nlargest(self, n, columns, keep="first"): The first `n` rows ordered by the given columns in descending order. - Notes - ----- - Difference from pandas: - - Only a single column is supported in *columns* - Examples -------- >>> import cudf @@ -3774,6 +3770,11 @@ def nlargest(self, n, columns, keep="first"): France 65000000 2583560 FR Italy 59000000 1937894 IT Brunei 434000 12128 BN + + .. pandas-compat:: + **DataFrame.nlargest** + + - Only a single column is supported in *columns* """ return self._n_largest_or_smallest(True, n, columns, keep) @@ -3800,11 +3801,6 @@ def nsmallest(self, n, columns, keep="first"): ------- DataFrame - Notes - ----- - Difference from pandas: - - Only a single column is supported in *columns* - Examples -------- >>> import cudf @@ -3846,6 +3842,11 @@ def nsmallest(self, n, columns, keep="first"): Anguilla 11300 311 AI Tuvalu 11300 38 TV Nauru 337000 182 NR + + .. pandas-compat:: + **DataFrame.nsmallest** + + - Only a single column is supported in *columns* """ return self._n_largest_or_smallest(False, n, columns, keep) @@ -3923,10 +3924,11 @@ def transpose(self): ------- a new (ncol x nrow) dataframe. self is (nrow x ncol) - Notes - ----- - Difference from pandas: - Not supporting *copy* because default and only behavior is copy=True + .. pandas-compat:: + **DataFrame.transpose, DataFrame.T** + + Not supporting *copy* because default and only behavior is + copy=True """ index = self._data.to_pandas_index() @@ -4078,10 +4080,6 @@ def merge( ------- merged : DataFrame - Notes - ----- - **DataFrames merges in cuDF result in non-deterministic row ordering.** - Examples -------- >>> import cudf @@ -4117,6 +4115,12 @@ def merge( right dtype respectively. This extends to semi and anti joins. - For outer joins, the result will be the union of categories from both sides. + + .. pandas-compat:: + **DataFrame.merge** + + DataFrames merges in cuDF result in non-deterministic row + ordering. """ if indicator: raise NotImplementedError( @@ -4187,12 +4191,11 @@ def join( ------- joined : DataFrame - Notes - ----- - Difference from pandas: + .. pandas-compat:: + **DataFrame.join** - - *other* must be a single DataFrame for now. - - *on* is not supported yet due to lack of multi-index support. + - *other* must be a single DataFrame for now. + - *on* is not supported yet due to lack of multi-index support. """ if on is not None: raise NotImplementedError("The on parameter is not yet supported") @@ -5327,11 +5330,6 @@ def from_arrow(cls, table): ------- cudf DataFrame - Notes - ----- - - Does not support automatically setting index column(s) similar - to how ``to_pandas`` works for PyArrow Tables. - Examples -------- >>> import cudf @@ -5342,6 +5340,12 @@ def from_arrow(cls, table): 0 1 4 1 2 5 2 3 6 + + .. pandas-compat:: + **DataFrame.from_arrow** + + - Does not support automatically setting index column(s) similar + to how ``to_pandas`` works for PyArrow Tables. """ index_col = None col_index_names = None @@ -5701,14 +5705,6 @@ def quantile( If q is a float, a Series will be returned where the index is the columns of self and the values are the quantiles. - .. pandas-compat:: - **DataFrame.quantile** - - One notable difference from Pandas is when DataFrame is of - non-numeric types and result is expected to be a Series in case of - Pandas. cuDF will return a DataFrame as it doesn't support mixed - types under Series. - Examples -------- >>> import cupy as cp @@ -5729,6 +5725,14 @@ def quantile( a b 0.1 1.3 3.7 0.5 2.5 55.0 + + .. pandas-compat:: + **DataFrame.quantile** + + One notable difference from Pandas is when DataFrame is of + non-numeric types and result is expected to be a Series in case of + Pandas. cuDF will return a DataFrame as it doesn't support mixed + types under Series. """ # noqa: E501 if axis not in (0, None): raise NotImplementedError("axis is not implemented yet") @@ -6001,10 +6005,6 @@ def count(self, axis=0, level=None, numeric_only=False, **kwargs): Series For each column/row the number of non-NA/null entries. - Notes - ----- - Parameters currently not supported are `axis`, `level`, `numeric_only`. - Examples -------- >>> import cudf @@ -6018,6 +6018,12 @@ def count(self, axis=0, level=None, numeric_only=False, **kwargs): Age 4 Single 5 dtype: int64 + + .. pandas-compat:: + **DataFrame.count** + + Parameters currently not supported are `axis`, `level`, + `numeric_only`. """ axis = self._get_axis_from_axis_arg(axis) if axis != 0: @@ -6191,10 +6197,6 @@ def mode(self, axis=0, numeric_only=False, dropna=True): cudf.Series.value_counts : Return the counts of values in a Series. - Notes - ----- - ``axis`` parameter is currently not supported. - Examples -------- >>> import cudf @@ -6233,6 +6235,11 @@ def mode(self, axis=0, numeric_only=False, dropna=True): legs wings 0 2 0.0 1 2.0 + + .. pandas-compat:: + **DataFrame.mode** + + ``axis`` parameter is currently not supported. """ if axis not in (0, "index"): raise NotImplementedError("Only axis=0 is currently supported") @@ -7007,7 +7014,7 @@ def to_struct(self, name=None): Notes ----- - Note that a copy of the columns is made. + Note: a copy of the columns is made. """ if not all(isinstance(name, str) for name in self._data.names): warnings.warn( @@ -7112,22 +7119,18 @@ def append( ------- DataFrame + Notes + ----- + Iteratively appending rows to a cudf DataFrame can be more + computationally intensive than a single concatenate. A better solution + is to append those rows to a list and then concatenate the list with + the original DataFrame all at once. + See Also -------- cudf.concat : General function to concatenate DataFrame or objects. - Notes - ----- - If a list of dict/series is passed and the keys are all contained in - the DataFrame's index, the order of the columns in the resulting - DataFrame will be unchanged. - Iteratively appending rows to a cudf DataFrame can be more - computationally intensive than a single concatenate. A better - solution is to append those rows to a list and then concatenate - the list with the original DataFrame all at once. - `verify_integrity` parameter is not supported yet. - Examples -------- >>> import cudf @@ -7182,6 +7185,14 @@ def append( 2 2 3 3 4 4 + + .. pandas-compat:: + **DataFrame.append** + + * If a list of dict/series is passed and the keys are all contained + in the DataFrame's index, the order of the columns in the + resulting DataFrame will be unchanged. + * The `verify_integrity` parameter is not supported yet. """ if isinstance(other, dict): if not ignore_index: @@ -7503,22 +7514,6 @@ def eval(self, expr: str, inplace: bool = False, **kwargs): DataFrame if any assignment statements are included in ``expr``, or None if ``inplace=True``. - Notes - ----- - Difference from pandas: - * Additional kwargs are not supported. - * Bitwise and logical operators are not dtype-dependent. - Specifically, `&` must be used for bitwise operators on integers, - not `and`, which is specifically for the logical and between - booleans. - * Only numerical types currently support all operators. - * String types currently support comparison operators. - * Operators generally will not cast automatically. Users are - responsible for casting columns to suitable types before - evaluating a function. - * Multiple assignments to the same name (i.e. a sequence of - assignment statements where later statements are conditioned upon - the output of earlier statements) is not supported. Examples -------- @@ -7581,6 +7576,22 @@ def eval(self, expr: str, inplace: bool = False, **kwargs): 2 3 6 9 -3 3 4 4 8 0 4 5 2 7 3 + + .. pandas-compat:: + **DataFrame.eval** + + * Additional kwargs are not supported. + * Bitwise and logical operators are not dtype-dependent. + Specifically, `&` must be used for bitwise operators on integers, + not `and`, which is specifically for the logical and between + booleans. + * Only numerical types are currently supported. + * Operators generally will not cast automatically. Users are + responsible for casting columns to suitable types before + evaluating a function. + * Multiple assignments to the same name (i.e. a sequence of + assignment statements where later statements are conditioned upon + the output of earlier statements) is not supported. """ if kwargs: raise ValueError( diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index eb14a8948af..1e6ff118626 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -251,7 +251,6 @@ def size(self) -> int: """ return self._num_columns * self._num_rows - @_cudf_nvtx_annotate def memory_usage(self, deep=False): """Return the memory usage of an object. @@ -597,6 +596,8 @@ def where(self, cond, other=None, inplace: bool = False) -> Optional[Self]: dtype: int64 .. pandas-compat:: + **DataFrame.where, Series.where** + Note that ``where`` treats missing values as falsy, in parallel with pandas treatment of nullable data: @@ -1948,10 +1949,6 @@ def min( ------- Series - Notes - ----- - Parameters currently not supported are `level`, `numeric_only`. - Examples -------- >>> import cudf @@ -1960,6 +1957,11 @@ def min( a 1 b 7 dtype: int64 + + .. pandas-compat:: + **DataFrame.min, Series.min** + + Parameters currently not supported are `level`, `numeric_only`. """ return self._reduce( "min", @@ -1999,10 +2001,6 @@ def max( ------- Series - Notes - ----- - Parameters currently not supported are `level`, `numeric_only`. - Examples -------- >>> import cudf @@ -2011,6 +2009,11 @@ def max( a 4 b 10 dtype: int64 + + .. pandas-compat:: + **DataFrame.max, Series.max** + + Parameters currently not supported are `level`, `numeric_only`. """ return self._reduce( "max", @@ -2055,10 +2058,6 @@ def sum( ------- Series - Notes - ----- - Parameters currently not supported are `level`, `numeric_only`. - Examples -------- >>> import cudf @@ -2067,6 +2066,11 @@ def sum( a 10 b 34 dtype: int64 + + .. pandas-compat:: + **DataFrame.sum, Series.sum** + + Parameters currently not supported are `level`, `numeric_only`. """ return self._reduce( "sum", @@ -2113,10 +2117,6 @@ def product( ------- Series - Notes - ----- - Parameters currently not supported are level`, `numeric_only`. - Examples -------- >>> import cudf @@ -2125,6 +2125,11 @@ def product( a 24 b 5040 dtype: int64 + + .. pandas-compat:: + **DataFrame.product, Series.product** + + Parameters currently not supported are level`, `numeric_only`. """ return self._reduce( @@ -2224,11 +2229,6 @@ def std( ------- Series - Notes - ----- - Parameters currently not supported are `level` and - `numeric_only` - Examples -------- >>> import cudf @@ -2237,6 +2237,12 @@ def std( a 1.290994 b 1.290994 dtype: float64 + + .. pandas-compat:: + **DataFrame.std, Series.std** + + Parameters currently not supported are `level` and + `numeric_only` """ return self._reduce( @@ -2280,11 +2286,6 @@ def var( ------- scalar - Notes - ----- - Parameters currently not supported are `level` and - `numeric_only` - Examples -------- >>> import cudf @@ -2293,6 +2294,12 @@ def var( a 1.666667 b 1.666667 dtype: float64 + + .. pandas-compat:: + **DataFrame.var, Series.var** + + Parameters currently not supported are `level` and + `numeric_only` """ return self._reduce( "var", @@ -2330,10 +2337,6 @@ def kurtosis( ------- Series or scalar - Notes - ----- - Parameters currently not supported are `level` and `numeric_only` - Examples -------- **Series** @@ -2351,6 +2354,11 @@ def kurtosis( a -1.2 b -1.2 dtype: float64 + + .. pandas-compat:: + **DataFrame.kurtosis** + + Parameters currently not supported are `level` and `numeric_only` """ if axis not in (0, "index", None, no_default): raise NotImplementedError("Only axis=0 is currently supported.") @@ -2388,11 +2396,6 @@ def skew( ------- Series - Notes - ----- - Parameters currently not supported are `axis`, `level` and - `numeric_only` - Examples -------- **Series** @@ -2417,6 +2420,12 @@ def skew( a 0.00000 b -0.37037 dtype: float64 + + .. pandas-compat:: + **DataFrame.skew, Series.skew, Frame.skew** + + Parameters currently not supported are `axis`, `level` and + `numeric_only` """ if axis not in (0, "index", None, no_default): raise NotImplementedError("Only axis=0 is currently supported.") @@ -2469,6 +2478,18 @@ def all(self, axis=0, skipna=True, level=None, **kwargs): a True b False dtype: bool + + .. pandas-compat:: + **DataFrame.all, Series.all** + + Parameters currently not supported are `axis`, `bool_only`, + `level`. + + .. pandas-compat:: + **DataFrame.all, Series.all** + + Parameters currently not supported are `axis`, `bool_only`, + `level`. """ return self._reduce( "all", @@ -2517,6 +2538,18 @@ def any(self, axis=0, skipna=True, level=None, **kwargs): a True b True dtype: bool + + .. pandas-compat:: + **DataFrame.any, Series.any** + + Parameters currently not supported are `axis`, `bool_only`, + `level`. + + .. pandas-compat:: + **DataFrame.any, Series.any** + + Parameters currently not supported are `axis`, `bool_only`, + `level`. """ return self._reduce( "any", @@ -2542,10 +2575,6 @@ def median( ------- scalar - Notes - ----- - Parameters currently not supported are `level` and `numeric_only`. - Examples -------- >>> import cudf @@ -2560,6 +2589,16 @@ def median( dtype: int64 >>> ser.median() 17.0 + + .. pandas-compat:: + **DataFrame.median, Series.median** + + Parameters currently not supported are `level` and `numeric_only`. + + .. pandas-compat:: + **DataFrame.median, Series.median** + + Parameters currently not supported are `level` and `numeric_only`. """ return self._reduce( "median", diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index e28ba233c56..c4d92b84c99 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -685,10 +685,10 @@ def _reduce( Series or DataFrame Computed {op} of values within each group. - Notes - ----- - Difference from pandas: - * Not supporting: numeric_only, min_count + .. pandas-compat:: + **{cls}.{op}** + + The numeric_only, min_count """ if numeric_only: raise NotImplementedError( @@ -1382,7 +1382,7 @@ def mult(df): 6 2 6 12 .. pandas-compat:: - **groupby.apply** + **GroupBy.apply** cuDF's ``groupby.apply`` is limited compared to pandas. In some situations, Pandas returns the grouped keys as part of @@ -2283,9 +2283,10 @@ def shift(self, periods=1, freq=None, axis=0, fill_value=None): Series or DataFrame Object shifted within each group. - Notes - ----- - Parameter ``freq`` is unsupported. + .. pandas-compat:: + **GroupBy.shift** + + Parameter ``freq`` is unsupported. """ if freq is not None: diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 3e564919090..6c0aba34970 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -446,11 +446,6 @@ def empty(self): out : bool If DataFrame/Series is empty, return True, if not return False. - Notes - ----- - If DataFrame/Series contains only `null` values, it is still not - considered empty. See the example below. - Examples -------- >>> import cudf @@ -491,6 +486,12 @@ def empty(self): Series([], dtype: float64) >>> s.empty True + + .. pandas-compat:: + **DataFrame.empty, Series.empty** + + If DataFrame/Series contains only `null` values, it is still not + considered empty. See the example above. """ return self.size == 0 @@ -638,11 +639,6 @@ def replace( result : Series Series after replacement. The mask and index are preserved. - Notes - ----- - Parameters that are currently not supported are: `limit`, `regex`, - `method` - Examples -------- **Series** @@ -785,6 +781,12 @@ def replace( 2 2 7 c 3 3 8 d 4 4 9 e + + .. pandas-compat:: + **DataFrame.replace, Series.replace** + + Parameters that are currently not supported are: `limit`, `regex`, + `method` """ if limit is not None: raise NotImplementedError("limit parameter is not implemented yet") @@ -1125,13 +1127,6 @@ def truncate(self, before=None, after=None, axis=0, copy=True): `before` and `after` may be specified as strings instead of Timestamps. - .. pandas-compat:: - **DataFrame.truncate, Series.truncate** - - The ``copy`` parameter is only present for API compatibility, but - ``copy=False`` is not supported. This method always generates a - copy. - Examples -------- **Series** @@ -1273,6 +1268,13 @@ def truncate(self, before=None, after=None, axis=0, copy=True): 2021-01-01 23:45:25 1 2 2021-01-01 23:45:26 1 2 2021-01-01 23:45:27 1 2 + + .. pandas-compat:: + **DataFrame.truncate, Series.truncate** + + The ``copy`` parameter is only present for API compatibility, but + ``copy=False`` is not supported. This method always generates a + copy. """ if not copy: raise ValueError("Truncating with copy=False is not supported.") @@ -1527,11 +1529,6 @@ def sort_index( ------- Frame or None - Notes - ----- - Difference from pandas: - * Not supporting: kind, sort_remaining=False - Examples -------- **Series** @@ -1574,6 +1571,11 @@ def sort_index( 1 2 3 3 1 2 2 3 1 + + .. pandas-compat:: + **DataFrame.sort_index, Series.sort_index** + + * Not supporting: kind, sort_remaining=False """ if kind is not None: raise NotImplementedError("kind is not yet supported") @@ -2383,12 +2385,6 @@ def sort_values( ------- Frame : Frame with sorted values. - Notes - ----- - Difference from pandas: - * Support axis='index' only. - * Not supporting: inplace, kind - Examples -------- >>> import cudf @@ -2400,6 +2396,12 @@ def sort_values( 0 0 -3 2 2 0 1 1 2 + + .. pandas-compat:: + **DataFrame.sort_values, Series.sort_values** + + * Support axis='index' only. + * Not supporting: inplace, kind """ if na_position not in {"first", "last"}: raise ValueError(f"invalid na_position: {na_position}") @@ -2923,13 +2925,14 @@ def resample( 2018-02-28 18.0 63.333333 - Notes - ----- - Note that the dtype of the index (or the 'on' column if using - 'on=') in the result will be of a frequency closest to the - resampled frequency. For example, if resampling from - nanoseconds to milliseconds, the index will be of dtype - 'datetime64[ms]'. + .. pandas-compat:: + **DataFrame.resample, Series.resample** + + Note that the dtype of the index (or the 'on' column if using + 'on=') in the result will be of a frequency closest to the + resampled frequency. For example, if resampling from + nanoseconds to milliseconds, the index will be of dtype + 'datetime64[ms]'. """ import cudf.core.resample @@ -3405,18 +3408,6 @@ def sample( provided via the `random_state` parameter. This function will always produce the same sample given an identical `random_state`. - Notes - ----- - When sampling from ``axis=0/'index'``, ``random_state`` can be either - a numpy random state (``numpy.random.RandomState``) or a cupy random - state (``cupy.random.RandomState``). When a numpy random state is - used, the output is guaranteed to match the output of the corresponding - pandas method call, but generating the sample may be slow. If exact - pandas equivalence is not required, using a cupy random state will - achieve better performance, especially when sampling large number of - items. It's advised to use the matching `ndarray` type to the random - state for the `weights` array. - Parameters ---------- n : int, optional @@ -3484,6 +3475,20 @@ def sample( a c 0 1 3 1 2 4 + + .. pandas-compat:: + **DataFrame.sample, Series.sample** + + When sampling from ``axis=0/'index'``, ``random_state`` can be + either a numpy random state (``numpy.random.RandomState``) + or a cupy random state (``cupy.random.RandomState``). When a numpy + random state is used, the output is guaranteed to match the output + of the corresponding pandas method call, but generating the sample + maybe slow. If exact pandas equivalence is not required, using a + cupy random state will achieve better performance, + especially when sampling large number of + items. It's advised to use the matching `ndarray` type to + the random state for the `weights` array. """ axis = 0 if axis is None else self._get_axis_from_axis_arg(axis) size = self.shape[axis] diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 55100343306..7e25713e63c 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -1356,10 +1356,11 @@ def map(self, arg, na_action=None) -> "Series": 4 dtype: int64 - Notes - ----- - Please note map currently only supports fixed-width numeric - type functions. + .. pandas-compat:: + **Series.map** + + Please note map currently only supports fixed-width numeric + type functions. """ if isinstance(arg, dict): if hasattr(arg, "__missing__"): @@ -2191,12 +2192,6 @@ def sort_values( ------- Series : Series with sorted values. - Notes - ----- - Difference from pandas: - * Support axis='index' only. - * Not supporting: inplace, kind - Examples -------- >>> import cudf @@ -2208,6 +2203,12 @@ def sort_values( 3 4 1 5 dtype: int64 + + .. pandas-compat:: + **Series.sort_values** + + * Support axis='index' only. + * The inplace and kind argument is currently unsupported """ return super().sort_values( by=self.name, @@ -2652,16 +2653,17 @@ def count(self, level=None): int Number of non-null values in the Series. - Notes - ----- - Parameters currently not supported is `level`. - Examples -------- >>> import cudf >>> ser = cudf.Series([1, 5, 2, 4, 3]) >>> ser.count() 5 + + .. pandas-compat:: + **Series.count** + + Parameters currently not supported is `level`. """ if level is not None: @@ -2765,10 +2767,6 @@ def cov(self, other, min_periods=None): Covariance between Series and other normalized by N-1 (unbiased estimator). - Notes - ----- - `min_periods` parameter is not yet supported. - Examples -------- >>> import cudf @@ -2776,6 +2774,11 @@ def cov(self, other, min_periods=None): >>> ser2 = cudf.Series([0.12, 0.26, 0.51]) >>> ser1.cov(ser2) -0.015750000000000004 + + .. pandas-compat:: + **Series.cov** + + `min_periods` parameter is not yet supported. """ if min_periods is not None: @@ -3521,12 +3524,6 @@ def rename(self, index=None, copy=True): ------- Series - Notes - ----- - Difference from pandas: - - Supports scalar values only for changing name attribute - - Not supporting : inplace, level - Examples -------- >>> import cudf @@ -3545,6 +3542,12 @@ def rename(self, index=None, copy=True): Name: numeric_series, dtype: int64 >>> renamed_series.name 'numeric_series' + + .. pandas-compat:: + **Series.rename** + + - Supports scalar values only for changing name attribute + - The ``inplace`` and ``level`` is not supported """ out_data = self._data.copy(deep=copy) return Series._from_data(out_data, self.index, name=index) @@ -4724,11 +4727,6 @@ def strftime(self, date_format, *args, **kwargs): Series Series of formatted strings. - Notes - ----- - The following date format identifiers are not yet - supported: ``%c``, ``%x``,``%X`` - Examples -------- >>> import cudf @@ -4755,6 +4753,12 @@ def strftime(self, date_format, *args, **kwargs): 1 2000 / 30 / 06 2 2000 / 30 / 09 dtype: object + + .. pandas-compat:: + **series.DatetimeProperties.strftime** + + The following date format identifiers are not yet + supported: ``%c``, ``%x``,``%X`` """ if not isinstance(date_format, str): diff --git a/python/cudf/cudf/core/tools/datetimes.py b/python/cudf/cudf/core/tools/datetimes.py index 14459c81966..0e0a32e21fe 100644 --- a/python/cudf/cudf/core/tools/datetimes.py +++ b/python/cudf/cudf/core/tools/datetimes.py @@ -825,7 +825,6 @@ def date_range( '2023-12-23 08:00:00', '2025-02-23 08:00:00', '2026-04-23 08:00:00'], dtype='datetime64[ns]') - """ if tz is not None: raise NotImplementedError("tz is currently unsupported.") diff --git a/python/cudf/cudf/core/tools/numeric.py b/python/cudf/cudf/core/tools/numeric.py index a28c679b8be..8991fbe1c13 100644 --- a/python/cudf/cudf/core/tools/numeric.py +++ b/python/cudf/cudf/core/tools/numeric.py @@ -54,13 +54,6 @@ def to_numeric(arg, errors="raise", downcast=None): Depending on the input, if series is passed in, series is returned, otherwise ndarray - Notes - ----- - An important difference from pandas is that this function does not accept - mixed numeric/non-numeric type sequences. For example ``[1, 'a']``. - A ``TypeError`` will be raised when such input is received, regardless of - ``errors`` parameter. - Examples -------- >>> s = cudf.Series(['1', '2.0', '3e3']) @@ -90,6 +83,14 @@ def to_numeric(arg, errors="raise", downcast=None): 1 1.0 2 3000.0 dtype: float64 + + .. pandas-compat:: + **cudf.to_numeric** + + An important difference from pandas is that this function does not + accept mixed numeric/non-numeric type sequences. + For example ``[1, 'a']``. A ``TypeError`` will be raised when such + input is received, regardless of ``errors`` parameter. """ if errors not in {"raise", "ignore", "coerce"}: From 8784551f84e06acb0486ddd72beed8fa6a197511 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 25 Jan 2024 01:29:42 +0530 Subject: [PATCH 109/384] Fix all reduction pytest failures (#14869) This PR fixes all the remaining one-off reduction pytest failures. This PR: ``` = 54 failed, 101872 passed, 2091 skipped, 977 xfailed, 312 xpassed in 1432.99s (0:23:52) = ``` On `pandas_2.0_feature_branch`: ``` = 61 failed, 101866 passed, 2091 skipped, 976 xfailed, 312 xpassed in 1446.19s (0:24:06) = ``` --- python/cudf/cudf/tests/test_stats.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/python/cudf/cudf/tests/test_stats.py b/python/cudf/cudf/tests/test_stats.py index c8357699350..edd7da3d42c 100644 --- a/python/cudf/cudf/tests/test_stats.py +++ b/python/cudf/cudf/tests/test_stats.py @@ -244,7 +244,7 @@ def test_misc_quantiles(data, q): "nan_as_null": False, }, {"data": [1.1032, 2.32, 43.4, 13, -312.0], "index": [0, 4, 3, 19, 6]}, - {"data": []}, + {"data": [], "dtype": "float64"}, {"data": [-3]}, ], ) @@ -274,13 +274,12 @@ def test_kurt_skew_error(op): gs = cudf.Series(["ab", "cd"]) ps = gs.to_pandas() - with pytest.warns(FutureWarning): - assert_exceptions_equal( - getattr(gs, op), - getattr(ps, op), - lfunc_args_and_kwargs=([], {"numeric_only": True}), - rfunc_args_and_kwargs=([], {"numeric_only": True}), - ) + assert_exceptions_equal( + getattr(gs, op), + getattr(ps, op), + lfunc_args_and_kwargs=([], {"numeric_only": True}), + rfunc_args_and_kwargs=([], {"numeric_only": True}), + ) @pytest.mark.parametrize( @@ -359,10 +358,17 @@ def test_series_median(dtype, num_na): @pytest.mark.parametrize( "fill_method", ["ffill", "bfill", "pad", "backfill", no_default, None] ) -def test_series_pct_change(data, periods, fill_method): +def test_series_pct_change(request, data, periods, fill_method): cs = cudf.Series(data) ps = cs.to_pandas() - + request.applymarker( + pytest.mark.xfail( + condition=( + len(cs) == 0 and periods == 0 and fill_method is no_default + ), + reason="https://github.com/pandas-dev/pandas/issues/57056", + ) + ) if np.abs(periods) <= len(cs): with expect_warning_if(fill_method not in (no_default, None)): got = cs.pct_change(periods=periods, fill_method=fill_method) From f800f5a2fa9a961699345e6febe740b4b8f4760e Mon Sep 17 00:00:00 2001 From: Shruti Shivakumar Date: Wed, 24 Jan 2024 12:14:05 -0800 Subject: [PATCH 110/384] JSON single quote normalization API (#14729) The goal of this PR is to address [10004](https://github.com/rapidsai/cudf/issues/10004) by supporting parsing of JSON files containing single quotes for field/value strings. This is a follow-up work to the POC [PR 14545](https://github.com/rapidsai/cudf/pull/14545) Authors: - Shruti Shivakumar (https://github.com/shrshi) Approvers: - Andy Grove (https://github.com/andygrove) - Vyas Ramasubramani (https://github.com/vyasr) - Vukasin Milovanovic (https://github.com/vuule) - Elias Stehle (https://github.com/elstehle) - Robert (Bobby) Evans (https://github.com/revans2) URL: https://github.com/rapidsai/cudf/pull/14729 --- cpp/CMakeLists.txt | 1 + cpp/include/cudf/io/detail/json.hpp | 14 +- cpp/include/cudf/io/json.hpp | 31 +++ .../io/json/json_quote_normalization.cu} | 204 +++-------------- cpp/src/io/json/read_json.cu | 21 +- cpp/src/io/json/read_json.hpp | 2 +- cpp/tests/CMakeLists.txt | 2 +- .../io/json_quote_normalization_test.cpp | 215 ++++++++++++++++++ .../main/java/ai/rapids/cudf/JSONOptions.java | 15 ++ java/src/main/java/ai/rapids/cudf/Table.java | 11 +- java/src/main/native/src/TableJni.cpp | 44 ++-- .../test/java/ai/rapids/cudf/TableTest.java | 33 +++ java/src/test/js | 0 java/src/test/resources/single_quotes.json | 2 + 14 files changed, 401 insertions(+), 194 deletions(-) rename cpp/{tests/io/fst/quote_normalization_test.cu => src/io/json/json_quote_normalization.cu} (56%) create mode 100644 cpp/tests/io/json_quote_normalization_test.cpp create mode 100644 java/src/test/js create mode 100644 java/src/test/resources/single_quotes.json diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 90eaec6804a..3925ac55d6b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -375,6 +375,7 @@ add_library( src/io/functions.cpp src/io/json/byte_range_info.cu src/io/json/json_column.cu + src/io/json/json_quote_normalization.cu src/io/json/json_tree.cu src/io/json/nested_json_gpu.cu src/io/json/read_json.cu diff --git a/cpp/include/cudf/io/detail/json.hpp b/cpp/include/cudf/io/detail/json.hpp index d0a9543397d..0eb0e17ea10 100644 --- a/cpp/include/cudf/io/detail/json.hpp +++ b/cpp/include/cudf/io/detail/json.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -51,4 +51,16 @@ void write_json(data_sink* sink, json_writer_options const& options, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr); + +/** + * @brief Normalize single quotes to double quotes using FST + * + * @param inbuf Input device buffer + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource to use for device memory allocation + */ +rmm::device_uvector normalize_single_quotes(rmm::device_uvector&& inbuf, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr); + } // namespace cudf::io::json::detail diff --git a/cpp/include/cudf/io/json.hpp b/cpp/include/cudf/io/json.hpp index 2a39a539cc7..f0c3d48ab7e 100644 --- a/cpp/include/cudf/io/json.hpp +++ b/cpp/include/cudf/io/json.hpp @@ -115,6 +115,9 @@ class json_reader_options { // Whether to keep the quote characters of string values bool _keep_quotes = false; + // Normalize single quotes + bool _normalize_single_quotes = false; + // Whether to recover after an invalid JSON line json_recovery_mode_t _recovery_mode = json_recovery_mode_t::FAIL; @@ -255,6 +258,13 @@ class json_reader_options { */ bool is_enabled_keep_quotes() const { return _keep_quotes; } + /** + * @brief Whether the reader should normalize single quotes around strings + * + * @returns true if the reader should normalize single quotes, false otherwise + */ + bool is_enabled_normalize_single_quotes() const { return _normalize_single_quotes; } + /** * @brief Queries the JSON reader's behavior on invalid JSON lines. * @@ -340,6 +350,14 @@ class json_reader_options { */ void enable_keep_quotes(bool val) { _keep_quotes = val; } + /** + * @brief Set whether the reader should enable normalization of single quotes around strings. + * + * @param val Boolean value to indicate whether the reader should normalize single quotes around + * strings + */ + void enable_normalize_single_quotes(bool val) { _normalize_single_quotes = val; } + /** * @brief Specifies the JSON reader's behavior on invalid JSON lines. * @@ -502,6 +520,19 @@ class json_reader_options_builder { return *this; } + /** + * @brief Set whether the reader should normalize single quotes around strings + * + * @param val Boolean value to indicate whether the reader should normalize single quotes + * of strings + * @return this for chaining + */ + json_reader_options_builder& normalize_single_quotes(bool val) + { + options._normalize_single_quotes = val; + return *this; + } + /** * @brief Specifies the JSON reader's behavior on invalid JSON lines. * diff --git a/cpp/tests/io/fst/quote_normalization_test.cu b/cpp/src/io/json/json_quote_normalization.cu similarity index 56% rename from cpp/tests/io/fst/quote_normalization_test.cu rename to cpp/src/io/json/json_quote_normalization.cu index d0794b8f17e..7c9466748cd 100644 --- a/cpp/tests/io/fst/quote_normalization_test.cu +++ b/cpp/src/io/json/json_quote_normalization.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * Copyright (c) 2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,19 +15,13 @@ */ #include -#include -#include -#include -#include - -#include -#include +#include #include #include #include -#include +#include #include #include @@ -36,17 +30,16 @@ #include #include -namespace { +namespace cudf::io::json { + +using SymbolT = char; +using StateT = char; +using SymbolOffsetT = uint32_t; -// Type used to represent the atomic symbol type used within the finite-state machine -// TODO: type aliasing to be declared in a common header for better maintainability and -// pre-empt future bugs -using SymbolT = char; -using StateT = char; +namespace normalize_quotes { // Type sufficiently large to index symbols within the input and output (may be unsigned) -using SymbolOffsetT = uint32_t; -enum class dfa_states : char { TT_OOS = 0U, TT_DQS, TT_SQS, TT_DEC, TT_SEC, TT_NUM_STATES }; +enum class dfa_states : StateT { TT_OOS = 0U, TT_DQS, TT_SQS, TT_DEC, TT_SEC, TT_NUM_STATES }; enum class dfa_symbol_group_id : uint32_t { DOUBLE_QUOTE_CHAR, ///< Quote character SG: " SINGLE_QUOTE_CHAR, ///< Quote character SG: ' @@ -62,7 +55,7 @@ constexpr auto TT_DQS = dfa_states::TT_DQS; constexpr auto TT_SQS = dfa_states::TT_SQS; constexpr auto TT_DEC = dfa_states::TT_DEC; constexpr auto TT_SEC = dfa_states::TT_SEC; -constexpr auto TT_NUM_STATES = static_cast(dfa_states::TT_NUM_STATES); +constexpr auto TT_NUM_STATES = static_cast(dfa_states::TT_NUM_STATES); constexpr auto NUM_SYMBOL_GROUPS = static_cast(dfa_symbol_group_id::NUM_SYMBOL_GROUPS); // The i-th string representing all the characters of a symbol group @@ -80,7 +73,7 @@ std::array, TT_NUM_STATES> const qna_s }}; // The DFA's starting state -constexpr char start_state = static_cast(TT_OOS); +constexpr auto start_state = static_cast(TT_OOS); struct TransduceToNormalizedQuotes { /** @@ -112,7 +105,7 @@ struct TransduceToNormalizedQuotes { // SEC | Sigma\{'} -> {\*} // Whether this transition translates to the escape sequence: \" - const bool outputs_escape_sequence = + bool const outputs_escape_sequence = (state_id == static_cast(dfa_states::TT_SQS)) && (match_id == static_cast(dfa_symbol_group_id::DOUBLE_QUOTE_CHAR)); // Case when a double quote needs to be replaced by the escape sequence: \" @@ -156,19 +149,19 @@ struct TransduceToNormalizedQuotes { SymbolT const read_symbol) const { // Whether this transition translates to the escape sequence: \" - const bool sqs_outputs_escape_sequence = + bool const sqs_outputs_escape_sequence = (state_id == static_cast(dfa_states::TT_SQS)) && (match_id == static_cast(dfa_symbol_group_id::DOUBLE_QUOTE_CHAR)); // Number of characters to output on this transition if (sqs_outputs_escape_sequence) { return 2; } // Whether this transition translates to the escape sequence \ or unescaped ' - const bool sec_outputs_escape_sequence = + bool const sec_outputs_escape_sequence = (state_id == static_cast(dfa_states::TT_SEC)) && (match_id != static_cast(dfa_symbol_group_id::SINGLE_QUOTE_CHAR)); // Number of characters to output on this transition if (sec_outputs_escape_sequence) { return 2; } // Whether this transition translates to no output - const bool sqs_outputs_nop = + bool const sqs_outputs_nop = (state_id == static_cast(dfa_states::TT_SQS)) && (match_id == static_cast(dfa_symbol_group_id::ESCAPE_CHAR)); // Number of characters to output on this transition @@ -177,156 +170,33 @@ struct TransduceToNormalizedQuotes { } }; -} // namespace +} // namespace normalize_quotes -// Base test fixture for tests -struct FstTest : public cudf::test::BaseFixture {}; +namespace detail { -void run_test(std::string& input, std::string& output) +rmm::device_uvector normalize_single_quotes(rmm::device_uvector&& inbuf, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) { - // Prepare cuda stream for data transfers & kernels - rmm::cuda_stream stream{}; - rmm::cuda_stream_view stream_view(stream); - - auto parser = cudf::io::fst::detail::make_fst( - cudf::io::fst::detail::make_symbol_group_lut(qna_sgs), - cudf::io::fst::detail::make_transition_table(qna_state_tt), - cudf::io::fst::detail::make_translation_functor(TransduceToNormalizedQuotes{}), + auto parser = fst::detail::make_fst( + fst::detail::make_symbol_group_lut(normalize_quotes::qna_sgs), + fst::detail::make_transition_table(normalize_quotes::qna_state_tt), + fst::detail::make_translation_functor(normalize_quotes::TransduceToNormalizedQuotes{}), stream); - auto d_input_scalar = cudf::make_string_scalar(input, stream_view); - auto& d_input = static_cast&>(*d_input_scalar); - - // Prepare input & output buffers - constexpr std::size_t single_item = 1; - cudf::detail::hostdevice_vector output_gpu(input.size() * 2, stream_view); - cudf::detail::hostdevice_vector output_gpu_size(single_item, stream_view); - - // Allocate device-side temporary storage & run algorithm - parser.Transduce(d_input.data(), - static_cast(d_input.size()), - output_gpu.device_ptr(), + rmm::device_uvector outbuf(inbuf.size() * 2, stream, mr); + rmm::device_scalar outbuf_size(stream, mr); + parser.Transduce(inbuf.data(), + static_cast(inbuf.size()), + outbuf.data(), thrust::make_discard_iterator(), - output_gpu_size.device_ptr(), - start_state, - stream_view); - - // Async copy results from device to host - output_gpu.device_to_host_async(stream_view); - output_gpu_size.device_to_host_async(stream_view); - - // Make sure results have been copied back to host - stream.synchronize(); - - // Verify results - ASSERT_EQ(output_gpu_size[0], output.size()); - CUDF_TEST_EXPECT_VECTOR_EQUAL(output_gpu, output, output.size()); -} - -TEST_F(FstTest, GroundTruth_QuoteNormalization1) -{ - std::string input = R"({"A":'TEST"'})"; - std::string output = R"({"A":"TEST\""})"; - run_test(input, output); -} - -TEST_F(FstTest, GroundTruth_QuoteNormalization2) -{ - std::string input = R"({'A':"TEST'"} ['OTHER STUFF'])"; - std::string output = R"({"A":"TEST'"} ["OTHER STUFF"])"; - run_test(input, output); -} - -TEST_F(FstTest, GroundTruth_QuoteNormalization3) -{ - std::string input = R"(['{"A": "B"}',"{'A': 'B'}"])"; - std::string output = R"(["{\"A\": \"B\"}","{'A': 'B'}"])"; - run_test(input, output); -} - -TEST_F(FstTest, GroundTruth_QuoteNormalization4) -{ - std::string input = R"({"ain't ain't a word and you ain't supposed to say it":'"""""""""""'})"; - std::string output = - R"({"ain't ain't a word and you ain't supposed to say it":"\"\"\"\"\"\"\"\"\"\"\""})"; - run_test(input, output); -} - -TEST_F(FstTest, GroundTruth_QuoteNormalization5) -{ - std::string input = R"({"\"'\"'\"'\"'":'"\'"\'"\'"\'"'})"; - std::string output = R"({"\"'\"'\"'\"'":"\"'\"'\"'\"'\""})"; - run_test(input, output); -} - -TEST_F(FstTest, GroundTruth_QuoteNormalization6) -{ - std::string input = R"([{"ABC':'CBA":'XYZ":"ZXY'}])"; - std::string output = R"([{"ABC':'CBA":"XYZ\":\"ZXY"}])"; - run_test(input, output); -} - -TEST_F(FstTest, GroundTruth_QuoteNormalization7) -{ - std::string input = R"(["\t","\\t","\\","\\\'\"\\\\","\n","\b"])"; - std::string output = R"(["\t","\\t","\\","\\\'\"\\\\","\n","\b"])"; - run_test(input, output); -} - -TEST_F(FstTest, GroundTruth_QuoteNormalization8) -{ - std::string input = R"(['\t','\\t','\\','\\\"\'\\\\','\n','\b','\u0012'])"; - std::string output = R"(["\t","\\t","\\","\\\"'\\\\","\n","\b","\u0012"])"; - run_test(input, output); -} - -TEST_F(FstTest, GroundTruth_QuoteNormalization_Invalid1) -{ - std::string input = R"(["THIS IS A TEST'])"; - std::string output = R"(["THIS IS A TEST'])"; - run_test(input, output); -} - -TEST_F(FstTest, GroundTruth_QuoteNormalization_Invalid2) -{ - std::string input = R"(['THIS IS A TEST"])"; - std::string output = R"(["THIS IS A TEST\"])"; - run_test(input, output); -} + outbuf_size.data(), + normalize_quotes::start_state, + stream); -TEST_F(FstTest, GroundTruth_QuoteNormalization_Invalid3) -{ - std::string input = R"({"MORE TEST'N":'RESUL})"; - std::string output = R"({"MORE TEST'N":"RESUL})"; - run_test(input, output); -} - -TEST_F(FstTest, GroundTruth_QuoteNormalization_Invalid4) -{ - std::string input = R"({"NUMBER":100'0,'STRING':'SOMETHING'})"; - std::string output = R"({"NUMBER":100"0,"STRING":"SOMETHING"})"; - run_test(input, output); -} - -TEST_F(FstTest, GroundTruth_QuoteNormalization_Invalid5) -{ - std::string input = R"({'NUMBER':100"0,"STRING":"SOMETHING"})"; - std::string output = R"({"NUMBER":100"0,"STRING":"SOMETHING"})"; - run_test(input, output); -} - -TEST_F(FstTest, GroundTruth_QuoteNormalization_Invalid6) -{ - std::string input = R"({'a':'\\''})"; - std::string output = R"({"a":"\\""})"; - run_test(input, output); -} - -TEST_F(FstTest, GroundTruth_QuoteNormalization_Invalid7) -{ - std::string input = R"(}'a': 'b'{)"; - std::string output = R"(}"a": "b"{)"; - run_test(input, output); + outbuf.resize(outbuf_size.value(stream), stream); + return outbuf; } -CUDF_TEST_PROGRAM_MAIN() +} // namespace detail +} // namespace cudf::io::json diff --git a/cpp/src/io/json/read_json.cu b/cpp/src/io/json/read_json.cu index 080da7800f4..2cfb5fa03c9 100644 --- a/cpp/src/io/json/read_json.cu +++ b/cpp/src/io/json/read_json.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -45,6 +46,15 @@ size_t sources_size(host_span> const sources, }); } +/** + * @brief Read from array of data sources into RMM buffer + * + * @param sources Array of data sources + * @param compression Compression format of source + * @param range_offset Number of bytes to skip from source start + * @param range_size Number of bytes to read from source + * @param stream CUDA stream used for device memory operations and kernel launches + */ rmm::device_uvector ingest_raw_input(host_span> sources, compression_type compression, size_t range_offset, @@ -217,7 +227,14 @@ table_with_metadata read_json(host_span> sources, "Multiple inputs are supported only for JSON Lines format"); } - auto const buffer = get_record_range_raw_input(sources, reader_opts, stream); + auto buffer = get_record_range_raw_input(sources, reader_opts, stream); + + // If input JSON buffer has single quotes and option to normalize single quotes is enabled, + // invoke pre-processing FST + if (reader_opts.is_enabled_normalize_single_quotes()) { + buffer = + normalize_single_quotes(std::move(buffer), stream, rmm::mr::get_current_device_resource()); + } return device_parse_nested_json(buffer, reader_opts, stream, mr); // For debug purposes, use host_parse_nested_json() diff --git a/cpp/src/io/json/read_json.hpp b/cpp/src/io/json/read_json.hpp index db37e7abcdb..d05134fa837 100644 --- a/cpp/src/io/json/read_json.hpp +++ b/cpp/src/io/json/read_json.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index eee736613fe..24085eb5e10 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -313,13 +313,13 @@ ConfigureTest(JSON_TYPE_CAST_TEST io/json_type_cast_test.cu) ConfigureTest(NESTED_JSON_TEST io/nested_json_test.cpp io/json_tree.cpp) ConfigureTest(ARROW_IO_SOURCE_TEST io/arrow_io_source_test.cpp) ConfigureTest(MULTIBYTE_SPLIT_TEST io/text/multibyte_split_test.cpp) +ConfigureTest(JSON_QUOTE_NORMALIZATION io/json_quote_normalization_test.cpp) ConfigureTest( DATA_CHUNK_SOURCE_TEST io/text/data_chunk_source_test.cpp GPUS 1 PERCENT 30 ) target_link_libraries(DATA_CHUNK_SOURCE_TEST PRIVATE ZLIB::ZLIB) -ConfigureTest(QUOTE_NORMALIZATION_TEST io/fst/quote_normalization_test.cu) ConfigureTest(LOGICAL_STACK_TEST io/fst/logical_stack_test.cu) ConfigureTest(FST_TEST io/fst/fst_test.cu) ConfigureTest(TYPE_INFERENCE_TEST io/type_inference_test.cu) diff --git a/cpp/tests/io/json_quote_normalization_test.cpp b/cpp/tests/io/json_quote_normalization_test.cpp new file mode 100644 index 00000000000..50faea5e4d8 --- /dev/null +++ b/cpp/tests/io/json_quote_normalization_test.cpp @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +// Base test fixture for tests +struct JsonNormalizationTest : public cudf::test::BaseFixture {}; + +void run_test(const std::string& host_input, const std::string& expected_host_output) +{ + // RMM memory resource + std::shared_ptr rsc = + std::make_shared(); + + rmm::device_uvector device_input( + host_input.size(), cudf::test::get_default_stream(), rsc.get()); + CUDF_CUDA_TRY(cudaMemcpyAsync(device_input.data(), + host_input.data(), + host_input.size(), + cudaMemcpyHostToDevice, + cudf::test::get_default_stream().value())); + // Preprocessing FST + auto device_fst_output = cudf::io::json::detail::normalize_single_quotes( + std::move(device_input), cudf::test::get_default_stream(), rsc.get()); + + std::string preprocessed_host_output(device_fst_output.size(), 0); + CUDF_CUDA_TRY(cudaMemcpyAsync(preprocessed_host_output.data(), + device_fst_output.data(), + preprocessed_host_output.size(), + cudaMemcpyDeviceToHost, + cudf::test::get_default_stream().value())); + CUDF_TEST_EXPECT_VECTOR_EQUAL( + preprocessed_host_output, expected_host_output, preprocessed_host_output.size()); +} + +TEST_F(JsonNormalizationTest, GroundTruth_QuoteNormalization1) +{ + std::string input = R"({"A":'TEST"'})"; + std::string output = R"({"A":"TEST\""})"; + run_test(input, output); +} + +TEST_F(JsonNormalizationTest, GroundTruth_QuoteNormalization2) +{ + std::string input = R"({'A':"TEST'"} ['OTHER STUFF'])"; + std::string output = R"({"A":"TEST'"} ["OTHER STUFF"])"; + run_test(input, output); +} + +TEST_F(JsonNormalizationTest, GroundTruth_QuoteNormalization3) +{ + std::string input = R"(['{"A": "B"}',"{'A': 'B'}"])"; + std::string output = R"(["{\"A\": \"B\"}","{'A': 'B'}"])"; + run_test(input, output); +} + +TEST_F(JsonNormalizationTest, GroundTruth_QuoteNormalization4) +{ + std::string input = R"({"ain't ain't a word and you ain't supposed to say it":'"""""""""""'})"; + std::string output = + R"({"ain't ain't a word and you ain't supposed to say it":"\"\"\"\"\"\"\"\"\"\"\""})"; + run_test(input, output); +} + +TEST_F(JsonNormalizationTest, GroundTruth_QuoteNormalization5) +{ + std::string input = R"({"\"'\"'\"'\"'":'"\'"\'"\'"\'"'})"; + std::string output = R"({"\"'\"'\"'\"'":"\"'\"'\"'\"'\""})"; + run_test(input, output); +} + +TEST_F(JsonNormalizationTest, GroundTruth_QuoteNormalization6) +{ + std::string input = R"([{"ABC':'CBA":'XYZ":"ZXY'}])"; + std::string output = R"([{"ABC':'CBA":"XYZ\":\"ZXY"}])"; + run_test(input, output); +} + +TEST_F(JsonNormalizationTest, GroundTruth_QuoteNormalization7) +{ + std::string input = R"(["\t","\\t","\\","\\\'\"\\\\","\n","\b"])"; + std::string output = R"(["\t","\\t","\\","\\\'\"\\\\","\n","\b"])"; + run_test(input, output); +} + +TEST_F(JsonNormalizationTest, GroundTruth_QuoteNormalization8) +{ + std::string input = R"(['\t','\\t','\\','\\\"\'\\\\','\n','\b','\u0012'])"; + std::string output = R"(["\t","\\t","\\","\\\"'\\\\","\n","\b","\u0012"])"; + run_test(input, output); +} + +TEST_F(JsonNormalizationTest, GroundTruth_QuoteNormalization_Invalid1) +{ + std::string input = R"(["THIS IS A TEST'])"; + std::string output = R"(["THIS IS A TEST'])"; + run_test(input, output); +} + +TEST_F(JsonNormalizationTest, GroundTruth_QuoteNormalization_Invalid2) +{ + std::string input = R"(['THIS IS A TEST"])"; + std::string output = R"(["THIS IS A TEST\"])"; + run_test(input, output); +} + +TEST_F(JsonNormalizationTest, GroundTruth_QuoteNormalization_Invalid3) +{ + std::string input = R"({"MORE TEST'N":'RESUL})"; + std::string output = R"({"MORE TEST'N":"RESUL})"; + run_test(input, output); +} + +TEST_F(JsonNormalizationTest, GroundTruth_QuoteNormalization_Invalid4) +{ + std::string input = R"({"NUMBER":100'0,'STRING':'SOMETHING'})"; + std::string output = R"({"NUMBER":100"0,"STRING":"SOMETHING"})"; + run_test(input, output); +} + +TEST_F(JsonNormalizationTest, GroundTruth_QuoteNormalization_Invalid5) +{ + std::string input = R"({'NUMBER':100"0,"STRING":"SOMETHING"})"; + std::string output = R"({"NUMBER":100"0,"STRING":"SOMETHING"})"; + run_test(input, output); +} + +TEST_F(JsonNormalizationTest, GroundTruth_QuoteNormalization_Invalid6) +{ + std::string input = R"({'a':'\\''})"; + std::string output = R"({"a":"\\""})"; + run_test(input, output); +} + +TEST_F(JsonNormalizationTest, GroundTruth_QuoteNormalization_Invalid7) +{ + std::string input = R"(}'a': 'b'{)"; + std::string output = R"(}"a": "b"{)"; + run_test(input, output); +} + +TEST_F(JsonNormalizationTest, ReadJsonOption) +{ + // RMM memory resource + std::shared_ptr rsc = + std::make_shared(); + + // Test input + std::string const host_input = R"({"A":'TEST"'})"; + cudf::io::json_reader_options input_options = + cudf::io::json_reader_options::builder( + cudf::io::source_info{host_input.data(), host_input.size()}) + .lines(true) + .normalize_single_quotes(true); + + cudf::io::table_with_metadata processed_table = + cudf::io::read_json(input_options, cudf::test::get_default_stream(), rsc.get()); + + // Expected table + std::string const expected_input = R"({"A":"TEST\""})"; + cudf::io::json_reader_options expected_input_options = + cudf::io::json_reader_options::builder( + cudf::io::source_info{expected_input.data(), expected_input.size()}) + .lines(true); + + cudf::io::table_with_metadata expected_table = + cudf::io::read_json(expected_input_options, cudf::test::get_default_stream(), rsc.get()); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_table.tbl->view(), processed_table.tbl->view()); +} + +TEST_F(JsonNormalizationTest, ErrorCheck) +{ + // RMM memory resource + std::shared_ptr rsc = + std::make_shared(); + + // Test input + std::string const host_input = R"({"A":'TEST"'})"; + cudf::io::json_reader_options input_options = + cudf::io::json_reader_options::builder( + cudf::io::source_info{host_input.data(), host_input.size()}) + .lines(true); + + EXPECT_THROW(cudf::io::read_json(input_options, cudf::test::get_default_stream(), rsc.get()), + cudf::logic_error); +} + +CUDF_TEST_PROGRAM_MAIN() diff --git a/java/src/main/java/ai/rapids/cudf/JSONOptions.java b/java/src/main/java/ai/rapids/cudf/JSONOptions.java index 523d594f8ba..35165c18c7a 100644 --- a/java/src/main/java/ai/rapids/cudf/JSONOptions.java +++ b/java/src/main/java/ai/rapids/cudf/JSONOptions.java @@ -30,6 +30,7 @@ public final class JSONOptions extends ColumnFilterOptions { private final boolean dayFirst; private final boolean lines; private final boolean recoverWithNull; + private final boolean normalizeSingleQuotes; private final boolean mixedTypesAsStrings; private JSONOptions(Builder builder) { @@ -37,6 +38,7 @@ private JSONOptions(Builder builder) { dayFirst = builder.dayFirst; lines = builder.lines; recoverWithNull = builder.recoverWithNull; + normalizeSingleQuotes = builder.normalizeSingleQuotes; mixedTypesAsStrings = builder.mixedTypesAsStrings; } @@ -53,6 +55,10 @@ public boolean isRecoverWithNull() { return recoverWithNull; } + public boolean isNormalizeSingleQuotes() { + return normalizeSingleQuotes; + } + public boolean isMixedTypesAsStrings() { return mixedTypesAsStrings; } @@ -71,6 +77,7 @@ public static final class Builder extends ColumnFilterOptions.Builder= 0 && offset < buffer.length; return new TableWithMeta(readAndInferJSON(buffer.getAddress() + offset, len, opts.isDayFirst(), opts.isLines(), opts.isRecoverWithNull(), + opts.isNormalizeSingleQuotes(), opts.isMixedTypesAsStrings())); } @@ -1166,7 +1170,8 @@ public static Table readJSON(Schema schema, JSONOptions opts, HostMemoryBuffer b try (TableWithMeta twm = new TableWithMeta(readJSON(schema.getColumnNames(), schema.getTypeIds(), schema.getTypeScales(), null, buffer.getAddress() + offset, len, opts.isDayFirst(), opts.isLines(), - opts.isRecoverWithNull(), opts.isMixedTypesAsStrings()))) { + opts.isRecoverWithNull(), opts.isNormalizeSingleQuotes(), + opts.isMixedTypesAsStrings()))) { return gatherJSONColumns(schema, twm); } } @@ -1182,7 +1187,7 @@ public static Table readJSON(Schema schema, JSONOptions opts, DataSource ds) { long dsHandle = DataSourceHelper.createWrapperDataSource(ds); try (TableWithMeta twm = new TableWithMeta(readJSONFromDataSource(schema.getColumnNames(), schema.getTypeIds(), schema.getTypeScales(), opts.isDayFirst(), opts.isLines(), - opts.isRecoverWithNull(), opts.isMixedTypesAsStrings(), dsHandle))) { + opts.isRecoverWithNull(), opts.isNormalizeSingleQuotes(), opts.isMixedTypesAsStrings(), dsHandle))) { return gatherJSONColumns(schema, twm); } finally { DataSourceHelper.destroyWrapperDataSource(dsHandle); diff --git a/java/src/main/native/src/TableJni.cpp b/java/src/main/native/src/TableJni.cpp index 1ac15a3023c..cef18b245e7 100644 --- a/java/src/main/native/src/TableJni.cpp +++ b/java/src/main/native/src/TableJni.cpp @@ -1392,7 +1392,7 @@ JNIEXPORT void JNICALL Java_ai_rapids_cudf_Table_endWriteCSVToBuffer(JNIEnv *env JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readAndInferJSON( JNIEnv *env, jclass, jlong buffer, jlong buffer_length, jboolean day_first, jboolean lines, - jboolean recover_with_null, jboolean mixed_types_as_string) { + jboolean recover_with_null, jboolean normalize_single_quotes, jboolean mixed_types_as_string) { JNI_NULL_CHECK(env, buffer, "buffer cannot be null", 0); if (buffer_length <= 0) { @@ -1408,11 +1408,13 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readAndInferJSON( auto const recovery_mode = recover_with_null ? cudf::io::json_recovery_mode_t::RECOVER_WITH_NULL : cudf::io::json_recovery_mode_t::FAIL; - cudf::io::json_reader_options_builder opts = cudf::io::json_reader_options::builder(source) - .dayfirst(static_cast(day_first)) - .lines(static_cast(lines)) - .recovery_mode(recovery_mode) - .mixed_types_as_string(mixed_types_as_string); + cudf::io::json_reader_options_builder opts = + cudf::io::json_reader_options::builder(source) + .dayfirst(static_cast(day_first)) + .lines(static_cast(lines)) + .recovery_mode(recovery_mode) + .normalize_single_quotes(static_cast(normalize_single_quotes)) + .mixed_types_as_string(mixed_types_as_string); auto result = std::make_unique(cudf::io::read_json(opts.build())); @@ -1470,8 +1472,8 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_TableWithMeta_releaseTable(JNIE JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readJSONFromDataSource( JNIEnv *env, jclass, jobjectArray col_names, jintArray j_types, jintArray j_scales, - jboolean day_first, jboolean lines, jboolean recover_with_null, jboolean mixed_types_as_string, - jlong ds_handle) { + jboolean day_first, jboolean lines, jboolean recover_with_null, + jboolean normalize_single_quotes, jboolean mixed_types_as_string, jlong ds_handle) { JNI_NULL_CHECK(env, ds_handle, "no data source handle given", 0); @@ -1503,11 +1505,13 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readJSONFromDataSource( cudf::io::json_recovery_mode_t recovery_mode = recover_with_null ? cudf::io::json_recovery_mode_t::RECOVER_WITH_NULL : cudf::io::json_recovery_mode_t::FAIL; - cudf::io::json_reader_options_builder opts = cudf::io::json_reader_options::builder(source) - .dayfirst(static_cast(day_first)) - .lines(static_cast(lines)) - .recovery_mode(recovery_mode) - .mixed_types_as_string(mixed_types_as_string); + cudf::io::json_reader_options_builder opts = + cudf::io::json_reader_options::builder(source) + .dayfirst(static_cast(day_first)) + .lines(static_cast(lines)) + .recovery_mode(recovery_mode) + .normalize_single_quotes(static_cast(normalize_single_quotes)) + .mixed_types_as_string(mixed_types_as_string); if (!n_col_names.is_null() && data_types.size() > 0) { if (n_col_names.size() != n_types.size()) { @@ -1539,7 +1543,7 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readJSONFromDataSource( JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readJSON( JNIEnv *env, jclass, jobjectArray col_names, jintArray j_types, jintArray j_scales, jstring inputfilepath, jlong buffer, jlong buffer_length, jboolean day_first, jboolean lines, - jboolean recover_with_null, jboolean mixed_types_as_string) { + jboolean recover_with_null, jboolean normalize_single_quotes, jboolean mixed_types_as_string) { bool read_buffer = true; if (buffer == 0) { @@ -1586,11 +1590,13 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readJSON( cudf::io::json_recovery_mode_t recovery_mode = recover_with_null ? cudf::io::json_recovery_mode_t::RECOVER_WITH_NULL : cudf::io::json_recovery_mode_t::FAIL; - cudf::io::json_reader_options_builder opts = cudf::io::json_reader_options::builder(source) - .dayfirst(static_cast(day_first)) - .lines(static_cast(lines)) - .recovery_mode(recovery_mode) - .mixed_types_as_string(mixed_types_as_string); + cudf::io::json_reader_options_builder opts = + cudf::io::json_reader_options::builder(source) + .dayfirst(static_cast(day_first)) + .lines(static_cast(lines)) + .recovery_mode(recovery_mode) + .normalize_single_quotes(static_cast(normalize_single_quotes)) + .mixed_types_as_string(mixed_types_as_string); if (!n_col_names.is_null() && data_types.size() > 0) { if (n_col_names.size() != n_types.size()) { diff --git a/java/src/test/java/ai/rapids/cudf/TableTest.java b/java/src/test/java/ai/rapids/cudf/TableTest.java index 73002644858..f1c4d0803a3 100644 --- a/java/src/test/java/ai/rapids/cudf/TableTest.java +++ b/java/src/test/java/ai/rapids/cudf/TableTest.java @@ -87,6 +87,7 @@ public class TableTest extends CudfTestBase { private static final File TEST_SIMPLE_CSV_FILE = TestUtils.getResourceAsFile("simple.csv"); private static final File TEST_SIMPLE_JSON_FILE = TestUtils.getResourceAsFile("people.json"); private static final File TEST_JSON_ERROR_FILE = TestUtils.getResourceAsFile("people_with_invalid_lines.json"); + private static final File TEST_JSON_SINGLE_QUOTES_FILE = TestUtils.getResourceAsFile("single_quotes.json"); private static final File TEST_MIXED_TYPE_1_JSON = TestUtils.getResourceAsFile("mixed_types_1.json"); private static final File TEST_MIXED_TYPE_2_JSON = TestUtils.getResourceAsFile("mixed_types_2.json"); @@ -330,6 +331,23 @@ void testReadJSONFile() { } @Test + void testReadSingleQuotesJSONFile() throws IOException { + Schema schema = Schema.builder() + .column(DType.STRING, "A") + .build(); + JSONOptions opts = JSONOptions.builder() + .withLines(true) + .withNormalizeSingleQuotes(true) + .build(); + try (Table expected = new Table.TestBuilder() + .column("TEST\"", "TESTER'") + .build(); + MultiBufferDataSource source = sourceFrom(TEST_JSON_SINGLE_QUOTES_FILE); + Table table = Table.readJSON(schema, opts, source)) { + assertTablesAreEqual(expected, table); + } + } + void testReadMixedType2JSONFileFeatureDisabled() { Schema schema = Schema.builder() .column(DType.STRING, "a") @@ -377,6 +395,21 @@ void testReadMixedType2JSONFile() throws IOException { } } + @Test + void testReadSingleQuotesJSONFileFeatureDisabled() throws IOException { + Schema schema = Schema.builder() + .column(DType.STRING, "A") + .build(); + JSONOptions opts = JSONOptions.builder() + .withLines(true) + .withNormalizeSingleQuotes(false) + .build(); + try (MultiBufferDataSource source = sourceFrom(TEST_JSON_SINGLE_QUOTES_FILE)) { + assertThrows(CudfException.class, () -> + Table.readJSON(schema, opts, source)); + } + } + @Test void testReadJSONFromDataSource() throws IOException { Schema schema = Schema.builder() diff --git a/java/src/test/js b/java/src/test/js new file mode 100644 index 00000000000..e69de29bb2d diff --git a/java/src/test/resources/single_quotes.json b/java/src/test/resources/single_quotes.json new file mode 100644 index 00000000000..cb432fbc643 --- /dev/null +++ b/java/src/test/resources/single_quotes.json @@ -0,0 +1,2 @@ +{"A":'TEST"'} +{'A':"TESTER'"} From 807318b5c0f10219291bf10db497018c7f42d591 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 24 Jan 2024 15:54:04 -0600 Subject: [PATCH 111/384] Update conda-cpp-post-build-checks to branch-24.04. (#14854) Fixes some merge issues with outdated versions from #14768. I also made a minor tweak to `update-version.sh` that double-quotes some outputs to make pre-commit happier. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Jake Awe (https://github.com/AyodeAwe) URL: https://github.com/rapidsai/cudf/pull/14854 --- .github/workflows/pr.yaml | 2 +- .github/workflows/test.yaml | 2 +- ci/release/update-version.sh | 8 ++++---- docs/dask_cudf/source/conf.py | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 41d4e42891c..14a74618413 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -47,7 +47,7 @@ jobs: conda-cpp-checks: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.04 with: build_type: pull-request enable_check_symbols: true diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a1674e691cd..e044d69c6d8 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.04 with: build_type: nightly branch: ${{ inputs.branch }} diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 47e3f887d7d..02dba0d09e4 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -67,10 +67,10 @@ sed_runner 's/'"branch-.*\/cmake-format-rapids-cmake.json"'/'"branch-${NEXT_SHOR sed_runner 's/PROJECT_NUMBER = .*/PROJECT_NUMBER = '${NEXT_FULL_TAG}'/g' cpp/doxygen/Doxyfile # sphinx docs update -sed_runner 's/version = .*/version = '"'${NEXT_SHORT_TAG}'"'/g' docs/cudf/source/conf.py -sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/cudf/source/conf.py -sed_runner 's/version = .*/version = '"'${NEXT_SHORT_TAG}'"'/g' docs/dask_cudf/source/conf.py -sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/dask_cudf/source/conf.py +sed_runner 's/version = .*/version = "'${NEXT_SHORT_TAG}'"/g' docs/cudf/source/conf.py +sed_runner 's/release = .*/release = "'${NEXT_FULL_TAG}'"/g' docs/cudf/source/conf.py +sed_runner 's/version = .*/version = "'${NEXT_SHORT_TAG}'"/g' docs/dask_cudf/source/conf.py +sed_runner 's/release = .*/release = "'${NEXT_FULL_TAG}'"/g' docs/dask_cudf/source/conf.py DEPENDENCIES=( cudf diff --git a/docs/dask_cudf/source/conf.py b/docs/dask_cudf/source/conf.py index f1f28ccd752..25f0eb41ed5 100644 --- a/docs/dask_cudf/source/conf.py +++ b/docs/dask_cudf/source/conf.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2023, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. # Configuration file for the Sphinx documentation builder. # @@ -11,8 +11,8 @@ project = "dask-cudf" copyright = "2018-2023, NVIDIA Corporation" author = "NVIDIA Corporation" -version = '24.04' -release = '24.04.00' +version = "24.04" +release = "24.04.00" language = "en" From 258d9ee28311df406c16b61e12bfc592d57149b0 Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Wed, 24 Jan 2024 15:13:25 -0800 Subject: [PATCH 112/384] Add row index and stripe size options to Python ORC chunked writer (#14785) Adds the APIs that control the stripe/row group size when using the chunked writer. This functions are already present in to_orc (non-chunked version of the same API). Adding this options to facilitate smaller unit tests. Authors: - Vukasin Milovanovic (https://github.com/vuule) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/14785 --- python/cudf/cudf/_lib/orc.pyx | 25 ++++++++++++++++++++----- python/cudf/cudf/tests/test_orc.py | 22 ++++++++++++++++++++++ 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/python/cudf/cudf/_lib/orc.pyx b/python/cudf/cudf/_lib/orc.pyx index c64296eb7da..2cbdf76030b 100644 --- a/python/cudf/cudf/_lib/orc.pyx +++ b/python/cudf/cudf/_lib/orc.pyx @@ -375,13 +375,19 @@ cdef class ORCWriter: cdef object index cdef table_input_metadata tbl_meta cdef object cols_as_map_type + cdef object stripe_size_bytes + cdef object stripe_size_rows + cdef object row_index_stride def __cinit__(self, object path, object index=None, object compression="snappy", object statistics="ROWGROUP", - object cols_as_map_type=None): + object cols_as_map_type=None, + object stripe_size_bytes=None, + object stripe_size_rows=None, + object row_index_stride=None): self.sink = make_sink_info(path, self._data_sink) self.stat_freq = _get_orc_stat_freq(statistics) @@ -389,6 +395,9 @@ cdef class ORCWriter: self.index = index self.cols_as_map_type = cols_as_map_type \ if cols_as_map_type is None else set(cols_as_map_type) + self.stripe_size_bytes = stripe_size_bytes + self.stripe_size_rows = stripe_size_rows + self.row_index_stride = row_index_stride self.initialized = False def write_table(self, table): @@ -456,9 +465,7 @@ cdef class ORCWriter: pandas_metadata = generate_pandas_metadata(table, self.index) user_data[str.encode("pandas")] = str.encode(pandas_metadata) - cdef chunked_orc_writer_options args - with nogil: - args = move( + cdef chunked_orc_writer_options c_opts = move( chunked_orc_writer_options.builder(self.sink) .metadata(self.tbl_meta) .key_value_metadata(move(user_data)) @@ -466,7 +473,15 @@ cdef class ORCWriter: .enable_statistics(self.stat_freq) .build() ) - self.writer.reset(new orc_chunked_writer(args)) + if self.stripe_size_bytes is not None: + c_opts.set_stripe_size_bytes(self.stripe_size_bytes) + if self.stripe_size_rows is not None: + c_opts.set_stripe_size_rows(self.stripe_size_rows) + if self.row_index_stride is not None: + c_opts.set_row_index_stride(self.row_index_stride) + + with nogil: + self.writer.reset(new orc_chunked_writer(c_opts)) self.initialized = True diff --git a/python/cudf/cudf/tests/test_orc.py b/python/cudf/cudf/tests/test_orc.py index 4630b6eef0a..6b7f86098a0 100644 --- a/python/cudf/cudf/tests/test_orc.py +++ b/python/cudf/cudf/tests/test_orc.py @@ -1911,3 +1911,25 @@ def test_orc_reader_empty_deeply_nested_level(datadir): got = cudf.read_orc(path) assert_eq(expect, got) + + +def test_orc_chunked_writer_stripe_size(datadir): + from pyarrow import orc + + df = cudf.DataFrame({"col": gen_rand_series("int", 100000)}) + + buffer = BytesIO() + writer = ORCWriter(buffer, stripe_size_bytes=64 * 1024) + writer.write_table(df) + writer.close() + + orc_file = orc.ORCFile(buffer) + assert_eq(orc_file.nstripes, 10) + + buffer = BytesIO() + writer = ORCWriter(buffer, stripe_size_rows=20000) + writer.write_table(df) + writer.close() + + orc_file = orc.ORCFile(buffer) + assert_eq(orc_file.nstripes, 5) From 5b1eef31ed4c5935285ef780dc74d35cea086b49 Mon Sep 17 00:00:00 2001 From: nvdbaranec <56695930+nvdbaranec@users.noreply.github.com> Date: Wed, 24 Jan 2024 18:40:15 -0600 Subject: [PATCH 113/384] Parquet sub-rowgroup reading. (#14360) closes #14270 Implementation of sub-rowgroup reading of Parquet files. This PR implements an additional layer on top of the existing chunking system. Currently, the reader takes two parameters: `input_pass_read_limit` which specifies a limit on temporary memory usage when reading and decompressing file data; and `output_pass_read_limit` which specifies a limit on how large an output chunk (a table) can be. Currently when the user specifies a limit via `input_pass_read_limit`, the reader will perform multiple `passes` over the file at row-group granularity. That is, it will control how many row groups it will read at once to conform to the specified limit. However, there are cases where this is not sufficient. So this PR changes things so that we now have `subpasses` below the top level `passes`. It works as follows: - We read a set of input chunks based on the `input_pass_read_limit` but we do not decompress them immediately. This constitutes a `pass`. - Within each pass of compressed data, we progressively decompress batches of pages as `subpasses`. - Within each `subpass` we apply the output limit to produce `chunks`. So the overall structure of the reader is: (read) `pass` -> (decompress) `subpass` -> (decode) `chunk` Major sections of code changes: - Previously the incoming page data in the file was unsorted. To handle this we later on produced a `page_index` that could be applied to the array to get them in schema-sorted order. This was getting very unwieldy so I just sort the pages up front now and the `page_index` array has gone away. - There are now two sets of pages to be aware of in the code. Within each `pass_intermediate_data` there is the set of all pages within the current set of loaded row groups. And then within the `subpass_intermediate_data` struct there is a separate array of pages representing the current batch of decompressed data we are processing. To keep the confusion down I changed a good amount of code to always reference it's array though it's associated struct. Ie, `pass.pages` or `subpass.pages`. In addition, I removed the `page_info` from `ColumnChunkDesc` to help prevent the kernels from getting confused. `ColumnChunkDesc` now only has a `dict_page` field which is constant across all subpasses. - The primary entry point for the chunking mechanism is in `handle_chunking`. Here we iterate through passes, subpasses and output chunks. Successive subpasses are computed and preprocessed through here. - The volume of diffs you'll see in `reader_impl_chunking.cu` is a little deceptive. A lot of this is just functions (or pieces of functions) that have been moved over from either `reader_impl_preprocess.cu` or `reader_impl_helpers.cpp`. The most relevant actual changes are in: ` handle_chunking`, `compute_input_passes`, `compute_next_subpass`, and `compute_chunks_for_subpass`. Note on tests: I renamed `parquet_chunked_reader_tests.cpp` to `parquet_chunked_reader_test.cu` as I needed to use thrust. The only actual changes in the file are the addition of the `ParquetChunkedReaderInputLimitConstrainedTest` and `ParquetChunkedReaderInputLimitTest` test suites at the bottom. Authors: - https://github.com/nvdbaranec - Nghia Truong (https://github.com/ttnghia) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Nghia Truong (https://github.com/ttnghia) - Vukasin Milovanovic (https://github.com/vuule) URL: https://github.com/rapidsai/cudf/pull/14360 --- cpp/src/io/comp/nvcomp_adapter.hpp | 24 +- cpp/src/io/parquet/page_decode.cuh | 9 +- cpp/src/io/parquet/page_hdr.cu | 27 +- cpp/src/io/parquet/page_string_decode.cu | 10 +- cpp/src/io/parquet/parquet_gpu.hpp | 60 +- cpp/src/io/parquet/reader_impl.cpp | 206 ++- cpp/src/io/parquet/reader_impl.hpp | 128 +- cpp/src/io/parquet/reader_impl_chunking.cu | 1410 ++++++++++++++--- cpp/src/io/parquet/reader_impl_chunking.hpp | 89 +- cpp/src/io/parquet/reader_impl_preprocess.cu | 1107 +++++++------ cpp/src/io/utilities/column_buffer.cpp | 31 + cpp/src/io/utilities/column_buffer.hpp | 12 +- cpp/tests/CMakeLists.txt | 2 +- ...est.cpp => parquet_chunked_reader_test.cu} | 342 +++- 14 files changed, 2387 insertions(+), 1070 deletions(-) rename cpp/tests/io/{parquet_chunked_reader_test.cpp => parquet_chunked_reader_test.cu} (73%) diff --git a/cpp/src/io/comp/nvcomp_adapter.hpp b/cpp/src/io/comp/nvcomp_adapter.hpp index 1393b70f058..69a278757ce 100644 --- a/cpp/src/io/comp/nvcomp_adapter.hpp +++ b/cpp/src/io/comp/nvcomp_adapter.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -99,8 +99,8 @@ inline bool operator==(feature_status_parameters const& lhs, feature_status_para * @param[in] inputs List of input buffers * @param[out] outputs List of output buffers * @param[out] results List of output status structures - * @param[in] max_uncomp_chunk_size maximum size of uncompressed chunk - * @param[in] max_total_uncomp_size maximum total size of uncompressed data + * @param[in] max_uncomp_chunk_size Maximum size of any single uncompressed chunk + * @param[in] max_total_uncomp_size Maximum total size of uncompressed data * @param[in] stream CUDA stream to use */ void batched_decompress(compression_type compression, @@ -111,6 +111,24 @@ void batched_decompress(compression_type compression, size_t max_total_uncomp_size, rmm::cuda_stream_view stream); +/** + * @brief Return the amount of temporary space required in bytes for a given decompression + * operation. + * + * The size returned reflects the size of the scratch buffer to be passed to + * `batched_decompress_async` + * + * @param[in] compression Compression type + * @param[in] num_chunks The number of decompression chunks to be processed + * @param[in] max_uncomp_chunk_size Maximum size of any single uncompressed chunk + * @param[in] max_total_uncomp_size Maximum total size of uncompressed data + * @returns The total required size in bytes + */ +size_t batched_decompress_temp_size(compression_type compression, + size_t num_chunks, + size_t max_uncomp_chunk_size, + size_t max_total_uncomp_size); + /** * @brief Gets the maximum size any chunk could compress to in the batch. * diff --git a/cpp/src/io/parquet/page_decode.cuh b/cpp/src/io/parquet/page_decode.cuh index 8f256cd1f97..409b1464cd1 100644 --- a/cpp/src/io/parquet/page_decode.cuh +++ b/cpp/src/io/parquet/page_decode.cuh @@ -1301,16 +1301,15 @@ inline __device__ bool setupLocalPageInfo(page_state_s* const s, if (((s->col.data_type & 7) == BYTE_ARRAY) && (s->col.str_dict_index)) { // String dictionary: use index s->dict_base = reinterpret_cast(s->col.str_dict_index); - s->dict_size = s->col.page_info[0].num_input_values * sizeof(string_index_pair); + s->dict_size = s->col.dict_page->num_input_values * sizeof(string_index_pair); } else { - s->dict_base = - s->col.page_info[0].page_data; // dictionary is always stored in the first page - s->dict_size = s->col.page_info[0].uncompressed_page_size; + s->dict_base = s->col.dict_page->page_data; + s->dict_size = s->col.dict_page->uncompressed_page_size; } s->dict_run = 0; s->dict_val = 0; s->dict_bits = (cur < end) ? *cur++ : 0; - if (s->dict_bits > 32 || !s->dict_base) { + if (s->dict_bits > 32 || (!s->dict_base && s->col.dict_page->num_input_values > 0)) { s->set_error_code(decode_error::INVALID_DICT_WIDTH); } break; diff --git a/cpp/src/io/parquet/page_hdr.cu b/cpp/src/io/parquet/page_hdr.cu index 4be4f45497d..888d9452612 100644 --- a/cpp/src/io/parquet/page_hdr.cu +++ b/cpp/src/io/parquet/page_hdr.cu @@ -348,9 +348,11 @@ struct gpuParsePageHeader { * @param[in] num_chunks Number of column chunks */ // blockDim {128,1,1} -CUDF_KERNEL void __launch_bounds__(128) gpuDecodePageHeaders(ColumnChunkDesc* chunks, - int32_t num_chunks, - kernel_error::pointer error_code) +CUDF_KERNEL +void __launch_bounds__(128) gpuDecodePageHeaders(ColumnChunkDesc* chunks, + chunk_page_info* chunk_pages, + int32_t num_chunks, + kernel_error::pointer error_code) { using cudf::detail::warp_size; gpuParsePageHeader parse_page_header; @@ -392,11 +394,10 @@ CUDF_KERNEL void __launch_bounds__(128) gpuDecodePageHeaders(ColumnChunkDesc* ch bs->page.temp_string_buf = nullptr; bs->page.kernel_mask = decode_kernel_mask::NONE; } - num_values = bs->ck.num_values; - page_info = bs->ck.page_info; - num_dict_pages = bs->ck.num_dict_pages; - max_num_pages = (page_info) ? bs->ck.max_num_pages : 0; - values_found = 0; + num_values = bs->ck.num_values; + page_info = chunk_pages ? chunk_pages[chunk].pages : nullptr; + max_num_pages = page_info ? bs->ck.max_num_pages : 0; + values_found = 0; __syncwarp(); while (values_found < num_values && bs->cur < bs->end) { int index_out = -1; @@ -495,9 +496,9 @@ CUDF_KERNEL void __launch_bounds__(128) if (!lane_id && ck->num_dict_pages > 0 && ck->str_dict_index) { // Data type to describe a string string_index_pair* dict_index = ck->str_dict_index; - uint8_t const* dict = ck->page_info[0].page_data; - int dict_size = ck->page_info[0].uncompressed_page_size; - int num_entries = ck->page_info[0].num_input_values; + uint8_t const* dict = ck->dict_page->page_data; + int dict_size = ck->dict_page->uncompressed_page_size; + int num_entries = ck->dict_page->num_input_values; int pos = 0, cur = 0; for (int i = 0; i < num_entries; i++) { int len = 0; @@ -518,13 +519,15 @@ CUDF_KERNEL void __launch_bounds__(128) } void __host__ DecodePageHeaders(ColumnChunkDesc* chunks, + chunk_page_info* chunk_pages, int32_t num_chunks, kernel_error::pointer error_code, rmm::cuda_stream_view stream) { dim3 dim_block(128, 1); dim3 dim_grid((num_chunks + 3) >> 2, 1); // 1 chunk per warp, 4 warps per block - gpuDecodePageHeaders<<>>(chunks, num_chunks, error_code); + gpuDecodePageHeaders<<>>( + chunks, chunk_pages, num_chunks, error_code); } void __host__ BuildStringDictionaryIndex(ColumnChunkDesc* chunks, diff --git a/cpp/src/io/parquet/page_string_decode.cu b/cpp/src/io/parquet/page_string_decode.cu index 37a8cabc182..d652a43d097 100644 --- a/cpp/src/io/parquet/page_string_decode.cu +++ b/cpp/src/io/parquet/page_string_decode.cu @@ -868,14 +868,16 @@ CUDF_KERNEL void __launch_bounds__(preprocess_block_size) gpuComputePageStringSi if (col.str_dict_index) { // String dictionary: use index dict_base = reinterpret_cast(col.str_dict_index); - dict_size = col.page_info[0].num_input_values * sizeof(string_index_pair); + dict_size = col.dict_page->num_input_values * sizeof(string_index_pair); } else { - dict_base = col.page_info[0].page_data; // dictionary is always stored in the first page - dict_size = col.page_info[0].uncompressed_page_size; + dict_base = col.dict_page->page_data; + dict_size = col.dict_page->uncompressed_page_size; } // FIXME: need to return an error condition...this won't actually do anything - if (s->dict_bits > 32 || !dict_base) { CUDF_UNREACHABLE("invalid dictionary bit size"); } + if (s->dict_bits > 32 || (!dict_base && col.dict_page->num_input_values > 0)) { + CUDF_UNREACHABLE("invalid dictionary bit size"); + } str_bytes = totalDictEntriesSize( data, dict_base, s->dict_bits, dict_size, (end - data), start_value, end_value); diff --git a/cpp/src/io/parquet/parquet_gpu.hpp b/cpp/src/io/parquet/parquet_gpu.hpp index 18d282be855..d58c7f95389 100644 --- a/cpp/src/io/parquet/parquet_gpu.hpp +++ b/cpp/src/io/parquet/parquet_gpu.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -339,6 +339,21 @@ struct PageInfo { decode_kernel_mask kernel_mask; }; +/** + * @brief Return the column schema id as the key for a PageInfo struct. + */ +struct get_page_key { + __device__ int32_t operator()(PageInfo const& page) const { return page.src_col_schema; } +}; + +/** + * @brief Return an iterator that returns they keys for a vector of pages. + */ +inline auto make_page_key_iterator(device_span pages) +{ + return thrust::make_transform_iterator(pages.begin(), get_page_key{}); +} + /** * @brief Struct describing a particular chunk of column data */ @@ -362,7 +377,8 @@ struct ColumnChunkDesc { int8_t decimal_precision_, int32_t ts_clock_rate_, int32_t src_col_index_, - int32_t src_col_schema_) + int32_t src_col_schema_, + float list_bytes_per_row_est_) : compressed_data(compressed_data_), compressed_size(compressed_size_), num_values(num_values_), @@ -375,7 +391,7 @@ struct ColumnChunkDesc { num_data_pages(0), num_dict_pages(0), max_num_pages(0), - page_info(nullptr), + dict_page(nullptr), str_dict_index(nullptr), valid_map_base{nullptr}, column_data_base{nullptr}, @@ -386,26 +402,25 @@ struct ColumnChunkDesc { decimal_precision(decimal_precision_), ts_clock_rate(ts_clock_rate_), src_col_index(src_col_index_), - src_col_schema(src_col_schema_) + src_col_schema(src_col_schema_), + list_bytes_per_row_est(list_bytes_per_row_est_) { } - uint8_t const* compressed_data{}; // pointer to compressed column chunk data - size_t compressed_size{}; // total compressed data size for this chunk - size_t num_values{}; // total number of values in this column - size_t start_row{}; // starting row of this chunk - uint32_t num_rows{}; // number of rows in this chunk + uint8_t const* compressed_data{}; // pointer to compressed column chunk data + size_t compressed_size{}; // total compressed data size for this chunk + size_t num_values{}; // total number of values in this column + size_t start_row{}; // file-wide, absolute starting row of this chunk + uint32_t num_rows{}; // number of rows in this chunk int16_t max_level[level_type::NUM_LEVEL_TYPES]{}; // max definition/repetition level int16_t max_nesting_depth{}; // max nesting depth of the output - uint16_t data_type{}; // basic column data type, ((type_length << 3) | - // parquet::Type) + uint16_t data_type{}; // basic column data type, ((type_length << 3) | // parquet::Type) uint8_t - level_bits[level_type::NUM_LEVEL_TYPES]{}; // bits to encode max definition/repetition levels - int32_t num_data_pages{}; // number of data pages - int32_t num_dict_pages{}; // number of dictionary pages - int32_t max_num_pages{}; // size of page_info array - PageInfo* page_info{}; // output page info for up to num_dict_pages + - // num_data_pages (dictionary pages first) + level_bits[level_type::NUM_LEVEL_TYPES]{}; // bits to encode max definition/repetition levels + int32_t num_data_pages{}; // number of data pages + int32_t num_dict_pages{}; // number of dictionary pages + int32_t max_num_pages{}; // size of page_info array + PageInfo const* dict_page{}; string_index_pair* str_dict_index{}; // index for string dictionary bitmask_type** valid_map_base{}; // base pointers of valid bit map for this column void** column_data_base{}; // base pointers of column data @@ -418,6 +433,15 @@ struct ColumnChunkDesc { int32_t src_col_index{}; // my input column index int32_t src_col_schema{}; // my schema index in the file + + float list_bytes_per_row_est{}; // for LIST columns, an estimate on number of bytes per row +}; + +/** + * @brief A utility structure for use in decoding page headers. + */ +struct chunk_page_info { + PageInfo* pages; }; /** @@ -578,11 +602,13 @@ constexpr bool is_string_col(ColumnChunkDesc const& chunk) * @brief Launches kernel for parsing the page headers in the column chunks * * @param[in] chunks List of column chunks + * @param[in] chunk_pages List of pages associated with the chunks, in chunk-sorted order * @param[in] num_chunks Number of column chunks * @param[out] error_code Error code for kernel failures * @param[in] stream CUDA stream to use */ void DecodePageHeaders(ColumnChunkDesc* chunks, + chunk_page_info* chunk_pages, int32_t num_chunks, kernel_error::pointer error_code, rmm::cuda_stream_view stream); diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index c1082c0305a..24d46d91dbb 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,26 +29,28 @@ namespace cudf::io::parquet::detail { void reader::impl::decode_page_data(size_t skip_rows, size_t num_rows) { - auto& chunks = _pass_itm_data->chunks; - auto& pages = _pass_itm_data->pages_info; - auto& page_nesting = _pass_itm_data->page_nesting_info; - auto& page_nesting_decode = _pass_itm_data->page_nesting_decode_info; - auto const level_type_size = _pass_itm_data->level_type_size; + auto& pass = *_pass_itm_data; + auto& subpass = *pass.subpass; + + auto& page_nesting = subpass.page_nesting_info; + auto& page_nesting_decode = subpass.page_nesting_decode_info; + + auto const level_type_size = pass.level_type_size; // temporary space for DELTA_BYTE_ARRAY decoding. this only needs to live until // gpu::DecodeDeltaByteArray returns. rmm::device_uvector delta_temp_buf(0, _stream); // Should not reach here if there is no page data. - CUDF_EXPECTS(pages.size() > 0, "There is no page to decode"); + CUDF_EXPECTS(subpass.pages.size() > 0, "There are no pages to decode"); size_t const sum_max_depths = std::accumulate( - chunks.begin(), chunks.end(), 0, [&](size_t cursum, ColumnChunkDesc const& chunk) { + pass.chunks.begin(), pass.chunks.end(), 0, [&](size_t cursum, ColumnChunkDesc const& chunk) { return cursum + _metadata->get_output_nesting_depth(chunk.src_col_schema); }); // figure out which kernels to run - auto const kernel_mask = GetAggregatedDecodeKernelMask(pages, _stream); + auto const kernel_mask = GetAggregatedDecodeKernelMask(subpass.pages, _stream); // Check to see if there are any string columns present. If so, then we need to get size info // for each string page. This size info will be used to pre-allocate memory for the column, @@ -59,8 +61,14 @@ void reader::impl::decode_page_data(size_t skip_rows, size_t num_rows) auto const has_strings = (kernel_mask & STRINGS_MASK) != 0; std::vector col_sizes(_input_columns.size(), 0L); if (has_strings) { - ComputePageStringSizes( - pages, chunks, delta_temp_buf, skip_rows, num_rows, level_type_size, kernel_mask, _stream); + ComputePageStringSizes(subpass.pages, + pass.chunks, + delta_temp_buf, + skip_rows, + num_rows, + level_type_size, + kernel_mask, + _stream); col_sizes = calculate_page_string_offsets(); @@ -83,26 +91,26 @@ void reader::impl::decode_page_data(size_t skip_rows, size_t num_rows) cudf::detail::hostdevice_vector(has_strings ? sum_max_depths : 0, _stream); // Update chunks with pointers to column data. - for (size_t c = 0, page_count = 0, chunk_off = 0; c < chunks.size(); c++) { - input_column_info const& input_col = _input_columns[chunks[c].src_col_index]; - CUDF_EXPECTS(input_col.schema_idx == chunks[c].src_col_schema, + for (size_t c = 0, chunk_off = 0; c < pass.chunks.size(); c++) { + input_column_info const& input_col = _input_columns[pass.chunks[c].src_col_index]; + CUDF_EXPECTS(input_col.schema_idx == pass.chunks[c].src_col_schema, "Column/page schema index mismatch"); - size_t max_depth = _metadata->get_output_nesting_depth(chunks[c].src_col_schema); + size_t max_depth = _metadata->get_output_nesting_depth(pass.chunks[c].src_col_schema); chunk_offsets.push_back(chunk_off); // get a slice of size `nesting depth` from `chunk_nested_valids` to store an array of pointers // to validity data - auto valids = chunk_nested_valids.host_ptr(chunk_off); - chunks[c].valid_map_base = chunk_nested_valids.device_ptr(chunk_off); + auto valids = chunk_nested_valids.host_ptr(chunk_off); + pass.chunks[c].valid_map_base = chunk_nested_valids.device_ptr(chunk_off); // get a slice of size `nesting depth` from `chunk_nested_data` to store an array of pointers to // out data - auto data = chunk_nested_data.host_ptr(chunk_off); - chunks[c].column_data_base = chunk_nested_data.device_ptr(chunk_off); + auto data = chunk_nested_data.host_ptr(chunk_off); + pass.chunks[c].column_data_base = chunk_nested_data.device_ptr(chunk_off); auto str_data = has_strings ? chunk_nested_str_data.host_ptr(chunk_off) : nullptr; - chunks[c].column_string_base = + pass.chunks[c].column_string_base = has_strings ? chunk_nested_str_data.device_ptr(chunk_off) : nullptr; chunk_off += max_depth; @@ -148,8 +156,8 @@ void reader::impl::decode_page_data(size_t skip_rows, size_t num_rows) valids[idx] = out_buf.null_mask(); data[idx] = out_buf.data(); // only do string buffer for leaf - if (out_buf.string_size() == 0 && col_sizes[chunks[c].src_col_index] > 0) { - out_buf.create_string_data(col_sizes[chunks[c].src_col_index], _stream); + if (out_buf.string_size() == 0 && col_sizes[pass.chunks[c].src_col_index] > 0) { + out_buf.create_string_data(col_sizes[pass.chunks[c].src_col_index], _stream); } if (has_strings) { str_data[idx] = out_buf.string_data(); } out_buf.user_data |= @@ -159,12 +167,9 @@ void reader::impl::decode_page_data(size_t skip_rows, size_t num_rows) data[idx] = nullptr; } } - - // column_data_base will always point to leaf data, even for nested types. - page_count += chunks[c].max_num_pages; } - chunks.host_to_device_async(_stream); + pass.chunks.host_to_device_async(_stream); chunk_nested_valids.host_to_device_async(_stream); chunk_nested_data.host_to_device_async(_stream); if (has_strings) { chunk_nested_str_data.host_to_device_async(_stream); } @@ -179,44 +184,71 @@ void reader::impl::decode_page_data(size_t skip_rows, size_t num_rows) // launch string decoder int s_idx = 0; if (BitAnd(kernel_mask, decode_kernel_mask::STRING) != 0) { - DecodeStringPageData( - pages, chunks, num_rows, skip_rows, level_type_size, error_code.data(), streams[s_idx++]); + DecodeStringPageData(subpass.pages, + pass.chunks, + num_rows, + skip_rows, + level_type_size, + error_code.data(), + streams[s_idx++]); } // launch delta byte array decoder if (BitAnd(kernel_mask, decode_kernel_mask::DELTA_BYTE_ARRAY) != 0) { - DecodeDeltaByteArray( - pages, chunks, num_rows, skip_rows, level_type_size, error_code.data(), streams[s_idx++]); + DecodeDeltaByteArray(subpass.pages, + pass.chunks, + num_rows, + skip_rows, + level_type_size, + error_code.data(), + streams[s_idx++]); } // launch delta length byte array decoder if (BitAnd(kernel_mask, decode_kernel_mask::DELTA_LENGTH_BA) != 0) { - DecodeDeltaLengthByteArray( - pages, chunks, num_rows, skip_rows, level_type_size, error_code.data(), streams[s_idx++]); + DecodeDeltaLengthByteArray(subpass.pages, + pass.chunks, + num_rows, + skip_rows, + level_type_size, + error_code.data(), + streams[s_idx++]); } // launch delta binary decoder if (BitAnd(kernel_mask, decode_kernel_mask::DELTA_BINARY) != 0) { - DecodeDeltaBinary( - pages, chunks, num_rows, skip_rows, level_type_size, error_code.data(), streams[s_idx++]); + DecodeDeltaBinary(subpass.pages, + pass.chunks, + num_rows, + skip_rows, + level_type_size, + error_code.data(), + streams[s_idx++]); } // launch the catch-all page decoder if (BitAnd(kernel_mask, decode_kernel_mask::GENERAL) != 0) { - DecodePageData( - pages, chunks, num_rows, skip_rows, level_type_size, error_code.data(), streams[s_idx++]); + DecodePageData(subpass.pages, + pass.chunks, + num_rows, + skip_rows, + level_type_size, + error_code.data(), + streams[s_idx++]); } // synchronize the streams cudf::detail::join_streams(streams, _stream); - pages.device_to_host_async(_stream); + subpass.pages.device_to_host_async(_stream); page_nesting.device_to_host_async(_stream); page_nesting_decode.device_to_host_async(_stream); if (error_code.value() != 0) { CUDF_FAIL("Parquet data decode failed with code(s) " + error_code.str()); } + // error_code.value() is synchronous; explicitly sync here for better visibility + _stream.synchronize(); // for list columns, add the final offset to every offset buffer. // TODO : make this happen in more efficiently. Maybe use thrust::for_each @@ -259,10 +291,10 @@ void reader::impl::decode_page_data(size_t skip_rows, size_t num_rows) } // update null counts in the final column buffers - for (size_t idx = 0; idx < pages.size(); idx++) { - PageInfo* pi = &pages[idx]; + for (size_t idx = 0; idx < subpass.pages.size(); idx++) { + PageInfo* pi = &subpass.pages[idx]; if (pi->flags & PAGEINFO_FLAGS_DICTIONARY) { continue; } - ColumnChunkDesc* col = &chunks[pi->chunk_idx]; + ColumnChunkDesc* col = &pass.chunks[pi->chunk_idx]; input_column_info const& input_col = _input_columns[col->src_col_index]; int index = pi->nesting_decode - page_nesting_decode.device_ptr(); @@ -344,60 +376,16 @@ void reader::impl::prepare_data(int64_t skip_rows, { // if we have not preprocessed at the whole-file level, do that now if (!_file_preprocessed) { - // if filter is not empty, then create output types as vector and pass for filtering. - std::vector output_types; - if (filter.has_value()) { - std::transform(_output_buffers.cbegin(), - _output_buffers.cend(), - std::back_inserter(output_types), - [](auto const& col) { return col.type; }); - } - std::tie( - _file_itm_data.global_skip_rows, _file_itm_data.global_num_rows, _file_itm_data.row_groups) = - _metadata->select_row_groups( - row_group_indices, skip_rows, num_rows, output_types, filter, _stream); - - if (_file_itm_data.global_num_rows > 0 && not _file_itm_data.row_groups.empty() && - not _input_columns.empty()) { - // fills in chunk information without physically loading or decompressing - // the associated data - create_global_chunk_info(); - - // compute schedule of input reads. Each rowgroup contains 1 chunk per column. For now - // we will read an entire row group at a time. However, it is possible to do - // sub-rowgroup reads if we made some estimates on individual chunk sizes (tricky) and - // changed the high level structure such that we weren't always reading an entire table's - // worth of columns at once. - compute_input_passes(); - } - - _file_preprocessed = true; + // setup file level information + // - read row group information + // - setup information on (parquet) chunks + // - compute schedule of input passes + preprocess_file(skip_rows, num_rows, row_group_indices, filter); } - // if we have to start a new pass, do that now - if (!_pass_preprocessed) { - auto const num_passes = _file_itm_data.input_pass_row_group_offsets.size() - 1; - - // always create the pass struct, even if we end up with no passes. - // this will also cause the previous pass information to be deleted - _pass_itm_data = std::make_unique(); - - if (_file_itm_data.global_num_rows > 0 && not _file_itm_data.row_groups.empty() && - not _input_columns.empty() && _current_input_pass < num_passes) { - // setup the pass_intermediate_info for this pass. - setup_next_pass(); - - load_and_decompress_data(); - preprocess_pages(uses_custom_row_bounds, _output_chunk_read_limit); - - if (_output_chunk_read_limit == 0) { // read the whole file at once - CUDF_EXPECTS(_pass_itm_data->output_chunk_read_info.size() == 1, - "Reading the whole file should yield only one chunk."); - } - } - - _pass_preprocessed = true; - } + // handle any chunking work (ratcheting through the subpasses and chunks within + // our current pass) + if (_file_itm_data.num_passes() > 0) { handle_chunking(uses_custom_row_bounds); } } void reader::impl::populate_metadata(table_metadata& out_metadata) @@ -427,12 +415,12 @@ table_with_metadata reader::impl::read_chunk_internal( auto out_columns = std::vector>{}; out_columns.reserve(_output_buffers.size()); - if (!has_next() || _pass_itm_data->output_chunk_read_info.empty()) { - return finalize_output(out_metadata, out_columns, filter); - } + // no work to do (this can happen on the first pass if we have no rows to read) + if (!has_more_work()) { return finalize_output(out_metadata, out_columns, filter); } - auto const& read_info = - _pass_itm_data->output_chunk_read_info[_pass_itm_data->current_output_chunk]; + auto& pass = *_pass_itm_data; + auto& subpass = *pass.subpass; + auto const& read_info = subpass.output_chunk_read_info[subpass.current_output_chunk]; // Allocate memory buffers for the output columns. allocate_columns(read_info.skip_rows, read_info.num_rows, uses_custom_row_bounds); @@ -485,15 +473,12 @@ table_with_metadata reader::impl::finalize_output( _output_metadata = std::make_unique(out_metadata); } - // advance chunks/passes as necessary - _pass_itm_data->current_output_chunk++; - _chunk_count++; - if (_pass_itm_data->current_output_chunk >= _pass_itm_data->output_chunk_read_info.size()) { - _pass_itm_data->current_output_chunk = 0; - _pass_itm_data->output_chunk_read_info.clear(); - - _current_input_pass++; - _pass_preprocessed = false; + // advance output chunk/subpass/pass info + if (_file_itm_data.num_passes() > 0) { + auto& pass = *_pass_itm_data; + auto& subpass = *pass.subpass; + subpass.current_output_chunk++; + _file_itm_data._output_chunk_count++; } if (filter.has_value()) { @@ -530,7 +515,7 @@ table_with_metadata reader::impl::read_chunk() { // Reset the output buffers to their original states (right after reader construction). // Don't need to do it if we read the file all at once. - if (_chunk_count > 0) { + if (_file_itm_data._output_chunk_count > 0) { _output_buffers.resize(0); for (auto const& buff : _output_buffers_template) { _output_buffers.emplace_back(cudf::io::detail::inline_column_buffer::empty_like(buff)); @@ -553,10 +538,9 @@ bool reader::impl::has_next() {} /*row_group_indices, empty means read all row groups*/, std::nullopt /*filter*/); - size_t const num_input_passes = std::max( - int64_t{0}, static_cast(_file_itm_data.input_pass_row_group_offsets.size()) - 1); - return (_pass_itm_data->current_output_chunk < _pass_itm_data->output_chunk_read_info.size()) || - (_current_input_pass < num_input_passes); + // current_input_pass will only be incremented to be == num_passes after + // the last chunk in the last subpass in the last pass has been returned + return has_more_work(); } namespace { diff --git a/cpp/src/io/parquet/reader_impl.hpp b/cpp/src/io/parquet/reader_impl.hpp index cea4ba35606..67c56c9c2d7 100644 --- a/cpp/src/io/parquet/reader_impl.hpp +++ b/cpp/src/io/parquet/reader_impl.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -120,6 +120,8 @@ class reader::impl { */ table_with_metadata read_chunk(); + // top level functions involved with ratcheting through the passes, subpasses + // and output chunks of the read process private: /** * @brief Perform the necessary data preprocessing for parsing file later on. @@ -138,20 +140,101 @@ class reader::impl { std::optional> filter); /** - * @brief Create chunk information and start file reads + * @brief Preprocess step for the entire file. + * + * Only ever called once. This function reads in rowgroup and associated chunk + * information and computes the schedule of top level passes (see `pass_intermediate_data`). + * + * @param skip_rows The number of rows to skip in the requested set of rowgroups to be read + * @param num_rows The total number of rows to read out of the selected rowgroups + * @param row_group_indices Lists of row groups to read, one per source + * @param filter Optional AST expression to filter output rows + */ + void preprocess_file(int64_t skip_rows, + std::optional const& num_rows, + host_span const> row_group_indices, + std::optional> filter); + + /** + * @brief Ratchet the pass/subpass/chunk process forward. + * + * @param uses_custom_row_bounds Whether or not num_rows and skip_rows represents user-specified + * bounds + */ + void handle_chunking(bool uses_custom_row_bounds); + + /** + * @brief Setup step for the next input read pass. + * + * A 'pass' is defined as a subset of row groups read out of the globally + * requested set of all row groups. + * + * @param uses_custom_row_bounds Whether or not num_rows and skip_rows represents user-specific + * bounds + */ + void setup_next_pass(bool uses_custom_row_bounds); + + /** + * @brief Setup step for the next decompression subpass. + * + * @param uses_custom_row_bounds Whether or not num_rows and skip_rows represents user-specific + * bounds + * + * A 'subpass' is defined as a subset of pages within a pass that are + * decompressed and decoded as a batch. Subpasses may be further subdivided + * into output chunks. + */ + void setup_next_subpass(bool uses_custom_row_bounds); + + /** + * @brief Read a chunk of data and return an output table. + * + * This function is called internally and expects all preprocessing steps have already been done. + * + * @param uses_custom_row_bounds Whether or not num_rows and skip_rows represents user-specific + * bounds + * @param filter Optional AST expression to filter output rows + * @return The output table along with columns' metadata + */ + table_with_metadata read_chunk_internal( + bool uses_custom_row_bounds, + std::optional> filter); + + // utility functions + private: + /** + * @brief Read the set of column chunks to be processed for this pass. + * + * Does not decompress the chunk data. * * @return pair of boolean indicating if compressed chunks were found and a vector of futures for * read completion */ - std::pair>> read_and_decompress_column_chunks(); + std::pair>> read_column_chunks(); /** - * @brief Load and decompress the input file(s) into memory. + * @brief Read compressed data and page information for the current pass. */ - void load_and_decompress_data(); + void read_compressed_data(); /** - * @brief Perform some preprocessing for page data and also compute the split locations + * @brief Build string dictionary indices for a pass. + * + */ + void build_string_dict_indices(); + + /** + * @brief For list columns, generate estimated row counts for pages in the current pass. + * + * The row counts in the pages that come out of the file only reflect the number of values in + * all of the rows in the page, not the number of rows themselves. In order to do subpass reading + * more accurately, we would like to have a more accurate guess of the real number of rows per + * page. + */ + void generate_list_column_row_count_estimates(); + + /** + * @brief Perform some preprocessing for subpass page data and also compute the split locations * {skip_rows, num_rows} for chunked reading. * * There are several pieces of information we can't compute directly from row counts in @@ -166,7 +249,7 @@ class reader::impl { * @param chunk_read_limit Limit on total number of bytes to be returned per read, * or `0` if there is no limit */ - void preprocess_pages(bool uses_custom_row_bounds, size_t chunk_read_limit); + void preprocess_subpass_pages(bool uses_custom_row_bounds, size_t chunk_read_limit); /** * @brief Allocate nesting information storage for all pages and set pointers to it. @@ -194,20 +277,6 @@ class reader::impl { */ void populate_metadata(table_metadata& out_metadata); - /** - * @brief Read a chunk of data and return an output table. - * - * This function is called internally and expects all preprocessing steps have already been done. - * - * @param uses_custom_row_bounds Whether or not num_rows and skip_rows represents user-specific - * bounds - * @param filter Optional AST expression to filter output rows - * @return The output table along with columns' metadata - */ - table_with_metadata read_chunk_internal( - bool uses_custom_row_bounds, - std::optional> filter); - /** * @brief Finalize the output table by adding empty columns for the non-selected columns in * schema. @@ -260,17 +329,18 @@ class reader::impl { */ void compute_input_passes(); - /** - * @brief Close out the existing pass (if any) and prepare for the next pass. - */ - void setup_next_pass(); - /** * @brief Given a set of pages that have had their sizes computed by nesting level and * a limit on total read size, generate a set of {skip_rows, num_rows} pairs representing * a set of reads that will generate output columns of total size <= `chunk_read_limit` bytes. */ - void compute_splits_for_pass(); + void compute_output_chunks_for_subpass(); + + [[nodiscard]] bool has_more_work() const + { + return _file_itm_data.num_passes() > 0 && + _file_itm_data._current_input_pass < _file_itm_data.num_passes(); + } private: rmm::cuda_stream_view _stream; @@ -311,13 +381,9 @@ class reader::impl { bool _file_preprocessed{false}; std::unique_ptr _pass_itm_data; - bool _pass_preprocessed{false}; std::size_t _output_chunk_read_limit{0}; // output chunk size limit in bytes std::size_t _input_pass_read_limit{0}; // input pass memory usage limit in bytes - - std::size_t _current_input_pass{0}; // current input pass index - std::size_t _chunk_count{0}; // how many output chunks we have produced }; } // namespace cudf::io::parquet::detail diff --git a/cpp/src/io/parquet/reader_impl_chunking.cu b/cpp/src/io/parquet/reader_impl_chunking.cu index 213fc380a34..1bfe5745b9e 100644 --- a/cpp/src/io/parquet/reader_impl_chunking.cu +++ b/cpp/src/io/parquet/reader_impl_chunking.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,7 +19,11 @@ #include #include +#include +#include + +#include #include #include @@ -27,37 +31,61 @@ #include #include #include +#include #include +#include + +#include + +#include namespace cudf::io::parquet::detail { namespace { -struct cumulative_row_info { - size_t row_count; // cumulative row count +struct split_info { + row_range rows; + int64_t split_pos; +}; + +struct cumulative_page_info { + size_t row_index; // row index size_t size_bytes; // cumulative size in bytes int key; // schema index }; +// the minimum amount of memory we can safely expect to be enough to +// do a subpass decode. if the difference between the user specified limit and +// the actual memory used for compressed/temp data is > than this value, we will still use +// at least this many additional bytes. +// Example: +// - user has specified 1 GB limit +// - we have read in 900 MB of compressed data +// - that leaves us 100 MB of space for decompression batches +// - to keep the gpu busy, we really don't want to do less than 200 MB at a time so we're just going +// to use 200 MB of space +// even if that goes past the user-specified limit. +constexpr size_t minimum_subpass_expected_size = 200 * 1024 * 1024; + +// percentage of the total available input read limit that should be reserved for compressed +// data vs uncompressed data. +constexpr float input_limit_compression_reserve = 0.3f; + #if defined(CHUNKING_DEBUG) -void print_cumulative_page_info(cudf::detail::hostdevice_vector& pages, - rmm::device_uvector const& page_index, - rmm::device_uvector const& c_info, +void print_cumulative_page_info(device_span d_pages, + device_span d_chunks, + device_span d_c_info, rmm::cuda_stream_view stream) { - pages.device_to_host_sync(stream); + std::vector pages = cudf::detail::make_std_vector_sync(d_pages, stream); + std::vector chunks = cudf::detail::make_std_vector_sync(d_chunks, stream); + std::vector c_info = cudf::detail::make_std_vector_sync(d_c_info, stream); printf("------------\nCumulative sizes by page\n"); std::vector schemas(pages.size()); - std::vector h_page_index(pages.size()); - CUDF_CUDA_TRY(cudaMemcpy( - h_page_index.data(), page_index.data(), sizeof(int) * pages.size(), cudaMemcpyDefault)); - std::vector h_cinfo(pages.size()); - CUDF_CUDA_TRY(cudaMemcpy( - h_cinfo.data(), c_info.data(), sizeof(cumulative_row_info) * pages.size(), cudaMemcpyDefault)); auto schema_iter = cudf::detail::make_counting_transform_iterator( - 0, [&](size_type i) { return pages[h_page_index[i]].src_col_schema; }); + 0, [&](size_type i) { return pages[i].src_col_schema; }); thrust::copy(thrust::seq, schema_iter, schema_iter + pages.size(), schemas.begin()); auto last = thrust::unique(thrust::seq, schemas.begin(), schemas.end()); schemas.resize(last - schemas.begin()); @@ -66,38 +94,44 @@ void print_cumulative_page_info(cudf::detail::hostdevice_vector& pages for (size_t idx = 0; idx < schemas.size(); idx++) { printf("Schema %d\n", schemas[idx]); for (size_t pidx = 0; pidx < pages.size(); pidx++) { - auto const& page = pages[h_page_index[pidx]]; + auto const& page = pages[pidx]; if (page.flags & PAGEINFO_FLAGS_DICTIONARY || page.src_col_schema != schemas[idx]) { continue; } - printf("\tP: {%lu, %lu}\n", h_cinfo[pidx].row_count, h_cinfo[pidx].size_bytes); + bool const is_list = chunks[page.chunk_idx].max_level[level_type::REPETITION] > 0; + printf("\tP %s: {%lu, %lu, %lu}\n", + is_list ? "(L)" : "", + pidx, + c_info[pidx].row_index, + c_info[pidx].size_bytes); } } } -void print_cumulative_row_info(host_span sizes, +void print_cumulative_row_info(host_span sizes, std::string const& label, - std::optional> splits = std::nullopt) + std::optional> splits = std::nullopt) { if (splits.has_value()) { - printf("------------\nSplits\n"); + printf("------------\nSplits (skip_rows, num_rows)\n"); for (size_t idx = 0; idx < splits->size(); idx++) { printf("{%lu, %lu}\n", splits.value()[idx].skip_rows, splits.value()[idx].num_rows); } } - printf("------------\nCumulative sizes %s\n", label.c_str()); + printf("------------\nCumulative sizes %s (index, row_index, size_bytes, page_key)\n", + label.c_str()); for (size_t idx = 0; idx < sizes.size(); idx++) { - printf("{%lu, %lu, %d}", sizes[idx].row_count, sizes[idx].size_bytes, sizes[idx].key); + printf("{%lu, %lu, %lu, %d}", idx, sizes[idx].row_index, sizes[idx].size_bytes, sizes[idx].key); if (splits.has_value()) { // if we have a split at this row count and this is the last instance of this row count - auto start = thrust::make_transform_iterator( - splits->begin(), [](chunk_read_info const& i) { return i.skip_rows; }); + auto start = thrust::make_transform_iterator(splits->begin(), + [](row_range const& i) { return i.skip_rows; }); auto end = start + splits->size(); - auto split = std::find(start, end, sizes[idx].row_count); + auto split = std::find(start, end, sizes[idx].row_index); auto const split_index = [&]() -> int { if (split != end && - ((idx == sizes.size() - 1) || (sizes[idx + 1].row_count > sizes[idx].row_count))) { + ((idx == sizes.size() - 1) || (sizes[idx + 1].row_index > sizes[idx].row_index))) { return static_cast(std::distance(start, split)); } return idx == 0 ? 0 : -1; @@ -114,13 +148,13 @@ void print_cumulative_row_info(host_span sizes, #endif // CHUNKING_DEBUG /** - * @brief Functor which reduces two cumulative_row_info structs of the same key. + * @brief Functor which reduces two cumulative_page_info structs of the same key. */ -struct cumulative_row_sum { - cumulative_row_info operator() - __device__(cumulative_row_info const& a, cumulative_row_info const& b) const +struct cumulative_page_sum { + cumulative_page_info operator() + __device__(cumulative_page_info const& a, cumulative_page_info const& b) const { - return cumulative_row_info{a.row_count + b.row_count, a.size_bytes + b.size_bytes, a.key}; + return cumulative_page_info{0, a.size_bytes + b.size_bytes, a.key}; } }; @@ -178,32 +212,57 @@ __device__ size_t row_size_functor::operator()(size_t num_rows, boo * * Sums across all nesting levels. */ -struct get_cumulative_row_info { - PageInfo const* const pages; - - __device__ cumulative_row_info operator()(size_type index) +struct get_page_output_size { + __device__ cumulative_page_info operator()(PageInfo const& page) const { - auto const& page = pages[index]; if (page.flags & PAGEINFO_FLAGS_DICTIONARY) { - return cumulative_row_info{0, 0, page.src_col_schema}; + return cumulative_page_info{0, 0, page.src_col_schema}; } // total nested size, not counting string data - auto iter = - cudf::detail::make_counting_transform_iterator(0, [page, index] __device__(size_type i) { + auto iter = cudf::detail::make_counting_transform_iterator( + 0, cuda::proclaim_return_type([page] __device__(size_type i) { auto const& pni = page.nesting[i]; return cudf::type_dispatcher( data_type{pni.type}, row_size_functor{}, pni.size, pni.nullable); - }); - - size_t const row_count = static_cast(page.nesting[0].size); + })); return { - row_count, + 0, thrust::reduce(thrust::seq, iter, iter + page.num_output_nesting_levels) + page.str_bytes, page.src_col_schema}; } }; +/** + * @brief Functor which sets the (uncompressed) size of a page. + */ +struct get_page_input_size { + __device__ cumulative_page_info operator()(PageInfo const& page) const + { + // we treat dictionary page sizes as 0 for subpasses because we have already paid the price for + // them at the pass level. + if (page.flags & PAGEINFO_FLAGS_DICTIONARY) { return {0, 0, page.src_col_schema}; } + return {0, static_cast(page.uncompressed_page_size), page.src_col_schema}; + } +}; + +/** + * @brief Functor which sets the absolute row index of a page in a cumulative_page_info struct + */ +struct set_row_index { + device_span chunks; + device_span pages; + device_span c_info; + + __device__ void operator()(size_t i) + { + auto const& page = pages[i]; + auto const& chunk = chunks[page.chunk_idx]; + size_t const page_start_row = chunk.start_row + page.chunk_row + page.num_rows; + c_info[i].row_index = page_start_row; + } +}; + /** * @brief Functor which computes the effective size of all input columns by page. * @@ -219,12 +278,12 @@ struct get_cumulative_row_info { * at that point. So we have to proceed as if we are taking the bytes from all 200 rows of that * page. Essentially, a conservative over-estimate of the real size. */ -struct row_total_size { - cumulative_row_info const* c_info; +struct page_total_size { + cumulative_page_info const* c_info; size_type const* key_offsets; size_t num_keys; - __device__ cumulative_row_info operator()(cumulative_row_info const& i) + __device__ cumulative_page_info operator()(cumulative_page_info const& i) const { // sum sizes for each input column at this row size_t sum = 0; @@ -232,71 +291,81 @@ struct row_total_size { auto const start = key_offsets[idx]; auto const end = key_offsets[idx + 1]; auto iter = cudf::detail::make_counting_transform_iterator( - 0, [&] __device__(size_type i) { return c_info[i].row_count; }); + 0, cuda::proclaim_return_type([&] __device__(size_type i) { + return c_info[i].row_index; + })); auto const page_index = - thrust::lower_bound(thrust::seq, iter + start, iter + end, i.row_count) - iter; + thrust::lower_bound(thrust::seq, iter + start, iter + end, i.row_index) - iter; sum += c_info[page_index].size_bytes; } - return {i.row_count, sum, i.key}; + return {i.row_index, sum, i.key}; } }; /** - * @brief Given a vector of cumulative {row_count, byte_size} pairs and a chunk read - * limit, determine the set of splits. + * @brief Functor which returns the compressed data size for a chunk + */ +struct get_chunk_compressed_size { + __device__ size_t operator()(ColumnChunkDesc const& chunk) const { return chunk.compressed_size; } +}; + +/** + * @brief Find the first entry in the aggreggated_info that corresponds to the specified row * - * @param sizes Vector of cumulative {row_count, byte_size} pairs - * @param num_rows Total number of rows to read - * @param chunk_read_limit Limit on total number of bytes to be returned per read, for all columns */ -std::vector find_splits(std::vector const& sizes, - size_t num_rows, - size_t chunk_read_limit) +size_t find_start_index(cudf::host_span aggregated_info, + size_t start_row) { - // now we have an array of {row_count, real output bytes}. just walk through it and generate - // splits. - // TODO: come up with a clever way to do this entirely in parallel. For now, as long as batch - // sizes are reasonably large, this shouldn't iterate too many times - std::vector splits; - { - size_t cur_pos = 0; - size_t cur_cumulative_size = 0; - size_t cur_row_count = 0; - auto start = thrust::make_transform_iterator(sizes.begin(), [&](cumulative_row_info const& i) { - return i.size_bytes - cur_cumulative_size; - }); - auto end = start + sizes.size(); - while (cur_row_count < num_rows) { - int64_t split_pos = - thrust::lower_bound(thrust::seq, start + cur_pos, end, chunk_read_limit) - start; - - // if we're past the end, or if the returned bucket is > than the chunk_read_limit, move back - // one. - if (static_cast(split_pos) >= sizes.size() || - (sizes[split_pos].size_bytes - cur_cumulative_size > chunk_read_limit)) { - split_pos--; - } + auto start = thrust::make_transform_iterator( + aggregated_info.begin(), [&](cumulative_page_info const& i) { return i.row_index; }); + auto start_index = + thrust::lower_bound(thrust::host, start, start + aggregated_info.size(), start_row) - start; + + // cumulative_page_info.row_index is the -end- of the rows of a given page. so move forward until + // we find the next group of pages + while (start_index < (static_cast(aggregated_info.size()) - 1) && + (start_index < 0 || aggregated_info[start_index].row_index == start_row)) { + start_index++; + } - // best-try. if we can't find something that'll fit, we have to go bigger. we're doing this in - // a loop because all of the cumulative sizes for all the pages are sorted into one big list. - // so if we had two columns, both of which had an entry {1000, 10000}, that entry would be in - // the list twice. so we have to iterate until we skip past all of them. The idea is that we - // either do this, or we have to call unique() on the input first. - while (split_pos < (static_cast(sizes.size()) - 1) && - (split_pos < 0 || sizes[split_pos].row_count == cur_row_count)) { - split_pos++; - } + return start_index; +} - auto const start_row = cur_row_count; - cur_row_count = sizes[split_pos].row_count; - splits.push_back(chunk_read_info{start_row, cur_row_count - start_row}); - cur_pos = split_pos; - cur_cumulative_size = sizes[split_pos].size_bytes; - } +/** + * @brief Given a current position and row index, find the next split based on the + * specified size limit + * + * @returns The inclusive index within `sizes` where the next split should happen + * + */ +int64_t find_next_split(int64_t cur_pos, + size_t cur_row_index, + size_t cur_cumulative_size, + cudf::host_span sizes, + size_t size_limit) +{ + auto const start = thrust::make_transform_iterator( + sizes.begin(), + [&](cumulative_page_info const& i) { return i.size_bytes - cur_cumulative_size; }); + auto const end = start + sizes.size(); + + int64_t split_pos = thrust::lower_bound(thrust::seq, start + cur_pos, end, size_limit) - start; + + // if we're past the end, or if the returned bucket is > than the chunk_read_limit, move back + // one. + if (static_cast(split_pos) >= sizes.size() || + (sizes[split_pos].size_bytes - cur_cumulative_size > size_limit)) { + split_pos--; } - // print_cumulative_row_info(sizes, "adjusted", splits); - return splits; + // cumulative_page_info.row_index is the -end- of the rows of a given page. so move forward until + // we find the next group of pages + while (split_pos < (static_cast(sizes.size()) - 1) && + (split_pos < 0 || sizes[split_pos].row_index == cur_row_index)) { + split_pos++; + } + + return split_pos; } /** @@ -340,15 +409,969 @@ template return static_cast(CompactProtocolReader::NumRequiredBits(max_level)); } -struct row_count_compare { - __device__ bool operator()(cumulative_row_info const& a, cumulative_row_info const& b) +struct row_count_less { + __device__ bool operator()(cumulative_page_info const& a, cumulative_page_info const& b) const + { + return a.row_index < b.row_index; + } +}; + +/** + * @brief return compressed and total size of the data in a row group + * + */ +std::pair get_row_group_size(RowGroup const& rg) +{ + auto compressed_size_iter = thrust::make_transform_iterator( + rg.columns.begin(), [](ColumnChunk const& c) { return c.meta_data.total_compressed_size; }); + + // the trick is that total temp space needed is tricky to know + auto const compressed_size = + std::reduce(compressed_size_iter, compressed_size_iter + rg.columns.size()); + auto const total_size = compressed_size + rg.total_byte_size; + return {compressed_size, total_size}; +} + +/** + * @brief For a set of cumulative_page_info data, adjust the size_bytes field + * such that it reflects the worst case for all pages that span the same rows. + * + * By doing this, we can now look at row X and know the total + * byte cost for all pages that span row X, not just the cost up to row X itself. + * + * This function is asynchronous. Call stream.synchronize() before using the + * results. + */ +std::pair, rmm::device_uvector> +adjust_cumulative_sizes(device_span c_info, + device_span pages, + rmm::cuda_stream_view stream) +{ + // sort by row count + rmm::device_uvector c_info_sorted = + make_device_uvector_async(c_info, stream, rmm::mr::get_current_device_resource()); + thrust::sort( + rmm::exec_policy_nosync(stream), c_info_sorted.begin(), c_info_sorted.end(), row_count_less{}); + + // page keys grouped by split. + rmm::device_uvector page_keys_by_split{c_info.size(), stream}; + thrust::transform(rmm::exec_policy_nosync(stream), + c_info_sorted.begin(), + c_info_sorted.end(), + page_keys_by_split.begin(), + cuda::proclaim_return_type( + [] __device__(cumulative_page_info const& c) { return c.key; })); + + // generate key offsets (offsets to the start of each partition of keys). worst case is 1 page per + // key + rmm::device_uvector key_offsets(pages.size() + 1, stream); + auto page_keys = make_page_key_iterator(pages); + auto const key_offsets_end = thrust::reduce_by_key(rmm::exec_policy(stream), + page_keys, + page_keys + pages.size(), + thrust::make_constant_iterator(1), + thrust::make_discard_iterator(), + key_offsets.begin()) + .second; + size_t const num_unique_keys = key_offsets_end - key_offsets.begin(); + thrust::exclusive_scan( + rmm::exec_policy_nosync(stream), key_offsets.begin(), key_offsets.end(), key_offsets.begin()); + + // adjust the cumulative info such that for each row count, the size includes any pages that span + // that row count. this is so that if we have this case: + // page row counts + // Column A: 0 <----> 100 <----> 200 + // Column B: 0 <---------------> 200 <--------> 400 + // | + // if we decide to split at row 100, we don't really know the actual amount of bytes in column B + // at that point. So we have to proceed as if we are taking the bytes from all 200 rows of that + // page. + // + rmm::device_uvector aggregated_info(c_info.size(), stream); + thrust::transform(rmm::exec_policy_nosync(stream), + c_info_sorted.begin(), + c_info_sorted.end(), + aggregated_info.begin(), + page_total_size{c_info.data(), key_offsets.data(), num_unique_keys}); + return {std::move(aggregated_info), std::move(page_keys_by_split)}; +} + +struct page_span { + size_t start, end; +}; + +struct get_page_row_index { + device_span c_info; + + __device__ size_t operator()(size_t i) const { return c_info[i].row_index; } +}; + +/** + * @brief Return the span of page indices for a given column index that spans start_row and end_row + * + */ +template +struct get_page_span { + device_span page_offsets; + RowIndexIter page_row_index; + size_t const start_row; + size_t const end_row; + + get_page_span(device_span _page_offsets, + RowIndexIter _page_row_index, + size_t _start_row, + size_t _end_row) + : page_offsets(_page_offsets), + page_row_index(_page_row_index), + start_row(_start_row), + end_row(_end_row) + { + } + + __device__ page_span operator()(size_t column_index) const + { + auto const first_page_index = page_offsets[column_index]; + auto const column_page_start = page_row_index + first_page_index; + auto const column_page_end = page_row_index + page_offsets[column_index + 1]; + auto const num_pages = column_page_end - column_page_start; + + auto start_page = + (thrust::lower_bound(thrust::seq, column_page_start, column_page_end, start_row) - + column_page_start) + + first_page_index; + if (page_row_index[start_page] == start_row) { start_page++; } + + auto end_page = (thrust::lower_bound(thrust::seq, column_page_start, column_page_end, end_row) - + column_page_start) + + first_page_index; + if (end_page < (first_page_index + num_pages)) { end_page++; } + + return {static_cast(start_page), static_cast(end_page)}; + } +}; + +struct get_span_size { + __device__ size_t operator()(page_span const& s) const { return s.end - s.start; } +}; + +/** + * @brief Computes the next subpass within the current pass. + * + * A subpass is a subset of the pages within the parent pass that is decompressed + * as a batch and decoded. Subpasses are the level at which we control memory intermediate + * memory usage. A pass consists of >= 1 subpass. We cannot compute all subpasses in one + * shot because we do not know how many rows we actually have in the pages of list columns. + * So we have to make an educated guess that fits within the memory limits, and then adjust + * for subsequent subpasses when we see how many rows we actually receive. + * + * @param c_info The cumulative page size information (row count and byte size) per column + * @param pages All of the pages in the pass + * @param page_offsets Offsets into the pages array representing the first page for each column + * @param start_row The row to start the subpass at + * @param size_limit The size limit in bytes of the subpass + * @param num_columns The number of columns + * @param stream The stream to execute cuda operations on + * @returns A tuple containing a vector of page_span structs indicating the page indices to include + * for each column to be processed, the total number of pages over all columns, and the total + * expected memory usage (including scratch space) + * + */ +std::tuple, size_t, size_t> compute_next_subpass( + device_span c_info, + device_span pages, + device_span page_offsets, + size_t start_row, + size_t size_limit, + size_t num_columns, + rmm::cuda_stream_view stream) +{ + auto [aggregated_info, page_keys_by_split] = adjust_cumulative_sizes(c_info, pages, stream); + + // bring back to the cpu + auto const h_aggregated_info = cudf::detail::make_std_vector_sync(aggregated_info, stream); + // print_cumulative_row_info(h_aggregated_info, "adjusted"); + + // TODO: if the user has explicitly specified skip_rows/num_rows we could be more intelligent + // about skipping subpasses/pages that do not fall within the range of values, but only if the + // data does not contain lists (because our row counts are only estimates in that case) + + // find the next split + auto const start_index = find_start_index(h_aggregated_info, start_row); + auto const cumulative_size = + start_row == 0 || start_index == 0 ? 0 : h_aggregated_info[start_index - 1].size_bytes; + auto const end_index = + find_next_split(start_index, start_row, cumulative_size, h_aggregated_info, size_limit); + auto const end_row = h_aggregated_info[end_index].row_index; + + // for each column, collect the set of pages that spans start_row / end_row + rmm::device_uvector page_bounds(num_columns, stream); + auto iter = thrust::make_counting_iterator(size_t{0}); + auto page_row_index = + cudf::detail::make_counting_transform_iterator(0, get_page_row_index{c_info}); + thrust::transform(rmm::exec_policy_nosync(stream), + iter, + iter + num_columns, + page_bounds.begin(), + get_page_span{page_offsets, page_row_index, start_row, end_row}); + + // total page count over all columns + auto page_count_iter = thrust::make_transform_iterator(page_bounds.begin(), get_span_size{}); + size_t const total_pages = + thrust::reduce(rmm::exec_policy(stream), page_count_iter, page_count_iter + num_columns); + + return {cudf::detail::make_std_vector_sync(page_bounds, stream), + total_pages, + h_aggregated_info[end_index].size_bytes - cumulative_size}; +} + +std::vector compute_page_splits_by_row(device_span c_info, + device_span pages, + size_t skip_rows, + size_t num_rows, + size_t size_limit, + rmm::cuda_stream_view stream) +{ + auto [aggregated_info, page_keys_by_split] = adjust_cumulative_sizes(c_info, pages, stream); + + // bring back to the cpu + std::vector h_aggregated_info = + cudf::detail::make_std_vector_sync(aggregated_info, stream); + // print_cumulative_row_info(h_aggregated_info, "adjusted"); + + std::vector splits; + // note: we are working with absolute row indices so skip_rows represents the absolute min row + // index we care about + size_t cur_pos = find_start_index(h_aggregated_info, skip_rows); + size_t cur_row_index = skip_rows; + size_t cur_cumulative_size = 0; + auto const max_row = min(skip_rows + num_rows, h_aggregated_info.back().row_index); + while (cur_row_index < max_row) { + auto const split_pos = + find_next_split(cur_pos, cur_row_index, cur_cumulative_size, h_aggregated_info, size_limit); + + auto const start_row = cur_row_index; + cur_row_index = min(max_row, h_aggregated_info[split_pos].row_index); + splits.push_back({start_row, cur_row_index - start_row}); + cur_pos = split_pos; + cur_cumulative_size = h_aggregated_info[split_pos].size_bytes; + } + // print_cumulative_row_info(h_aggregated_info, "adjusted w/splits", splits); + + return splits; +} + +/** + * @brief Decompresses a set of pages contained in the set of chunks. + * + * This function handles the case where `pages` is only a subset of all available + * pages in `chunks`. + * + * @param chunks List of column chunk descriptors + * @param pages List of page information + * @param dict_pages If true, decompress dictionary pages only. Otherwise decompress non-dictionary + * pages only. + * @param stream CUDA stream used for device memory operations and kernel launches + * + * @return Device buffer to decompressed page data + */ +[[nodiscard]] rmm::device_buffer decompress_page_data( + cudf::detail::hostdevice_vector const& chunks, + cudf::detail::hostdevice_vector& pages, + bool dict_pages, + rmm::cuda_stream_view stream) +{ + auto for_each_codec_page = [&](Compression codec, std::function const& f) { + for (size_t p = 0; p < pages.size(); p++) { + if (chunks[pages[p].chunk_idx].codec == codec && + ((dict_pages && (pages[p].flags & PAGEINFO_FLAGS_DICTIONARY)) || + (!dict_pages && !(pages[p].flags & PAGEINFO_FLAGS_DICTIONARY)))) { + f(p); + } + } + }; + + // Brotli scratch memory for decompressing + rmm::device_buffer debrotli_scratch; + + // Count the exact number of compressed pages + size_t num_comp_pages = 0; + size_t total_decomp_size = 0; + + struct codec_stats { + Compression compression_type = UNCOMPRESSED; + size_t num_pages = 0; + int32_t max_decompressed_size = 0; + size_t total_decomp_size = 0; + }; + + std::array codecs{codec_stats{GZIP}, codec_stats{SNAPPY}, codec_stats{BROTLI}, codec_stats{ZSTD}}; + + auto is_codec_supported = [&codecs](int8_t codec) { + if (codec == UNCOMPRESSED) return true; + return std::find_if(codecs.begin(), codecs.end(), [codec](auto& cstats) { + return codec == cstats.compression_type; + }) != codecs.end(); + }; + CUDF_EXPECTS(std::all_of(chunks.begin(), + chunks.end(), + [&is_codec_supported](auto const& chunk) { + return is_codec_supported(chunk.codec); + }), + "Unsupported compression type"); + + for (auto& codec : codecs) { + for_each_codec_page(codec.compression_type, [&](size_t page) { + auto page_uncomp_size = pages[page].uncompressed_page_size; + total_decomp_size += page_uncomp_size; + codec.total_decomp_size += page_uncomp_size; + codec.max_decompressed_size = std::max(codec.max_decompressed_size, page_uncomp_size); + codec.num_pages++; + num_comp_pages++; + }); + if (codec.compression_type == BROTLI && codec.num_pages > 0) { + debrotli_scratch.resize(get_gpu_debrotli_scratch_size(codec.num_pages), stream); + } + } + + // Dispatch batches of pages to decompress for each codec. + // Buffer needs to be padded, required by `gpuDecodePageData`. + rmm::device_buffer decomp_pages( + cudf::util::round_up_safe(total_decomp_size, BUFFER_PADDING_MULTIPLE), stream); + + std::vector> comp_in; + comp_in.reserve(num_comp_pages); + std::vector> comp_out; + comp_out.reserve(num_comp_pages); + + // vectors to save v2 def and rep level data, if any + std::vector> copy_in; + copy_in.reserve(num_comp_pages); + std::vector> copy_out; + copy_out.reserve(num_comp_pages); + + rmm::device_uvector comp_res(num_comp_pages, stream); + thrust::fill(rmm::exec_policy_nosync(stream), + comp_res.begin(), + comp_res.end(), + compression_result{0, compression_status::FAILURE}); + + size_t decomp_offset = 0; + int32_t start_pos = 0; + for (auto const& codec : codecs) { + if (codec.num_pages == 0) { continue; } + + for_each_codec_page(codec.compression_type, [&](size_t page_idx) { + auto const dst_base = static_cast(decomp_pages.data()) + decomp_offset; + auto& page = pages[page_idx]; + // offset will only be non-zero for V2 pages + auto const offset = + page.lvl_bytes[level_type::DEFINITION] + page.lvl_bytes[level_type::REPETITION]; + // for V2 need to copy def and rep level info into place, and then offset the + // input and output buffers. otherwise we'd have to keep both the compressed + // and decompressed data. + if (offset != 0) { + copy_in.emplace_back(page.page_data, offset); + copy_out.emplace_back(dst_base, offset); + } + comp_in.emplace_back(page.page_data + offset, + static_cast(page.compressed_page_size - offset)); + comp_out.emplace_back(dst_base + offset, + static_cast(page.uncompressed_page_size - offset)); + page.page_data = dst_base; + decomp_offset += page.uncompressed_page_size; + }); + + host_span const> comp_in_view{comp_in.data() + start_pos, + codec.num_pages}; + auto const d_comp_in = cudf::detail::make_device_uvector_async( + comp_in_view, stream, rmm::mr::get_current_device_resource()); + host_span const> comp_out_view(comp_out.data() + start_pos, + codec.num_pages); + auto const d_comp_out = cudf::detail::make_device_uvector_async( + comp_out_view, stream, rmm::mr::get_current_device_resource()); + device_span d_comp_res_view(comp_res.data() + start_pos, codec.num_pages); + + switch (codec.compression_type) { + case GZIP: + gpuinflate(d_comp_in, d_comp_out, d_comp_res_view, gzip_header_included::YES, stream); + break; + case SNAPPY: + if (cudf::io::detail::nvcomp_integration::is_stable_enabled()) { + nvcomp::batched_decompress(nvcomp::compression_type::SNAPPY, + d_comp_in, + d_comp_out, + d_comp_res_view, + codec.max_decompressed_size, + codec.total_decomp_size, + stream); + } else { + gpu_unsnap(d_comp_in, d_comp_out, d_comp_res_view, stream); + } + break; + case ZSTD: + nvcomp::batched_decompress(nvcomp::compression_type::ZSTD, + d_comp_in, + d_comp_out, + d_comp_res_view, + codec.max_decompressed_size, + codec.total_decomp_size, + stream); + break; + case BROTLI: + gpu_debrotli(d_comp_in, + d_comp_out, + d_comp_res_view, + debrotli_scratch.data(), + debrotli_scratch.size(), + stream); + break; + default: CUDF_FAIL("Unexpected decompression dispatch"); break; + } + start_pos += codec.num_pages; + } + + CUDF_EXPECTS(thrust::all_of(rmm::exec_policy(stream), + comp_res.begin(), + comp_res.end(), + cuda::proclaim_return_type([] __device__(auto const& res) { + return res.status == compression_status::SUCCESS; + })), + "Error during decompression"); + + // now copy the uncompressed V2 def and rep level data + if (not copy_in.empty()) { + auto const d_copy_in = cudf::detail::make_device_uvector_async( + copy_in, stream, rmm::mr::get_current_device_resource()); + auto const d_copy_out = cudf::detail::make_device_uvector_async( + copy_out, stream, rmm::mr::get_current_device_resource()); + + gpu_copy_uncompressed_blocks(d_copy_in, d_copy_out, stream); + stream.synchronize(); + } + + pages.host_to_device_async(stream); + + stream.synchronize(); + return decomp_pages; +} + +struct flat_column_num_rows { + ColumnChunkDesc const* chunks; + + __device__ size_type operator()(PageInfo const& page) const + { + // ignore dictionary pages and pages belonging to any column containing repetition (lists) + if ((page.flags & PAGEINFO_FLAGS_DICTIONARY) || + (chunks[page.chunk_idx].max_level[level_type::REPETITION] > 0)) { + return 0; + } + return page.num_rows; + } +}; + +struct row_counts_nonzero { + __device__ bool operator()(size_type count) const { return count > 0; } +}; + +struct row_counts_different { + size_type const expected; + __device__ bool operator()(size_type count) const { return (count != 0) && (count != expected); } +}; + +/** + * @brief Detect malformed parquet input data. + * + * We have seen cases where parquet files can be oddly malformed. This function specifically + * detects one case in particular: + * + * - When you have a file containing N rows + * - For some reason, the sum total of the number of rows over all pages for a given column + * is != N + * + * @param pages All pages to be decoded + * @param chunks Chunk data + * @param expected_row_count Expected row count, if applicable + * @param stream CUDA stream used for device memory operations and kernel launches + */ +void detect_malformed_pages(device_span pages, + device_span chunks, + std::optional expected_row_count, + rmm::cuda_stream_view stream) +{ + // sum row counts for all non-dictionary, non-list columns. other columns will be indicated as 0 + rmm::device_uvector row_counts(pages.size(), + stream); // worst case: num keys == num pages + auto const size_iter = + thrust::make_transform_iterator(pages.begin(), flat_column_num_rows{chunks.data()}); + auto const row_counts_begin = row_counts.begin(); + auto page_keys = make_page_key_iterator(pages); + auto const row_counts_end = thrust::reduce_by_key(rmm::exec_policy(stream), + page_keys, + page_keys + pages.size(), + size_iter, + thrust::make_discard_iterator(), + row_counts_begin) + .second; + + // make sure all non-zero row counts are the same + rmm::device_uvector compacted_row_counts(pages.size(), stream); + auto const compacted_row_counts_begin = compacted_row_counts.begin(); + auto const compacted_row_counts_end = thrust::copy_if(rmm::exec_policy(stream), + row_counts_begin, + row_counts_end, + compacted_row_counts_begin, + row_counts_nonzero{}); + if (compacted_row_counts_end != compacted_row_counts_begin) { + size_t const found_row_count = static_cast(compacted_row_counts.element(0, stream)); + + // if we somehow don't match the expected row count from the row groups themselves + if (expected_row_count.has_value()) { + CUDF_EXPECTS(expected_row_count.value() == found_row_count, + "Encountered malformed parquet page data (unexpected row count in page data)"); + } + + // all non-zero row counts must be the same + auto const chk = + thrust::count_if(rmm::exec_policy(stream), + compacted_row_counts_begin, + compacted_row_counts_end, + row_counts_different{static_cast(found_row_count)}); + CUDF_EXPECTS(chk == 0, + "Encountered malformed parquet page data (row count mismatch in page data)"); + } +} + +struct decompression_info { + Compression codec; + size_t num_pages; + size_t max_page_decompressed_size; + size_t total_decompressed_size; +}; + +/** + * @brief Functor which retrieves per-page decompression information. + * + */ +struct get_decomp_info { + device_span chunks; + + __device__ decompression_info operator()(PageInfo const& p) const + { + return {static_cast(chunks[p.chunk_idx].codec), + 1, + static_cast(p.uncompressed_page_size), + static_cast(p.uncompressed_page_size)}; + } +}; + +/** + * @brief Functor which accumulates per-page decompression information. + * + */ +struct decomp_sum { + __device__ decompression_info operator()(decompression_info const& a, + decompression_info const& b) const + { + return {a.codec, + a.num_pages + b.num_pages, + std::max(a.max_page_decompressed_size, b.max_page_decompressed_size), + a.total_decompressed_size + b.total_decompressed_size}; + } +}; + +/** + * @brief Functor which returns total scratch space required based on computed decompression_info + * data. + * + */ +struct get_decomp_scratch { + size_t operator()(decompression_info const& di) const { - return a.row_count < b.row_count; + switch (di.codec) { + case UNCOMPRESSED: + case GZIP: return 0; + + case BROTLI: return get_gpu_debrotli_scratch_size(di.num_pages); + + case SNAPPY: + if (cudf::io::detail::nvcomp_integration::is_stable_enabled()) { + return cudf::io::nvcomp::batched_decompress_temp_size( + cudf::io::nvcomp::compression_type::SNAPPY, + di.num_pages, + di.max_page_decompressed_size, + di.total_decompressed_size); + } else { + return 0; + } + break; + + case ZSTD: + return cudf::io::nvcomp::batched_decompress_temp_size( + cudf::io::nvcomp::compression_type::ZSTD, + di.num_pages, + di.max_page_decompressed_size, + di.total_decompressed_size); + + default: CUDF_FAIL("Invalid compression codec for parquet decompression"); + } } }; +/** + * @brief Add the cost of decompression codec scratch space to the per-page cumulative + * size information. + * + */ +void include_decompression_scratch_size(device_span chunks, + device_span pages, + device_span c_info, + rmm::cuda_stream_view stream) +{ + CUDF_EXPECTS(pages.size() == c_info.size(), + "Encountered page/cumulative_page_info size mismatch"); + + auto page_keys = make_page_key_iterator(pages); + + // per-codec page counts and decompression sizes + rmm::device_uvector decomp_info(pages.size(), stream); + auto decomp_iter = thrust::make_transform_iterator(pages.begin(), get_decomp_info{chunks}); + thrust::inclusive_scan_by_key(rmm::exec_policy_nosync(stream), + page_keys, + page_keys + pages.size(), + decomp_iter, + decomp_info.begin(), + thrust::equal_to{}, + decomp_sum{}); + + // retrieve to host so we can call nvcomp to get compression scratch sizes + std::vector h_decomp_info = + cudf::detail::make_std_vector_sync(decomp_info, stream); + std::vector temp_cost(pages.size()); + thrust::transform(thrust::host, + h_decomp_info.begin(), + h_decomp_info.end(), + temp_cost.begin(), + get_decomp_scratch{}); + + // add to the cumulative_page_info data + rmm::device_uvector d_temp_cost = cudf::detail::make_device_uvector_async( + temp_cost, stream, rmm::mr::get_current_device_resource()); + auto iter = thrust::make_counting_iterator(size_t{0}); + thrust::for_each(rmm::exec_policy_nosync(stream), + iter, + iter + pages.size(), + [temp_cost = d_temp_cost.begin(), c_info = c_info.begin()] __device__(size_t i) { + c_info[i].size_bytes += temp_cost[i]; + }); + stream.synchronize(); +} + } // anonymous namespace +void reader::impl::handle_chunking(bool uses_custom_row_bounds) +{ + // if this is our first time in here, setup the first pass. + if (!_pass_itm_data) { + // setup the next pass + setup_next_pass(uses_custom_row_bounds); + } + + auto& pass = *_pass_itm_data; + + // if we already have a subpass in flight. + if (pass.subpass != nullptr) { + // if it still has more chunks in flight, there's nothing more to do + if (pass.subpass->current_output_chunk < pass.subpass->output_chunk_read_info.size()) { + return; + } + + // increment rows processed + pass.processed_rows += pass.subpass->num_rows; + + // release the old subpass (will free memory) + pass.subpass.reset(); + + // otherwise we are done with the pass entirely + if (pass.processed_rows == pass.num_rows) { + // release the old pass + _pass_itm_data.reset(); + + _file_itm_data._current_input_pass++; + // no more passes. we are absolutely done with this file. + if (_file_itm_data._current_input_pass == _file_itm_data.num_passes()) { return; } + + // setup the next pass + setup_next_pass(uses_custom_row_bounds); + } + } + + // setup the next sub pass + setup_next_subpass(uses_custom_row_bounds); +} + +void reader::impl::setup_next_pass(bool uses_custom_row_bounds) +{ + auto const num_passes = _file_itm_data.num_passes(); + + // always create the pass struct, even if we end up with no work. + // this will also cause the previous pass information to be deleted + _pass_itm_data = std::make_unique(); + + if (_file_itm_data.global_num_rows > 0 && not _file_itm_data.row_groups.empty() && + not _input_columns.empty() && _file_itm_data._current_input_pass < num_passes) { + auto& pass = *_pass_itm_data; + + // setup row groups to be loaded for this pass + auto const row_group_start = + _file_itm_data.input_pass_row_group_offsets[_file_itm_data._current_input_pass]; + auto const row_group_end = + _file_itm_data.input_pass_row_group_offsets[_file_itm_data._current_input_pass + 1]; + auto const num_row_groups = row_group_end - row_group_start; + pass.row_groups.resize(num_row_groups); + std::copy(_file_itm_data.row_groups.begin() + row_group_start, + _file_itm_data.row_groups.begin() + row_group_end, + pass.row_groups.begin()); + + CUDF_EXPECTS(_file_itm_data._current_input_pass < num_passes, + "Encountered an invalid read pass index"); + + auto const chunks_per_rowgroup = _input_columns.size(); + auto const num_chunks = chunks_per_rowgroup * num_row_groups; + + auto chunk_start = _file_itm_data.chunks.begin() + (row_group_start * chunks_per_rowgroup); + auto chunk_end = _file_itm_data.chunks.begin() + (row_group_end * chunks_per_rowgroup); + + pass.chunks = cudf::detail::hostdevice_vector(num_chunks, _stream); + std::copy(chunk_start, chunk_end, pass.chunks.begin()); + + // compute skip_rows / num_rows for this pass. + if (num_passes == 1) { + pass.skip_rows = _file_itm_data.global_skip_rows; + pass.num_rows = _file_itm_data.global_num_rows; + } else { + auto const global_start_row = _file_itm_data.global_skip_rows; + auto const global_end_row = global_start_row + _file_itm_data.global_num_rows; + auto const start_row = + std::max(_file_itm_data.input_pass_start_row_count[_file_itm_data._current_input_pass], + global_start_row); + auto const end_row = + std::min(_file_itm_data.input_pass_start_row_count[_file_itm_data._current_input_pass + 1], + global_end_row); + + // skip_rows is always global in the sense that it is relative to the first row of + // everything we will be reading, regardless of what pass we are on. + // num_rows is how many rows we are reading this pass. + pass.skip_rows = + global_start_row + + _file_itm_data.input_pass_start_row_count[_file_itm_data._current_input_pass]; + pass.num_rows = end_row - start_row; + } + + // load page information for the chunk. this retrieves the compressed bytes for all the + // pages, and their headers (which we can access without decompressing) + read_compressed_data(); + + // detect malformed columns. + // - we have seen some cases in the wild where we have a row group containing N + // rows, but the total number of rows in the pages for column X is != N. while it + // is possible to load this by just capping the number of rows read, we cannot tell + // which rows are invalid so we may be returning bad data. in addition, this mismatch + // confuses the chunked reader + detect_malformed_pages( + pass.pages, + pass.chunks, + uses_custom_row_bounds ? std::nullopt : std::make_optional(pass.num_rows), + _stream); + + // decompress dictionary data if applicable. + if (pass.has_compressed_data) { + pass.decomp_dict_data = decompress_page_data(pass.chunks, pass.pages, true, _stream); + } + + // store off how much memory we've used so far. This includes the compressed page data and the + // decompressed dictionary data. we will subtract this from the available total memory for the + // subpasses + auto chunk_iter = + thrust::make_transform_iterator(pass.chunks.d_begin(), get_chunk_compressed_size{}); + pass.base_mem_size = + pass.decomp_dict_data.size() + + thrust::reduce(rmm::exec_policy(_stream), chunk_iter, chunk_iter + pass.chunks.size()); + + // since there is only ever 1 dictionary per chunk (the first page), do it at the + // pass level. + build_string_dict_indices(); + + // if we are doing subpass reading, generate more accurate num_row estimates for list columns. + // this helps us to generate more accurate subpass splits. + if (_input_pass_read_limit != 0) { generate_list_column_row_count_estimates(); } + +#if defined(PARQUET_CHUNK_LOGGING) + printf("Pass: row_groups(%'lu), chunks(%'lu), pages(%'lu)\n", + pass.row_groups.size(), + pass.chunks.size(), + pass.pages.size()); + printf("\tskip_rows: %'lu\n", pass.skip_rows); + printf("\tnum_rows: %'lu\n", pass.num_rows); + printf("\tbase mem usage: %'lu\n", pass.base_mem_size); + auto const num_columns = _input_columns.size(); + for (size_t c_idx = 0; c_idx < num_columns; c_idx++) { + printf("\t\tColumn %'lu: num_pages(%'d)\n", + c_idx, + pass.page_offsets[c_idx + 1] - pass.page_offsets[c_idx]); + } +#endif + + _stream.synchronize(); + } +} + +void reader::impl::setup_next_subpass(bool uses_custom_row_bounds) +{ + auto& pass = *_pass_itm_data; + pass.subpass = std::make_unique(); + auto& subpass = *pass.subpass; + + auto const num_columns = _input_columns.size(); + + // if the user has passed a very small value (under the hardcoded minimum_subpass_expected_size), + // respect it. + auto const min_subpass_size = std::min(_input_pass_read_limit, minimum_subpass_expected_size); + + // what do we do if the base memory size (the compressed data) itself is approaching or larger + // than the overall read limit? we are still going to be decompressing in subpasses, but we have + // to assume some reasonable minimum size needed to safely decompress a single subpass. so always + // reserve at least that much space. this can result in using up to 2x the specified user limit + // but should only ever happen with unrealistically low numbers. + size_t const remaining_read_limit = + _input_pass_read_limit == 0 ? 0 + : pass.base_mem_size + min_subpass_size >= _input_pass_read_limit + ? min_subpass_size + : _input_pass_read_limit - pass.base_mem_size; + + auto [page_indices, total_pages, total_expected_size] = + [&]() -> std::tuple, size_t, size_t> { + // special case: if we contain no compressed data, or if we have no input limit, we can always + // just do 1 subpass since what we already have loaded is all the temporary memory we will ever + // use. + if (!pass.has_compressed_data || _input_pass_read_limit == 0) { + std::vector page_indices; + page_indices.reserve(num_columns); + auto iter = thrust::make_counting_iterator(0); + std::transform( + iter, iter + num_columns, std::back_inserter(page_indices), [&](size_t i) -> page_span { + return {static_cast(pass.page_offsets[i]), + static_cast(pass.page_offsets[i + 1])}; + }); + return {page_indices, pass.pages.size(), 0}; + } + // otherwise we have to look forward and choose a batch of pages + + // as subpasses get decoded, the initial estimates we have for list row counts + // get updated with accurate data, so regenerate cumulative size info and row + // indices + rmm::device_uvector c_info(pass.pages.size(), _stream); + auto page_keys = make_page_key_iterator(pass.pages); + auto page_size = thrust::make_transform_iterator(pass.pages.d_begin(), get_page_input_size{}); + thrust::inclusive_scan_by_key(rmm::exec_policy_nosync(_stream), + page_keys, + page_keys + pass.pages.size(), + page_size, + c_info.begin(), + thrust::equal_to{}, + cumulative_page_sum{}); + + // include scratch space needed for decompression. for certain codecs (eg ZSTD) this + // can be considerable. + include_decompression_scratch_size(pass.chunks, pass.pages, c_info, _stream); + + auto iter = thrust::make_counting_iterator(0); + thrust::for_each(rmm::exec_policy_nosync(_stream), + iter, + iter + pass.pages.size(), + set_row_index{pass.chunks, pass.pages, c_info}); + // print_cumulative_page_info(pass.pages, pass.chunks, c_info, _stream); + + // get the next batch of pages + return compute_next_subpass(c_info, + pass.pages, + pass.page_offsets, + pass.processed_rows + pass.skip_rows, + remaining_read_limit, + num_columns, + _stream); + }(); + + // fill out the subpass struct + subpass.pages = cudf::detail::hostdevice_vector(0, total_pages, _stream); + subpass.page_src_index = + cudf::detail::hostdevice_vector(total_pages, total_pages, _stream); + // copy the appropriate subset of pages from each column + size_t page_count = 0; + for (size_t c_idx = 0; c_idx < num_columns; c_idx++) { + auto const num_column_pages = page_indices[c_idx].end - page_indices[c_idx].start; + subpass.column_page_count.push_back(num_column_pages); + std::copy(pass.pages.begin() + page_indices[c_idx].start, + pass.pages.begin() + page_indices[c_idx].end, + std::back_inserter(subpass.pages)); + + // mapping back to original pages in the pass + thrust::sequence(thrust::host, + subpass.page_src_index.begin() + page_count, + subpass.page_src_index.begin() + page_count + num_column_pages, + page_indices[c_idx].start); + page_count += num_column_pages; + } + // print_hostdevice_vector(subpass.page_src_index); + + // decompress the data for the pages in this subpass. + if (pass.has_compressed_data) { + subpass.decomp_page_data = decompress_page_data(pass.chunks, subpass.pages, false, _stream); + } + + subpass.pages.host_to_device_async(_stream); + subpass.page_src_index.host_to_device_async(_stream); + _stream.synchronize(); + + // buffers needed by the decode kernels + { + // nesting information (sizes, etc) stored -per page- + // note : even for flat schemas, we allocate 1 level of "nesting" info + allocate_nesting_info(); + + // level decode space + allocate_level_decode_space(); + } + subpass.pages.host_to_device_async(_stream); + + // preprocess pages (computes row counts for lists, computes output chunks and computes + // the actual row counts we will be able load out of this subpass) + preprocess_subpass_pages(uses_custom_row_bounds, _output_chunk_read_limit); + +#if defined(PARQUET_CHUNK_LOGGING) + printf("\tSubpass: skip_rows(%'lu), num_rows(%'lu), remaining read limit(%'lu)\n", + subpass.skip_rows, + subpass.num_rows, + remaining_read_limit); + printf("\t\tDecompressed size: %'lu\n", subpass.decomp_page_data.size()); + printf("\t\tTotal expected usage: %'lu\n", + total_expected_size == 0 ? subpass.decomp_page_data.size() + pass.base_mem_size + : total_expected_size + pass.base_mem_size); + for (size_t c_idx = 0; c_idx < num_columns; c_idx++) { + printf("\t\tColumn %'lu: pages(%'lu - %'lu)\n", + c_idx, + page_indices[c_idx].start, + page_indices[c_idx].end); + } + printf("\t\tOutput chunks:\n"); + for (size_t idx = 0; idx < subpass.output_chunk_read_info.size(); idx++) { + printf("\t\t\t%'lu: skip_rows(%'lu) num_rows(%'lu)\n", + idx, + subpass.output_chunk_read_info[idx].skip_rows, + subpass.output_chunk_read_info[idx].num_rows); + } +#endif +} + void reader::impl::create_global_chunk_info() { auto const num_rows = _file_itm_data.global_num_rows; @@ -380,6 +1403,14 @@ void reader::impl::create_global_chunk_info() schema.converted_type, schema.type_length); + // for lists, estimate the number of bytes per row. this is used by the subpass reader to + // determine where to split the decompression boundaries + float const list_bytes_per_row_est = + schema.max_repetition_level > 0 && row_group.num_rows > 0 + ? static_cast(col_meta.total_uncompressed_size) / + static_cast(row_group.num_rows) + : 0.0f; + chunks.push_back(ColumnChunkDesc(col_meta.total_compressed_size, nullptr, col_meta.num_values, @@ -398,7 +1429,8 @@ void reader::impl::create_global_chunk_info() schema.decimal_precision, clock_rate, i, - col.schema_idx)); + col.schema_idx, + list_bytes_per_row_est)); } remaining_rows -= row_group_rows; @@ -415,185 +1447,101 @@ void reader::impl::compute_input_passes() if (_input_pass_read_limit == 0) { _file_itm_data.input_pass_row_group_offsets.push_back(0); _file_itm_data.input_pass_row_group_offsets.push_back(row_groups_info.size()); + _file_itm_data.input_pass_start_row_count.push_back(0); + auto rg_row_count = cudf::detail::make_counting_transform_iterator(0, [&](size_t i) { + auto const& rgi = row_groups_info[i]; + auto const& row_group = _metadata->get_row_group(rgi.index, rgi.source_index); + return row_group.num_rows; + }); + _file_itm_data.input_pass_start_row_count.push_back( + std::reduce(rg_row_count, rg_row_count + row_groups_info.size())); return; } // generate passes. make sure to account for the case where a single row group doesn't fit within // - std::size_t const read_limit = - _input_pass_read_limit > 0 ? _input_pass_read_limit : std::numeric_limits::max(); + std::size_t const comp_read_limit = + _input_pass_read_limit > 0 + ? static_cast(_input_pass_read_limit * input_limit_compression_reserve) + : std::numeric_limits::max(); std::size_t cur_pass_byte_size = 0; std::size_t cur_rg_start = 0; std::size_t cur_row_count = 0; _file_itm_data.input_pass_row_group_offsets.push_back(0); - _file_itm_data.input_pass_row_count.push_back(0); + _file_itm_data.input_pass_start_row_count.push_back(0); for (size_t cur_rg_index = 0; cur_rg_index < row_groups_info.size(); cur_rg_index++) { auto const& rgi = row_groups_info[cur_rg_index]; auto const& row_group = _metadata->get_row_group(rgi.index, rgi.source_index); + // total compressed size and total size (compressed + uncompressed) for + auto const [compressed_rg_size, _ /*compressed + uncompressed*/] = + get_row_group_size(row_group); + // can we add this row group - if (cur_pass_byte_size + row_group.total_byte_size >= read_limit) { + if (cur_pass_byte_size + compressed_rg_size >= comp_read_limit) { // A single row group (the current one) is larger than the read limit: // We always need to include at least one row group, so end the pass at the end of the current // row group if (cur_rg_start == cur_rg_index) { _file_itm_data.input_pass_row_group_offsets.push_back(cur_rg_index + 1); - _file_itm_data.input_pass_row_count.push_back(cur_row_count + row_group.num_rows); + _file_itm_data.input_pass_start_row_count.push_back(cur_row_count + row_group.num_rows); cur_rg_start = cur_rg_index + 1; cur_pass_byte_size = 0; } // End the pass at the end of the previous row group else { _file_itm_data.input_pass_row_group_offsets.push_back(cur_rg_index); - _file_itm_data.input_pass_row_count.push_back(cur_row_count); + _file_itm_data.input_pass_start_row_count.push_back(cur_row_count); cur_rg_start = cur_rg_index; - cur_pass_byte_size = row_group.total_byte_size; + cur_pass_byte_size = compressed_rg_size; } } else { - cur_pass_byte_size += row_group.total_byte_size; + cur_pass_byte_size += compressed_rg_size; } cur_row_count += row_group.num_rows; } + // add the last pass if necessary if (_file_itm_data.input_pass_row_group_offsets.back() != row_groups_info.size()) { _file_itm_data.input_pass_row_group_offsets.push_back(row_groups_info.size()); - _file_itm_data.input_pass_row_count.push_back(cur_row_count); - } -} - -void reader::impl::setup_next_pass() -{ - // this will also cause the previous pass information to be deleted - _pass_itm_data = std::make_unique(); - - // setup row groups to be loaded for this pass - auto const row_group_start = _file_itm_data.input_pass_row_group_offsets[_current_input_pass]; - auto const row_group_end = _file_itm_data.input_pass_row_group_offsets[_current_input_pass + 1]; - auto const num_row_groups = row_group_end - row_group_start; - _pass_itm_data->row_groups.resize(num_row_groups); - std::copy(_file_itm_data.row_groups.begin() + row_group_start, - _file_itm_data.row_groups.begin() + row_group_end, - _pass_itm_data->row_groups.begin()); - - auto const num_passes = _file_itm_data.input_pass_row_group_offsets.size() - 1; - CUDF_EXPECTS(_current_input_pass < num_passes, "Encountered an invalid read pass index"); - - auto const chunks_per_rowgroup = _input_columns.size(); - auto const num_chunks = chunks_per_rowgroup * num_row_groups; - - auto chunk_start = _file_itm_data.chunks.begin() + (row_group_start * chunks_per_rowgroup); - auto chunk_end = _file_itm_data.chunks.begin() + (row_group_end * chunks_per_rowgroup); - - _pass_itm_data->chunks = cudf::detail::hostdevice_vector(num_chunks, _stream); - std::copy(chunk_start, chunk_end, _pass_itm_data->chunks.begin()); - - // adjust skip_rows and num_rows by what's available in the row groups we are processing - if (num_passes == 1) { - _pass_itm_data->skip_rows = _file_itm_data.global_skip_rows; - _pass_itm_data->num_rows = _file_itm_data.global_num_rows; - } else { - auto const global_start_row = _file_itm_data.global_skip_rows; - auto const global_end_row = global_start_row + _file_itm_data.global_num_rows; - auto const start_row = - std::max(_file_itm_data.input_pass_row_count[_current_input_pass], global_start_row); - auto const end_row = - std::min(_file_itm_data.input_pass_row_count[_current_input_pass + 1], global_end_row); - - // skip_rows is always global in the sense that it is relative to the first row of - // everything we will be reading, regardless of what pass we are on. - // num_rows is how many rows we are reading this pass. - _pass_itm_data->skip_rows = - global_start_row + _file_itm_data.input_pass_row_count[_current_input_pass]; - _pass_itm_data->num_rows = end_row - start_row; + _file_itm_data.input_pass_start_row_count.push_back(cur_row_count); } } -void reader::impl::compute_splits_for_pass() +void reader::impl::compute_output_chunks_for_subpass() { - auto const skip_rows = _pass_itm_data->skip_rows; - auto const num_rows = _pass_itm_data->num_rows; + auto& pass = *_pass_itm_data; + auto& subpass = *pass.subpass; // simple case : no chunk size, no splits if (_output_chunk_read_limit <= 0) { - _pass_itm_data->output_chunk_read_info = std::vector{{skip_rows, num_rows}}; + subpass.output_chunk_read_info.push_back({subpass.skip_rows, subpass.num_rows}); return; } - auto& pages = _pass_itm_data->pages_info; - - auto const& page_keys = _pass_itm_data->page_keys; - auto const& page_index = _pass_itm_data->page_index; - - // generate cumulative row counts and sizes - rmm::device_uvector c_info(page_keys.size(), _stream); - // convert PageInfo to cumulative_row_info - auto page_input = thrust::make_transform_iterator(page_index.begin(), - get_cumulative_row_info{pages.device_ptr()}); - thrust::inclusive_scan_by_key(rmm::exec_policy(_stream), - page_keys.begin(), - page_keys.end(), + // generate row_indices and cumulative output sizes for all pages + rmm::device_uvector c_info(subpass.pages.size(), _stream); + auto page_input = + thrust::make_transform_iterator(subpass.pages.d_begin(), get_page_output_size{}); + auto page_keys = make_page_key_iterator(subpass.pages); + thrust::inclusive_scan_by_key(rmm::exec_policy_nosync(_stream), + page_keys, + page_keys + subpass.pages.size(), page_input, c_info.begin(), thrust::equal_to{}, - cumulative_row_sum{}); - // print_cumulative_page_info(pages, page_index, c_info, stream); - - // sort by row count - rmm::device_uvector c_info_sorted{c_info, _stream}; - thrust::sort( - rmm::exec_policy(_stream), c_info_sorted.begin(), c_info_sorted.end(), row_count_compare{}); - - // std::vector h_c_info_sorted(c_info_sorted.size()); - // CUDF_CUDA_TRY(cudaMemcpy(h_c_info_sorted.data(), - // c_info_sorted.data(), - // sizeof(cumulative_row_info) * c_info_sorted.size(), - // cudaMemcpyDefault)); - // print_cumulative_row_info(h_c_info_sorted, "raw"); - - // generate key offsets (offsets to the start of each partition of keys). worst case is 1 page per - // key - rmm::device_uvector key_offsets(page_keys.size() + 1, _stream); - auto const key_offsets_end = thrust::reduce_by_key(rmm::exec_policy(_stream), - page_keys.begin(), - page_keys.end(), - thrust::make_constant_iterator(1), - thrust::make_discard_iterator(), - key_offsets.begin()) - .second; - size_t const num_unique_keys = key_offsets_end - key_offsets.begin(); - thrust::exclusive_scan( - rmm::exec_policy(_stream), key_offsets.begin(), key_offsets.end(), key_offsets.begin()); - - // adjust the cumulative info such that for each row count, the size includes any pages that span - // that row count. this is so that if we have this case: - // page row counts - // Column A: 0 <----> 100 <----> 200 - // Column B: 0 <---------------> 200 <--------> 400 - // | - // if we decide to split at row 100, we don't really know the actual amount of bytes in column B - // at that point. So we have to proceed as if we are taking the bytes from all 200 rows of that - // page. - // - rmm::device_uvector aggregated_info(c_info.size(), _stream); - thrust::transform(rmm::exec_policy(_stream), - c_info_sorted.begin(), - c_info_sorted.end(), - aggregated_info.begin(), - row_total_size{c_info.data(), key_offsets.data(), num_unique_keys}); - - // bring back to the cpu - std::vector h_aggregated_info(aggregated_info.size()); - CUDF_CUDA_TRY(cudaMemcpyAsync(h_aggregated_info.data(), - aggregated_info.data(), - sizeof(cumulative_row_info) * c_info.size(), - cudaMemcpyDefault, - _stream.value())); - _stream.synchronize(); - - // generate the actual splits - _pass_itm_data->output_chunk_read_info = - find_splits(h_aggregated_info, num_rows, _output_chunk_read_limit); + cumulative_page_sum{}); + auto iter = thrust::make_counting_iterator(0); + thrust::for_each(rmm::exec_policy_nosync(_stream), + iter, + iter + subpass.pages.size(), + set_row_index{pass.chunks, subpass.pages, c_info}); + // print_cumulative_page_info(subpass.pages, c_info, _stream); + + // compute the splits + subpass.output_chunk_read_info = compute_page_splits_by_row( + c_info, subpass.pages, subpass.skip_rows, subpass.num_rows, _output_chunk_read_limit, _stream); } } // namespace cudf::io::parquet::detail diff --git a/cpp/src/io/parquet/reader_impl_chunking.hpp b/cpp/src/io/parquet/reader_impl_chunking.hpp index dfc239d8451..a9cf0e94ec8 100644 --- a/cpp/src/io/parquet/reader_impl_chunking.hpp +++ b/cpp/src/io/parquet/reader_impl_chunking.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,58 +30,105 @@ struct file_intermediate_data { // all row groups to read std::vector row_groups{}; - // all chunks from the selected row groups. We may end up reading these chunks progressively - // instead of all at once + // all chunks from the selected row groups. std::vector chunks{}; // an array of offsets into _file_itm_data::global_chunks. Each pair of offsets represents // the start/end of the chunks to be loaded for a given pass. std::vector input_pass_row_group_offsets{}; - // row counts per input-pass - std::vector input_pass_row_count{}; - // skip_rows/num_rows values for the entire file. these need to be adjusted per-pass because we - // may not be visiting every row group that contains these bounds + // start row counts per input-pass. this includes all rows in the row groups of the pass and + // is not capped by global_skip_rows and global_num_rows. + std::vector input_pass_start_row_count{}; + + size_t _current_input_pass{0}; // current input pass index + size_t _output_chunk_count{0}; // how many output chunks we have produced + + // skip_rows/num_rows values for the entire file. size_t global_skip_rows; size_t global_num_rows; + + [[nodiscard]] size_t num_passes() const + { + return input_pass_row_group_offsets.size() == 0 ? 0 : input_pass_row_group_offsets.size() - 1; + } }; /** - * @brief Struct to identify the range for each chunk of rows during a chunked reading pass. + * @brief Struct to identify a range of rows. */ -struct chunk_read_info { +struct row_range { + size_t skip_rows; + size_t num_rows; +}; + +/** + * @brief Passes are broken down into subpasses based on temporary memory constraints. + */ +struct subpass_intermediate_data { + rmm::device_buffer decomp_page_data; + + rmm::device_buffer level_decode_data{}; + cudf::detail::hostdevice_vector pages{}; + // for each page in the subpass, the index of our source page in the pass + cudf::detail::hostdevice_vector page_src_index{}; + // for each column in the file (indexed by _input_columns.size()) + // the number of associated pages for this subpass + std::vector column_page_count; + cudf::detail::hostdevice_vector page_nesting_info{}; + cudf::detail::hostdevice_vector page_nesting_decode_info{}; + + std::vector output_chunk_read_info; + std::size_t current_output_chunk{0}; + + // skip_rows and num_rows values for this particular subpass. in absolute row indices. size_t skip_rows; size_t num_rows; }; /** * @brief Struct to store pass-level data that remains constant for a single pass. + * + * A pass is defined as a set of rowgroups read but not yet decompressed. This set of + * rowgroups may represent less than all of the rowgroups to be read for the file. */ struct pass_intermediate_data { std::vector> raw_page_data; - rmm::device_buffer decomp_page_data; // rowgroup, chunk and page information for the current pass. + bool has_compressed_data{false}; std::vector row_groups{}; cudf::detail::hostdevice_vector chunks{}; - cudf::detail::hostdevice_vector pages_info{}; - cudf::detail::hostdevice_vector page_nesting_info{}; - cudf::detail::hostdevice_vector page_nesting_decode_info{}; + cudf::detail::hostdevice_vector pages{}; - rmm::device_uvector page_keys{0, rmm::cuda_stream_default}; - rmm::device_uvector page_index{0, rmm::cuda_stream_default}; - rmm::device_uvector str_dict_index{0, rmm::cuda_stream_default}; + // base memory used for the pass itself (compressed data in the loaded chunks and any + // decompressed dictionary pages) + size_t base_mem_size{0}; - std::vector output_chunk_read_info; - std::size_t current_output_chunk{0}; + // offsets to each group of input pages (by column/schema, indexed by _input_columns.size()) + // so if we had 2 columns/schemas, with page keys + // + // 1 1 1 1 1 2 2 2 + // + // page_offsets would be 0, 5, 8 + cudf::detail::hostdevice_vector page_offsets{}; + + rmm::device_buffer decomp_dict_data{0, rmm::cuda_stream_default}; + rmm::device_uvector str_dict_index{0, rmm::cuda_stream_default}; - rmm::device_buffer level_decode_data{}; int level_type_size{0}; - // skip_rows and num_rows values for this particular pass. these may be adjusted values from the - // global values stored in file_intermediate_data. + // skip_rows / num_rows for this pass. + // NOTE: skip_rows is the absolute row index in the file. size_t skip_rows; size_t num_rows; + // number of rows we have processed so far (out of num_rows). note that this + // only includes the number of rows we have processed before starting the current + // subpass. it does not get updated as a subpass iterates through output chunks. + size_t processed_rows{0}; + + // currently active subpass + std::unique_ptr subpass{}; }; } // namespace cudf::io::parquet::detail diff --git a/cpp/src/io/parquet/reader_impl_preprocess.cu b/cpp/src/io/parquet/reader_impl_preprocess.cu index e10f2c00f40..ee3b1c466e0 100644 --- a/cpp/src/io/parquet/reader_impl_preprocess.cu +++ b/cpp/src/io/parquet/reader_impl_preprocess.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,9 +17,6 @@ #include "error.hpp" #include "reader_impl.hpp" -#include -#include - #include #include #include @@ -49,6 +46,28 @@ namespace cudf::io::parquet::detail { namespace { +#if defined(PREPROCESS_DEBUG) +void print_pages(cudf::detail::hostdevice_vector& pages, rmm::cuda_stream_view _stream) +{ + pages.device_to_host_sync(_stream); + for (size_t idx = 0; idx < pages.size(); idx++) { + auto const& p = pages[idx]; + // skip dictionary pages + if (p.flags & PAGEINFO_FLAGS_DICTIONARY) { continue; } + printf( + "P(%lu, s:%d): chunk_row(%d), num_rows(%d), skipped_values(%d), skipped_leaf_values(%d), " + "str_bytes(%d)\n", + idx, + p.src_col_schema, + p.chunk_row, + p.num_rows, + p.skipped_values, + p.skipped_leaf_values, + p.str_bytes); + } +} +#endif // PREPROCESS_DEBUG + /** * @brief Generate depth remappings for repetition and definition levels. * @@ -269,7 +288,7 @@ void generate_depth_remappings(std::map, std::ve kernel_error error_code(stream); chunks.host_to_device_async(stream); - DecodePageHeaders(chunks.device_ptr(), chunks.size(), error_code.data(), stream); + DecodePageHeaders(chunks.device_ptr(), nullptr, chunks.size(), error_code.data(), stream); chunks.device_to_host_sync(stream); // It's required to ignore unsupported encodings in this function @@ -351,33 +370,37 @@ std::string encoding_to_string(Encoding encoding) } /** - * @brief Decode the page information from the given column chunks. + * @brief Decode the page information for a given pass. * - * @param chunks List of column chunk descriptors - * @param pages List of page information - * @param stream CUDA stream used for device memory operations and kernel launches - * @returns The size in bytes of level type data required + * @param pass_intermediate_data The struct containing pass information */ -int decode_page_headers(cudf::detail::hostdevice_vector& chunks, - cudf::detail::hostdevice_vector& pages, - rmm::cuda_stream_view stream) +void decode_page_headers(pass_intermediate_data& pass, + device_span unsorted_pages, + rmm::cuda_stream_view stream) { + cudf::detail::hostdevice_vector chunk_page_info(pass.chunks.size(), stream); + // IMPORTANT : if you change how pages are stored within a chunk (dist pages, then data pages), // please update preprocess_nested_columns to reflect this. - for (size_t c = 0, page_count = 0; c < chunks.size(); c++) { - chunks[c].max_num_pages = chunks[c].num_data_pages + chunks[c].num_dict_pages; - chunks[c].page_info = pages.device_ptr(page_count); - page_count += chunks[c].max_num_pages; + for (size_t c = 0, page_count = 0; c < pass.chunks.size(); c++) { + pass.chunks[c].max_num_pages = pass.chunks[c].num_data_pages + pass.chunks[c].num_dict_pages; + chunk_page_info[c].pages = &unsorted_pages[page_count]; + page_count += pass.chunks[c].max_num_pages; } kernel_error error_code(stream); - chunks.host_to_device_async(stream); - DecodePageHeaders(chunks.device_ptr(), chunks.size(), error_code.data(), stream); + pass.chunks.host_to_device_async(stream); + chunk_page_info.host_to_device_async(stream); + DecodePageHeaders(pass.chunks.device_ptr(), + chunk_page_info.device_ptr(), + pass.chunks.size(), + error_code.data(), + stream); if (error_code.value() != 0) { if (BitAnd(error_code.value(), decode_error::UNSUPPORTED_ENCODING) != 0) { auto const unsupported_str = - ". With unsupported encodings found: " + list_unsupported_encodings(pages, stream); + ". With unsupported encodings found: " + list_unsupported_encodings(pass.pages, stream); CUDF_FAIL("Parquet header parsing failed with code(s) " + error_code.str() + unsupported_str); } else { CUDF_FAIL("Parquet header parsing failed with code(s) " + error_code.str()); @@ -386,7 +409,7 @@ int decode_page_headers(cudf::detail::hostdevice_vector& chunks // compute max bytes needed for level data auto level_bit_size = cudf::detail::make_counting_transform_iterator( - 0, cuda::proclaim_return_type([chunks = chunks.d_begin()] __device__(int i) { + 0, cuda::proclaim_return_type([chunks = pass.chunks.d_begin()] __device__(int i) { auto c = chunks[i]; return static_cast( max(c.level_bits[level_type::REPETITION], c.level_bits[level_type::DEFINITION])); @@ -394,223 +417,243 @@ int decode_page_headers(cudf::detail::hostdevice_vector& chunks // max level data bit size. int const max_level_bits = thrust::reduce(rmm::exec_policy(stream), level_bit_size, - level_bit_size + chunks.size(), + level_bit_size + pass.chunks.size(), 0, thrust::maximum()); + pass.level_type_size = std::max(1, cudf::util::div_rounding_up_safe(max_level_bits, 8)); - return std::max(1, cudf::util::div_rounding_up_safe(max_level_bits, 8)); -} + // sort the pages in chunk/schema order. we use chunk.src_col_index instead of + // chunk.src_col_schema because the user may have reordered them (reading columns, "a" and "b" but + // returning them as "b" and "a") + // + // ordering of pages is by input column schema, repeated across row groups. so + // if we had 3 columns, each with 2 pages, and 1 row group, our schema values might look like + // + // 1, 1, 2, 2, 3, 3 + // + // However, if we had more than one row group, the pattern would be + // + // 1, 1, 2, 2, 3, 3, 1, 1, 2, 2, 3, 3 + // ^ row group 0 | + // ^ row group 1 + // + // To process pages by key (exclusive_scan_by_key, reduce_by_key, etc), the ordering we actually + // want is + // + // 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3 + // + // We also need to preserve key-relative page ordering, so we need to use a stable sort. + { + rmm::device_uvector page_keys{unsorted_pages.size(), stream}; + thrust::transform(rmm::exec_policy_nosync(stream), + unsorted_pages.begin(), + unsorted_pages.end(), + page_keys.begin(), + [chunks = pass.chunks.d_begin()] __device__(PageInfo const& page) { + return chunks[page.chunk_idx].src_col_index; + }); + // we are doing this by sorting indices first and then transforming the output because nvcc + // started generating kernels using too much shared memory when trying to sort the pages + // directly. + rmm::device_uvector sort_indices(unsorted_pages.size(), stream); + thrust::sequence(rmm::exec_policy_nosync(stream), sort_indices.begin(), sort_indices.end(), 0); + thrust::stable_sort_by_key(rmm::exec_policy_nosync(stream), + page_keys.begin(), + page_keys.end(), + sort_indices.begin(), + thrust::less()); + pass.pages = cudf::detail::hostdevice_vector( + unsorted_pages.size(), unsorted_pages.size(), stream); + thrust::transform(rmm::exec_policy_nosync(stream), + sort_indices.begin(), + sort_indices.end(), + pass.pages.d_begin(), + [unsorted_pages = unsorted_pages.begin()] __device__(int32_t i) { + return unsorted_pages[i]; + }); + } -/** - * @brief Decompresses the page data, at page granularity. - * - * @param chunks List of column chunk descriptors - * @param pages List of page information - * @param stream CUDA stream used for device memory operations and kernel launches - * - * @return Device buffer to decompressed page data - */ -[[nodiscard]] rmm::device_buffer decompress_page_data( - cudf::detail::hostdevice_vector& chunks, - cudf::detail::hostdevice_vector& pages, - rmm::cuda_stream_view stream) -{ - auto for_each_codec_page = [&](Compression codec, std::function const& f) { - for (size_t c = 0, page_count = 0; c < chunks.size(); c++) { - const auto page_stride = chunks[c].max_num_pages; - if (chunks[c].codec == codec) { - for (int k = 0; k < page_stride; k++) { - f(page_count + k); - } - } - page_count += page_stride; - } - }; + // compute offsets to each group of input pages. + // page_keys: 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3 + // + // result: 0, 4, 8 + rmm::device_uvector page_counts(pass.pages.size() + 1, stream); + auto page_keys = make_page_key_iterator(pass.pages); + auto const page_counts_end = thrust::reduce_by_key(rmm::exec_policy(stream), + page_keys, + page_keys + pass.pages.size(), + thrust::make_constant_iterator(1), + thrust::make_discard_iterator(), + page_counts.begin()) + .second; + auto const num_page_counts = page_counts_end - page_counts.begin(); + pass.page_offsets = cudf::detail::hostdevice_vector(num_page_counts + 1, stream); + thrust::exclusive_scan(rmm::exec_policy_nosync(stream), + page_counts.begin(), + page_counts.begin() + num_page_counts + 1, + pass.page_offsets.d_begin()); + + // setup dict_page for each chunk if necessary + thrust::for_each(rmm::exec_policy_nosync(stream), + pass.pages.d_begin(), + pass.pages.d_end(), + [chunks = pass.chunks.d_begin()] __device__(PageInfo const& p) { + if (p.flags & PAGEINFO_FLAGS_DICTIONARY) { + chunks[p.chunk_idx].dict_page = &p; + } + }); - // Brotli scratch memory for decompressing - rmm::device_buffer debrotli_scratch; + pass.page_offsets.device_to_host_async(stream); + pass.pages.device_to_host_async(stream); + pass.chunks.device_to_host_async(stream); + stream.synchronize(); +} - // Count the exact number of compressed pages - size_t num_comp_pages = 0; - size_t total_decomp_size = 0; +struct set_str_dict_index_count { + device_span str_dict_index_count; + device_span chunks; - struct codec_stats { - Compression compression_type = UNCOMPRESSED; - size_t num_pages = 0; - int32_t max_decompressed_size = 0; - size_t total_decomp_size = 0; - }; + __device__ void operator()(PageInfo const& page) + { + auto const& chunk = chunks[page.chunk_idx]; + if ((page.flags & PAGEINFO_FLAGS_DICTIONARY) && (chunk.data_type & 0x7) == BYTE_ARRAY && + (chunk.num_dict_pages > 0)) { + // there is only ever one dictionary page per chunk, so this is safe to do in parallel. + str_dict_index_count[page.chunk_idx] = page.num_input_values; + } + } +}; - std::array codecs{codec_stats{GZIP}, codec_stats{SNAPPY}, codec_stats{BROTLI}, codec_stats{ZSTD}}; +struct set_str_dict_index_ptr { + string_index_pair* const base; + device_span str_dict_index_offsets; + device_span chunks; - auto is_codec_supported = [&codecs](int8_t codec) { - if (codec == UNCOMPRESSED) return true; - return std::find_if(codecs.begin(), codecs.end(), [codec](auto& cstats) { - return codec == cstats.compression_type; - }) != codecs.end(); - }; - CUDF_EXPECTS(std::all_of(chunks.begin(), - chunks.end(), - [&is_codec_supported](auto const& chunk) { - return is_codec_supported(chunk.codec); - }), - "Unsupported compression type"); - - for (auto& codec : codecs) { - for_each_codec_page(codec.compression_type, [&](size_t page) { - auto page_uncomp_size = pages[page].uncompressed_page_size; - total_decomp_size += page_uncomp_size; - codec.total_decomp_size += page_uncomp_size; - codec.max_decompressed_size = std::max(codec.max_decompressed_size, page_uncomp_size); - codec.num_pages++; - num_comp_pages++; - }); - if (codec.compression_type == BROTLI && codec.num_pages > 0) { - debrotli_scratch.resize(get_gpu_debrotli_scratch_size(codec.num_pages), stream); + __device__ void operator()(size_t i) + { + auto& chunk = chunks[i]; + if ((chunk.data_type & 0x7) == BYTE_ARRAY && (chunk.num_dict_pages > 0)) { + chunk.str_dict_index = base + str_dict_index_offsets[i]; } } +}; - // Dispatch batches of pages to decompress for each codec. - // Buffer needs to be padded, required by `gpuDecodePageData`. - rmm::device_buffer decomp_pages( - cudf::util::round_up_safe(total_decomp_size, BUFFER_PADDING_MULTIPLE), stream); - - std::vector> comp_in; - comp_in.reserve(num_comp_pages); - std::vector> comp_out; - comp_out.reserve(num_comp_pages); - - // vectors to save v2 def and rep level data, if any - std::vector> copy_in; - copy_in.reserve(num_comp_pages); - std::vector> copy_out; - copy_out.reserve(num_comp_pages); - - rmm::device_uvector comp_res(num_comp_pages, stream); - thrust::fill(rmm::exec_policy(stream), - comp_res.begin(), - comp_res.end(), - compression_result{0, compression_status::FAILURE}); - - size_t decomp_offset = 0; - int32_t start_pos = 0; - for (auto const& codec : codecs) { - if (codec.num_pages == 0) { continue; } - - for_each_codec_page(codec.compression_type, [&](size_t page_idx) { - auto const dst_base = static_cast(decomp_pages.data()) + decomp_offset; - auto& page = pages[page_idx]; - // offset will only be non-zero for V2 pages - auto const offset = - page.lvl_bytes[level_type::DEFINITION] + page.lvl_bytes[level_type::REPETITION]; - // for V2 need to copy def and rep level info into place, and then offset the - // input and output buffers. otherwise we'd have to keep both the compressed - // and decompressed data. - if (offset != 0) { - copy_in.emplace_back(page.page_data, offset); - copy_out.emplace_back(dst_base, offset); - } - comp_in.emplace_back(page.page_data + offset, - static_cast(page.compressed_page_size - offset)); - comp_out.emplace_back(dst_base + offset, - static_cast(page.uncompressed_page_size - offset)); - page.page_data = dst_base; - decomp_offset += page.uncompressed_page_size; - }); +/** + * @brief Functor which computes an estimated row count for list pages. + * + */ +struct set_list_row_count_estimate { + device_span chunks; - host_span const> comp_in_view{comp_in.data() + start_pos, - codec.num_pages}; - auto const d_comp_in = cudf::detail::make_device_uvector_async( - comp_in_view, stream, rmm::mr::get_current_device_resource()); - host_span const> comp_out_view(comp_out.data() + start_pos, - codec.num_pages); - auto const d_comp_out = cudf::detail::make_device_uvector_async( - comp_out_view, stream, rmm::mr::get_current_device_resource()); - device_span d_comp_res_view(comp_res.data() + start_pos, codec.num_pages); - - switch (codec.compression_type) { - case GZIP: - gpuinflate(d_comp_in, d_comp_out, d_comp_res_view, gzip_header_included::YES, stream); - break; - case SNAPPY: - if (cudf::io::detail::nvcomp_integration::is_stable_enabled()) { - nvcomp::batched_decompress(nvcomp::compression_type::SNAPPY, - d_comp_in, - d_comp_out, - d_comp_res_view, - codec.max_decompressed_size, - codec.total_decomp_size, - stream); - } else { - gpu_unsnap(d_comp_in, d_comp_out, d_comp_res_view, stream); - } - break; - case ZSTD: - nvcomp::batched_decompress(nvcomp::compression_type::ZSTD, - d_comp_in, - d_comp_out, - d_comp_res_view, - codec.max_decompressed_size, - codec.total_decomp_size, - stream); - break; - case BROTLI: - gpu_debrotli(d_comp_in, - d_comp_out, - d_comp_res_view, - debrotli_scratch.data(), - debrotli_scratch.size(), - stream); - break; - default: CUDF_FAIL("Unexpected decompression dispatch"); break; - } - start_pos += codec.num_pages; + __device__ void operator()(PageInfo& page) + { + if (page.flags & PAGEINFO_FLAGS_DICTIONARY) { return; } + auto const& chunk = chunks[page.chunk_idx]; + auto const is_list = chunk.max_level[level_type::REPETITION] > 0; + if (!is_list) { return; } + + // For LIST pages that we have not yet decoded, page.num_rows is not an accurate number. + // so we instead estimate the number of rows as follows: + // - each chunk stores an estimated number of bytes per row E + // - estimate number of rows in a page = page.uncompressed_page_size / E + // + // it is not required that this number is accurate. we just want it to be somewhat close so that + // we get reasonable results as we choose subpass splits. + // + // all other columns can use page.num_rows directly as it will be accurate. + page.num_rows = static_cast(static_cast(page.uncompressed_page_size) / + chunk.list_bytes_per_row_est); } +}; + +/** + * @brief Set the expected row count on the final page for all columns. + * + */ +struct set_final_row_count { + device_span pages; + device_span chunks; + device_span page_offsets; + size_t const max_row; - CUDF_EXPECTS(thrust::all_of(rmm::exec_policy(stream), - comp_res.begin(), - comp_res.end(), - [] __device__(auto const& res) { - return res.status == compression_status::SUCCESS; - }), - "Error during decompression"); - - // now copy the uncompressed V2 def and rep level data - if (not copy_in.empty()) { - auto const d_copy_in = cudf::detail::make_device_uvector_async( - copy_in, stream, rmm::mr::get_current_device_resource()); - auto const d_copy_out = cudf::detail::make_device_uvector_async( - copy_out, stream, rmm::mr::get_current_device_resource()); - - gpu_copy_uncompressed_blocks(d_copy_in, d_copy_out, stream); - stream.synchronize(); + __device__ void operator()(size_t i) + { + auto const last_page_index = page_offsets[i + 1] - 1; + auto const& page = pages[last_page_index]; + auto const& chunk = chunks[page.chunk_idx]; + size_t const page_start_row = chunk.start_row + page.chunk_row; + pages[last_page_index].num_rows = max_row - page_start_row; } +}; - // Update the page information in device memory with the updated value of - // page_data; it now points to the uncompressed data buffer - pages.host_to_device_async(stream); +} // anonymous namespace - return decomp_pages; +void reader::impl::build_string_dict_indices() +{ + auto& pass = *_pass_itm_data; + + // compute number of indices per chunk and a summed total + rmm::device_uvector str_dict_index_count(pass.chunks.size() + 1, _stream); + thrust::fill( + rmm::exec_policy_nosync(_stream), str_dict_index_count.begin(), str_dict_index_count.end(), 0); + thrust::for_each(rmm::exec_policy_nosync(_stream), + pass.pages.begin(), + pass.pages.end(), + set_str_dict_index_count{str_dict_index_count, pass.chunks}); + + size_t const total_str_dict_indexes = thrust::reduce( + rmm::exec_policy(_stream), str_dict_index_count.begin(), str_dict_index_count.end()); + if (total_str_dict_indexes == 0) { return; } + + // convert to offsets + rmm::device_uvector& str_dict_index_offsets = str_dict_index_count; + thrust::exclusive_scan(rmm::exec_policy_nosync(_stream), + str_dict_index_offsets.begin(), + str_dict_index_offsets.end(), + str_dict_index_offsets.begin(), + 0); + + // allocate and distribute pointers + pass.str_dict_index = cudf::detail::make_zeroed_device_uvector_async( + total_str_dict_indexes, _stream, rmm::mr::get_current_device_resource()); + + auto iter = thrust::make_counting_iterator(0); + thrust::for_each( + rmm::exec_policy_nosync(_stream), + iter, + iter + pass.chunks.size(), + set_str_dict_index_ptr{pass.str_dict_index.data(), str_dict_index_offsets, pass.chunks}); + + // compute the indices + BuildStringDictionaryIndex(pass.chunks.device_ptr(), pass.chunks.size(), _stream); + pass.chunks.device_to_host_sync(_stream); } -} // namespace - void reader::impl::allocate_nesting_info() { - auto const& chunks = _pass_itm_data->chunks; - auto& pages = _pass_itm_data->pages_info; - auto& page_nesting_info = _pass_itm_data->page_nesting_info; - auto& page_nesting_decode_info = _pass_itm_data->page_nesting_decode_info; + auto& pass = *_pass_itm_data; + auto& subpass = *pass.subpass; + + auto const num_columns = _input_columns.size(); + auto& pages = subpass.pages; + auto& page_nesting_info = subpass.page_nesting_info; + auto& page_nesting_decode_info = subpass.page_nesting_decode_info; + + // generate the number of nesting info structs needed per-page, by column + std::vector per_page_nesting_info_size(num_columns); + auto iter = thrust::make_counting_iterator(size_type{0}); + std::transform(iter, iter + num_columns, per_page_nesting_info_size.begin(), [&](size_type i) { + auto const schema_idx = _input_columns[i].schema_idx; + auto const& schema = _metadata->get_schema(schema_idx); + return max(schema.max_definition_level + 1, _metadata->get_output_nesting_depth(schema_idx)); + }); // compute total # of page_nesting infos needed and allocate space. doing this in one // buffer to keep it to a single gpu allocation - size_t const total_page_nesting_infos = std::accumulate( - chunks.host_ptr(), chunks.host_ptr() + chunks.size(), 0, [&](int total, auto& chunk) { - // the schema of the input column - auto const& schema = _metadata->get_schema(chunk.src_col_schema); - auto const per_page_nesting_info_size = max( - schema.max_definition_level + 1, _metadata->get_output_nesting_depth(chunk.src_col_schema)); - return total + (per_page_nesting_info_size * chunk.num_data_pages); + auto counting_iter = thrust::make_counting_iterator(size_t{0}); + size_t const total_page_nesting_infos = + std::accumulate(counting_iter, counting_iter + num_columns, 0, [&](int total, size_t index) { + return total + (per_page_nesting_info_size[index] * subpass.column_page_count[index]); }); page_nesting_info = @@ -621,41 +664,33 @@ void reader::impl::allocate_nesting_info() // update pointers in the PageInfos int target_page_index = 0; int src_info_index = 0; - for (size_t idx = 0; idx < chunks.size(); idx++) { - int src_col_schema = chunks[idx].src_col_schema; - auto& schema = _metadata->get_schema(src_col_schema); - auto const per_page_nesting_info_size = std::max( - schema.max_definition_level + 1, _metadata->get_output_nesting_depth(src_col_schema)); - - // skip my dict pages - target_page_index += chunks[idx].num_dict_pages; - for (int p_idx = 0; p_idx < chunks[idx].num_data_pages; p_idx++) { + for (size_t idx = 0; idx < _input_columns.size(); idx++) { + auto const src_col_schema = _input_columns[idx].schema_idx; + + for (size_t p_idx = 0; p_idx < subpass.column_page_count[idx]; p_idx++) { pages[target_page_index + p_idx].nesting = page_nesting_info.device_ptr() + src_info_index; pages[target_page_index + p_idx].nesting_decode = page_nesting_decode_info.device_ptr() + src_info_index; - pages[target_page_index + p_idx].nesting_info_size = per_page_nesting_info_size; + pages[target_page_index + p_idx].nesting_info_size = per_page_nesting_info_size[idx]; pages[target_page_index + p_idx].num_output_nesting_levels = _metadata->get_output_nesting_depth(src_col_schema); - src_info_index += per_page_nesting_info_size; + src_info_index += per_page_nesting_info_size[idx]; } - target_page_index += chunks[idx].num_data_pages; + target_page_index += subpass.column_page_count[idx]; } // fill in int nesting_info_index = 0; std::map, std::vector>> depth_remapping; - for (size_t idx = 0; idx < chunks.size(); idx++) { - int src_col_schema = chunks[idx].src_col_schema; + for (size_t idx = 0; idx < _input_columns.size(); idx++) { + auto const src_col_schema = _input_columns[idx].schema_idx; // schema of the input column auto& schema = _metadata->get_schema(src_col_schema); // real depth of the output cudf column hierarchy (1 == no nesting, 2 == 1 level, etc) - int max_depth = _metadata->get_output_nesting_depth(src_col_schema); - - // # of nesting infos stored per page for this column - auto const per_page_nesting_info_size = std::max(schema.max_definition_level + 1, max_depth); + int const max_output_depth = _metadata->get_output_nesting_depth(src_col_schema); // if this column has lists, generate depth remapping std::map, std::vector>> depth_remapping; @@ -666,18 +701,19 @@ void reader::impl::allocate_nesting_info() // fill in host-side nesting info int schema_idx = src_col_schema; auto cur_schema = _metadata->get_schema(schema_idx); - int cur_depth = max_depth - 1; + int cur_depth = max_output_depth - 1; while (schema_idx > 0) { - // stub columns (basically the inner field of a list scheme element) are not real columns. + // stub columns (basically the inner field of a list schema element) are not real columns. // we can ignore them for the purposes of output nesting info if (!cur_schema.is_stub()) { // initialize each page within the chunk - for (int p_idx = 0; p_idx < chunks[idx].num_data_pages; p_idx++) { + for (size_t p_idx = 0; p_idx < subpass.column_page_count[idx]; p_idx++) { PageNestingInfo* pni = - &page_nesting_info[nesting_info_index + (p_idx * per_page_nesting_info_size)]; + &page_nesting_info[nesting_info_index + (p_idx * per_page_nesting_info_size[idx])]; PageNestingDecodeInfo* nesting_info = - &page_nesting_decode_info[nesting_info_index + (p_idx * per_page_nesting_info_size)]; + &page_nesting_decode_info[nesting_info_index + + (p_idx * per_page_nesting_info_size[idx])]; // if we have lists, set our start and end depth remappings if (schema.max_repetition_level > 0) { @@ -712,7 +748,7 @@ void reader::impl::allocate_nesting_info() cur_schema = _metadata->get_schema(schema_idx); } - nesting_info_index += (per_page_nesting_info_size * chunks[idx].num_data_pages); + nesting_info_index += (per_page_nesting_info_size[idx] * subpass.column_page_count[idx]); } // copy nesting info to the device @@ -722,32 +758,33 @@ void reader::impl::allocate_nesting_info() void reader::impl::allocate_level_decode_space() { - auto& pages = _pass_itm_data->pages_info; + auto& pass = *_pass_itm_data; + auto& subpass = *pass.subpass; + + auto& pages = subpass.pages; // TODO: this could be made smaller if we ignored dictionary pages and pages with no // repetition data. - size_t const per_page_decode_buf_size = - LEVEL_DECODE_BUF_SIZE * 2 * _pass_itm_data->level_type_size; - auto const decode_buf_size = per_page_decode_buf_size * pages.size(); - _pass_itm_data->level_decode_data = + size_t const per_page_decode_buf_size = LEVEL_DECODE_BUF_SIZE * 2 * pass.level_type_size; + auto const decode_buf_size = per_page_decode_buf_size * pages.size(); + subpass.level_decode_data = rmm::device_buffer(decode_buf_size, _stream, rmm::mr::get_current_device_resource()); // distribute the buffers - uint8_t* buf = static_cast(_pass_itm_data->level_decode_data.data()); + uint8_t* buf = static_cast(subpass.level_decode_data.data()); for (size_t idx = 0; idx < pages.size(); idx++) { auto& p = pages[idx]; p.lvl_decode_buf[level_type::DEFINITION] = buf; - buf += (LEVEL_DECODE_BUF_SIZE * _pass_itm_data->level_type_size); + buf += (LEVEL_DECODE_BUF_SIZE * pass.level_type_size); p.lvl_decode_buf[level_type::REPETITION] = buf; - buf += (LEVEL_DECODE_BUF_SIZE * _pass_itm_data->level_type_size); + buf += (LEVEL_DECODE_BUF_SIZE * pass.level_type_size); } } -std::pair>> reader::impl::read_and_decompress_column_chunks() +std::pair>> reader::impl::read_column_chunks() { auto const& row_groups_info = _pass_itm_data->row_groups; - auto const num_rows = _pass_itm_data->num_rows; auto& raw_page_data = _pass_itm_data->raw_page_data; auto& chunks = _pass_itm_data->chunks; @@ -767,13 +804,14 @@ std::pair>> reader::impl::read_and_decompres // Initialize column chunk information size_t total_decompressed_size = 0; - auto remaining_rows = num_rows; + // TODO: make this respect the pass-wide skip_rows/num_rows instead of the file-wide + // skip_rows/num_rows + // auto remaining_rows = num_rows; std::vector> read_chunk_tasks; size_type chunk_count = 0; for (auto const& rg : row_groups_info) { auto const& row_group = _metadata->get_row_group(rg.index, rg.source_index); auto const row_group_source = rg.source_index; - auto const row_group_rows = std::min(remaining_rows, row_group.num_rows); // generate ColumnChunkDesc objects for everything to be decoded (all input columns) for (size_t i = 0; i < num_input_columns; ++i) { @@ -795,7 +833,6 @@ std::pair>> reader::impl::read_and_decompres chunk_count++; } - remaining_rows -= row_group_rows; } // Read compressed chunk data to device memory @@ -808,22 +845,20 @@ std::pair>> reader::impl::read_and_decompres chunk_source_map, _stream)); - CUDF_EXPECTS(remaining_rows == 0, "All rows data must be read."); - return {total_decompressed_size > 0, std::move(read_chunk_tasks)}; } -void reader::impl::load_and_decompress_data() +void reader::impl::read_compressed_data() { + auto& pass = *_pass_itm_data; + // This function should never be called if `num_rows == 0`. CUDF_EXPECTS(_pass_itm_data->num_rows > 0, "Number of reading rows must not be zero."); - auto& raw_page_data = _pass_itm_data->raw_page_data; - auto& decomp_page_data = _pass_itm_data->decomp_page_data; - auto& chunks = _pass_itm_data->chunks; - auto& pages = _pass_itm_data->pages_info; + auto& chunks = pass.chunks; - auto const [has_compressed_data, read_chunks_tasks] = read_and_decompress_column_chunks(); + auto const [has_compressed_data, read_chunks_tasks] = read_column_chunks(); + pass.has_compressed_data = has_compressed_data; for (auto& task : read_chunks_tasks) { task.wait(); @@ -832,44 +867,12 @@ void reader::impl::load_and_decompress_data() // Process dataset chunk pages into output columns auto const total_pages = count_page_headers(chunks, _stream); if (total_pages <= 0) { return; } - pages = cudf::detail::hostdevice_vector(total_pages, total_pages, _stream); + rmm::device_uvector unsorted_pages(total_pages, _stream); // decoding of column/page information - _pass_itm_data->level_type_size = decode_page_headers(chunks, pages, _stream); - pages.device_to_host_sync(_stream); - if (has_compressed_data) { - decomp_page_data = decompress_page_data(chunks, pages, _stream); - // Free compressed data - for (size_t c = 0; c < chunks.size(); c++) { - if (chunks[c].codec != Compression::UNCOMPRESSED) { raw_page_data[c].reset(); } - } - } - - // build output column info - // walk the schema, building out_buffers that mirror what our final cudf columns will look - // like. important : there is not necessarily a 1:1 mapping between input columns and output - // columns. For example, parquet does not explicitly store a ColumnChunkDesc for struct - // columns. The "structiness" is simply implied by the schema. For example, this schema: - // required group field_id=1 name { - // required binary field_id=2 firstname (String); - // required binary field_id=3 middlename (String); - // required binary field_id=4 lastname (String); - // } - // will only contain 3 columns of data (firstname, middlename, lastname). But of course - // "name" is a struct column that we want to return, so we have to make sure that we - // create it ourselves. - // std::vector output_info = build_output_column_info(); - - // the following two allocate functions modify the page data - { - // nesting information (sizes, etc) stored -per page- - // note : even for flat schemas, we allocate 1 level of "nesting" info - allocate_nesting_info(); - - // level decode space - allocate_level_decode_space(); - } - pages.host_to_device_async(_stream); + decode_page_headers(pass, unsorted_pages, _stream); + CUDF_EXPECTS(pass.page_offsets.size() - 1 == static_cast(_input_columns.size()), + "Encountered page_offsets / num_columns mismatch"); } namespace { @@ -880,28 +883,6 @@ struct cumulative_row_info { int key; // schema index }; -#if defined(PREPROCESS_DEBUG) -void print_pages(cudf::detail::hostdevice_vector& pages, rmm::cuda_stream_view _stream) -{ - pages.device_to_host_sync(_stream); - for (size_t idx = 0; idx < pages.size(); idx++) { - auto const& p = pages[idx]; - // skip dictionary pages - if (p.flags & PAGEINFO_FLAGS_DICTIONARY) { continue; } - printf( - "P(%lu, s:%d): chunk_row(%d), num_rows(%d), skipped_values(%d), skipped_leaf_values(%d), " - "str_bytes(%d)\n", - idx, - p.src_col_schema, - p.chunk_row, - p.num_rows, - p.skipped_values, - p.skipped_leaf_values, - p.str_bytes); - } -} -#endif // PREPROCESS_DEBUG - struct get_page_chunk_idx { __device__ size_type operator()(PageInfo const& page) { return page.chunk_idx; } }; @@ -910,14 +891,6 @@ struct get_page_num_rows { __device__ size_type operator()(PageInfo const& page) { return page.num_rows; } }; -struct get_page_column_index { - ColumnChunkDesc const* chunks; - __device__ size_type operator()(PageInfo const& page) - { - return chunks[page.chunk_idx].src_col_index; - } -}; - struct input_col_info { int const schema_idx; size_type const nesting_depth; @@ -950,13 +923,12 @@ struct get_page_nesting_size { size_type const max_depth; size_t const num_pages; PageInfo const* const pages; - int const* page_indices; __device__ size_type operator()(size_t index) const { auto const indices = reduction_indices{index, max_depth, num_pages}; - auto const& page = pages[page_indices[indices.page_idx]]; + auto const& page = pages[indices.page_idx]; if (page.src_col_schema != input_cols[indices.col_idx].schema_idx || page.flags & PAGEINFO_FLAGS_DICTIONARY || indices.depth_idx >= input_cols[indices.col_idx].nesting_depth) { @@ -995,12 +967,14 @@ struct chunk_row_output_iter { __device__ reference operator*() { return p->chunk_row; } }; +/** + * @brief Writes to the page_start_value field of the PageNestingInfo struct, keyed by schema. + */ /** * @brief Writes to the page_start_value field of the PageNestingInfo struct, keyed by schema. */ struct start_offset_output_iterator { PageInfo const* pages; - int const* page_indices; size_t cur_index; input_col_info const* input_cols; size_type max_depth; @@ -1014,17 +988,16 @@ struct start_offset_output_iterator { constexpr void operator=(start_offset_output_iterator const& other) { - pages = other.pages; - page_indices = other.page_indices; - cur_index = other.cur_index; - input_cols = other.input_cols; - max_depth = other.max_depth; - num_pages = other.num_pages; + pages = other.pages; + cur_index = other.cur_index; + input_cols = other.input_cols; + max_depth = other.max_depth; + num_pages = other.num_pages; } constexpr start_offset_output_iterator operator+(size_t i) { - return {pages, page_indices, cur_index + i, input_cols, max_depth, num_pages}; + return start_offset_output_iterator{pages, cur_index + i, input_cols, max_depth, num_pages}; } constexpr start_offset_output_iterator& operator++() @@ -1041,7 +1014,7 @@ struct start_offset_output_iterator { { auto const indices = reduction_indices{index, max_depth, num_pages}; - PageInfo const& p = pages[page_indices[indices.page_idx]]; + PageInfo const& p = pages[indices.page_idx]; if (p.src_col_schema != input_cols[indices.col_idx].schema_idx || p.flags & PAGEINFO_FLAGS_DICTIONARY || indices.depth_idx >= input_cols[indices.col_idx].nesting_depth) { @@ -1051,114 +1024,20 @@ struct start_offset_output_iterator { } }; -struct flat_column_num_rows { - PageInfo const* pages; - ColumnChunkDesc const* chunks; - - __device__ size_type operator()(size_type pindex) const - { - PageInfo const& page = pages[pindex]; - // ignore dictionary pages and pages belonging to any column containing repetition (lists) - if ((page.flags & PAGEINFO_FLAGS_DICTIONARY) || - (chunks[page.chunk_idx].max_level[level_type::REPETITION] > 0)) { - return 0; - } - return page.num_rows; - } -}; - -struct row_counts_nonzero { - __device__ bool operator()(size_type count) const { return count > 0; } -}; - -struct row_counts_different { - size_type const expected; - __device__ bool operator()(size_type count) const { return (count != 0) && (count != expected); } -}; - -/** - * @brief Detect malformed parquet input data. - * - * We have seen cases where parquet files can be oddly malformed. This function specifically - * detects one case in particular: - * - * - When you have a file containing N rows - * - For some reason, the sum total of the number of rows over all pages for a given column - * is != N - * - * @param pages All pages to be decoded - * @param chunks Chunk data - * @param page_keys Keys (schema id) associated with each page, sorted by column - * @param page_index Page indices for iteration, sorted by column - * @param expected_row_count Expected row count, if applicable - * @param stream CUDA stream used for device memory operations and kernel launches - */ -void detect_malformed_pages(cudf::detail::hostdevice_vector& pages, - cudf::detail::hostdevice_vector const& chunks, - device_span page_keys, - device_span page_index, - std::optional expected_row_count, - rmm::cuda_stream_view stream) -{ - // sum row counts for all non-dictionary, non-list columns. other columns will be indicated as 0 - rmm::device_uvector row_counts(pages.size(), - stream); // worst case: num keys == num pages - auto const size_iter = thrust::make_transform_iterator( - page_index.begin(), flat_column_num_rows{pages.device_ptr(), chunks.device_ptr()}); - auto const row_counts_begin = row_counts.begin(); - auto const row_counts_end = thrust::reduce_by_key(rmm::exec_policy(stream), - page_keys.begin(), - page_keys.end(), - size_iter, - thrust::make_discard_iterator(), - row_counts_begin) - .second; - - // make sure all non-zero row counts are the same - rmm::device_uvector compacted_row_counts(pages.size(), stream); - auto const compacted_row_counts_begin = compacted_row_counts.begin(); - auto const compacted_row_counts_end = thrust::copy_if(rmm::exec_policy(stream), - row_counts_begin, - row_counts_end, - compacted_row_counts_begin, - row_counts_nonzero{}); - if (compacted_row_counts_end != compacted_row_counts_begin) { - size_t const found_row_count = static_cast(compacted_row_counts.element(0, stream)); - - // if we somehow don't match the expected row count from the row groups themselves - if (expected_row_count.has_value()) { - CUDF_EXPECTS(expected_row_count.value() == found_row_count, - "Encountered malformed parquet page data (unexpected row count in page data)"); - } - - // all non-zero row counts must be the same - auto const chk = - thrust::count_if(rmm::exec_policy(stream), - compacted_row_counts_begin, - compacted_row_counts_end, - row_counts_different{static_cast(found_row_count)}); - CUDF_EXPECTS(chk == 0, - "Encountered malformed parquet page data (row count mismatch in page data)"); - } -} - struct page_to_string_size { - PageInfo* pages; ColumnChunkDesc const* chunks; - __device__ size_t operator()(size_type page_idx) const + __device__ size_t operator()(PageInfo const& page) const { - auto const page = pages[page_idx]; auto const chunk = chunks[page.chunk_idx]; if (not is_string_col(chunk) || (page.flags & PAGEINFO_FLAGS_DICTIONARY) != 0) { return 0; } - return pages[page_idx].str_bytes; + return page.str_bytes; } }; struct page_offset_output_iter { PageInfo* p; - size_type const* index; using value_type = size_type; using difference_type = size_type; @@ -1166,78 +1045,148 @@ struct page_offset_output_iter { using reference = size_type&; using iterator_category = thrust::output_device_iterator_tag; - __host__ __device__ page_offset_output_iter operator+(int i) { return {p, index + i}; } + __host__ __device__ page_offset_output_iter operator+(int i) { return {p + i}; } __host__ __device__ page_offset_output_iter& operator++() { - index++; + p++; return *this; } - __device__ reference operator[](int i) { return p[index[i]].str_offset; } - __device__ reference operator*() { return p[*index].str_offset; } + __device__ reference operator[](int i) { return p[i].str_offset; } + __device__ reference operator*() { return p->str_offset; } }; +// update chunk_row field in subpass page from pass page +struct update_subpass_chunk_row { + device_span pass_pages; + device_span subpass_pages; + device_span page_src_index; -} // anonymous namespace + __device__ void operator()(size_t i) + { + subpass_pages[i].chunk_row = pass_pages[page_src_index[i]].chunk_row; + } +}; -void reader::impl::preprocess_pages(bool uses_custom_row_bounds, size_t chunk_read_limit) -{ - auto const skip_rows = _pass_itm_data->skip_rows; - auto const num_rows = _pass_itm_data->num_rows; - auto& chunks = _pass_itm_data->chunks; - auto& pages = _pass_itm_data->pages_info; +// update num_rows field from pass page to subpass page +struct update_pass_num_rows { + device_span pass_pages; + device_span subpass_pages; + device_span page_src_index; - // compute page ordering. - // - // ordering of pages is by input column schema, repeated across row groups. so - // if we had 3 columns, each with 2 pages, and 1 row group, our schema values might look like - // - // 1, 1, 2, 2, 3, 3 - // - // However, if we had more than one row group, the pattern would be - // - // 1, 1, 2, 2, 3, 3, 1, 1, 2, 2, 3, 3 - // ^ row group 0 | - // ^ row group 1 - // - // To process pages by key (exclusive_scan_by_key, reduce_by_key, etc), the ordering we actually - // want is - // - // 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3 - // - // We also need to preserve key-relative page ordering, so we need to use a stable sort. - rmm::device_uvector page_keys(pages.size(), _stream); - rmm::device_uvector page_index(pages.size(), _stream); + __device__ void operator()(size_t i) { - thrust::transform(rmm::exec_policy(_stream), - pages.device_ptr(), - pages.device_ptr() + pages.size(), - page_keys.begin(), - get_page_column_index{chunks.device_ptr()}); + pass_pages[page_src_index[i]].num_rows = subpass_pages[i].num_rows; + } +}; - thrust::sequence(rmm::exec_policy(_stream), page_index.begin(), page_index.end()); - thrust::stable_sort_by_key(rmm::exec_policy(_stream), - page_keys.begin(), - page_keys.end(), - page_index.begin(), - thrust::less()); +} // anonymous namespace + +void reader::impl::preprocess_file( + int64_t skip_rows, + std::optional const& num_rows, + host_span const> row_group_indices, + std::optional> filter) +{ + CUDF_EXPECTS(!_file_preprocessed, "Attempted to preprocess file more than once"); + + // if filter is not empty, then create output types as vector and pass for filtering. + std::vector output_types; + if (filter.has_value()) { + std::transform(_output_buffers.cbegin(), + _output_buffers.cend(), + std::back_inserter(output_types), + [](auto const& col) { return col.type; }); + } + std::tie( + _file_itm_data.global_skip_rows, _file_itm_data.global_num_rows, _file_itm_data.row_groups) = + _metadata->select_row_groups( + row_group_indices, skip_rows, num_rows, output_types, filter, _stream); + + if (_file_itm_data.global_num_rows > 0 && not _file_itm_data.row_groups.empty() && + not _input_columns.empty()) { + // fills in chunk information without physically loading or decompressing + // the associated data + create_global_chunk_info(); + + // compute schedule of input reads. + compute_input_passes(); + } + +#if defined(PARQUET_CHUNK_LOGGING) + printf("==============================================\n"); + setlocale(LC_NUMERIC, ""); + printf("File: skip_rows(%'lu), num_rows(%'lu), input_read_limit(%'lu), output_read_limit(%'lu)\n", + _file_itm_data.global_skip_rows, + _file_itm_data.global_num_rows, + _input_pass_read_limit, + _output_chunk_read_limit); + printf("# Row groups: %'lu\n", _file_itm_data.row_groups.size()); + printf("# Input passes: %'lu\n", _file_itm_data.num_passes()); + printf("# Input columns: %'lu\n", _input_columns.size()); + for (size_t idx = 0; idx < _input_columns.size(); idx++) { + auto const& schema = _metadata->get_schema(_input_columns[idx].schema_idx); + auto const type_id = to_type_id(schema, _strings_to_categorical, _timestamp_type.id()); + printf("\tC(%'lu, %s): %s\n", + idx, + _input_columns[idx].name.c_str(), + cudf::type_to_name(cudf::data_type{type_id}).c_str()); + } + printf("# Output columns: %'lu\n", _output_buffers.size()); + for (size_t idx = 0; idx < _output_buffers.size(); idx++) { + printf("\tC(%'lu): %s\n", idx, cudf::io::detail::type_to_name(_output_buffers[idx]).c_str()); } +#endif + + _file_preprocessed = true; +} + +void reader::impl::generate_list_column_row_count_estimates() +{ + auto& pass = *_pass_itm_data; + thrust::for_each(rmm::exec_policy(_stream), + pass.pages.d_begin(), + pass.pages.d_end(), + set_list_row_count_estimate{pass.chunks}); + + // computes: + // PageInfo::chunk_row (the chunk-relative row index) for all pages in the pass. The start_row + // field in ColumnChunkDesc is the absolute row index for the whole file. chunk_row in PageInfo is + // relative to the beginning of the chunk. so in the kernels, chunk.start_row + page.chunk_row + // gives us the absolute row index + auto key_input = thrust::make_transform_iterator(pass.pages.d_begin(), get_page_chunk_idx{}); + auto page_input = thrust::make_transform_iterator(pass.pages.d_begin(), get_page_num_rows{}); + thrust::exclusive_scan_by_key(rmm::exec_policy_nosync(_stream), + key_input, + key_input + pass.pages.size(), + page_input, + chunk_row_output_iter{pass.pages.device_ptr()}); + + // finally, fudge the last page for each column such that it ends on the real known row count + // for the pass. this is so that as we march through the subpasses, we will find that every column + // cleanly ends up the expected row count at the row group boundary. + auto const& last_chunk = pass.chunks[pass.chunks.size() - 1]; + auto const num_columns = _input_columns.size(); + size_t const max_row = last_chunk.start_row + last_chunk.num_rows; + auto iter = thrust::make_counting_iterator(0); + thrust::for_each(rmm::exec_policy_nosync(_stream), + iter, + iter + num_columns, + set_final_row_count{pass.pages, pass.chunks, pass.page_offsets, max_row}); + + pass.chunks.device_to_host_async(_stream); + pass.pages.device_to_host_async(_stream); + _stream.synchronize(); +} + +void reader::impl::preprocess_subpass_pages(bool uses_custom_row_bounds, size_t chunk_read_limit) +{ + auto& pass = *_pass_itm_data; + auto& subpass = *pass.subpass; - // detect malformed columns. - // - we have seen some cases in the wild where we have a row group containing N - // rows, but the total number of rows in the pages for column X is != N. while it - // is possible to load this by just capping the number of rows read, we cannot tell - // which rows are invalid so we may be returning bad data. in addition, this mismatch - // confuses the chunked reader - detect_malformed_pages(pages, - chunks, - page_keys, - page_index, - uses_custom_row_bounds ? std::nullopt : std::make_optional(num_rows), - _stream); - - // iterate over all input columns and determine if they contain lists so we can further - // preprocess them. + // iterate over all input columns and determine if they contain lists. + // TODO: we could do this once at the file level instead of every time we get in here. the set of + // columns we are processing does not change over multiple passes/subpasses/output chunks. bool has_lists = false; for (size_t idx = 0; idx < _input_columns.size(); idx++) { auto const& input_col = _input_columns[idx]; @@ -1258,49 +1207,9 @@ void reader::impl::preprocess_pages(bool uses_custom_row_bounds, size_t chunk_re if (has_lists) { break; } } - // generate string dict indices if necessary - { - auto is_dict_chunk = [](ColumnChunkDesc const& chunk) { - return (chunk.data_type & 0x7) == BYTE_ARRAY && chunk.num_dict_pages > 0; - }; - - // Count the number of string dictionary entries - // NOTE: Assumes first page in the chunk is always the dictionary page - size_t total_str_dict_indexes = 0; - for (size_t c = 0, page_count = 0; c < chunks.size(); c++) { - if (is_dict_chunk(chunks[c])) { - total_str_dict_indexes += pages[page_count].num_input_values; - } - page_count += chunks[c].max_num_pages; - } - - // Build index for string dictionaries since they can't be indexed - // directly due to variable-sized elements - _pass_itm_data->str_dict_index = - cudf::detail::make_zeroed_device_uvector_async( - total_str_dict_indexes, _stream, rmm::mr::get_current_device_resource()); - - // Update chunks with pointers to string dict indices - for (size_t c = 0, page_count = 0, str_ofs = 0; c < chunks.size(); c++) { - input_column_info const& input_col = _input_columns[chunks[c].src_col_index]; - CUDF_EXPECTS(input_col.schema_idx == chunks[c].src_col_schema, - "Column/page schema index mismatch"); - if (is_dict_chunk(chunks[c])) { - chunks[c].str_dict_index = _pass_itm_data->str_dict_index.data() + str_ofs; - str_ofs += pages[page_count].num_input_values; - } - - // column_data_base will always point to leaf data, even for nested types. - page_count += chunks[c].max_num_pages; - } - - if (total_str_dict_indexes > 0) { - chunks.host_to_device_async(_stream); - BuildStringDictionaryIndex(chunks.device_ptr(), chunks.size(), _stream); - } - } - - // intermediate data we will need for further chunked reads + // in some cases we will need to do further preprocessing of pages. + // - if we have lists, the num_rows field in PageInfo will be incorrect coming out of the file + // - if we are doing a chunked read, we need to compute the size of all string data if (has_lists || chunk_read_limit > 0) { // computes: // PageNestingInfo::num_rows for each page. the true number of rows (taking repetition into @@ -1311,48 +1220,92 @@ void reader::impl::preprocess_pages(bool uses_custom_row_bounds, size_t chunk_re // if: // - user has passed custom row bounds // - we will be doing a chunked read - ComputePageSizes(pages, - chunks, + ComputePageSizes(subpass.pages, + pass.chunks, 0, // 0-max size_t. process all possible rows std::numeric_limits::max(), true, // compute num_rows chunk_read_limit > 0, // compute string sizes _pass_itm_data->level_type_size, _stream); + } - // computes: - // PageInfo::chunk_row (the absolute start row index) for all pages - // Note: this is doing some redundant work for pages in flat hierarchies. chunk_row has already - // been computed during header decoding. the overall amount of work here is very small though. - auto key_input = thrust::make_transform_iterator(pages.device_ptr(), get_page_chunk_idx{}); - auto page_input = thrust::make_transform_iterator(pages.device_ptr(), get_page_num_rows{}); - thrust::exclusive_scan_by_key(rmm::exec_policy(_stream), - key_input, - key_input + pages.size(), - page_input, - chunk_row_output_iter{pages.device_ptr()}); - - // retrieve pages back - pages.device_to_host_sync(_stream); + // copy our now-correct row counts back to the base pages stored in the pass. + auto iter = thrust::make_counting_iterator(0); + thrust::for_each(rmm::exec_policy_nosync(_stream), + iter, + iter + subpass.pages.size(), + update_pass_num_rows{pass.pages, subpass.pages, subpass.page_src_index}); - // print_pages(pages, _stream); - } + // computes: + // PageInfo::chunk_row (the chunk-relative row index) for all pages in the pass. The start_row + // field in ColumnChunkDesc is the absolute row index for the whole file. chunk_row in PageInfo is + // relative to the beginning of the chunk. so in the kernels, chunk.start_row + page.chunk_row + // gives us the absolute row index + auto key_input = thrust::make_transform_iterator(pass.pages.d_begin(), get_page_chunk_idx{}); + auto page_input = thrust::make_transform_iterator(pass.pages.d_begin(), get_page_num_rows{}); + thrust::exclusive_scan_by_key(rmm::exec_policy_nosync(_stream), + key_input, + key_input + pass.pages.size(), + page_input, + chunk_row_output_iter{pass.pages.device_ptr()}); + + // copy chunk row into the subpass pages + thrust::for_each(rmm::exec_policy_nosync(_stream), + iter, + iter + subpass.pages.size(), + update_subpass_chunk_row{pass.pages, subpass.pages, subpass.page_src_index}); + + // retrieve pages back + pass.pages.device_to_host_async(_stream); + subpass.pages.device_to_host_async(_stream); + _stream.synchronize(); - // preserve page ordering data for string decoder - _pass_itm_data->page_keys = std::move(page_keys); - _pass_itm_data->page_index = std::move(page_index); + // at this point we have an accurate row count so we can compute how many rows we will actually be + // able to decode for this pass. we will have selected a set of pages for each column in the + // row group, but not every page will have the same number of rows. so, we can only read as many + // rows as the smallest batch (by column) we have decompressed. + size_t page_index = 0; + size_t max_row = std::numeric_limits::max(); + auto const last_pass_row = + _file_itm_data.input_pass_start_row_count[_file_itm_data._current_input_pass + 1]; + for (size_t idx = 0; idx < subpass.column_page_count.size(); idx++) { + auto const& last_page = subpass.pages[page_index + (subpass.column_page_count[idx] - 1)]; + auto const& chunk = pass.chunks[last_page.chunk_idx]; + + size_t max_col_row = + static_cast(chunk.start_row + last_page.chunk_row + last_page.num_rows); + // special case. list rows can span page boundaries, but we can't tell if that is happening + // here because we have not yet decoded the pages. the very last row starting in the page may + // not terminate in the page. to handle this, only decode up to the second to last row in the + // subpass since we know that will safely completed. + bool const is_list = chunk.max_level[level_type::REPETITION] > 0; + if (is_list && max_col_row < last_pass_row) { + size_t const min_col_row = static_cast(chunk.start_row + last_page.chunk_row); + CUDF_EXPECTS((max_col_row - min_col_row) > 1, "Unexpected short subpass"); + max_col_row--; + } + + max_row = min(max_row, max_col_row); + + page_index += subpass.column_page_count[idx]; + } + subpass.skip_rows = pass.skip_rows + pass.processed_rows; + auto const pass_end = pass.skip_rows + pass.num_rows; + max_row = min(max_row, pass_end); + subpass.num_rows = max_row - subpass.skip_rows; - // compute splits for the pass - compute_splits_for_pass(); + // now split up the output into chunks as necessary + compute_output_chunks_for_subpass(); } void reader::impl::allocate_columns(size_t skip_rows, size_t num_rows, bool uses_custom_row_bounds) { - auto const& chunks = _pass_itm_data->chunks; - auto& pages = _pass_itm_data->pages_info; + auto& pass = *_pass_itm_data; + auto& subpass = *pass.subpass; // Should not reach here if there is no page data. - CUDF_EXPECTS(pages.size() > 0, "There is no page to parse"); + CUDF_EXPECTS(subpass.pages.size() > 0, "There are no pages present in the subpass"); // computes: // PageNestingInfo::batch_size for each level of nesting, for each page, taking row bounds into @@ -1360,13 +1313,13 @@ void reader::impl::allocate_columns(size_t skip_rows, size_t num_rows, bool uses // respect the user bounds. It is only necessary to do this second pass if uses_custom_row_bounds // is set (if the user has specified artificial bounds). if (uses_custom_row_bounds) { - ComputePageSizes(pages, - chunks, + ComputePageSizes(subpass.pages, + pass.chunks, skip_rows, num_rows, false, // num_rows is already computed false, // no need to compute string sizes - _pass_itm_data->level_type_size, + pass.level_type_size, _stream); // print_pages(pages, _stream); @@ -1403,8 +1356,6 @@ void reader::impl::allocate_columns(size_t skip_rows, size_t num_rows, bool uses // compute output column sizes by examining the pages of the -input- columns if (has_lists) { - auto& page_index = _pass_itm_data->page_index; - std::vector h_cols_info; h_cols_info.reserve(_input_columns.size()); std::transform(_input_columns.cbegin(), @@ -1423,7 +1374,7 @@ void reader::impl::allocate_columns(size_t skip_rows, size_t num_rows, bool uses auto const d_cols_info = cudf::detail::make_device_uvector_async( h_cols_info, _stream, rmm::mr::get_current_device_resource()); - auto const num_keys = _input_columns.size() * max_depth * pages.size(); + auto const num_keys = _input_columns.size() * max_depth * subpass.pages.size(); // size iterator. indexes pages by sorted order rmm::device_uvector size_input{num_keys, _stream}; thrust::transform( @@ -1432,9 +1383,9 @@ void reader::impl::allocate_columns(size_t skip_rows, size_t num_rows, bool uses thrust::make_counting_iterator(num_keys), size_input.begin(), get_page_nesting_size{ - d_cols_info.data(), max_depth, pages.size(), pages.device_ptr(), page_index.begin()}); + d_cols_info.data(), max_depth, subpass.pages.size(), subpass.pages.d_begin()}); auto const reduction_keys = - cudf::detail::make_counting_transform_iterator(0, get_reduction_key{pages.size()}); + cudf::detail::make_counting_transform_iterator(0, get_reduction_key{subpass.pages.size()}); cudf::detail::hostdevice_vector sizes{_input_columns.size() * max_depth, _stream}; // find the size of each column @@ -1452,7 +1403,7 @@ void reader::impl::allocate_columns(size_t skip_rows, size_t num_rows, bool uses reduction_keys + num_keys, size_input.cbegin(), start_offset_output_iterator{ - pages.device_ptr(), page_index.begin(), 0, d_cols_info.data(), max_depth, pages.size()}); + subpass.pages.d_begin(), 0, d_cols_info.data(), max_depth, subpass.pages.size()}); sizes.device_to_host_sync(_stream); for (size_type idx = 0; idx < static_cast(_input_columns.size()); idx++) { @@ -1483,30 +1434,30 @@ void reader::impl::allocate_columns(size_t skip_rows, size_t num_rows, bool uses std::vector reader::impl::calculate_page_string_offsets() { - auto& chunks = _pass_itm_data->chunks; - auto& pages = _pass_itm_data->pages_info; - auto const& page_keys = _pass_itm_data->page_keys; - auto const& page_index = _pass_itm_data->page_index; + auto& pass = *_pass_itm_data; + auto& subpass = *pass.subpass; + + auto page_keys = make_page_key_iterator(subpass.pages); std::vector col_sizes(_input_columns.size(), 0L); rmm::device_uvector d_col_sizes(col_sizes.size(), _stream); // use page_index to fetch page string sizes in the proper order - auto val_iter = thrust::make_transform_iterator( - page_index.begin(), page_to_string_size{pages.device_ptr(), chunks.device_ptr()}); + auto val_iter = thrust::make_transform_iterator(subpass.pages.d_begin(), + page_to_string_size{pass.chunks.d_begin()}); // do scan by key to calculate string offsets for each page thrust::exclusive_scan_by_key(rmm::exec_policy_nosync(_stream), - page_keys.begin(), - page_keys.end(), + page_keys, + page_keys + subpass.pages.size(), val_iter, - page_offset_output_iter{pages.device_ptr(), page_index.data()}); + page_offset_output_iter{subpass.pages.device_ptr()}); // now sum up page sizes rmm::device_uvector reduce_keys(col_sizes.size(), _stream); thrust::reduce_by_key(rmm::exec_policy_nosync(_stream), - page_keys.begin(), - page_keys.end(), + page_keys, + page_keys + subpass.pages.size(), val_iter, reduce_keys.begin(), d_col_sizes.begin()); diff --git a/cpp/src/io/utilities/column_buffer.cpp b/cpp/src/io/utilities/column_buffer.cpp index 36303a60aa9..951217dc442 100644 --- a/cpp/src/io/utilities/column_buffer.cpp +++ b/cpp/src/io/utilities/column_buffer.cpp @@ -26,6 +26,9 @@ #include +#include +#include + namespace cudf::io::detail { void gather_column_buffer::allocate_strings_data(rmm::cuda_stream_view stream) @@ -129,6 +132,30 @@ string_policy column_buffer_base::empty_like(string_policy const& return new_buff; } +template +std::string type_to_name(column_buffer_base const& buffer) +{ + if (buffer.type.id() == cudf::type_id::LIST) { + return "List<" + (type_to_name(buffer.children[0])) + ">"; + } + + if (buffer.type.id() == cudf::type_id::STRUCT) { + std::ostringstream out; + + out << "Struct<"; + auto iter = thrust::make_counting_iterator(0); + std::transform( + iter, + iter + buffer.children.size(), + std::ostream_iterator(out, ","), + [&buffer](size_type i) { return type_to_name(buffer.children[i]); }); + out << ">"; + return out.str(); + } + + return cudf::type_to_name(buffer.type); +} + template std::unique_ptr make_column(column_buffer_base& buffer, column_name_info* schema_info, @@ -336,6 +363,10 @@ template std::unique_ptr empty_like(pointer_column_buffer& rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr); +template std::string type_to_name(string_column_buffer const& buffer); +template std::string type_to_name(pointer_column_buffer const& buffer); + template class column_buffer_base; template class column_buffer_base; + } // namespace cudf::io::detail diff --git a/cpp/src/io/utilities/column_buffer.hpp b/cpp/src/io/utilities/column_buffer.hpp index 2ee7c17e480..57ee1043ee9 100644 --- a/cpp/src/io/utilities/column_buffer.hpp +++ b/cpp/src/io/utilities/column_buffer.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -253,6 +253,16 @@ std::unique_ptr empty_like(column_buffer_base& buffer, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr); +/** + * @brief Given a column_buffer, produce a formatted name string describing the type. + * + * @param buffer The column buffer + * + * @return A string describing the type of the buffer suitable for printing + */ +template +std::string type_to_name(column_buffer_base const& buffer); + } // namespace detail } // namespace io } // namespace cudf diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 24085eb5e10..d40b2410ca3 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -293,7 +293,7 @@ ConfigureTest( ConfigureTest( PARQUET_TEST io/parquet_test.cpp - io/parquet_chunked_reader_test.cpp + io/parquet_chunked_reader_test.cu io/parquet_chunked_writer_test.cpp io/parquet_common.cpp io/parquet_misc_test.cpp diff --git a/cpp/tests/io/parquet_chunked_reader_test.cpp b/cpp/tests/io/parquet_chunked_reader_test.cu similarity index 73% rename from cpp/tests/io/parquet_chunked_reader_test.cpp rename to cpp/tests/io/parquet_chunked_reader_test.cu index 05fb9a3ec48..dea44f0e7c3 100644 --- a/cpp/tests/io/parquet_chunked_reader_test.cpp +++ b/cpp/tests/io/parquet_chunked_reader_test.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,8 @@ * limitations under the License. */ +#include "parquet_common.hpp" + #include #include #include @@ -44,14 +46,12 @@ #include #include +#include #include #include namespace { -// Global environment for temporary files -auto const temp_env = static_cast( - ::testing::AddGlobalTestEnvironment(new cudf::test::TempDirTestEnvironment)); using int32s_col = cudf::test::fixed_width_column_wrapper; using int64s_col = cudf::test::fixed_width_column_wrapper; @@ -953,64 +953,296 @@ TEST_F(ParquetChunkedReaderTest, TestChunkedReadNullCount) } while (reader.has_next()); } -TEST_F(ParquetChunkedReaderTest, InputLimitSimple) +constexpr size_t input_limit_expected_file_count = 4; + +std::vector input_limit_get_test_names(std::string const& base_filename) { - auto const filepath = temp_env->get_temp_filepath("input_limit_10_rowgroups.parquet"); - - // This results in 10 grow groups, at 4001150 bytes per row group - constexpr int num_rows = 25'000'000; - auto value_iter = cudf::detail::make_counting_transform_iterator(0, [](int i) { return i; }); - cudf::test::fixed_width_column_wrapper expected(value_iter, value_iter + num_rows); - cudf::io::parquet_writer_options opts = - cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, - cudf::table_view{{expected}}) - // note: it is unnecessary to force compression to NONE here because the size we are using in - // the row group is the uncompressed data size. But forcing the dictionary policy to - // dictionary_policy::NEVER is necessary to prevent changes in the - // decompressed-but-not-yet-decoded data. - .dictionary_policy(cudf::io::dictionary_policy::NEVER); - - cudf::io::write_parquet(opts); - - { - // no chunking - auto const [result, num_chunks] = chunked_read(filepath, 0, 0); - EXPECT_EQ(num_chunks, 1); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->get_column(0)); - } + return {base_filename + "_a.parquet", + base_filename + "_b.parquet", + base_filename + "_c.parquet", + base_filename + "_d.parquet"}; +} - { - // 25 chunks of 100k rows each - auto const [result, num_chunks] = chunked_read(filepath, 0, 1); - EXPECT_EQ(num_chunks, 25); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->get_column(0)); - } +void input_limit_test_write_one(std::string const& filepath, + cudf::table_view const& t, + cudf::io::compression_type compression, + cudf::io::dictionary_policy dict_policy) +{ + cudf::io::parquet_writer_options out_opts = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, t) + .compression(compression) + .dictionary_policy(dict_policy); + cudf::io::write_parquet(out_opts); +} - { - // 25 chunks of 100k rows each - auto const [result, num_chunks] = chunked_read(filepath, 0, 4000000); - EXPECT_EQ(num_chunks, 25); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->get_column(0)); - } +void input_limit_test_write(std::vector const& test_filenames, + cudf::table_view const& t) +{ + CUDF_EXPECTS(test_filenames.size() == 4, "Unexpected count of test filenames"); + CUDF_EXPECTS(test_filenames.size() == input_limit_expected_file_count, + "Unexpected count of test filenames"); + + // no compression + input_limit_test_write_one( + test_filenames[0], t, cudf::io::compression_type::NONE, cudf::io::dictionary_policy::NEVER); + // compression with a codec that uses a lot of scratch space at decode time (2.5x the total + // decompressed buffer size) + input_limit_test_write_one( + test_filenames[1], t, cudf::io::compression_type::ZSTD, cudf::io::dictionary_policy::NEVER); + // compression with a codec that uses no scratch space at decode time + input_limit_test_write_one( + test_filenames[2], t, cudf::io::compression_type::SNAPPY, cudf::io::dictionary_policy::NEVER); + input_limit_test_write_one( + test_filenames[3], t, cudf::io::compression_type::SNAPPY, cudf::io::dictionary_policy::ALWAYS); +} - { - // 25 chunks of 100k rows each - auto const [result, num_chunks] = chunked_read(filepath, 0, 4100000); - EXPECT_EQ(num_chunks, 25); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->get_column(0)); - } +void input_limit_test_read(std::vector const& test_filenames, + cudf::table_view const& t, + size_t output_limit, + size_t input_limit, + int const expected_chunk_counts[input_limit_expected_file_count]) +{ + CUDF_EXPECTS(test_filenames.size() == input_limit_expected_file_count, + "Unexpected count of test filenames"); - { - // 12 chunks of 200k rows each, plus 1 final chunk of 100k rows. - auto const [result, num_chunks] = chunked_read(filepath, 0, 8002301); - EXPECT_EQ(num_chunks, 13); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->get_column(0)); + for (size_t idx = 0; idx < test_filenames.size(); idx++) { + auto result = chunked_read(test_filenames[idx], output_limit, input_limit); + CUDF_EXPECTS(result.second == expected_chunk_counts[idx], + "Unexpected number of chunks produced in chunk read"); + CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*result.first, t); } +} + +struct ParquetChunkedReaderInputLimitConstrainedTest : public cudf::test::BaseFixture {}; + +TEST_F(ParquetChunkedReaderInputLimitConstrainedTest, SingleFixedWidthColumn) +{ + auto base_path = temp_env->get_temp_filepath("single_col_fixed_width"); + auto test_filenames = input_limit_get_test_names(base_path); + + constexpr auto num_rows = 1'000'000; + auto iter1 = thrust::make_constant_iterator(15); + cudf::test::fixed_width_column_wrapper col1(iter1, iter1 + num_rows); + auto tbl = cudf::table_view{{col1}}; + + input_limit_test_write(test_filenames, tbl); + + // semi-reasonable limit + constexpr int expected_a[] = {1, 17, 4, 1}; + input_limit_test_read(test_filenames, tbl, 0, 2 * 1024 * 1024, expected_a); + // an unreasonable limit + constexpr int expected_b[] = {1, 50, 50, 1}; + input_limit_test_read(test_filenames, tbl, 0, 1, expected_b); +} + +TEST_F(ParquetChunkedReaderInputLimitConstrainedTest, MixedColumns) +{ + auto base_path = temp_env->get_temp_filepath("mixed_columns"); + auto test_filenames = input_limit_get_test_names(base_path); + + constexpr auto num_rows = 1'000'000; + + auto iter1 = thrust::make_counting_iterator(0); + cudf::test::fixed_width_column_wrapper col1(iter1, iter1 + num_rows); + + auto iter2 = thrust::make_counting_iterator(0); + cudf::test::fixed_width_column_wrapper col2(iter2, iter2 + num_rows); + + auto const strings = std::vector{"abc", "de", "fghi"}; + auto const str_iter = cudf::detail::make_counting_transform_iterator(0, [&](int32_t i) { + if (i < 250000) { return strings[0]; } + if (i < 750000) { return strings[1]; } + return strings[2]; + }); + auto col3 = strings_col(str_iter, str_iter + num_rows); + + auto tbl = cudf::table_view{{col1, col2, col3}}; + + input_limit_test_write(test_filenames, tbl); + constexpr int expected_a[] = {1, 50, 10, 7}; + input_limit_test_read(test_filenames, tbl, 0, 2 * 1024 * 1024, expected_a); + constexpr int expected_b[] = {1, 50, 50, 50}; + input_limit_test_read(test_filenames, tbl, 0, 1, expected_b); +} + +struct ParquetChunkedReaderInputLimitTest : public cudf::test::BaseFixture {}; + +struct offset_gen { + int const group_size; + __device__ int operator()(int i) { return i * group_size; } +}; + +template +struct value_gen { + __device__ T operator()(int i) { return i % 1024; } +}; +TEST_F(ParquetChunkedReaderInputLimitTest, List) +{ + auto base_path = temp_env->get_temp_filepath("list"); + auto test_filenames = input_limit_get_test_names(base_path); + + constexpr int num_rows = 50'000'000; + constexpr int list_size = 4; + + auto const stream = cudf::get_default_stream(); + + auto offset_iter = cudf::detail::make_counting_transform_iterator(0, offset_gen{list_size}); + auto offset_col = cudf::make_fixed_width_column( + cudf::data_type{cudf::type_id::INT32}, num_rows + 1, cudf::mask_state::UNALLOCATED); + thrust::copy(rmm::exec_policy(stream), + offset_iter, + offset_iter + num_rows + 1, + offset_col->mutable_view().begin()); + + // list + constexpr int num_ints = num_rows * list_size; + auto value_iter = cudf::detail::make_counting_transform_iterator(0, value_gen{}); + auto value_col = cudf::make_fixed_width_column( + cudf::data_type{cudf::type_id::INT32}, num_ints, cudf::mask_state::UNALLOCATED); + thrust::copy(rmm::exec_policy(stream), + value_iter, + value_iter + num_ints, + value_col->mutable_view().begin()); + auto col1 = + cudf::make_lists_column(num_rows, + std::move(offset_col), + std::move(value_col), + 0, + cudf::create_null_mask(num_rows, cudf::mask_state::UNALLOCATED), + stream); + + auto tbl = cudf::table_view{{*col1}}; + + input_limit_test_write(test_filenames, tbl); + + // even though we have a very large limit here, there are two cases where we actually produce + // splits. + // - uncompressed data (with no dict). This happens because the code has to make a guess at how + // much + // space to reserve for compressed/uncompressed data prior to reading. It does not know that + // everything it will be reading in this case is uncompressed already, so this guess ends up + // causing it to generate two top level passes. in practice, this shouldn't matter because we + // never really see uncompressed data in the wild. + // + // - ZSTD (with no dict). In this case, ZSTD simple requires a huge amount of temporary + // space: 2.5x the total + // size of the decompressed data. so 2 GB is actually not enough to hold the whole thing at + // once. + // + // Note that in the dictionary cases, both of these revert down to 1 chunk because the + // dictionaries dramatically shrink the size of the uncompressed data. + constexpr int expected_a[] = {2, 2, 1, 1}; + input_limit_test_read(test_filenames, tbl, 0, size_t{2} * 1024 * 1024 * 1024, expected_a); + // smaller limit + constexpr int expected_b[] = {6, 6, 2, 1}; + input_limit_test_read(test_filenames, tbl, 0, 512 * 1024 * 1024, expected_b); + // include output chunking as well + constexpr int expected_c[] = {11, 11, 9, 8}; + input_limit_test_read(test_filenames, tbl, 128 * 1024 * 1024, 512 * 1024 * 1024, expected_c); +} + +struct char_values { + __device__ int8_t operator()(int i) { - // 1 big chunk - auto const [result, num_chunks] = chunked_read(filepath, 0, size_t{1} * 1024 * 1024 * 1024); - EXPECT_EQ(num_chunks, 1); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->get_column(0)); + int const index = (i / 2) % 3; + // generate repeating 3-runs of 2 values each. aabbcc + return index == 0 ? 'a' : (index == 1 ? 'b' : 'c'); } +}; +TEST_F(ParquetChunkedReaderInputLimitTest, Mixed) +{ + auto base_path = temp_env->get_temp_filepath("mixed_types"); + auto test_filenames = input_limit_get_test_names(base_path); + + constexpr int num_rows = 50'000'000; + constexpr int list_size = 4; + constexpr int str_size = 3; + + auto const stream = cudf::get_default_stream(); + + auto offset_iter = cudf::detail::make_counting_transform_iterator(0, offset_gen{list_size}); + auto offset_col = cudf::make_fixed_width_column( + cudf::data_type{cudf::type_id::INT32}, num_rows + 1, cudf::mask_state::UNALLOCATED); + thrust::copy(rmm::exec_policy(stream), + offset_iter, + offset_iter + num_rows + 1, + offset_col->mutable_view().begin()); + + // list + constexpr int num_ints = num_rows * list_size; + auto value_iter = cudf::detail::make_counting_transform_iterator(0, value_gen{}); + auto value_col = cudf::make_fixed_width_column( + cudf::data_type{cudf::type_id::INT32}, num_ints, cudf::mask_state::UNALLOCATED); + thrust::copy(rmm::exec_policy(stream), + value_iter, + value_iter + num_ints, + value_col->mutable_view().begin()); + auto col1 = + cudf::make_lists_column(num_rows, + std::move(offset_col), + std::move(value_col), + 0, + cudf::create_null_mask(num_rows, cudf::mask_state::UNALLOCATED), + stream); + + // strings + constexpr int num_chars = num_rows * str_size; + auto str_offset_iter = cudf::detail::make_counting_transform_iterator(0, offset_gen{str_size}); + auto str_offset_col = cudf::make_fixed_width_column( + cudf::data_type{cudf::type_id::INT32}, num_rows + 1, cudf::mask_state::UNALLOCATED); + thrust::copy(rmm::exec_policy(stream), + str_offset_iter, + str_offset_iter + num_rows + 1, + str_offset_col->mutable_view().begin()); + auto str_iter = cudf::detail::make_counting_transform_iterator(0, char_values{}); + rmm::device_buffer str_chars(num_chars, stream); + thrust::copy(rmm::exec_policy(stream), + str_iter, + str_iter + num_chars, + static_cast(str_chars.data())); + auto col2 = + cudf::make_strings_column(num_rows, + std::move(str_offset_col), + std::move(str_chars), + 0, + cudf::create_null_mask(num_rows, cudf::mask_state::UNALLOCATED)); + + // doubles + auto double_iter = cudf::detail::make_counting_transform_iterator(0, value_gen{}); + auto col3 = cudf::make_fixed_width_column( + cudf::data_type{cudf::type_id::FLOAT64}, num_rows, cudf::mask_state::UNALLOCATED); + thrust::copy(rmm::exec_policy(stream), + double_iter, + double_iter + num_rows, + col3->mutable_view().begin()); + + auto tbl = cudf::table_view{{*col1, *col2, *col3}}; + + input_limit_test_write(test_filenames, tbl); + + // even though we have a very large limit here, there are two cases where we actually produce + // splits. + // - uncompressed data (with no dict). This happens because the code has to make a guess at how + // much + // space to reserve for compressed/uncompressed data prior to reading. It does not know that + // everything it will be reading in this case is uncompressed already, so this guess ends up + // causing it to generate two top level passes. in practice, this shouldn't matter because we + // never really see uncompressed data in the wild. + // + // - ZSTD (with no dict). In this case, ZSTD simple requires a huge amount of temporary + // space: 2.5x the total + // size of the decompressed data. so 2 GB is actually not enough to hold the whole thing at + // once. + // + // Note that in the dictionary cases, both of these revert down to 1 chunk because the + // dictionaries dramatically shrink the size of the uncompressed data. + constexpr int expected_a[] = {3, 3, 1, 1}; + input_limit_test_read(test_filenames, tbl, 0, size_t{2} * 1024 * 1024 * 1024, expected_a); + // smaller limit + constexpr int expected_b[] = {10, 11, 4, 1}; + input_limit_test_read(test_filenames, tbl, 0, 512 * 1024 * 1024, expected_b); + // include output chunking as well + constexpr int expected_c[] = {20, 21, 15, 14}; + input_limit_test_read(test_filenames, tbl, 128 * 1024 * 1024, 512 * 1024 * 1024, expected_c); } From 59199da40881cb392a0496ba89f865a5a0b0bdb1 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 24 Jan 2024 20:20:17 -0600 Subject: [PATCH 114/384] Update pre-commit hooks (#14837) This PR updates pre-commit hook versions and reformats the YAML so its spacing is more similar to the YAML format elsewhere in cudf and in other RAPIDS repos. Feel free to review this as two separate commits: a content change, and a format change. The changes outside of `.pre-commit-config.yaml` are from minor updates in `black`, which removed some extraneous blank lines, and a few fixes requested by `ruff`. Also, the newer version of nbqa in this PR supports Python 3.12. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Matthew Roeschke (https://github.com/mroeschke) - Richard (Rick) Zamora (https://github.com/rjzamora) URL: https://github.com/rapidsai/cudf/pull/14837 --- .pre-commit-config.yaml | 308 +++++++++--------- python/cudf/benchmarks/conftest.py | 3 +- python/cudf/benchmarks/internal/conftest.py | 3 +- python/cudf/cudf/_fuzz_testing/fuzzer.py | 4 +- .../cudf/cudf/core/buffer/spillable_buffer.py | 1 - python/cudf/cudf/core/dataframe.py | 3 +- python/cudf/cudf/core/df_protocol.py | 1 - python/cudf/cudf/core/resample.py | 8 +- python/cudf/cudf/core/scalar.py | 3 +- python/cudf/cudf/core/series.py | 1 - python/cudf/cudf/core/subword_tokenizer.py | 3 +- python/cudf/cudf/core/udf/strings_typing.py | 4 +- python/cudf/cudf/io/parquet.py | 4 +- python/cudf/cudf/pandas/_wrappers/pandas.py | 3 +- python/cudf/cudf/testing/testing.py | 3 +- .../test_avro_reader_fastavro_integration.py | 8 +- python/cudf/cudf/tests/test_binops.py | 9 +- python/cudf/cudf/tests/test_categorical.py | 5 - python/cudf/cudf/tests/test_column.py | 2 - python/cudf/cudf/tests/test_concat.py | 3 +- python/cudf/cudf/tests/test_cut.py | 8 +- python/cudf/cudf/tests/test_dataframe.py | 1 - python/cudf/cudf/tests/test_datetime.py | 6 - python/cudf/cudf/tests/test_dropna.py | 3 +- python/cudf/cudf/tests/test_duplicates.py | 3 +- .../cudf/tests/test_extension_compilation.py | 5 +- python/cudf/cudf/tests/test_factorize.py | 3 +- python/cudf/cudf/tests/test_groupby.py | 1 - python/cudf/cudf/tests/test_hdf.py | 3 +- python/cudf/cudf/tests/test_interval.py | 3 +- python/cudf/cudf/tests/test_joining.py | 9 +- python/cudf/cudf/tests/test_replace.py | 6 +- python/cudf/cudf/tests/test_repr.py | 3 +- python/cudf/cudf/tests/test_reshape.py | 4 +- python/cudf/cudf/tests/test_rolling.py | 5 +- python/cudf/cudf/tests/test_scalar.py | 4 +- python/cudf/cudf/tests/test_search.py | 4 +- python/cudf/cudf/tests/test_series.py | 2 - python/cudf/cudf/tests/test_sorting.py | 5 +- python/cudf/cudf/tests/test_stats.py | 5 +- python/cudf/cudf/tests/test_string.py | 6 +- python/cudf/cudf/tests/test_testing.py | 1 - python/cudf/cudf/tests/test_timedelta.py | 1 - python/cudf/cudf/tests/test_transform.py | 3 +- python/cudf/cudf/tests/test_udf_masked_ops.py | 1 - python/cudf/cudf/utils/applyutils.py | 3 +- python/cudf/cudf/utils/hash_vocab_utils.py | 5 +- python/cudf/cudf/utils/ioutils.py | 6 +- .../cudf_pandas_tests/test_cudf_pandas.py | 1 - python/custreamz/custreamz/tests/conftest.py | 3 +- python/dask_cudf/dask_cudf/backends.py | 4 +- python/dask_cudf/dask_cudf/core.py | 2 - python/dask_cudf/dask_cudf/io/parquet.py | 9 +- .../dask_cudf/dask_cudf/io/tests/test_csv.py | 3 +- .../dask_cudf/dask_cudf/io/tests/test_orc.py | 4 +- python/dask_cudf/dask_cudf/io/text.py | 3 +- .../dask_cudf/tests/test_accessor.py | 4 +- python/dask_cudf/dask_cudf/tests/test_core.py | 6 - .../dask_cudf/tests/test_reductions.py | 3 +- 59 files changed, 203 insertions(+), 327 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4fab4ddc6bd..9ac373db309 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,159 +1,159 @@ -# Copyright (c) 2019-2022, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 - hooks: - - id: trailing-whitespace - exclude: | - (?x)^( - ^python/cudf/cudf/tests/data/subword_tokenizer_data/.* - ) - - id: end-of-file-fixer - exclude: | - (?x)^( - ^python/cudf/cudf/tests/data/subword_tokenizer_data/.* - ) - - repo: https://github.com/PyCQA/isort - rev: 5.12.0 - hooks: - - id: isort - # Use the config file specific to each subproject so that each - # project can specify its own first/third-party packages. - args: ["--config-root=python/", "--resolve-all-configs"] - files: python/.* - types_or: [python, cython, pyi] - - repo: https://github.com/psf/black - rev: 22.3.0 - hooks: - - id: black - files: python/.* - # Explicitly specify the pyproject.toml at the repo root, not per-project. - args: ["--config", "pyproject.toml"] - - repo: https://github.com/MarcoGorelli/cython-lint - rev: v0.15.0 - hooks: - - id: cython-lint - - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v1.3.0' - hooks: - - id: mypy - additional_dependencies: [types-cachetools] - args: ["--config-file=pyproject.toml", - "python/cudf/cudf", - "python/custreamz/custreamz", - "python/cudf_kafka/cudf_kafka", - "python/dask_cudf/dask_cudf"] - pass_filenames: false - - repo: https://github.com/PyCQA/pydocstyle - rev: 6.1.1 - hooks: - - id: pydocstyle - # https://github.com/PyCQA/pydocstyle/issues/603 - additional_dependencies: [toml] - args: ["--config=pyproject.toml"] - exclude: | - (?x)^( - ^python/cudf/cudf/pandas/scripts/.*| - ^python/cudf/cudf_pandas_tests/.* - ) - - repo: https://github.com/nbQA-dev/nbQA - rev: 1.6.3 - hooks: - - id: nbqa-isort - # Use the cudf_kafka isort orderings in notebooks so that dask - # and RAPIDS packages have their own sections. - args: ["--settings-file=python/cudf_kafka/pyproject.toml"] - - id: nbqa-black - # Explicitly specify the pyproject.toml at the repo root, not per-project. - args: ["--config=pyproject.toml"] - - repo: https://github.com/pre-commit/mirrors-clang-format - rev: v16.0.6 - hooks: - - id: clang-format - types_or: [c, c++, cuda] - args: ["-fallback-style=none", "-style=file", "-i"] - - repo: https://github.com/sirosen/texthooks - rev: 0.4.0 - hooks: - - id: fix-smartquotes - exclude: | - (?x)^( - ^cpp/include/cudf_test/cxxopts.hpp| - ^python/cudf/cudf/tests/data/subword_tokenizer_data/.*| - ^python/cudf/cudf/tests/text/test_text_methods.py - ) - - repo: local - hooks: - - id: no-deprecationwarning - name: no-deprecationwarning - description: 'Enforce that DeprecationWarning is not introduced (use FutureWarning instead)' - entry: '(category=|\s)DeprecationWarning[,)]' - language: pygrep - types_or: [python, cython] - - id: no-programmatic-xfail - name: no-programmatic-xfail - description: 'Enforce that pytest.xfail is not introduced (see dev docs for details)' - entry: 'pytest\.xfail' - language: pygrep - types: [python] - - id: cmake-format - name: cmake-format - entry: ./cpp/scripts/run-cmake-format.sh cmake-format - language: python - types: [cmake] - # Note that pre-commit autoupdate does not update the versions - # of dependencies, so we'll have to update this manually. - additional_dependencies: - - cmakelang==0.6.13 - verbose: true - require_serial: true - - id: cmake-lint - name: cmake-lint - entry: ./cpp/scripts/run-cmake-format.sh cmake-lint - language: python - types: [cmake] - # Note that pre-commit autoupdate does not update the versions - # of dependencies, so we'll have to update this manually. - additional_dependencies: - - cmakelang==0.6.13 - verbose: true - require_serial: true - - id: copyright-check - name: copyright-check - entry: python ./ci/checks/copyright.py --git-modified-only --update-current-year - language: python - pass_filenames: false - additional_dependencies: [gitpython] - - id: doxygen-check - name: doxygen-check - entry: ./ci/checks/doxygen.sh - files: ^cpp/include/ - types_or: [file] - language: system - pass_filenames: false - verbose: true - - repo: https://github.com/codespell-project/codespell - rev: v2.2.2 - hooks: - - id: codespell - additional_dependencies: [tomli] - args: ["--toml", "pyproject.toml"] - exclude: | - (?x)^( - .*test.*| - ^CHANGELOG.md$ - ) - - repo: https://github.com/rapidsai/dependency-file-generator - rev: v1.8.0 - hooks: - - id: rapids-dependency-file-generator - args: ["--clean"] - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.278 - hooks: - - id: ruff - files: python/.*$ + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + exclude: | + (?x)^( + ^python/cudf/cudf/tests/data/subword_tokenizer_data/.* + ) + - id: end-of-file-fixer + exclude: | + (?x)^( + ^python/cudf/cudf/tests/data/subword_tokenizer_data/.* + ) + - repo: https://github.com/PyCQA/isort + rev: 5.13.2 + hooks: + - id: isort + # Use the config file specific to each subproject so that each + # project can specify its own first/third-party packages. + args: ["--config-root=python/", "--resolve-all-configs"] + files: python/.* + types_or: [python, cython, pyi] + - repo: https://github.com/psf/black + rev: 23.12.1 + hooks: + - id: black + files: python/.* + # Explicitly specify the pyproject.toml at the repo root, not per-project. + args: ["--config", "pyproject.toml"] + - repo: https://github.com/MarcoGorelli/cython-lint + rev: v0.16.0 + hooks: + - id: cython-lint + - repo: https://github.com/pre-commit/mirrors-mypy + rev: 'v1.3.0' + hooks: + - id: mypy + additional_dependencies: [types-cachetools] + args: ["--config-file=pyproject.toml", + "python/cudf/cudf", + "python/custreamz/custreamz", + "python/cudf_kafka/cudf_kafka", + "python/dask_cudf/dask_cudf"] + pass_filenames: false + - repo: https://github.com/PyCQA/pydocstyle + rev: 6.3.0 + hooks: + - id: pydocstyle + # https://github.com/PyCQA/pydocstyle/issues/603 + additional_dependencies: [tomli] + args: ["--config=pyproject.toml"] + exclude: | + (?x)^( + ^python/cudf/cudf/pandas/scripts/.*| + ^python/cudf/cudf_pandas_tests/.* + ) + - repo: https://github.com/nbQA-dev/nbQA + rev: 1.7.1 + hooks: + - id: nbqa-isort + # Use the cudf_kafka isort orderings in notebooks so that dask + # and RAPIDS packages have their own sections. + args: ["--settings-file=python/cudf_kafka/pyproject.toml"] + - id: nbqa-black + # Explicitly specify the pyproject.toml at the repo root, not per-project. + args: ["--config=pyproject.toml"] + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v16.0.6 + hooks: + - id: clang-format + types_or: [c, c++, cuda] + args: ["-fallback-style=none", "-style=file", "-i"] + - repo: https://github.com/sirosen/texthooks + rev: 0.6.3 + hooks: + - id: fix-smartquotes + exclude: | + (?x)^( + ^cpp/include/cudf_test/cxxopts.hpp| + ^python/cudf/cudf/tests/data/subword_tokenizer_data/.*| + ^python/cudf/cudf/tests/text/test_text_methods.py + ) + - repo: local + hooks: + - id: no-deprecationwarning + name: no-deprecationwarning + description: 'Enforce that DeprecationWarning is not introduced (use FutureWarning instead)' + entry: '(category=|\s)DeprecationWarning[,)]' + language: pygrep + types_or: [python, cython] + - id: no-programmatic-xfail + name: no-programmatic-xfail + description: 'Enforce that pytest.xfail is not introduced (see dev docs for details)' + entry: 'pytest\.xfail' + language: pygrep + types: [python] + - id: cmake-format + name: cmake-format + entry: ./cpp/scripts/run-cmake-format.sh cmake-format + language: python + types: [cmake] + # Note that pre-commit autoupdate does not update the versions + # of dependencies, so we'll have to update this manually. + additional_dependencies: + - cmakelang==0.6.13 + verbose: true + require_serial: true + - id: cmake-lint + name: cmake-lint + entry: ./cpp/scripts/run-cmake-format.sh cmake-lint + language: python + types: [cmake] + # Note that pre-commit autoupdate does not update the versions + # of dependencies, so we'll have to update this manually. + additional_dependencies: + - cmakelang==0.6.13 + verbose: true + require_serial: true + - id: copyright-check + name: copyright-check + entry: python ./ci/checks/copyright.py --git-modified-only --update-current-year + language: python + pass_filenames: false + additional_dependencies: [gitpython] + - id: doxygen-check + name: doxygen-check + entry: ./ci/checks/doxygen.sh + files: ^cpp/include/ + types_or: [file] + language: system + pass_filenames: false + verbose: true + - repo: https://github.com/codespell-project/codespell + rev: v2.2.2 + hooks: + - id: codespell + additional_dependencies: [tomli] + args: ["--toml", "pyproject.toml"] + exclude: | + (?x)^( + .*test.*| + ^CHANGELOG.md$ + ) + - repo: https://github.com/rapidsai/dependency-file-generator + rev: v1.8.0 + hooks: + - id: rapids-dependency-file-generator + args: ["--clean"] + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.13 + hooks: + - id: ruff + files: python/.*$ default_language_version: diff --git a/python/cudf/benchmarks/conftest.py b/python/cudf/benchmarks/conftest.py index 4f2bb96061f..a70d2329625 100644 --- a/python/cudf/benchmarks/conftest.py +++ b/python/cudf/benchmarks/conftest.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. """Defines pytest fixtures for all benchmarks. @@ -206,7 +206,6 @@ def default_union_id(val): (r"_rows_\d+", ""), (r"_cols_\d+", ""), ]: - collapse_fixtures(fixtures, pat, repl, globals(), idfunc) num_new_fixtures = len(fixtures) - num_fixtures diff --git a/python/cudf/benchmarks/internal/conftest.py b/python/cudf/benchmarks/internal/conftest.py index 7351f1d1427..a710cf61753 100644 --- a/python/cudf/benchmarks/internal/conftest.py +++ b/python/cudf/benchmarks/internal/conftest.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. """Defines pytest fixtures for internal benchmarks.""" @@ -50,7 +50,6 @@ def column_nulls_true(request, nr=nr): ("_nulls_(true|false)", ""), (r"_rows_\d+", ""), ]: - collapse_fixtures(fixtures, pat, repl, globals()) num_new_fixtures = len(fixtures) - num_fixtures diff --git a/python/cudf/cudf/_fuzz_testing/fuzzer.py b/python/cudf/cudf/_fuzz_testing/fuzzer.py index 59d6f198681..ee1b2c1f1c4 100644 --- a/python/cudf/cudf/_fuzz_testing/fuzzer.py +++ b/python/cudf/cudf/_fuzz_testing/fuzzer.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. import datetime import json @@ -31,7 +31,6 @@ def __init__( max_lists_length=None, max_lists_nesting_depth=None, ): - self._target = target self._dirs = [] if dirs is None else dirs self._crash_dir = crash_reports_dir @@ -86,7 +85,6 @@ def write_crash(self, error): self._data_handler.write_data(error_file_name) def start(self): - while True: logging.info(f"Running test {self._total_executions}") file_name = self._data_handler.generate_input() diff --git a/python/cudf/cudf/core/buffer/spillable_buffer.py b/python/cudf/cudf/core/buffer/spillable_buffer.py index aeac4b76e58..b25af13679c 100644 --- a/python/cudf/cudf/core/buffer/spillable_buffer.py +++ b/python/cudf/cudf/core/buffer/spillable_buffer.py @@ -226,7 +226,6 @@ def spill(self, target: str = "cpu") -> None: color=_get_color_for_nvtx("SpillHtoD"), domain="cudf_python-spill", ): - dev_mem = rmm.DeviceBuffer.to_device( self._ptr_desc.pop("memoryview") ) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 7c48352d861..c61fd54db29 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -3517,7 +3517,7 @@ def rename( if index: if ( - any(type(item) == str for item in index.values()) + any(isinstance(item, str) for item in index.values()) and type(self.index) != cudf.StringIndex ): raise NotImplementedError( @@ -5274,7 +5274,6 @@ def from_pandas(cls, dataframe, nan_as_null=no_default): ) if isinstance(dataframe, pd.DataFrame): - if not dataframe.columns.is_unique: raise ValueError("Duplicate column names are not allowed") diff --git a/python/cudf/cudf/core/df_protocol.py b/python/cudf/cudf/core/df_protocol.py index c97d6dcdd2d..62ded8ac6f1 100644 --- a/python/cudf/cudf/core/df_protocol.py +++ b/python/cudf/cudf/core/df_protocol.py @@ -792,7 +792,6 @@ def _set_missing_values( cudf_col: cudf.core.column.ColumnBase, allow_copy: bool, ) -> cudf.core.column.ColumnBase: - valid_mask = protocol_col.get_buffers()["validity"] if valid_mask is not None: null, invalid = protocol_col.describe_null diff --git a/python/cudf/cudf/core/resample.py b/python/cudf/cudf/core/resample.py index 0226c778da3..5b0df97de71 100644 --- a/python/cudf/cudf/core/resample.py +++ b/python/cudf/cudf/core/resample.py @@ -1,6 +1,6 @@ -# SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & -# AFFILIATES. All rights reserved. SPDX-License-Identifier: -# Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -33,7 +33,6 @@ class _Resampler(GroupBy): - grouping: "_ResampleGrouping" def __init__(self, obj, by, axis=None, kind=None): @@ -118,7 +117,6 @@ class SeriesResampler(_Resampler, SeriesGroupBy): class _ResampleGrouping(_Grouping): - bin_labels: cudf.core.index.Index def __init__(self, obj, by=None, level=None): diff --git a/python/cudf/cudf/core/scalar.py b/python/cudf/cudf/core/scalar.py index a20628f6601..f7d05e53ce7 100644 --- a/python/cudf/cudf/core/scalar.py +++ b/python/cudf/cudf/core/scalar.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. import decimal import operator @@ -114,7 +114,6 @@ class Scalar(BinaryOperand, metaclass=CachedScalarInstanceMeta): _VALID_BINARY_OPERATIONS = BinaryOperand._SUPPORTED_BINARY_OPERATIONS def __init__(self, value, dtype=None): - self._host_value = None self._host_dtype = None self._device_value = None diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 7e25713e63c..d7249d1a781 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -201,7 +201,6 @@ def __getitem__(self, arg): @_cudf_nvtx_annotate def __setitem__(self, key, value): - if isinstance(key, tuple): key = list(key) diff --git a/python/cudf/cudf/core/subword_tokenizer.py b/python/cudf/cudf/core/subword_tokenizer.py index 821afa2ebe2..24c49e3662a 100644 --- a/python/cudf/cudf/core/subword_tokenizer.py +++ b/python/cudf/cudf/core/subword_tokenizer.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2023, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. from __future__ import annotations @@ -50,7 +50,6 @@ class SubwordTokenizer: """ def __init__(self, hash_file: str, do_lower_case: bool = True): - self.do_lower_case = do_lower_case self.vocab_file = cpp_hashed_vocabulary(hash_file) diff --git a/python/cudf/cudf/core/udf/strings_typing.py b/python/cudf/cudf/core/udf/strings_typing.py index 50d34be40a0..43604ab21a7 100644 --- a/python/cudf/cudf/core/udf/strings_typing.py +++ b/python/cudf/cudf/core/udf/strings_typing.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. import operator @@ -17,7 +17,6 @@ # String object definitions class UDFString(types.Type): - np_dtype = np.dtype("object") def __init__(self): @@ -29,7 +28,6 @@ def return_type(self): class StringView(types.Type): - np_dtype = np.dtype("object") def __init__(self): diff --git a/python/cudf/cudf/io/parquet.py b/python/cudf/cudf/io/parquet.py index bcc24a85cf9..bac919182c0 100644 --- a/python/cudf/cudf/io/parquet.py +++ b/python/cudf/cudf/io/parquet.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2023, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. from __future__ import annotations import itertools @@ -1259,7 +1259,7 @@ def write_table(self, df): """ Write a dataframe to the file/dataset """ - (part_names, grouped_df, part_offsets,) = _get_groups_and_offsets( + part_names, grouped_df, part_offsets = _get_groups_and_offsets( df=df, partition_cols=self.partition_cols, preserve_index=self.common_args["index"], diff --git a/python/cudf/cudf/pandas/_wrappers/pandas.py b/python/cudf/cudf/pandas/_wrappers/pandas.py index 5ea2af7d002..afcfc13a9c4 100644 --- a/python/cudf/cudf/pandas/_wrappers/pandas.py +++ b/python/cudf/cudf/pandas/_wrappers/pandas.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. # All rights reserved. # SPDX-License-Identifier: Apache-2.0 import copyreg @@ -1323,6 +1323,7 @@ def _df_query_method(self, *args, local_dict=None, global_dict=None, **kwargs): typ, ) + # timestamps and timedeltas are not proxied, but non-proxied # pandas types are currently not picklable. Thus, we define # custom reducer/unpicker functions for these types: diff --git a/python/cudf/cudf/testing/testing.py b/python/cudf/cudf/testing/testing.py index 6c2f073b7ac..39fdac0f71a 100644 --- a/python/cudf/cudf/testing/testing.py +++ b/python/cudf/cudf/testing/testing.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from __future__ import annotations @@ -47,7 +47,6 @@ def _check_isinstance(left, right, obj): def raise_assert_detail(obj, message, left, right, diff=None): - msg = f"""{obj} are different {message} diff --git a/python/cudf/cudf/tests/test_avro_reader_fastavro_integration.py b/python/cudf/cudf/tests/test_avro_reader_fastavro_integration.py index 2272231fec1..0e38b10ed52 100644 --- a/python/cudf/cudf/tests/test_avro_reader_fastavro_integration.py +++ b/python/cudf/cudf/tests/test_avro_reader_fastavro_integration.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2023, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -125,7 +125,6 @@ def test_can_detect_dtype_from_avro_type_nested( ], ) def test_can_parse_single_value(avro_type, cudf_type, avro_val, cudf_val): - schema_root = { "name": "root", "type": "record", @@ -147,7 +146,6 @@ def test_can_parse_single_value(avro_type, cudf_type, avro_val, cudf_val): @pytest.mark.parametrize("avro_type, cudf_type", avro_type_params) def test_can_parse_single_null(avro_type, cudf_type): - schema_root = { "name": "root", "type": "record", @@ -167,7 +165,6 @@ def test_can_parse_single_null(avro_type, cudf_type): @pytest.mark.parametrize("avro_type, cudf_type", avro_type_params) def test_can_parse_no_data(avro_type, cudf_type): - schema_root = { "name": "root", "type": "record", @@ -188,7 +185,6 @@ def test_can_parse_no_data(avro_type, cudf_type): ) @pytest.mark.parametrize("avro_type, cudf_type", avro_type_params) def test_can_parse_no_fields(avro_type, cudf_type): - schema_root = { "name": "root", "type": "record", @@ -205,7 +201,6 @@ def test_can_parse_no_fields(avro_type, cudf_type): def test_can_parse_no_schema(): - schema_root = None records = [] actual = cudf_from_avro_util(schema_root, records) @@ -307,7 +302,6 @@ def get_days_from_epoch(date: Optional[datetime.date]) -> Optional[int]: @pytest.mark.parametrize("nullable", [True, False]) @pytest.mark.parametrize("prepend_null", [True, False]) def test_can_parse_avro_date_logical_type(namespace, nullable, prepend_null): - avro_type = {"logicalType": "date", "type": "int"} if nullable: if prepend_null: diff --git a/python/cudf/cudf/tests/test_binops.py b/python/cudf/cudf/tests/test_binops.py index cd3e8f75950..9de7dac652c 100644 --- a/python/cudf/cudf/tests/test_binops.py +++ b/python/cudf/cudf/tests/test_binops.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2023, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. import decimal import operator @@ -605,7 +605,6 @@ def test_series_reflected_ops_cudf_scalar(funcs, dtype, obj_class): @pytest.mark.parametrize("binop", _binops) def test_different_shapes_and_columns(binop): - # TODO: support `pow()` on NaN values. Particularly, the cases: # `pow(1, NaN) == 1` and `pow(NaN, 0) == 1` if binop is operator.pow: @@ -639,7 +638,6 @@ def test_different_shapes_and_columns(binop): @pytest.mark.parametrize("binop", _binops) def test_different_shapes_and_same_columns(binop): - # TODO: support `pow()` on NaN values. Particularly, the cases: # `pow(1, NaN) == 1` and `pow(NaN, 0) == 1` if binop is operator.pow: @@ -658,7 +656,6 @@ def test_different_shapes_and_same_columns(binop): @pytest.mark.parametrize("binop", _binops) def test_different_shapes_and_columns_with_unaligned_indices(binop): - # TODO: support `pow()` on NaN values. Particularly, the cases: # `pow(1, NaN) == 1` and `pow(NaN, 0) == 1` if binop is operator.pow: @@ -791,7 +788,6 @@ def test_operator_func_series_and_scalar( def test_operator_func_between_series_logical( dtype, func, scalar_a, scalar_b, fill_value ): - gdf_series_a = Series([scalar_a], nan_as_null=False).astype(dtype) gdf_series_b = Series([scalar_b], nan_as_null=False).astype(dtype) @@ -1787,7 +1783,6 @@ def test_datetime_dateoffset_binaryop( ) @pytest.mark.parametrize("op", [operator.add, operator.sub]) def test_datetime_dateoffset_binaryop_multiple(date_col, kwargs, op): - gsr = cudf.Series(date_col, dtype="datetime64[ns]") psr = gsr.to_pandas() @@ -2294,7 +2289,6 @@ def test_binops_with_NA_consistent(dtype, op): ], ) def test_binops_decimal(op, lhs, l_dtype, rhs, r_dtype, expect, expect_dtype): - if isinstance(lhs, (int, float)): a = cudf.Scalar(lhs, l_dtype) else: @@ -2358,7 +2352,6 @@ def test_binops_decimal(op, lhs, l_dtype, rhs, r_dtype, expect, expect_dtype): def test_binops_reflect_decimal( op, lhs, l_dtype, rhs, r_dtype, expect, expect_dtype ): - a = utils._decimal_series(lhs, l_dtype) b = utils._decimal_series(rhs, r_dtype) expect = utils._decimal_series(expect, expect_dtype) diff --git a/python/cudf/cudf/tests/test_categorical.py b/python/cudf/cudf/tests/test_categorical.py index 52b7236b965..52c50ec58a8 100644 --- a/python/cudf/cudf/tests/test_categorical.py +++ b/python/cudf/cudf/tests/test_categorical.py @@ -364,7 +364,6 @@ def test_categorical_set_categories_preserves_order(): @pytest.mark.parametrize("inplace", [True, False]) def test_categorical_as_ordered(pd_str_cat, inplace): - pd_sr = pd.Series(pd_str_cat.copy().set_ordered(False)) cd_sr = cudf.Series(pd_str_cat.copy().set_ordered(False)) @@ -388,7 +387,6 @@ def test_categorical_as_ordered(pd_str_cat, inplace): @pytest.mark.parametrize("inplace", [True, False]) def test_categorical_as_unordered(pd_str_cat, inplace): - pd_sr = pd.Series(pd_str_cat.copy().set_ordered(True)) cd_sr = cudf.Series(pd_str_cat.copy().set_ordered(True)) @@ -428,7 +426,6 @@ def test_categorical_as_unordered(pd_str_cat, inplace): def test_categorical_reorder_categories( pd_str_cat, from_ordered, to_ordered, inplace ): - pd_sr = pd.Series(pd_str_cat.copy().set_ordered(from_ordered)) cd_sr = cudf.Series(pd_str_cat.copy().set_ordered(from_ordered)) @@ -469,7 +466,6 @@ def test_categorical_reorder_categories( ], ) def test_categorical_add_categories(pd_str_cat, inplace): - pd_sr = pd.Series(pd_str_cat.copy()) cd_sr = cudf.Series(pd_str_cat.copy()) @@ -510,7 +506,6 @@ def test_categorical_add_categories(pd_str_cat, inplace): ], ) def test_categorical_remove_categories(pd_str_cat, inplace): - pd_sr = pd.Series(pd_str_cat.copy()) cd_sr = cudf.Series(pd_str_cat.copy()) diff --git a/python/cudf/cudf/tests/test_column.py b/python/cudf/cudf/tests/test_column.py index 3d21994a8d5..c3623f495c0 100644 --- a/python/cudf/cudf/tests/test_column.py +++ b/python/cudf/cudf/tests/test_column.py @@ -333,7 +333,6 @@ def test_column_view_valid_string_to_numeric(data, to_dtype): def test_column_view_nulls_widths_even(): - data = [1, 2, None, 4, None] expect_data = [ np.int32(val).view("float32") if val is not None else np.nan @@ -361,7 +360,6 @@ def test_column_view_nulls_widths_even(): @pytest.mark.parametrize("slc", [slice(1, 5), slice(0, 4), slice(2, 4)]) def test_column_view_numeric_slice(slc): - data = np.array([1, 2, 3, 4, 5], dtype="int32") sr = cudf.Series(data) diff --git a/python/cudf/cudf/tests/test_concat.py b/python/cudf/cudf/tests/test_concat.py index df743a96759..466455eb48c 100644 --- a/python/cudf/cudf/tests/test_concat.py +++ b/python/cudf/cudf/tests/test_concat.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2023, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. from decimal import Decimal @@ -1057,7 +1057,6 @@ def test_concat_join_no_overlapping_columns_many_and_empty2( def test_concat_join_no_overlapping_columns_empty_df_basic( ignore_index, sort, join, axis ): - pdf6 = pd.DataFrame( { "x": range(10), diff --git a/python/cudf/cudf/tests/test_cut.py b/python/cudf/cudf/tests/test_cut.py index 02e48f2639b..24c1eaa8f02 100644 --- a/python/cudf/cudf/tests/test_cut.py +++ b/python/cudf/cudf/tests/test_cut.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. """ Test related to Cut @@ -60,7 +60,6 @@ def test_cut_basic(x, bins, right, include_lowest, ordered, precision): def test_cut_labels( x, bins, right, include_lowest, ordered, precision, labels ): - pcat = pd.cut( x=x, bins=bins, @@ -98,7 +97,6 @@ def test_cut_labels( def test_cut_labels_non_unique( x, bins, right, include_lowest, ordered, precision, labels ): - pcat = pd.cut( x=x, bins=bins, @@ -138,7 +136,6 @@ def test_cut_labels_non_unique( @pytest.mark.parametrize("right", [True, False]) @pytest.mark.parametrize("precision", [3]) def test_cut_right(x, bins, right, precision): - pcat = pd.cut( x=x, bins=bins, @@ -177,7 +174,6 @@ def test_cut_right(x, bins, right, precision): def test_cut_drop_duplicates( x, bins, right, precision, duplicates, ordered, include_lowest ): - pcat = pd.cut( x=x, bins=bins, @@ -264,7 +260,6 @@ def test_cut_drop_duplicates_raises( @pytest.mark.parametrize("precision", [1, 2, 3]) @pytest.mark.parametrize("duplicates", ["drop", "raise"]) def test_cut_intervalindex_bin(x, bins, right, precision, duplicates): - pcat = pd.cut( x=x, bins=bins, @@ -294,7 +289,6 @@ def test_cut_intervalindex_bin(x, bins, right, precision, duplicates): @pytest.mark.parametrize("ordered", [True]) @pytest.mark.parametrize("precision", [3]) def test_cut_series(x, bins, right, include_lowest, ordered, precision): - pcat = pd.cut( x=x, bins=bins, diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 37c115a47d9..0664e7991b5 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -4004,7 +4004,6 @@ def test_diff(dtype, period, data_empty): @pytest.mark.parametrize("df", _dataframe_na_data()) @pytest.mark.parametrize("nan_as_null", [True, False, None]) def test_dataframe_isnull_isna(df, nan_as_null): - if nan_as_null is False and ( df.select_dtypes(object).isna().any().any() and not df.select_dtypes(object).isna().all().all() diff --git a/python/cudf/cudf/tests/test_datetime.py b/python/cudf/cudf/tests/test_datetime.py index deddedbe3e8..ab1fb2eedd5 100644 --- a/python/cudf/cudf/tests/test_datetime.py +++ b/python/cudf/cudf/tests/test_datetime.py @@ -702,7 +702,6 @@ def test_to_datetime_errors(data): def test_to_datetime_not_implemented(): - with pytest.raises(NotImplementedError): cudf.to_datetime([], exact=False) @@ -815,7 +814,6 @@ def test_to_datetime_different_formats_notimplemented(): def test_datetime_can_cast_safely(): - sr = cudf.Series( ["1679-01-01", "2000-01-31", "2261-01-01"], dtype="datetime64[ms]" ) @@ -933,7 +931,6 @@ def test_str_to_datetime_error(): @pytest.mark.parametrize("data_dtype", DATETIME_TYPES) @pytest.mark.parametrize("other_dtype", DATETIME_TYPES) def test_datetime_subtract(data, other, data_dtype, other_dtype): - gsr = cudf.Series(data, dtype=data_dtype) psr = gsr.to_pandas() @@ -1985,7 +1982,6 @@ def test_error_values(): "resolution", ["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"] ) def test_ceil(data, time_type, resolution): - gs = cudf.Series(data, dtype=time_type) ps = gs.to_pandas() @@ -2016,7 +2012,6 @@ def test_ceil(data, time_type, resolution): "resolution", ["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"] ) def test_floor(data, time_type, resolution): - gs = cudf.Series(data, dtype=time_type) ps = gs.to_pandas() @@ -2047,7 +2042,6 @@ def test_floor(data, time_type, resolution): "resolution", ["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"] ) def test_round(data, time_type, resolution): - gs = cudf.Series(data, dtype=time_type) ps = gs.to_pandas() diff --git a/python/cudf/cudf/tests/test_dropna.py b/python/cudf/cudf/tests/test_dropna.py index d53d24cd6c6..ac104b7e513 100644 --- a/python/cudf/cudf/tests/test_dropna.py +++ b/python/cudf/cudf/tests/test_dropna.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. import numpy as np import pandas as pd @@ -24,7 +24,6 @@ @pytest.mark.parametrize("nulls", ["one", "some", "all", "none"]) @pytest.mark.parametrize("inplace", [True, False]) def test_dropna_series(data, nulls, inplace): - psr = _create_pandas_series_float64_default(data) if len(data) > 0: diff --git a/python/cudf/cudf/tests/test_duplicates.py b/python/cudf/cudf/tests/test_duplicates.py index ddbfdf5eee2..ad513ea3cd5 100644 --- a/python/cudf/cudf/tests/test_duplicates.py +++ b/python/cudf/cudf/tests/test_duplicates.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. import itertools import random @@ -386,7 +386,6 @@ def test_dataframe_drop_duplicates_method(): def test_datetime_drop_duplicates(): - date_df = cudf.DataFrame() date_df["date"] = pd.date_range("11/20/2018", periods=6, freq="D") date_df["value"] = np.random.sample(len(date_df)) diff --git a/python/cudf/cudf/tests/test_extension_compilation.py b/python/cudf/cudf/tests/test_extension_compilation.py index 857cc114ffa..4272c70f898 100644 --- a/python/cudf/cudf/tests/test_extension_compilation.py +++ b/python/cudf/cudf/tests/test_extension_compilation.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2023, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. import operator import cupy as cp @@ -198,7 +198,6 @@ def func_na_is_x(x): @pytest.mark.parametrize("fn", (func_x_is_na, func_na_is_x)) def test_is_na(fn): - valid = Masked(1, True) invalid = Masked(1, False) @@ -288,7 +287,6 @@ def func_na_le(x): @pytest.mark.parametrize("fn", na_comparison_funcs) @pytest.mark.parametrize("ty", number_types, ids=number_ids) def test_na_masked_comparisons(fn, ty): - device_fn = cuda.jit(device=True)(fn) @cuda.jit @@ -317,7 +315,6 @@ def test_kernel(err): @pytest.mark.parametrize("fn", na_comparison_funcs) @pytest.mark.parametrize("ty", number_types, ids=number_ids) def test_na_scalar_comparisons(fn, ty): - device_fn = cuda.jit(device=True)(fn) @cuda.jit diff --git a/python/cudf/cudf/tests/test_factorize.py b/python/cudf/cudf/tests/test_factorize.py index bf409b30090..f8782681f62 100644 --- a/python/cudf/cudf/tests/test_factorize.py +++ b/python/cudf/cudf/tests/test_factorize.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2023, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. import cupy as cp import numpy as np @@ -124,7 +124,6 @@ def test_cudf_factorize_array(): @pytest.mark.parametrize("pandas_compatibility", [True, False]) def test_factorize_code_pandas_compatibility(pandas_compatibility): - psr = pd.Series([1, 2, 3, 4, 5]) gsr = cudf.from_pandas(psr) diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index b46949faa06..b757f8acb6e 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -625,7 +625,6 @@ def func(group): ], ) def test_groupby_apply_jit_correlation(dataset, groupby_jit_datasets, dtype): - dataset = groupby_jit_datasets[dataset] dataset["val1"] = dataset["val1"].astype(dtype) diff --git a/python/cudf/cudf/tests/test_hdf.py b/python/cudf/cudf/tests/test_hdf.py index 71c94858cfe..063fffd948b 100644 --- a/python/cudf/cudf/tests/test_hdf.py +++ b/python/cudf/cudf/tests/test_hdf.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2023, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. import os from string import ascii_letters @@ -96,7 +96,6 @@ def test_hdf_reader(hdf_files, columns): ) for column in hdf_series.keys(): - expect_series = pd.read_hdf(hdf_series[column]) got_series = cudf.read_hdf(hdf_series[column]) diff --git a/python/cudf/cudf/tests/test_interval.py b/python/cudf/cudf/tests/test_interval.py index a27de60c2c5..ef853a23004 100644 --- a/python/cudf/cudf/tests/test_interval.py +++ b/python/cudf/cudf/tests/test_interval.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. import numpy as np @@ -16,7 +16,6 @@ @pytest.mark.parametrize("data3, data4", [(6, 10), (5.0, 9.0), (2, 6.0)]) @pytest.mark.parametrize("closed", ["left", "right", "both", "neither"]) def test_create_interval_series(data1, data2, data3, data4, closed): - expect = pd.Series(pd.Interval(data1, data2, closed), dtype="interval") got = cudf.Series(pd.Interval(data1, data2, closed), dtype="interval") assert_eq(expect, got) diff --git a/python/cudf/cudf/tests/test_joining.py b/python/cudf/cudf/tests/test_joining.py index 9c9c99a0cfa..ece676329bc 100644 --- a/python/cudf/cudf/tests/test_joining.py +++ b/python/cudf/cudf/tests/test_joining.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2023, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. from itertools import combinations, product, repeat @@ -502,7 +502,6 @@ def test_dataframe_pairs_of_triples(pairs, max, rows, how): def test_safe_merging_with_left_empty(): - np.random.seed(0) pairs = ("bcd", "b") @@ -910,7 +909,6 @@ def test_join_multi(how, column_a, column_b, column_c): ], ) def test_merge_multi(kwargs): - left = cudf.DataFrame( { "a": [1, 2, 3, 4, 3, 5, 6], @@ -1072,7 +1070,6 @@ def test_typecast_on_join_mixed_int_float(dtype_l, dtype_r): def test_typecast_on_join_no_float_round(): - other_data = ["a", "b", "c", "d", "e"] join_data_l = cudf.Series([1, 2, 3, 4, 5], dtype="int8") @@ -1530,7 +1527,6 @@ def test_categorical_typecast_outer(): @pytest.mark.parametrize("dtype", NUMERIC_TYPES + ["object"]) def test_categorical_typecast_inner_one_cat(dtype): - data = np.array([1, 2, 3], dtype=dtype) left = make_categorical_dataframe(data) @@ -1542,7 +1538,6 @@ def test_categorical_typecast_inner_one_cat(dtype): @pytest.mark.parametrize("dtype", NUMERIC_TYPES + ["object"]) def test_categorical_typecast_left_one_cat(dtype): - data = np.array([1, 2, 3], dtype=dtype) left = make_categorical_dataframe(data) @@ -1554,7 +1549,6 @@ def test_categorical_typecast_left_one_cat(dtype): @pytest.mark.parametrize("dtype", NUMERIC_TYPES + ["object"]) def test_categorical_typecast_outer_one_cat(dtype): - data = np.array([1, 2, 3], dtype=dtype) left = make_categorical_dataframe(data) @@ -1810,7 +1804,6 @@ def test_typecast_on_join_indexes_matching_categorical(): ], ) def test_series_dataframe_mixed_merging(lhs, rhs, how, kwargs): - if how in ("leftsemi", "leftanti") and ( kwargs.get("left_index") or kwargs.get("right_index") ): diff --git a/python/cudf/cudf/tests/test_replace.py b/python/cudf/cudf/tests/test_replace.py index 13e44e7cf59..e52bbe54072 100644 --- a/python/cudf/cudf/tests/test_replace.py +++ b/python/cudf/cudf/tests/test_replace.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. import re from decimal import Decimal @@ -863,7 +863,6 @@ def test_dataframe_with_nulls_where_with_scalars(fill_value): def test_dataframe_with_different_types(): - # Testing for int and float pdf = pd.DataFrame( {"A": [111, 22, 31, 410, 56], "B": [-10.12, 121.2, 45.7, 98.4, 87.6]} @@ -963,7 +962,6 @@ def test_numeric_series_replace_dtype(series_dtype, replacement): # to_replace is a list, replacement is a scalar if not can_replace: with pytest.raises(TypeError): - sr.replace([2, 3], replacement) else: expect = psr.replace([2, 3], replacement).astype(psr.dtype) @@ -1168,7 +1166,6 @@ def test_series_clip(data, lower, upper, inplace): def test_series_exceptions_for_clip(): - with pytest.raises(ValueError): cudf.Series([1, 2, 3, 4]).clip([1, 2], [2, 3]) @@ -1331,7 +1328,6 @@ def test_series_replace_errors(): ], ) def test_replace_nulls(gsr, old, new, expected): - actual = gsr.replace(old, new) assert_eq( expected.sort_values().reset_index(drop=True), diff --git a/python/cudf/cudf/tests/test_repr.py b/python/cudf/cudf/tests/test_repr.py index a36cc1b3819..efc738eec1f 100644 --- a/python/cudf/cudf/tests/test_repr.py +++ b/python/cudf/cudf/tests/test_repr.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2023, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. import textwrap @@ -382,7 +382,6 @@ def test_dataframe_sliced(gdf, slice, max_seq_items, max_rows): ], ) def test_generic_index_null(index, expected_repr): - actual_repr = repr(index) assert expected_repr == actual_repr diff --git a/python/cudf/cudf/tests/test_reshape.py b/python/cudf/cudf/tests/test_reshape.py index 0a07eecd096..b437c82bf6e 100644 --- a/python/cudf/cudf/tests/test_reshape.py +++ b/python/cudf/cudf/tests/test_reshape.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2023, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. import re from itertools import chain @@ -253,7 +253,6 @@ def test_df_stack_multiindex_column_axis_pd_example(level): ) @pytest.mark.parametrize("nulls", ["none", "some"]) def test_interleave_columns(nulls, num_cols, num_rows, dtype): - if dtype not in ["float32", "float64"] and nulls in ["some"]: pytest.skip(reason="nulls not supported in dtype: " + dtype) @@ -290,7 +289,6 @@ def test_interleave_columns(nulls, num_cols, num_rows, dtype): @pytest.mark.parametrize("dtype", ALL_TYPES) @pytest.mark.parametrize("nulls", ["none", "some"]) def test_tile(nulls, num_cols, num_rows, dtype, count): - if dtype not in ["float32", "float64"] and nulls in ["some"]: pytest.skip(reason="nulls not supported in dtype: " + dtype) diff --git a/python/cudf/cudf/tests/test_rolling.py b/python/cudf/cudf/tests/test_rolling.py index 19714b7b9d3..91643f21155 100644 --- a/python/cudf/cudf/tests/test_rolling.py +++ b/python/cudf/cudf/tests/test_rolling.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2023, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. import math from contextlib import contextmanager @@ -154,7 +154,6 @@ def test_rolling_with_offset(agg): @pytest.mark.parametrize("seed", [100, 2000]) @pytest.mark.parametrize("window_size", [2, 10, 100]) def test_rolling_var_std_large(agg, ddof, center, seed, window_size): - iupper_bound = math.sqrt(np.iinfo(np.int64).max / window_size) ilower_bound = -math.sqrt(abs(np.iinfo(np.int64).min) / window_size) @@ -315,7 +314,6 @@ def test_rolling_getitem_window(): ) @pytest.mark.parametrize("center", [True, False]) def test_rollling_series_numba_udf_basic(data, index, center): - psr = _create_pandas_series_float64_default(data, index=index) gsr = cudf.from_pandas(psr) @@ -352,7 +350,6 @@ def some_func(A): ) @pytest.mark.parametrize("center", [True, False]) def test_rolling_dataframe_numba_udf_basic(data, center): - pdf = pd.DataFrame(data) gdf = cudf.from_pandas(pdf) diff --git a/python/cudf/cudf/tests/test_scalar.py b/python/cudf/cudf/tests/test_scalar.py index d73a1d40aaa..05a91a8fea3 100644 --- a/python/cudf/cudf/tests/test_scalar.py +++ b/python/cudf/cudf/tests/test_scalar.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2023, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. import datetime import re @@ -178,7 +178,6 @@ def test_scalar_device_initialization_decimal(value, decimal_type): @pytest.mark.parametrize("value", SCALAR_VALUES + DECIMAL_VALUES) def test_scalar_roundtrip(value): - s = cudf.Scalar(value) assert s._is_host_value_current @@ -352,7 +351,6 @@ def test_scalar_implicit_int_conversion(value): @pytest.mark.parametrize("cls", [int, float, bool]) @pytest.mark.parametrize("dtype", sorted(set(ALL_TYPES) - {"category"})) def test_scalar_invalid_implicit_conversion(cls, dtype): - try: cls( pd.NaT diff --git a/python/cudf/cudf/tests/test_search.py b/python/cudf/cudf/tests/test_search.py index 17cf3cf8141..3ba652ff6c0 100644 --- a/python/cudf/cudf/tests/test_search.py +++ b/python/cudf/cudf/tests/test_search.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2023, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. import cupy import numpy as np import pandas as pd @@ -86,7 +86,6 @@ def test_search_sorted_dataframe_unequal_number_of_columns(): @pytest.mark.parametrize("side", ["left", "right"]) def test_searchsorted_categorical(side): - cat1 = pd.Categorical( ["a", "a", "b", "c", "a"], categories=["a", "b", "c"], ordered=True ) @@ -106,7 +105,6 @@ def test_searchsorted_categorical(side): @pytest.mark.parametrize("side", ["left", "right"]) def test_searchsorted_datetime(side): - psr1 = pd.Series( pd.date_range("20190101", "20200101", freq="400h", name="times") ) diff --git a/python/cudf/cudf/tests/test_series.py b/python/cudf/cudf/tests/test_series.py index 2e2b79386d7..7dcbf859f08 100644 --- a/python/cudf/cudf/tests/test_series.py +++ b/python/cudf/cudf/tests/test_series.py @@ -813,7 +813,6 @@ def test_round_nan_as_null_false(series, decimal): @pytest.mark.parametrize("ps", _series_na_data()) @pytest.mark.parametrize("nan_as_null", [True, False, None]) def test_series_isnull_isna(ps, nan_as_null): - if nan_as_null is False and ( ps.isna().any() and not ps.isna().all() and ps.dtype == object ): @@ -829,7 +828,6 @@ def test_series_isnull_isna(ps, nan_as_null): @pytest.mark.parametrize("ps", _series_na_data()) @pytest.mark.parametrize("nan_as_null", [True, False, None]) def test_series_notnull_notna(ps, nan_as_null): - if nan_as_null is False and ( ps.isna().any() and not ps.isna().all() and ps.dtype == object ): diff --git a/python/cudf/cudf/tests/test_sorting.py b/python/cudf/cudf/tests/test_sorting.py index b3db1310adb..8152c1bc03c 100644 --- a/python/cudf/cudf/tests/test_sorting.py +++ b/python/cudf/cudf/tests/test_sorting.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2023, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. import string from itertools import product @@ -205,7 +205,6 @@ def test_dataframe_nsmallest_sliced(counts, sliceobj): def test_dataframe_multi_column( num_cols, num_rows, dtype, ascending, na_position ): - np.random.seed(0) by = list(string.ascii_lowercase[:num_cols]) pdf = pd.DataFrame() @@ -234,7 +233,6 @@ def test_dataframe_multi_column( def test_dataframe_multi_column_nulls( num_cols, num_rows, dtype, nulls, ascending, na_position ): - np.random.seed(0) by = list(string.ascii_lowercase[:num_cols]) pdf = pd.DataFrame() @@ -298,7 +296,6 @@ def test_series_nlargest_nelem(nelem): @pytest.mark.parametrize("nelem", [1, 10, 100]) @pytest.mark.parametrize("keep", [True, False]) def test_dataframe_scatter_by_map(map_size, nelem, keep): - strlist = ["dog", "cat", "fish", "bird", "pig", "fox", "cow", "goat"] np.random.seed(0) df = DataFrame() diff --git a/python/cudf/cudf/tests/test_stats.py b/python/cudf/cudf/tests/test_stats.py index 5f010668383..8ff4dc73c4c 100644 --- a/python/cudf/cudf/tests/test_stats.py +++ b/python/cudf/cudf/tests/test_stats.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2023, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. from concurrent.futures import ThreadPoolExecutor @@ -182,7 +182,6 @@ def test_exact_quantiles_int(int_method): def test_approx_quantiles(): - arr = np.asarray([6.8, 0.15, 3.4, 4.17, 2.13, 1.11, -1.01, 0.8, 5.7]) quant_values = [0.0, 0.25, 0.33, 0.5, 1.0] @@ -222,7 +221,6 @@ def test_approx_quantiles_int(): ], ) def test_misc_quantiles(data, q): - pdf_series = _create_pandas_series_float64_default(data) gdf_series = _create_cudf_series_float64_default(data) @@ -485,7 +483,6 @@ def test_corr1d(data1, data2, method): @pytest.mark.parametrize("method", ["spearman", "pearson"]) def test_df_corr(method): - gdf = randomdata(100, {str(x): float for x in range(50)}) pdf = gdf.to_pandas() got = gdf.corr(method) diff --git a/python/cudf/cudf/tests/test_string.py b/python/cudf/cudf/tests/test_string.py index 198dfa9372c..4c5598b547e 100644 --- a/python/cudf/cudf/tests/test_string.py +++ b/python/cudf/cudf/tests/test_string.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2023, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. import json import re @@ -847,7 +847,6 @@ def test_string_contains_case(ps_gs): ], ) def test_string_like(pat, esc, expect): - expectation = does_not_raise() if len(esc) > 1: expectation = pytest.raises(ValueError) @@ -2402,7 +2401,6 @@ def test_string_str_translate(data): def test_string_str_filter_characters(): - data = [ "hello world", "A+B+C+D", @@ -2432,7 +2430,6 @@ def test_string_str_filter_characters(): def test_string_str_code_points(): - data = [ "abc", "Def", @@ -2598,7 +2595,6 @@ def test_string_typecast_error(data, obj_type, dtype): ], ) def test_string_hex_to_int(data): - gsr = cudf.Series(data) expected = cudf.Series([263988422296292, 0, 281474976710655]) diff --git a/python/cudf/cudf/tests/test_testing.py b/python/cudf/cudf/tests/test_testing.py index 3024c8e2e7b..091cd6b57a4 100644 --- a/python/cudf/cudf/tests/test_testing.py +++ b/python/cudf/cudf/tests/test_testing.py @@ -112,7 +112,6 @@ def test_basic_assert_series_equal( check_categorical, dtype, ): - p_left = pd.Series([1, 2, 3], name="a", dtype=dtype) p_right = pd.Series(rdata, name=rname, dtype=dtype) diff --git a/python/cudf/cudf/tests/test_timedelta.py b/python/cudf/cudf/tests/test_timedelta.py index d86612d3143..12f1ace7867 100644 --- a/python/cudf/cudf/tests/test_timedelta.py +++ b/python/cudf/cudf/tests/test_timedelta.py @@ -1185,7 +1185,6 @@ def test_timedelta_fillna(data, dtype, fill_value): ], ) def test_timedelta_str_roundtrip(gsr, expected_series): - actual_series = gsr.astype("str") assert_eq(expected_series, actual_series) diff --git a/python/cudf/cudf/tests/test_transform.py b/python/cudf/cudf/tests/test_transform.py index 723bbdf9371..88938457545 100644 --- a/python/cudf/cudf/tests/test_transform.py +++ b/python/cudf/cudf/tests/test_transform.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2022, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. import numpy as np @@ -23,7 +23,6 @@ def _generic_function(a): ], ) def test_apply_python_lambda(dtype, udf, testfunc): - size = 500 lhs_arr = np.random.random(size).astype(dtype) diff --git a/python/cudf/cudf/tests/test_udf_masked_ops.py b/python/cudf/cudf/tests/test_udf_masked_ops.py index 11970944a95..95ea4544917 100644 --- a/python/cudf/cudf/tests/test_udf_masked_ops.py +++ b/python/cudf/cudf/tests/test_udf_masked_ops.py @@ -74,7 +74,6 @@ def run_masked_udf_test(func, data, args=(), nullable=True, **kwargs): def run_masked_string_udf_test(func, data, args=(), **kwargs): - gdf = data pdf = data.to_pandas(nullable=True) diff --git a/python/cudf/cudf/utils/applyutils.py b/python/cudf/cudf/utils/applyutils.py index 66dbd731e69..d57303ca122 100644 --- a/python/cudf/cudf/utils/applyutils.py +++ b/python/cudf/cudf/utils/applyutils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2023, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. import functools from typing import Any, Dict @@ -108,7 +108,6 @@ def apply_chunks( @acquire_spill_lock() def make_aggregate_nullmask(df, columns=None, op="__and__"): - out_mask = None for k in columns or df._data: col = cudf.core.dataframe.extract_col(df, k) diff --git a/python/cudf/cudf/utils/hash_vocab_utils.py b/python/cudf/cudf/utils/hash_vocab_utils.py index a0915951240..ef078ed8c5d 100644 --- a/python/cudf/cudf/utils/hash_vocab_utils.py +++ b/python/cudf/cudf/utils/hash_vocab_utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. # This function is from the rapidsai/clx repo at below link # https://github.com/rapidsai/clx/blob/267c6d30805c9dcbf80840f222bf31c5c4b7068a/python/clx/analytics/_perfect_hash.py import numpy as np @@ -158,7 +158,6 @@ def _perfect_hash(integers, max_constant): def _pack_keys_and_values(flattened_hash_table, original_dict): - for i in range(len(flattened_hash_table)): if flattened_hash_table[i] in original_dict: value = original_dict[flattened_hash_table[i]] @@ -189,7 +188,6 @@ def _store_func( first_token_id, sep_token_id, ): - with open(out_name, mode="w+") as f: f.write(f"{outer_a}\n") f.write(f"{outer_b}\n") @@ -215,7 +213,6 @@ def _retrieve( inner_table_coeffs, offsets_into_ht, ): - bin_hash = _hash_func(k, outer_a, outer_b, num_outer_bins) start_offset_in_ht = offsets_into_ht[bin_hash] inner_table_values = inner_table_coeffs[bin_hash] diff --git a/python/cudf/cudf/utils/ioutils.py b/python/cudf/cudf/utils/ioutils.py index 6641bd8290a..57e657eb5c1 100644 --- a/python/cudf/cudf/utils/ioutils.py +++ b/python/cudf/cudf/utils/ioutils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2023, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. import datetime import os @@ -2028,7 +2028,7 @@ def _merge_ranges(byte_ranges, max_block=256_000_000, max_gap=64_000): return new_ranges offset, size = byte_ranges[0] - for (new_offset, new_size) in byte_ranges[1:]: + for new_offset, new_size in byte_ranges[1:]: gap = new_offset - (offset + size) if gap > max_gap or (size + new_size + gap) > max_block: # Gap is too large or total read is too large @@ -2068,7 +2068,7 @@ def _read_byte_ranges( # Simple utility to copy remote byte ranges # into a local buffer for IO in libcudf workers = [] - for (offset, nbytes) in ranges: + for offset, nbytes in ranges: if len(ranges) > 1: workers.append( Thread( diff --git a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py index 738ff24f374..df4bed0be0a 100644 --- a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py +++ b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py @@ -1135,7 +1135,6 @@ def test_index_new(): @pytest.mark.xfail(not LOADED, reason="Should not fail in accelerated mode") def test_groupby_apply_callable_referencing_pandas(dataframe): - pdf, df = dataframe class Callable1: diff --git a/python/custreamz/custreamz/tests/conftest.py b/python/custreamz/custreamz/tests/conftest.py index 5840ff710d5..1cda9b71387 100644 --- a/python/custreamz/custreamz/tests/conftest.py +++ b/python/custreamz/custreamz/tests/conftest.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. import socket import pytest @@ -7,7 +7,6 @@ @pytest.fixture(scope="session") def kafka_client(): - # Check for the existence of a kafka broker s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: diff --git a/python/dask_cudf/dask_cudf/backends.py b/python/dask_cudf/dask_cudf/backends.py index 387643587d1..5d951cec266 100644 --- a/python/dask_cudf/dask_cudf/backends.py +++ b/python/dask_cudf/dask_cudf/backends.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. import warnings from collections.abc import Iterator @@ -438,7 +438,6 @@ def hash_object_cudf(frame, index=True): @hash_object_dispatch.register(cudf.BaseIndex) @_dask_cudf_nvtx_annotate def hash_object_cudf_index(ind, index=None): - if isinstance(ind, cudf.MultiIndex): return ind.to_frame(index=False).hash_values() @@ -586,7 +585,6 @@ def from_dict( columns=None, constructor=cudf.DataFrame, ): - return _default_backend( dd.from_dict, data, diff --git a/python/dask_cudf/dask_cudf/core.py b/python/dask_cudf/dask_cudf/core.py index 08c03235484..c2b2428bf14 100644 --- a/python/dask_cudf/dask_cudf/core.py +++ b/python/dask_cudf/dask_cudf/core.py @@ -153,7 +153,6 @@ def set_index( shuffle_method=None, **kwargs, ): - pre_sorted = sorted del sorted @@ -165,7 +164,6 @@ def set_index( and cudf.api.types.is_string_dtype(self[other].dtype) ) ): - # Let upstream-dask handle "pre-sorted" case if pre_sorted: return dd.shuffle.set_sorted_index( diff --git a/python/dask_cudf/dask_cudf/io/parquet.py b/python/dask_cudf/dask_cudf/io/parquet.py index d82d539358d..fc962670c47 100644 --- a/python/dask_cudf/dask_cudf/io/parquet.py +++ b/python/dask_cudf/dask_cudf/io/parquet.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2023, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. import itertools import warnings from contextlib import ExitStack @@ -73,7 +73,6 @@ def _read_paths( dataset_kwargs=None, **kwargs, ): - # Simplify row_groups if all None if row_groups == [None for path in paths]: row_groups = None @@ -94,7 +93,6 @@ def _read_paths( dataset_kwargs = dataset_kwargs or {} dataset_kwargs["partitioning"] = partitioning or "hive" with ExitStack() as stack: - # Non-local filesystem handling paths_or_fobs = paths if not _is_local_filesystem(fs): @@ -153,7 +151,6 @@ def _read_paths( df = df[projected_columns] if partitions and partition_keys is None: - # Use `HivePartitioning` by default ds = pa_ds.dataset( paths, @@ -175,7 +172,6 @@ def _read_paths( raise ValueError("Must pass partition sets") for i, (name, index2) in enumerate(partition_keys): - if len(partitions[i].keys): # Build a categorical column from `codes` directly # (since the category is often a larger dtype) @@ -211,7 +207,6 @@ def read_partition( open_file_options=None, **kwargs, ): - if columns is not None: columns = [c for c in columns] if isinstance(index, list): @@ -241,7 +236,6 @@ def read_partition( # inform the user that the `read_parquet` partition # size is too large for the available memory try: - # Assume multi-piece read paths = [] rgs = [] @@ -249,7 +243,6 @@ def read_partition( dfs = [] for i, piece in enumerate(pieces): - (path, row_group, partition_keys) = piece row_group = None if row_group == [None] else row_group diff --git a/python/dask_cudf/dask_cudf/io/tests/test_csv.py b/python/dask_cudf/dask_cudf/io/tests/test_csv.py index 4ff630a89e8..5f1aa98e888 100644 --- a/python/dask_cudf/dask_cudf/io/tests/test_csv.py +++ b/python/dask_cudf/dask_cudf/io/tests/test_csv.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2023, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. import gzip import os @@ -248,7 +248,6 @@ def test_read_csv_nrows(csv_end_bad_lines): def test_read_csv_nrows_error(csv_end_bad_lines): - with pytest.raises(ValueError): dask_cudf.read_csv( csv_end_bad_lines, nrows=2, blocksize="100 MiB" diff --git a/python/dask_cudf/dask_cudf/io/tests/test_orc.py b/python/dask_cudf/dask_cudf/io/tests/test_orc.py index 5565a44c7d8..c2be75e8ddd 100644 --- a/python/dask_cudf/dask_cudf/io/tests/test_orc.py +++ b/python/dask_cudf/dask_cudf/io/tests/test_orc.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2022, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. import glob import os @@ -85,7 +85,6 @@ def test_read_orc_filtered(tmpdir, engine, predicate, expected_len): def test_read_orc_first_file_empty(tmpdir): - # Write a 3-file dataset where the first file is empty # See: https://github.com/rapidsai/cudf/issues/8011 path = str(tmpdir) @@ -112,7 +111,6 @@ def test_read_orc_first_file_empty(tmpdir): ], ) def test_to_orc(tmpdir, dtypes, compression, compute): - # Create cudf and dask_cudf dataframes df = cudf.datasets.randomdata(nrows=10, dtypes=dtypes, seed=1) df = df.set_index("index").sort_index() diff --git a/python/dask_cudf/dask_cudf/io/text.py b/python/dask_cudf/dask_cudf/io/text.py index 2adace565d5..9cdb7c5220b 100644 --- a/python/dask_cudf/dask_cudf/io/text.py +++ b/python/dask_cudf/dask_cudf/io/text.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. import os from glob import glob @@ -11,7 +11,6 @@ def read_text(path, chunksize="256 MiB", **kwargs): - if isinstance(chunksize, str): chunksize = parse_bytes(chunksize) diff --git a/python/dask_cudf/dask_cudf/tests/test_accessor.py b/python/dask_cudf/dask_cudf/tests/test_accessor.py index 7ed5d797822..3a54672c1d3 100644 --- a/python/dask_cudf/dask_cudf/tests/test_accessor.py +++ b/python/dask_cudf/dask_cudf/tests/test_accessor.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2023, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. import numpy as np import pandas as pd @@ -254,7 +254,6 @@ def test_string_slicing(data): def test_categorical_categories(): - df = DataFrame( {"a": ["a", "b", "c", "d", "e", "e", "a", "d"], "b": range(8)} ) @@ -283,7 +282,6 @@ def test_categorical_as_known(): def test_str_slice(): - df = DataFrame({"a": ["abc,def,123", "xyz,hi,bye"]}) ddf = dgd.from_cudf(df, 1) diff --git a/python/dask_cudf/dask_cudf/tests/test_core.py b/python/dask_cudf/dask_cudf/tests/test_core.py index 63fd6599496..5b11b337f21 100644 --- a/python/dask_cudf/dask_cudf/tests/test_core.py +++ b/python/dask_cudf/dask_cudf/tests/test_core.py @@ -106,7 +106,6 @@ def test_from_cudf(): def test_from_cudf_multiindex_raises(): - df = cudf.DataFrame({"x": list("abc"), "y": [1, 2, 3], "z": [1, 2, 3]}) with pytest.raises(NotImplementedError): @@ -115,7 +114,6 @@ def test_from_cudf_multiindex_raises(): def test_from_cudf_with_generic_idx(): - cdf = cudf.DataFrame( { "a": list(range(20)), @@ -641,7 +639,6 @@ def test_concat(gdf, gddf, series): def test_boolean_index(gdf, gddf): - gdf2 = gdf[gdf.x > 2] gddf2 = gddf[gddf.x > 2] @@ -658,7 +655,6 @@ def test_drop(gdf, gddf): @pytest.mark.parametrize("deep", [True, False]) @pytest.mark.parametrize("index", [True, False]) def test_memory_usage(gdf, gddf, index, deep): - dd.assert_eq( gdf.memory_usage(deep=deep, index=index), gddf.memory_usage(deep=deep, index=index), @@ -710,7 +706,6 @@ def test_hash_object_dispatch(index): ], ) def test_make_meta_backends(index): - dtypes = ["int8", "int32", "int64", "float64"] df = cudf.DataFrame( {dt: np.arange(start=0, stop=3, dtype=dt) for dt in dtypes} @@ -734,7 +729,6 @@ def test_make_meta_backends(index): # Check dask code path if not MultiIndex if not isinstance(df.index, cudf.MultiIndex): - ddf = dgd.from_cudf(df, npartitions=1) # Check "empty" metadata types diff --git a/python/dask_cudf/dask_cudf/tests/test_reductions.py b/python/dask_cudf/dask_cudf/tests/test_reductions.py index c34fbc3b0e7..e966e58f46e 100644 --- a/python/dask_cudf/dask_cudf/tests/test_reductions.py +++ b/python/dask_cudf/dask_cudf/tests/test_reductions.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. import numpy as np import pandas as pd @@ -66,7 +66,6 @@ def test_series_reduce(reducer): "op", ["max", "min", "sum", "prod", "mean", "var", "std"] ) def test_rowwise_reductions(data, op): - gddf = dgd.from_cudf(data, npartitions=10) pddf = gddf.to_dask_dataframe() From d7f9688bb58ceee32a9753dc2f3f6dd046a92257 Mon Sep 17 00:00:00 2001 From: Ashwin Srinath <3190405+shwina@users.noreply.github.com> Date: Wed, 24 Jan 2024 22:45:32 -0500 Subject: [PATCH 115/384] Fix empty groupby return types (#14871) Closes #14862 This PR fixes the errors in #14862 by ensuring we match the pandas return type when doing grouped count, size, idxmax, idxmin. --------- Co-authored-by: Ashwin Srinath --- python/cudf/cudf/core/groupby/groupby.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index bf470c29c99..6aba93855a7 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -596,7 +596,7 @@ def agg(self, func): # Structs lose their labels which we reconstruct here col = col._with_type_metadata(cudf.ListDtype(orig_dtype)) - if agg_kind in {"COUNT", "SIZE"}: + if agg_kind in {"COUNT", "SIZE", "ARGMIN", "ARGMAX"}: data[key] = col.astype("int64") elif ( self.obj.empty @@ -1449,9 +1449,11 @@ def mult(df): dtype: int64 """ - if self.obj.empty: - res = self.obj.copy(deep=True) + if function in {"count", "size", "idxmin", "idxmax"}: + res = cudf.Series([], dtype="int64") + else: + res = self.obj.copy(deep=True) res.index = self.grouping.keys if function in {"sum", "product"}: # For `sum` & `product`, boolean types From 8a25f70c13991f5bb9e904e4e11283e9020f9381 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 24 Jan 2024 18:13:14 -1000 Subject: [PATCH 116/384] Support kurt/skew(axis=None) for multi columns/low row count (#14874) closes #14866 @galipremsagar it appears the linked failing test in the issue test_reductions_axis_none_warning expected FutureWarning from these calls. Should they be expected for kurt/skew too? --- python/cudf/cudf/core/dataframe.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 1057fd0b716..a3642bcc43f 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -57,6 +57,7 @@ is_string_dtype, ) from cudf.core import column, df_protocol, indexing_utils, reshape +from cudf.core._compat import PANDAS_GE_200 from cudf.core.abc import Serializable from cudf.core.column import ( CategoricalColumn, @@ -95,11 +96,8 @@ min_scalar_type, numeric_normalize_types, ) - from cudf.utils.nvtx_annotation import _cudf_nvtx_annotate from cudf.utils.utils import GetAttrGetItemMixin, _external_only_api -from cudf.core._compat import PANDAS_GE_200 - _cupy_nan_methods_map = { "min": "nanmin", @@ -6112,8 +6110,13 @@ def _reduce( if axis == 0 else source.index ) - if axis in {0, 2}: + if axis == 2 and op in ("kurtosis", "kurt", "skew"): + # TODO: concat + op can probably be done in the general case + # for axis == 2. + return getattr(concat_columns(source._data.columns), op)( + **kwargs + ) try: result = [ getattr(source._data[col], op)(**kwargs) From 7bf4376d8067130d7e3c5eb5afc0b033e3658cd9 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 25 Jan 2024 16:54:10 +0530 Subject: [PATCH 117/384] Fix miscellaneous failures in pytests (#14879) --- python/cudf/cudf/core/series.py | 5 +---- python/cudf/cudf/tests/test_index.py | 4 +++- python/cudf/cudf/tests/test_joining.py | 11 +++++++++-- python/cudf/cudf/tests/test_series.py | 3 ++- python/dask_cudf/dask_cudf/backends.py | 7 +++---- 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 53218903ed2..e1015e53c88 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -606,10 +606,7 @@ def __init__( name_from_data = data.name column = as_column(data, nan_as_null=nan_as_null, dtype=dtype) if isinstance(data, (pd.Series, Series)): - if isinstance(data.index, pd.MultiIndex): - index = cudf.from_pandas(data.index) - else: - index = as_index(data.index) + index_from_data = as_index(data.index) elif isinstance(data, ColumnAccessor): raise TypeError( "Use cudf.Series._from_data for constructing a Series from " diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index 20af94576fe..e47b2f5d5d5 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -2800,7 +2800,9 @@ def test_rangeindex_join_user_option(default_integer_bitwidth): actual = idx1.join(idx2, how="inner", sort=True) expected = idx1.to_pandas().join(idx2.to_pandas(), how="inner", sort=True) assert actual.dtype == cudf.dtype(f"int{default_integer_bitwidth}") - assert_eq(expected, actual) + # exact=False to ignore dtype comparison, + # because `default_integer_bitwidth` is cudf only option + assert_eq(expected, actual, exact=False) def test_rangeindex_where_user_option(default_integer_bitwidth): diff --git a/python/cudf/cudf/tests/test_joining.py b/python/cudf/cudf/tests/test_joining.py index 42b466f486b..8ce2adae15b 100644 --- a/python/cudf/cudf/tests/test_joining.py +++ b/python/cudf/cudf/tests/test_joining.py @@ -2246,11 +2246,18 @@ def test_index_join_return_indexers_notimplemented(): @pytest.mark.parametrize("how", ["inner", "outer"]) -def test_index_join_names(how): +def test_index_join_names(request, how): idx1 = cudf.Index([10, 1, 2, 4, 2, 1], name="a") idx2 = cudf.Index([-10, 2, 3, 1, 2], name="b") + request.applymarker( + pytest.mark.xfail( + reason="https://github.com/pandas-dev/pandas/issues/57065", + ) + ) + pidx1 = idx1.to_pandas() + pidx2 = idx2.to_pandas() - expected = idx1.to_pandas().join(idx2.to_pandas(), how=how) + expected = pidx1.join(pidx2, how=how) actual = idx1.join(idx2, how=how) assert_join_results_equal(actual, expected, how=how) diff --git a/python/cudf/cudf/tests/test_series.py b/python/cudf/cudf/tests/test_series.py index 2772ce6ffee..623657d127f 100644 --- a/python/cudf/cudf/tests/test_series.py +++ b/python/cudf/cudf/tests/test_series.py @@ -2159,7 +2159,8 @@ def test_series_init_scalar_with_index(data, index): assert_eq( pandas_series, cudf_series, - check_index_type=False if data is None and index is None else True, + check_index_type=data is not None or index is not None, + check_dtype=data is not None, ) diff --git a/python/dask_cudf/dask_cudf/backends.py b/python/dask_cudf/dask_cudf/backends.py index 11e0f1e0e60..86283f57366 100644 --- a/python/dask_cudf/dask_cudf/backends.py +++ b/python/dask_cudf/dask_cudf/backends.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. import warnings from collections.abc import Iterator @@ -8,7 +8,6 @@ import pandas as pd import pyarrow as pa from pandas.api.types import is_scalar -from pandas.core.tools.datetimes import is_datetime64tz_dtype import dask.dataframe as dd from dask import config @@ -42,7 +41,7 @@ from dask.utils import Dispatch, is_arraylike import cudf -from cudf.api.types import is_string_dtype +from cudf.api.types import _is_datetime64tz_dtype, is_string_dtype from cudf.utils.nvtx_annotation import _dask_cudf_nvtx_annotate from .core import DataFrame, Index, Series @@ -127,7 +126,7 @@ def _get_non_empty_data(s): data = cudf.core.column.as_column(data, dtype=s.dtype) elif is_string_dtype(s.dtype): data = pa.array(["cat", "dog"]) - elif is_datetime64tz_dtype(s.dtype): + elif _is_datetime64tz_dtype(s.dtype): from cudf.utils.dtypes import get_time_unit data = cudf.date_range("2001-01-01", periods=2, freq=get_time_unit(s)) From f14ba2221efab298f064b4275f69b58a833b6374 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 25 Jan 2024 07:37:10 -0800 Subject: [PATCH 118/384] Move all core types to using enum class in Cython (#14876) This change is a necessary prerequisite for adding other APIs to pylibcudf that need these types. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/14876 --- python/cudf/cudf/_lib/cpp/types.pxd | 83 ++++++++++----------- python/cudf/cudf/_lib/lists.pyx | 4 +- python/cudf/cudf/_lib/stream_compaction.pyx | 4 +- 3 files changed, 42 insertions(+), 49 deletions(-) diff --git a/python/cudf/cudf/_lib/cpp/types.pxd b/python/cudf/cudf/_lib/cpp/types.pxd index 14bf8a83de0..13aebdff726 100644 --- a/python/cudf/cudf/_lib/cpp/types.pxd +++ b/python/cudf/cudf/_lib/cpp/types.pxd @@ -1,58 +1,56 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from libc.stdint cimport int32_t, uint32_t +from libcpp cimport bool cdef extern from "cudf/types.hpp" namespace "cudf" nogil: - # The declaration below is to work around - # https://github.com/cython/cython/issues/5637 - """ - #define __PYX_ENUM_CLASS_DECL enum - """ ctypedef int32_t size_type ctypedef uint32_t bitmask_type ctypedef uint32_t char_utf8 - ctypedef enum mask_state: - UNALLOCATED "cudf::mask_state::UNALLOCATED" - UNINITIALIZED "cudf::mask_state::UNINITIALIZED" - ALL_VALID "cudf::mask_state::ALL_VALID" - ALL_NULL "cudf::mask_state::ALL_NULL" + # A Hack to let cython compile with __int128_t symbol + # https://stackoverflow.com/a/27609033 + ctypedef int int128 "__int128_t" - ctypedef enum order "cudf::order": - ASCENDING "cudf::order::ASCENDING" - DESCENDING "cudf::order::DESCENDING" + cpdef enum class mask_state(int32_t): + UNALLOCATED + UNINITIALIZED + ALL_VALID + ALL_NULL - ctypedef enum null_order "cudf::null_order": - AFTER "cudf::null_order::AFTER" - BEFORE "cudf::null_order::BEFORE" + cpdef enum class order(bool): + ASCENDING + DESCENDING - ctypedef enum sorted "cudf::sorted": - NO "cudf::sorted::NO" - YES "cudf::sorted::YES" + cpdef enum class null_order(bool): + AFTER + BEFORE + + cpdef enum class sorted(bool): + NO + YES cdef cppclass order_info: sorted is_sorted order ordering null_order null_ordering - ctypedef enum null_policy "cudf::null_policy": - EXCLUDE "cudf::null_policy::EXCLUDE" - INCLUDE "cudf::null_policy::INCLUDE" + cpdef enum class null_policy(bool): + EXCLUDE + INCLUDE - ctypedef enum nan_policy "cudf::nan_policy": - NAN_IS_NULL "cudf::nan_policy::NAN_IS_NULL" - NAN_IS_VALID "cudf::nan_policy::NAN_IS_VALID" + cpdef enum class nan_policy(bool): + NAN_IS_NULL + NAN_IS_VALID - ctypedef enum null_equality "cudf::null_equality": - EQUAL "cudf::null_equality::EQUAL" - UNEQUAL "cudf::null_equality::UNEQUAL" + cpdef enum class null_equality(bool): + EQUAL + UNEQUAL - ctypedef enum nan_equality "cudf::nan_equality": - # These names differ from the C++ names due to Cython warnings if - # "UNEQUAL" is declared by both null_equality and nan_equality. - ALL_EQUAL "cudf::nan_equality::ALL_EQUAL" - NANS_UNEQUAL "cudf::nan_equality::UNEQUAL" + cpdef enum class nan_equality(bool): + ALL_EQUAL + UNEQUAL cpdef enum class type_id(int32_t): EMPTY @@ -93,14 +91,9 @@ cdef extern from "cudf/types.hpp" namespace "cudf" nogil: type_id id() except + int32_t scale() except + -cdef extern from "cudf/types.hpp" namespace "cudf" nogil: - ctypedef enum interpolation: - LINEAR "cudf::interpolation::LINEAR" - LOWER "cudf::interpolation::LOWER" - HIGHER "cudf::interpolation::HIGHER" - MIDPOINT "cudf::interpolation::MIDPOINT" - NEAREST "cudf::interpolation::NEAREST" - - # A Hack to let cython compile with __int128_t symbol - # https://stackoverflow.com/a/27609033 - ctypedef int int128 "__int128_t" + cpdef enum class interpolation(int32_t): + LINEAR + LOWER + HIGHER + MIDPOINT + NEAREST diff --git a/python/cudf/cudf/_lib/lists.pyx b/python/cudf/cudf/_lib/lists.pyx index 199641fd2ce..f76d7a9a388 100644 --- a/python/cudf/cudf/_lib/lists.pyx +++ b/python/cudf/cudf/_lib/lists.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2023, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. from cudf.core.buffer import acquire_spill_lock @@ -84,7 +84,7 @@ def distinct(Column col, bool nulls_equal, bool nans_all_equal): null_equality.EQUAL if nulls_equal else null_equality.UNEQUAL ) cdef nan_equality c_nans_equal = ( - nan_equality.ALL_EQUAL if nans_all_equal else nan_equality.NANS_UNEQUAL + nan_equality.ALL_EQUAL if nans_all_equal else nan_equality.UNEQUAL ) cdef unique_ptr[column] c_result diff --git a/python/cudf/cudf/_lib/stream_compaction.pyx b/python/cudf/cudf/_lib/stream_compaction.pyx index 9b22728d2f0..d7725e8df94 100644 --- a/python/cudf/cudf/_lib/stream_compaction.pyx +++ b/python/cudf/cudf/_lib/stream_compaction.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from cudf.core.buffer import acquire_spill_lock @@ -209,7 +209,7 @@ def distinct_indices( cdef nan_equality cpp_nans_equal = ( nan_equality.ALL_EQUAL if nans_equal - else nan_equality.NANS_UNEQUAL + else nan_equality.UNEQUAL ) cdef table_view source = table_view_from_columns(columns) cdef unique_ptr[column] c_result From 0cd58fbec63d5e461b487e7e37aa9942ebe0f116 Mon Sep 17 00:00:00 2001 From: AmirAli Mirian <37371367+amiralimi@users.noreply.github.com> Date: Thu, 25 Jan 2024 11:40:04 -0500 Subject: [PATCH 119/384] Fix index difference to follow the pandas format (#14789) This PR fixes an error in `Index.difference` where the function keeps duplicate elements while pandas removes the duplicates. The tests had no inputs with duplicates, so I added new tests too (I added the test from the original issue). - closes #14489 Authors: - AmirAli Mirian (https://github.com/amiralimi) - Ashwin Srinath (https://github.com/shwina) Approvers: - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/14789 --- python/cudf/cudf/core/_base_index.py | 4 ++-- python/cudf/cudf/tests/test_index.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index 2aef77b6c99..d7d8e26db1b 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -1040,11 +1040,11 @@ def difference(self, other, sort=None): res_name = _get_result_name(self.name, other.name) if is_mixed_with_object_dtype(self, other): - difference = self.copy() + difference = self.copy().unique() else: other = other.copy(deep=False) difference = cudf.core.index._index_from_data( - cudf.DataFrame._from_data({"None": self._column}) + cudf.DataFrame._from_data({"None": self._column.unique()}) .merge( cudf.DataFrame._from_data({"None": other._column}), how="leftanti", diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index a480a4624f7..e0a369d8d91 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2023, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. """ Test related to Index @@ -803,6 +803,7 @@ def test_index_to_series(data): pd.Series(["1", "2", "a", "3", None], dtype="category"), range(0, 10), [], + [1, 1, 2, 2], ], ) @pytest.mark.parametrize( @@ -819,6 +820,7 @@ def test_index_to_series(data): range(2, 4), pd.Series(["1", "a", "3", None], dtype="category"), [], + [2], ], ) @pytest.mark.parametrize("sort", [None, False]) From 35011dd13c93f2b4e7c46e9360a7c545eb40dd9b Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Thu, 25 Jan 2024 17:15:46 +0000 Subject: [PATCH 120/384] De-DOS line-endings (#14880) These are the only two files in the repo (other than the sphinx make.bat files, which should have DOS line-endings) that use \r\n as the line-ending. Let's fix that. Authors: - Lawrence Mitchell (https://github.com/wence-) Approvers: - David Wendt (https://github.com/davidwendt) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/14880 --- cpp/doxygen/unicode.md | 46 +-- cpp/src/search/contains_table.cu | 584 +++++++++++++++---------------- 2 files changed, 315 insertions(+), 315 deletions(-) diff --git a/cpp/doxygen/unicode.md b/cpp/doxygen/unicode.md index 1ab09e110c1..089bb944b42 100644 --- a/cpp/doxygen/unicode.md +++ b/cpp/doxygen/unicode.md @@ -1,23 +1,23 @@ -# Unicode Limitations - -The strings column currently supports only UTF-8 characters internally. -For functions that require character testing (e.g. cudf::strings::all_characters_of_type()) or -case conversion (e.g. cudf::strings::capitalize(), etc) only the 16-bit [Unicode 13.0](http://www.unicode.org/versions/Unicode13.0.0) -character code-points (0-65535) values are supported. -Case conversion and character testing on characters above code-point 65535 are not supported. - -Case conversions that are context-sensitive are not supported. Also, case conversions that result -in multiple characters are not reversible. That is, adjacent individual characters will not be case converted -to a single character. For example, converting character ß to upper case will result in the characters "SS". But converting "SS" to lower case will produce "ss". - -Strings case and type APIs: - -- cudf::strings::all_characters_of_type() -- cudf::strings::to_upper() -- cudf::strings::to_lower() -- cudf::strings::capitalize() -- cudf::strings::title() -- cudf::strings::swapcase() - -Also, using regex patterns that use the shorthand character classes `\d \D \w \W \s \S` will include only appropriate characters with -code-points between (0-65535). +# Unicode Limitations + +The strings column currently supports only UTF-8 characters internally. +For functions that require character testing (e.g. cudf::strings::all_characters_of_type()) or +case conversion (e.g. cudf::strings::capitalize(), etc) only the 16-bit [Unicode 13.0](http://www.unicode.org/versions/Unicode13.0.0) +character code-points (0-65535) values are supported. +Case conversion and character testing on characters above code-point 65535 are not supported. + +Case conversions that are context-sensitive are not supported. Also, case conversions that result +in multiple characters are not reversible. That is, adjacent individual characters will not be case converted +to a single character. For example, converting character ß to upper case will result in the characters "SS". But converting "SS" to lower case will produce "ss". + +Strings case and type APIs: + +- cudf::strings::all_characters_of_type() +- cudf::strings::to_upper() +- cudf::strings::to_lower() +- cudf::strings::capitalize() +- cudf::strings::title() +- cudf::strings::swapcase() + +Also, using regex patterns that use the shorthand character classes `\d \D \w \W \s \S` will include only appropriate characters with +code-points between (0-65535). diff --git a/cpp/src/search/contains_table.cu b/cpp/src/search/contains_table.cu index 09122b37d6f..b8ece03c4a0 100644 --- a/cpp/src/search/contains_table.cu +++ b/cpp/src/search/contains_table.cu @@ -1,292 +1,292 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include -#include - -#include -#include - -#include - -#include - -#include - -#include - -namespace cudf::detail { - -namespace { - -using cudf::experimental::row::lhs_index_type; -using cudf::experimental::row::rhs_index_type; - -/** - * @brief An hasher adapter wrapping both haystack hasher and needles hasher - */ -template -struct hasher_adapter { - hasher_adapter(HaystackHasher const& haystack_hasher, NeedleHasher const& needle_hasher) - : _haystack_hasher{haystack_hasher}, _needle_hasher{needle_hasher} - { - } - - __device__ constexpr auto operator()(lhs_index_type idx) const noexcept - { - return _haystack_hasher(static_cast(idx)); - } - - __device__ constexpr auto operator()(rhs_index_type idx) const noexcept - { - return _needle_hasher(static_cast(idx)); - } - - private: - HaystackHasher const _haystack_hasher; - NeedleHasher const _needle_hasher; -}; - -/** - * @brief An comparator adapter wrapping both self comparator and two table comparator - */ -template -struct comparator_adapter { - comparator_adapter(SelfEqual const& self_equal, TwoTableEqual const& two_table_equal) - : _self_equal{self_equal}, _two_table_equal{two_table_equal} - { - } - - __device__ constexpr auto operator()(lhs_index_type lhs_index, - lhs_index_type rhs_index) const noexcept - { - auto const lhs = static_cast(lhs_index); - auto const rhs = static_cast(rhs_index); - - return _self_equal(lhs, rhs); - } - - __device__ constexpr auto operator()(lhs_index_type lhs_index, - rhs_index_type rhs_index) const noexcept - { - return _two_table_equal(lhs_index, rhs_index); - } - - private: - SelfEqual const _self_equal; - TwoTableEqual const _two_table_equal; -}; - -/** - * @brief Build a row bitmask for the input table. - * - * The output bitmask will have invalid bits corresponding to the input rows having nulls (at - * any nested level) and vice versa. - * - * @param input The input table - * @param stream CUDA stream used for device memory operations and kernel launches - * @return A pair of pointer to the output bitmask and the buffer containing the bitmask - */ -std::pair build_row_bitmask(table_view const& input, - rmm::cuda_stream_view stream) -{ - auto const nullable_columns = get_nullable_columns(input); - CUDF_EXPECTS(nullable_columns.size() > 0, - "The input table has nulls thus it should have nullable columns."); - - // If there are more than one nullable column, we compute `bitmask_and` of their null masks. - // Otherwise, we have only one nullable column and can use its null mask directly. - if (nullable_columns.size() > 1) { - auto row_bitmask = - cudf::detail::bitmask_and( - table_view{nullable_columns}, stream, rmm::mr::get_current_device_resource()) - .first; - auto const row_bitmask_ptr = static_cast(row_bitmask.data()); - return std::pair(std::move(row_bitmask), row_bitmask_ptr); - } - - return std::pair(rmm::device_buffer{0, stream}, nullable_columns.front().null_mask()); -} - -/** - * @brief Invokes the given `func` with desired comparators based on the specified `compare_nans` - * parameter - * - * @tparam HasNested Flag indicating whether there are nested columns in haystack or needles - * @tparam Hasher Type of device hash function - * @tparam Func Type of the helper function doing `contains` check - * - * @param compare_nulls Control whether nulls should be compared as equal or not - * @param compare_nans Control whether floating-point NaNs values should be compared as equal or not - * @param haystack_has_nulls Flag indicating whether haystack has nulls or not - * @param has_any_nulls Flag indicating whether there are nested nulls is either haystack or needles - * @param self_equal Self table comparator - * @param two_table_equal Two table comparator - * @param d_hasher Device hash functor - * @param func The input functor to invoke - */ -template -void dispatch_nan_comparator( - null_equality compare_nulls, - nan_equality compare_nans, - bool haystack_has_nulls, - bool has_any_nulls, - cudf::experimental::row::equality::self_comparator self_equal, - cudf::experimental::row::equality::two_table_comparator two_table_equal, - Hasher const& d_hasher, - Func&& func) -{ - // Distinguish probing scheme CG sizes between nested and flat types for better performance - auto const probing_scheme = [&]() { - if constexpr (HasNested) { - return cuco::experimental::linear_probing<4, Hasher>{d_hasher}; - } else { - return cuco::experimental::linear_probing<1, Hasher>{d_hasher}; - } - }(); - - if (compare_nans == nan_equality::ALL_EQUAL) { - using nan_equal_comparator = - cudf::experimental::row::equality::nan_equal_physical_equality_comparator; - auto const d_self_equal = self_equal.equal_to( - nullate::DYNAMIC{haystack_has_nulls}, compare_nulls, nan_equal_comparator{}); - auto const d_two_table_equal = two_table_equal.equal_to( - nullate::DYNAMIC{has_any_nulls}, compare_nulls, nan_equal_comparator{}); - func(d_self_equal, d_two_table_equal, probing_scheme); - } else { - using nan_unequal_comparator = cudf::experimental::row::equality::physical_equality_comparator; - auto const d_self_equal = self_equal.equal_to( - nullate::DYNAMIC{haystack_has_nulls}, compare_nulls, nan_unequal_comparator{}); - auto const d_two_table_equal = two_table_equal.equal_to( - nullate::DYNAMIC{has_any_nulls}, compare_nulls, nan_unequal_comparator{}); - func(d_self_equal, d_two_table_equal, probing_scheme); - } -} - -} // namespace - -rmm::device_uvector contains(table_view const& haystack, - table_view const& needles, - null_equality compare_nulls, - nan_equality compare_nans, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) -{ - CUDF_EXPECTS(cudf::have_same_types(haystack, needles), "Column types mismatch"); - - auto const haystack_has_nulls = has_nested_nulls(haystack); - auto const needles_has_nulls = has_nested_nulls(needles); - auto const has_any_nulls = haystack_has_nulls || needles_has_nulls; - - auto const preprocessed_needles = - cudf::experimental::row::equality::preprocessed_table::create(needles, stream); - auto const preprocessed_haystack = - cudf::experimental::row::equality::preprocessed_table::create(haystack, stream); - - auto const haystack_hasher = cudf::experimental::row::hash::row_hasher(preprocessed_haystack); - auto const d_haystack_hasher = haystack_hasher.device_hasher(nullate::DYNAMIC{has_any_nulls}); - auto const needle_hasher = cudf::experimental::row::hash::row_hasher(preprocessed_needles); - auto const d_needle_hasher = needle_hasher.device_hasher(nullate::DYNAMIC{has_any_nulls}); - auto const d_hasher = hasher_adapter{d_haystack_hasher, d_needle_hasher}; - - auto const self_equal = cudf::experimental::row::equality::self_comparator(preprocessed_haystack); - auto const two_table_equal = cudf::experimental::row::equality::two_table_comparator( - preprocessed_haystack, preprocessed_needles); - - // The output vector. - auto contained = rmm::device_uvector(needles.num_rows(), stream, mr); - - auto const haystack_iter = cudf::detail::make_counting_transform_iterator( - size_type{0}, cuda::proclaim_return_type([] __device__(auto idx) { - return lhs_index_type{idx}; - })); - auto const needles_iter = cudf::detail::make_counting_transform_iterator( - size_type{0}, cuda::proclaim_return_type([] __device__(auto idx) { - return rhs_index_type{idx}; - })); - - auto const helper_func = - [&](auto const& d_self_equal, auto const& d_two_table_equal, auto const& probing_scheme) { - auto const d_equal = comparator_adapter{d_self_equal, d_two_table_equal}; - - auto set = cuco::experimental::static_set{ - cuco::experimental::extent{compute_hash_table_size(haystack.num_rows())}, - cuco::empty_key{lhs_index_type{-1}}, - d_equal, - probing_scheme, - detail::hash_table_allocator_type{default_allocator{}, stream}, - stream.value()}; - - if (haystack_has_nulls && compare_nulls == null_equality::UNEQUAL) { - auto const bitmask_buffer_and_ptr = build_row_bitmask(haystack, stream); - auto const row_bitmask_ptr = bitmask_buffer_and_ptr.second; - - // If the haystack table has nulls but they are compared unequal, don't insert them. - // Otherwise, it was known to cause performance issue: - // - https://github.com/rapidsai/cudf/pull/6943 - // - https://github.com/rapidsai/cudf/pull/8277 - set.insert_if_async(haystack_iter, - haystack_iter + haystack.num_rows(), - thrust::counting_iterator(0), // stencil - row_is_valid{row_bitmask_ptr}, - stream.value()); - } else { - set.insert_async(haystack_iter, haystack_iter + haystack.num_rows(), stream.value()); - } - - if (needles_has_nulls && compare_nulls == null_equality::UNEQUAL) { - auto const bitmask_buffer_and_ptr = build_row_bitmask(needles, stream); - auto const row_bitmask_ptr = bitmask_buffer_and_ptr.second; - set.contains_if_async(needles_iter, - needles_iter + needles.num_rows(), - thrust::counting_iterator(0), // stencil - row_is_valid{row_bitmask_ptr}, - contained.begin(), - stream.value()); - } else { - set.contains_async( - needles_iter, needles_iter + needles.num_rows(), contained.begin(), stream.value()); - } - }; - - if (cudf::detail::has_nested_columns(haystack)) { - dispatch_nan_comparator(compare_nulls, - compare_nans, - haystack_has_nulls, - has_any_nulls, - self_equal, - two_table_equal, - d_hasher, - helper_func); - } else { - dispatch_nan_comparator(compare_nulls, - compare_nans, - haystack_has_nulls, - has_any_nulls, - self_equal, - two_table_equal, - d_hasher, - helper_func); - } - - return contained; -} - -} // namespace cudf::detail +/* + * Copyright (c) 2022-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include + +#include +#include + +#include + +#include + +#include + +#include + +namespace cudf::detail { + +namespace { + +using cudf::experimental::row::lhs_index_type; +using cudf::experimental::row::rhs_index_type; + +/** + * @brief An hasher adapter wrapping both haystack hasher and needles hasher + */ +template +struct hasher_adapter { + hasher_adapter(HaystackHasher const& haystack_hasher, NeedleHasher const& needle_hasher) + : _haystack_hasher{haystack_hasher}, _needle_hasher{needle_hasher} + { + } + + __device__ constexpr auto operator()(lhs_index_type idx) const noexcept + { + return _haystack_hasher(static_cast(idx)); + } + + __device__ constexpr auto operator()(rhs_index_type idx) const noexcept + { + return _needle_hasher(static_cast(idx)); + } + + private: + HaystackHasher const _haystack_hasher; + NeedleHasher const _needle_hasher; +}; + +/** + * @brief An comparator adapter wrapping both self comparator and two table comparator + */ +template +struct comparator_adapter { + comparator_adapter(SelfEqual const& self_equal, TwoTableEqual const& two_table_equal) + : _self_equal{self_equal}, _two_table_equal{two_table_equal} + { + } + + __device__ constexpr auto operator()(lhs_index_type lhs_index, + lhs_index_type rhs_index) const noexcept + { + auto const lhs = static_cast(lhs_index); + auto const rhs = static_cast(rhs_index); + + return _self_equal(lhs, rhs); + } + + __device__ constexpr auto operator()(lhs_index_type lhs_index, + rhs_index_type rhs_index) const noexcept + { + return _two_table_equal(lhs_index, rhs_index); + } + + private: + SelfEqual const _self_equal; + TwoTableEqual const _two_table_equal; +}; + +/** + * @brief Build a row bitmask for the input table. + * + * The output bitmask will have invalid bits corresponding to the input rows having nulls (at + * any nested level) and vice versa. + * + * @param input The input table + * @param stream CUDA stream used for device memory operations and kernel launches + * @return A pair of pointer to the output bitmask and the buffer containing the bitmask + */ +std::pair build_row_bitmask(table_view const& input, + rmm::cuda_stream_view stream) +{ + auto const nullable_columns = get_nullable_columns(input); + CUDF_EXPECTS(nullable_columns.size() > 0, + "The input table has nulls thus it should have nullable columns."); + + // If there are more than one nullable column, we compute `bitmask_and` of their null masks. + // Otherwise, we have only one nullable column and can use its null mask directly. + if (nullable_columns.size() > 1) { + auto row_bitmask = + cudf::detail::bitmask_and( + table_view{nullable_columns}, stream, rmm::mr::get_current_device_resource()) + .first; + auto const row_bitmask_ptr = static_cast(row_bitmask.data()); + return std::pair(std::move(row_bitmask), row_bitmask_ptr); + } + + return std::pair(rmm::device_buffer{0, stream}, nullable_columns.front().null_mask()); +} + +/** + * @brief Invokes the given `func` with desired comparators based on the specified `compare_nans` + * parameter + * + * @tparam HasNested Flag indicating whether there are nested columns in haystack or needles + * @tparam Hasher Type of device hash function + * @tparam Func Type of the helper function doing `contains` check + * + * @param compare_nulls Control whether nulls should be compared as equal or not + * @param compare_nans Control whether floating-point NaNs values should be compared as equal or not + * @param haystack_has_nulls Flag indicating whether haystack has nulls or not + * @param has_any_nulls Flag indicating whether there are nested nulls is either haystack or needles + * @param self_equal Self table comparator + * @param two_table_equal Two table comparator + * @param d_hasher Device hash functor + * @param func The input functor to invoke + */ +template +void dispatch_nan_comparator( + null_equality compare_nulls, + nan_equality compare_nans, + bool haystack_has_nulls, + bool has_any_nulls, + cudf::experimental::row::equality::self_comparator self_equal, + cudf::experimental::row::equality::two_table_comparator two_table_equal, + Hasher const& d_hasher, + Func&& func) +{ + // Distinguish probing scheme CG sizes between nested and flat types for better performance + auto const probing_scheme = [&]() { + if constexpr (HasNested) { + return cuco::experimental::linear_probing<4, Hasher>{d_hasher}; + } else { + return cuco::experimental::linear_probing<1, Hasher>{d_hasher}; + } + }(); + + if (compare_nans == nan_equality::ALL_EQUAL) { + using nan_equal_comparator = + cudf::experimental::row::equality::nan_equal_physical_equality_comparator; + auto const d_self_equal = self_equal.equal_to( + nullate::DYNAMIC{haystack_has_nulls}, compare_nulls, nan_equal_comparator{}); + auto const d_two_table_equal = two_table_equal.equal_to( + nullate::DYNAMIC{has_any_nulls}, compare_nulls, nan_equal_comparator{}); + func(d_self_equal, d_two_table_equal, probing_scheme); + } else { + using nan_unequal_comparator = cudf::experimental::row::equality::physical_equality_comparator; + auto const d_self_equal = self_equal.equal_to( + nullate::DYNAMIC{haystack_has_nulls}, compare_nulls, nan_unequal_comparator{}); + auto const d_two_table_equal = two_table_equal.equal_to( + nullate::DYNAMIC{has_any_nulls}, compare_nulls, nan_unequal_comparator{}); + func(d_self_equal, d_two_table_equal, probing_scheme); + } +} + +} // namespace + +rmm::device_uvector contains(table_view const& haystack, + table_view const& needles, + null_equality compare_nulls, + nan_equality compare_nans, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) +{ + CUDF_EXPECTS(cudf::have_same_types(haystack, needles), "Column types mismatch"); + + auto const haystack_has_nulls = has_nested_nulls(haystack); + auto const needles_has_nulls = has_nested_nulls(needles); + auto const has_any_nulls = haystack_has_nulls || needles_has_nulls; + + auto const preprocessed_needles = + cudf::experimental::row::equality::preprocessed_table::create(needles, stream); + auto const preprocessed_haystack = + cudf::experimental::row::equality::preprocessed_table::create(haystack, stream); + + auto const haystack_hasher = cudf::experimental::row::hash::row_hasher(preprocessed_haystack); + auto const d_haystack_hasher = haystack_hasher.device_hasher(nullate::DYNAMIC{has_any_nulls}); + auto const needle_hasher = cudf::experimental::row::hash::row_hasher(preprocessed_needles); + auto const d_needle_hasher = needle_hasher.device_hasher(nullate::DYNAMIC{has_any_nulls}); + auto const d_hasher = hasher_adapter{d_haystack_hasher, d_needle_hasher}; + + auto const self_equal = cudf::experimental::row::equality::self_comparator(preprocessed_haystack); + auto const two_table_equal = cudf::experimental::row::equality::two_table_comparator( + preprocessed_haystack, preprocessed_needles); + + // The output vector. + auto contained = rmm::device_uvector(needles.num_rows(), stream, mr); + + auto const haystack_iter = cudf::detail::make_counting_transform_iterator( + size_type{0}, cuda::proclaim_return_type([] __device__(auto idx) { + return lhs_index_type{idx}; + })); + auto const needles_iter = cudf::detail::make_counting_transform_iterator( + size_type{0}, cuda::proclaim_return_type([] __device__(auto idx) { + return rhs_index_type{idx}; + })); + + auto const helper_func = + [&](auto const& d_self_equal, auto const& d_two_table_equal, auto const& probing_scheme) { + auto const d_equal = comparator_adapter{d_self_equal, d_two_table_equal}; + + auto set = cuco::experimental::static_set{ + cuco::experimental::extent{compute_hash_table_size(haystack.num_rows())}, + cuco::empty_key{lhs_index_type{-1}}, + d_equal, + probing_scheme, + detail::hash_table_allocator_type{default_allocator{}, stream}, + stream.value()}; + + if (haystack_has_nulls && compare_nulls == null_equality::UNEQUAL) { + auto const bitmask_buffer_and_ptr = build_row_bitmask(haystack, stream); + auto const row_bitmask_ptr = bitmask_buffer_and_ptr.second; + + // If the haystack table has nulls but they are compared unequal, don't insert them. + // Otherwise, it was known to cause performance issue: + // - https://github.com/rapidsai/cudf/pull/6943 + // - https://github.com/rapidsai/cudf/pull/8277 + set.insert_if_async(haystack_iter, + haystack_iter + haystack.num_rows(), + thrust::counting_iterator(0), // stencil + row_is_valid{row_bitmask_ptr}, + stream.value()); + } else { + set.insert_async(haystack_iter, haystack_iter + haystack.num_rows(), stream.value()); + } + + if (needles_has_nulls && compare_nulls == null_equality::UNEQUAL) { + auto const bitmask_buffer_and_ptr = build_row_bitmask(needles, stream); + auto const row_bitmask_ptr = bitmask_buffer_and_ptr.second; + set.contains_if_async(needles_iter, + needles_iter + needles.num_rows(), + thrust::counting_iterator(0), // stencil + row_is_valid{row_bitmask_ptr}, + contained.begin(), + stream.value()); + } else { + set.contains_async( + needles_iter, needles_iter + needles.num_rows(), contained.begin(), stream.value()); + } + }; + + if (cudf::detail::has_nested_columns(haystack)) { + dispatch_nan_comparator(compare_nulls, + compare_nans, + haystack_has_nulls, + has_any_nulls, + self_equal, + two_table_equal, + d_hasher, + helper_func); + } else { + dispatch_nan_comparator(compare_nulls, + compare_nans, + haystack_has_nulls, + has_any_nulls, + self_equal, + two_table_equal, + d_hasher, + helper_func); + } + + return contained; +} + +} // namespace cudf::detail From d83f12e37a2a42d3fce7f2302b104ee8f4b0619e Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 26 Jan 2024 01:29:38 +0530 Subject: [PATCH 121/384] Preserve columns dtype in dataframe constructor (#14878) This PR preserves columns dtype in DataFrame constructor. This PR: = 52 failed, 101872 passed, 2091 skipped, 977 xfailed, 312 xpassed in 1188.72s (0:19:48) = On pandas_2.0_feature_branch: = 61 failed, 101866 passed, 2091 skipped, 976 xfailed, 312 xpassed in 1446.19s (0:24:06) = --- python/cudf/cudf/core/dataframe.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index a3642bcc43f..5fa1956eaf1 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -795,6 +795,7 @@ def __init__( if is_list_like(data): if len(data) > 0 and is_scalar(data[0]): if columns is not None: + label_dtype = getattr(columns, "dtype", None) data = dict(zip(columns, [data])) rangeindex = isinstance( columns, (range, pd.RangeIndex, cudf.RangeIndex) @@ -802,6 +803,7 @@ def __init__( else: data = dict(enumerate([data])) rangeindex = True + label_dtype = None new_df = DataFrame(data=data, index=index) self._data = new_df._data @@ -812,6 +814,11 @@ def __init__( else self._data._level_names ) self._data.rangeindex = rangeindex + self._data.label_dtype = ( + cudf.dtype(label_dtype) + if label_dtype is not None + else None + ) elif len(data) > 0 and isinstance(data[0], Series): self._init_from_series_list( data=data, columns=columns, index=index From 8db3b706d1ff4c6182abe5d5c7374a8233772c96 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 25 Jan 2024 22:35:13 +0000 Subject: [PATCH 122/384] Disable style check --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index edcc140b191..25c1294d11a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -37,6 +37,7 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.02 with: enable_check_generated_files: false + enable_check_style: false conda-cpp-build: needs: checks secrets: inherit From 4b5b8af994956315836fdf252cd453620e5e9aea Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 25 Jan 2024 22:37:59 +0000 Subject: [PATCH 123/384] Pin pandas --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-120_arch-x86_64.yaml | 2 +- dependencies.yaml | 2 +- python/cudf/pyproject.toml | 2 +- python/dask_cudf/pyproject.toml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index a5e3ea4c531..0c58976de86 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -64,7 +64,7 @@ dependencies: - nvcomp==3.0.5 - nvtx>=0.2.1 - packaging -- pandas>=1.3,<1.6.0dev0 +- pandas==2.1.4 - pandoc - pip - pre-commit diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 579bbb6d52d..0671a1a8f98 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -62,7 +62,7 @@ dependencies: - nvcomp==3.0.5 - nvtx>=0.2.1 - packaging -- pandas>=1.3,<1.6.0dev0 +- pandas==2.1.4 - pandoc - pip - pre-commit diff --git a/dependencies.yaml b/dependencies.yaml index 20998847a75..3513c6161c8 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -500,7 +500,7 @@ dependencies: packages: - fsspec>=0.6.0 - *numpy - - pandas>=1.3,<1.6.0dev0 + - pandas==2.1.4 run_cudf: common: - output_types: [conda, requirements, pyproject] diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 18771804f61..62f218e86a4 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ "numpy>=1.21,<1.25", "nvtx>=0.2.1", "packaging", - "pandas>=1.3,<1.6.0dev0", + "pandas==2.1.4", "protobuf>=4.21,<5", "ptxcompiler", "pyarrow>=14.0.1,<15.0.0a0", diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index 33065da6e8d..6613a5f32e3 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "cupy-cuda11x>=12.0.0", "fsspec>=0.6.0", "numpy>=1.21,<1.25", - "pandas>=1.3,<1.6.0dev0", + "pandas==2.1.4", "rapids-dask-dependency==24.2.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ From d2cc4db4a46fd787122b2117ff8981d450b2a06a Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 25 Jan 2024 22:41:59 +0000 Subject: [PATCH 124/384] Disable some more jobs --- .github/workflows/pr.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 25c1294d11a..67396cb0274 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -13,19 +13,19 @@ jobs: pr-builder: needs: - checks - - conda-cpp-build - - conda-cpp-tests - - conda-python-build - - conda-python-cudf-tests - - conda-python-other-tests - - conda-java-tests - - conda-notebook-tests - - docs-build + #- conda-cpp-build + #- conda-cpp-tests + #- conda-python-build + #- conda-python-cudf-tests + #- conda-python-other-tests + #- conda-java-tests + #- conda-notebook-tests + #- docs-build - wheel-build-cudf - wheel-tests-cudf - wheel-build-dask-cudf - wheel-tests-dask-cudf - - devcontainer + #- devcontainer - unit-tests-cudf-pandas - pandas-tests #- pandas-tests-diff From 32e0982ed73319f2459581ad83e21d19d567ce56 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 25 Jan 2024 22:44:55 +0000 Subject: [PATCH 125/384] Actually remove the jobs --- .github/workflows/pr.yaml | 142 +++++++++++++++++++------------------- 1 file changed, 71 insertions(+), 71 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 67396cb0274..c570fea3ea4 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -38,69 +38,69 @@ jobs: with: enable_check_generated_files: false enable_check_style: false - conda-cpp-build: - needs: checks - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.02 - with: - build_type: pull-request - conda-cpp-tests: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.02 - with: - build_type: pull-request - conda-python-build: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.02 - with: - build_type: pull-request - conda-python-cudf-tests: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.02 - with: - build_type: pull-request - test_script: "ci/test_python_cudf.sh" - conda-python-other-tests: - # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.02 - with: - build_type: pull-request - test_script: "ci/test_python_other.sh" - conda-java-tests: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.02 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/test_java.sh" - conda-notebook-tests: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.02 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/test_notebooks.sh" - docs-build: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.02 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/build_docs.sh" + #conda-cpp-build: + # needs: checks + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.02 + # with: + # build_type: pull-request + #conda-cpp-tests: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.02 + # with: + # build_type: pull-request + #conda-python-build: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.02 + # with: + # build_type: pull-request + #conda-python-cudf-tests: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.02 + # with: + # build_type: pull-request + # test_script: "ci/test_python_cudf.sh" + #conda-python-other-tests: + # # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.02 + # with: + # build_type: pull-request + # test_script: "ci/test_python_other.sh" + #conda-java-tests: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.02 + # with: + # build_type: pull-request + # node_type: "gpu-v100-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:latest" + # run_script: "ci/test_java.sh" + #conda-notebook-tests: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.02 + # with: + # build_type: pull-request + # node_type: "gpu-v100-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:latest" + # run_script: "ci/test_notebooks.sh" + #docs-build: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.02 + # with: + # build_type: pull-request + # node_type: "gpu-v100-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:latest" + # run_script: "ci/build_docs.sh" wheel-build-cudf: needs: checks secrets: inherit @@ -132,14 +132,14 @@ jobs: matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) build_type: pull-request script: ci/test_wheel_dask_cudf.sh - devcontainer: - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.02 - with: - build_command: | - sccache -z; - build-all -DBUILD_BENCHMARKS=ON -DNVBench_ENABLE_CUPTI=OFF --verbose; - sccache -s; + #devcontainer: + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.02 + # with: + # build_command: | + # sccache -z; + # build-all -DBUILD_BENCHMARKS=ON -DNVBench_ENABLE_CUPTI=OFF --verbose; + # sccache -s; unit-tests-cudf-pandas: needs: wheel-build-cudf secrets: inherit From 821f4dea107db6a51fcbffff997fa6844ab5565f Mon Sep 17 00:00:00 2001 From: nvdbaranec <56695930+nvdbaranec@users.noreply.github.com> Date: Thu, 25 Jan 2024 17:44:58 -0600 Subject: [PATCH 126/384] Fixed an issue with output chunking computation stemming from input chunking. (#14889) Fixes https://github.com/rapidsai/cudf/issues/14883 The core issue was that the output chunking code was expecting all columns to have terminating pages that end in the same row count. Previously this was the case because we always processed entire row groups. But now with the subrowgroup reader, we can split on page boundaries that cause a jagged max row index for different columns. Example: ``` 0 100 200 Col A [-----------][--------------] 300 Col B [-----------][----------------------] ``` The input chunking would have computed a max row index of 200 for the subpass. But when computing the _output_ chunks, there was code that would have tried finding where row 300 was in column A, resulting in an out-of-bounds read. The fix is simply to cap the max row seen for column B to be the max expected row for the subpass. Authors: - https://github.com/nvdbaranec Approvers: - Nghia Truong (https://github.com/ttnghia) - Vukasin Milovanovic (https://github.com/vuule) - Mike Wilson (https://github.com/hyperbolic2346) URL: https://github.com/rapidsai/cudf/pull/14889 --- cpp/src/io/parquet/reader_impl_chunking.cu | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/cpp/src/io/parquet/reader_impl_chunking.cu b/cpp/src/io/parquet/reader_impl_chunking.cu index 1bfe5745b9e..e0cb2fbb4f4 100644 --- a/cpp/src/io/parquet/reader_impl_chunking.cu +++ b/cpp/src/io/parquet/reader_impl_chunking.cu @@ -253,13 +253,15 @@ struct set_row_index { device_span chunks; device_span pages; device_span c_info; + size_t max_row; __device__ void operator()(size_t i) { - auto const& page = pages[i]; - auto const& chunk = chunks[page.chunk_idx]; - size_t const page_start_row = chunk.start_row + page.chunk_row + page.num_rows; - c_info[i].row_index = page_start_row; + auto const& page = pages[i]; + auto const& chunk = chunks[page.chunk_idx]; + size_t const page_end_row = chunk.start_row + page.chunk_row + page.num_rows; + // if we have been passed in a cap, apply it + c_info[i].row_index = max_row > 0 ? min(max_row, page_end_row) : page_end_row; } }; @@ -1288,7 +1290,7 @@ void reader::impl::setup_next_subpass(bool uses_custom_row_bounds) thrust::for_each(rmm::exec_policy_nosync(_stream), iter, iter + pass.pages.size(), - set_row_index{pass.chunks, pass.pages, c_info}); + set_row_index{pass.chunks, pass.pages, c_info, 0}); // print_cumulative_page_info(pass.pages, pass.chunks, c_info, _stream); // get the next batch of pages @@ -1533,10 +1535,15 @@ void reader::impl::compute_output_chunks_for_subpass() thrust::equal_to{}, cumulative_page_sum{}); auto iter = thrust::make_counting_iterator(0); + // cap the max row in all pages by the max row we expect in the subpass. input chunking + // can cause "dangling" row counts where for example, only 1 column has a page whose + // maximum row is beyond our expected subpass max row, which will cause an out of + // bounds index in compute_page_splits_by_row. + auto const subpass_max_row = subpass.skip_rows + subpass.num_rows; thrust::for_each(rmm::exec_policy_nosync(_stream), iter, iter + subpass.pages.size(), - set_row_index{pass.chunks, subpass.pages, c_info}); + set_row_index{pass.chunks, subpass.pages, c_info, subpass_max_row}); // print_cumulative_page_info(subpass.pages, c_info, _stream); // compute the splits From 302c8760008afee155034afe5a7913b94bc899c2 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 25 Jan 2024 16:10:01 -0800 Subject: [PATCH 127/384] Unpin numpy<1.25 --- .../all_cuda-118_arch-x86_64.yaml | 2 +- .../all_cuda-120_arch-x86_64.yaml | 2 +- conda/recipes/cudf/meta.yaml | 3 +- dependencies.yaml | 3 +- python/cudf/cudf/core/column/categorical.py | 7 +++-- python/cudf/cudf/core/column/column.py | 7 ++--- python/cudf/cudf/core/column/numerical.py | 2 +- python/cudf/cudf/core/join/_join_helpers.py | 2 +- python/cudf/cudf/tests/test_array_function.py | 6 ++-- python/cudf/cudf/tests/test_datasets.py | 2 +- python/cudf/cudf/tests/test_joining.py | 8 ++--- python/cudf/cudf/utils/dtypes.py | 30 ++++++++++++++++++- python/cudf/pyproject.toml | 4 +-- python/cudf_kafka/pyproject.toml | 2 +- python/dask_cudf/pyproject.toml | 2 +- 15 files changed, 54 insertions(+), 28 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 0c58976de86..3d3830e1dc9 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -58,7 +58,7 @@ dependencies: - ninja - notebook - numba>=0.57 -- numpy>=1.21,<1.25 +- numpy>=1.21 - numpydoc - nvcc_linux-64=11.8 - nvcomp==3.0.5 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 0671a1a8f98..e441a0aac4b 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -57,7 +57,7 @@ dependencies: - ninja - notebook - numba>=0.57 -- numpy>=1.21,<1.25 +- numpy>=1.21 - numpydoc - nvcomp==3.0.5 - nvtx>=0.2.1 diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index bc91ee61f6f..89c3d8ecab2 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -80,8 +80,7 @@ requirements: - cupy >=12.0.0 # TODO: Pin to numba<0.58 until #14160 is resolved - numba >=0.57,<0.58 - # TODO: Pin to numpy<1.25 until cudf requires pandas 2 - - numpy >=1.21,<1.25 + - numpy >=1.21 - {{ pin_compatible('pyarrow', max_pin='x') }} - libcudf ={{ version }} - {{ pin_compatible('rmm', max_pin='x.x') }} diff --git a/dependencies.yaml b/dependencies.yaml index 3513c6161c8..9289a07083e 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -266,8 +266,7 @@ dependencies: - *cmake_ver - cython>=3.0.3 - *ninja - # TODO: Pin to numpy<1.25 until cudf requires pandas 2 - - &numpy numpy>=1.21,<1.25 + - &numpy numpy>=1.21 # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. - pyarrow==14.0.1.* diff --git a/python/cudf/cudf/core/column/categorical.py b/python/cudf/cudf/core/column/categorical.py index fb7f841c3f3..f036703a147 100644 --- a/python/cudf/cudf/core/column/categorical.py +++ b/python/cudf/cudf/core/column/categorical.py @@ -7,12 +7,12 @@ from typing import TYPE_CHECKING, Any, Mapping, Optional, Sequence, Tuple, cast import numpy as np +import pandas as pd import pyarrow as pa from numba import cuda from typing_extensions import Self import cudf -import pandas as pd from cudf import _lib as libcudf from cudf._lib.transform import bools_to_mask from cudf._typing import ColumnBinaryOperand, ColumnLike, Dtype, ScalarLike @@ -21,6 +21,7 @@ from cudf.core.column.methods import ColumnMethods from cudf.core.dtypes import CategoricalDtype, IntervalDtype from cudf.utils.dtypes import ( + find_common_type, is_mixed_with_object_dtype, min_signed_type, min_unsigned_type, @@ -265,8 +266,8 @@ def add_categories(self, new_categories: Any) -> Optional[SeriesOrIndex]: f"type-cast new_categories to the same type as " f"existing categories." ) - common_dtype = np.find_common_type( - [old_categories.dtype, new_categories.dtype], [] + common_dtype = find_common_type( + [old_categories.dtype, new_categories.dtype] ) new_categories = new_categories.astype(common_dtype) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 70e7717be33..d37f0d54c1a 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -5,7 +5,6 @@ import builtins import pickle import warnings - from collections import abc from functools import cached_property from itertools import chain @@ -25,6 +24,7 @@ import cupy import numpy as np +import pandas as pd import pyarrow as pa import pyarrow.compute as pc from numba import cuda @@ -33,7 +33,6 @@ import rmm import cudf -import pandas as pd from cudf import _lib as libcudf from cudf._lib.column import Column from cudf._lib.null_mask import ( @@ -87,6 +86,7 @@ from cudf.utils.dtypes import ( _maybe_convert_to_default_type, cudf_dtype_from_pa_type, + find_common_type, get_time_unit, is_mixed_with_object_dtype, min_scalar_type, @@ -2671,8 +2671,7 @@ def concat_columns(objs: "MutableSequence[ColumnBase]") -> ColumnBase: and np.issubdtype(dtyp, np.datetime64) for dtyp in not_null_col_dtypes ): - # Use NumPy to find a common dtype - common_dtype = np.find_common_type(not_null_col_dtypes, []) + common_dtype = find_common_type(not_null_col_dtypes) # Cast all columns to the common dtype objs = [obj.astype(common_dtype) for obj in objs] diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index 35de4d0ae7c..ae4ad9c5136 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -702,7 +702,7 @@ def _reduction_result_dtype(self, reduction_op: str) -> Dtype: col_dtype if col_dtype.kind == "f" else np.dtype("int64") ) elif reduction_op == "sum_of_squares": - col_dtype = np.find_common_type([col_dtype], [np.dtype("uint64")]) + col_dtype = np.result_dtype(col_dtype, np.dtype("uint64")) return col_dtype diff --git a/python/cudf/cudf/core/join/_join_helpers.py b/python/cudf/cudf/core/join/_join_helpers.py index 0aebc6453bc..6a619945e75 100644 --- a/python/cudf/cudf/core/join/_join_helpers.py +++ b/python/cudf/cudf/core/join/_join_helpers.py @@ -98,7 +98,7 @@ def _match_join_keys( common_type = ( max(ltype, rtype) if ltype.kind == rtype.kind - else np.find_common_type([], (ltype, rtype)) + else np.result_type(ltype, rtype) ) elif ( np.issubdtype(ltype, np.datetime64) diff --git a/python/cudf/cudf/tests/test_array_function.py b/python/cudf/cudf/tests/test_array_function.py index 758a8cbb535..db48a017138 100644 --- a/python/cudf/cudf/tests/test_array_function.py +++ b/python/cudf/cudf/tests/test_array_function.py @@ -69,9 +69,9 @@ def test_array_func_cudf_series(np_ar, func): lambda x: np.dot(x, x.transpose()), lambda x: np.all(x), lambda x: np.any(x), - lambda x: np.product(x), - lambda x: np.product(x, axis=0), - lambda x: np.product(x, axis=1), + lambda x: np.prod(x), + lambda x: np.prod(x, axis=0), + lambda x: np.prod(x, axis=1), ], ) def test_array_func_cudf_dataframe(pd_df, func): diff --git a/python/cudf/cudf/tests/test_datasets.py b/python/cudf/cudf/tests/test_datasets.py index 98f801d0cba..fdab8cb5edf 100644 --- a/python/cudf/cudf/tests/test_datasets.py +++ b/python/cudf/cudf/tests/test_datasets.py @@ -52,6 +52,6 @@ def test_make_bool(): n = 10 state = np.random.RandomState(12) arr = gd.datasets.make_bool(n, state) - assert np.alltrue(np.isin(arr, [True, False])) + assert np.all(np.isin(arr, [True, False])) assert arr.size == n assert arr.dtype == bool diff --git a/python/cudf/cudf/tests/test_joining.py b/python/cudf/cudf/tests/test_joining.py index 8ce2adae15b..2b35c808466 100644 --- a/python/cudf/cudf/tests/test_joining.py +++ b/python/cudf/cudf/tests/test_joining.py @@ -7,6 +7,7 @@ import pytest import cudf +from cudf.core._compat import PANDAS_GE_200 from cudf.core.dtypes import CategoricalDtype, Decimal64Dtype, Decimal128Dtype from cudf.testing._utils import ( INTEGER_TYPES, @@ -16,7 +17,6 @@ assert_exceptions_equal, expect_warning_if, ) -from cudf.core._compat import PANDAS_GE_200 _JOIN_TYPES = ("left", "inner", "outer", "right", "leftanti", "leftsemi") @@ -982,7 +982,7 @@ def test_typecast_on_join_int_to_int(dtype_l, dtype_r): gdf_l = cudf.DataFrame({"join_col": join_data_l, "B": other_data}) gdf_r = cudf.DataFrame({"join_col": join_data_r, "B": other_data}) - exp_dtype = np.find_common_type([], [np.dtype(dtype_l), np.dtype(dtype_r)]) + exp_dtype = np.result_type(np.dtype(dtype_l), np.dtype(dtype_r)) exp_join_data = [1, 2] exp_other_data = ["a", "b"] @@ -1012,7 +1012,7 @@ def test_typecast_on_join_float_to_float(dtype_l, dtype_r): gdf_l = cudf.DataFrame({"join_col": join_data_l, "B": other_data}) gdf_r = cudf.DataFrame({"join_col": join_data_r, "B": other_data}) - exp_dtype = np.find_common_type([], [np.dtype(dtype_l), np.dtype(dtype_r)]) + exp_dtype = np.result_type(np.dtype(dtype_l), np.dtype(dtype_r)) if dtype_l != dtype_r: exp_join_data = [1, 2, 3, 4.5] @@ -1053,7 +1053,7 @@ def test_typecast_on_join_mixed_int_float(dtype_l, dtype_r): gdf_l = cudf.DataFrame({"join_col": join_data_l, "B": other_data}) gdf_r = cudf.DataFrame({"join_col": join_data_r, "B": other_data}) - exp_dtype = np.find_common_type([], [np.dtype(dtype_l), np.dtype(dtype_r)]) + exp_dtype = np.result_type(np.dtype(dtype_l), np.dtype(dtype_r)) exp_join_data = [1, 2, 3] exp_other_data = ["a", "b", "c"] diff --git a/python/cudf/cudf/utils/dtypes.py b/python/cudf/cudf/utils/dtypes.py index 538678b47b0..345b7b0aad6 100644 --- a/python/cudf/cudf/utils/dtypes.py +++ b/python/cudf/cudf/utils/dtypes.py @@ -507,6 +507,34 @@ def get_allowed_combinations_for_operator(dtype_l, dtype_r, op): raise error +def np_find_common_type(*dtypes: np.dtype) -> np.dtype: + """ + np.find_common_type implementation pre-1.25 deprecation using np.result_type + https://github.com/pandas-dev/pandas/pull/49569#issuecomment-1308300065 + + Parameters + ---------- + dtypes : np.dtypes + + Returns + ------- + np.dtype + """ + # TODO: possibly raise the TypeError. Coercing to np.dtype("O") (string) + # might not make sense in cudf + try: + common_dtype = np.result_type(*dtypes) + if common_dtype.kind in "mMSU": + # NumPy promotion currently (1.25) misbehaves for for times and strings, + # so fall back to object (find_common_dtype did unless there + # was only one dtype) + common_dtype = np.dtype("O") + + except TypeError: + common_dtype = np.dtype("O") + return common_dtype + + def find_common_type(dtypes): """ Wrapper over np.find_common_type to handle special cases @@ -614,7 +642,7 @@ def find_common_type(dtypes): dtypes = dtypes - td_dtypes dtypes.add(np.result_type(*td_dtypes)) - common_dtype = np.find_common_type(list(dtypes), []) + common_dtype = np_find_common_type(*dtypes) if common_dtype == np.dtype("float16"): return cudf.dtype("float32") return cudf.dtype(common_dtype) diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 62f218e86a4..9f31c34cbf2 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -6,7 +6,7 @@ requires = [ "cmake>=3.26.4", "cython>=3.0.3", "ninja", - "numpy>=1.21,<1.25", + "numpy>=1.21", "protoc-wheel", "pyarrow==14.0.1.*", "rmm==24.2.*", @@ -30,7 +30,7 @@ dependencies = [ "cupy-cuda11x>=12.0.0", "fsspec>=0.6.0", "numba>=0.57", - "numpy>=1.21,<1.25", + "numpy>=1.21", "nvtx>=0.2.1", "packaging", "pandas==2.1.4", diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index 062a0224c1f..e7549eb7e10 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -6,7 +6,7 @@ requires = [ "cmake>=3.26.4", "cython>=3.0.3", "ninja", - "numpy>=1.21,<1.25", + "numpy>=1.21", "pyarrow==14.0.1.*", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index 6613a5f32e3..2567a6ec565 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "cudf==24.2.*", "cupy-cuda11x>=12.0.0", "fsspec>=0.6.0", - "numpy>=1.21,<1.25", + "numpy>=1.21", "pandas==2.1.4", "rapids-dask-dependency==24.2.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 481ea9cecb8721ebe13aca3650933d34e76de511 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 25 Jan 2024 18:15:51 -0800 Subject: [PATCH 128/384] Remove pandas shim and use result_type --- python/cudf/cudf/utils/dtypes.py | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/python/cudf/cudf/utils/dtypes.py b/python/cudf/cudf/utils/dtypes.py index 345b7b0aad6..2dc3a08eb8d 100644 --- a/python/cudf/cudf/utils/dtypes.py +++ b/python/cudf/cudf/utils/dtypes.py @@ -507,34 +507,6 @@ def get_allowed_combinations_for_operator(dtype_l, dtype_r, op): raise error -def np_find_common_type(*dtypes: np.dtype) -> np.dtype: - """ - np.find_common_type implementation pre-1.25 deprecation using np.result_type - https://github.com/pandas-dev/pandas/pull/49569#issuecomment-1308300065 - - Parameters - ---------- - dtypes : np.dtypes - - Returns - ------- - np.dtype - """ - # TODO: possibly raise the TypeError. Coercing to np.dtype("O") (string) - # might not make sense in cudf - try: - common_dtype = np.result_type(*dtypes) - if common_dtype.kind in "mMSU": - # NumPy promotion currently (1.25) misbehaves for for times and strings, - # so fall back to object (find_common_dtype did unless there - # was only one dtype) - common_dtype = np.dtype("O") - - except TypeError: - common_dtype = np.dtype("O") - return common_dtype - - def find_common_type(dtypes): """ Wrapper over np.find_common_type to handle special cases @@ -642,7 +614,7 @@ def find_common_type(dtypes): dtypes = dtypes - td_dtypes dtypes.add(np.result_type(*td_dtypes)) - common_dtype = np_find_common_type(*dtypes) + common_dtype = np.result_type(*dtypes) if common_dtype == np.dtype("float16"): return cudf.dtype("float32") return cudf.dtype(common_dtype) From a41238fd009403030b6f172025263633c913f82c Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 25 Jan 2024 18:55:37 -0800 Subject: [PATCH 129/384] Optimize doc builds (#14856) cudf docs are generally very slow to build. This problem was exacerbated by the recent addition of libcudf C++ API documentation to the Sphinx build. This PR aims to ameliorate this issue for both local and CI builds by making the following changes: - The XML parsing logic used to clean up doxygen XML now avoids rewriting files unless they are actually modified. This prevents Sphinx from doing extra work during a second (text) build after the first (HTML) build. - toctrees on the generated API pages are removed (see https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/performance.html#selectively-remove-pages-from-your-sidebar). - Text docs are disabled in PRs and only occur in nightly/branch builds. The net result is roughly a halving of the CI run time for the builds (~40 min to ~20 min). Further potential optimizations: - Reenabling parallel builds. We cannot fully revert https://github.com/rapidsai/cudf/pull/14796 until the theme is fixed, but if we can put in a warning filter we could reenable parallelism and have it work on just the reading steps of the build and not the writes. That would still improve performance. - Better caching of notebooks. [nbsphinx supports caching](https://myst-nb.readthedocs.io/en/latest/computation/execute.html#execute-cache), but there are various caveats w.r.t. 1) local vs CI builds, 2) proper cache invalidation, e.g. when notebook source does not change but underlying libraries do, and 3) forcing rebuilds. Alternatively, we could enable some environment variable that allows devs to turn off notebook execution locally. Making it opt-in would make the default behavior safe while providing an escape hatch for power users who want the builds to be fast. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/14856 --- ci/build_docs.sh | 18 ++++++++++++------ .../environments/all_cuda-118_arch-x86_64.yaml | 1 + .../environments/all_cuda-120_arch-x86_64.yaml | 1 + dependencies.yaml | 1 + docs/cudf/source/conf.py | 13 ++++++++++++- 5 files changed, 27 insertions(+), 7 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 5a4bf3e0dbc..2b55a9db8af 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -41,19 +41,25 @@ popd rapids-logger "Build Python docs" pushd docs/cudf make dirhtml -make text -mkdir -p "${RAPIDS_DOCS_DIR}/cudf/"{html,txt} +mkdir -p "${RAPIDS_DOCS_DIR}/cudf/html" mv build/dirhtml/* "${RAPIDS_DOCS_DIR}/cudf/html" -mv build/text/* "${RAPIDS_DOCS_DIR}/cudf/txt" +if [[ "${RAPIDS_BUILD_TYPE}" != "pull-request" ]]; then + make text + mkdir -p "${RAPIDS_DOCS_DIR}/cudf/txt" + mv build/text/* "${RAPIDS_DOCS_DIR}/cudf/txt" +fi popd rapids-logger "Build dask-cuDF Sphinx docs" pushd docs/dask_cudf make dirhtml -make text -mkdir -p "${RAPIDS_DOCS_DIR}/dask-cudf/"{html,txt} +mkdir -p "${RAPIDS_DOCS_DIR}/dask-cudf/html" mv build/dirhtml/* "${RAPIDS_DOCS_DIR}/dask-cudf/html" -mv build/text/* "${RAPIDS_DOCS_DIR}/dask-cudf/txt" +if [[ "${RAPIDS_BUILD_TYPE}" != "pull-request" ]]; then + make text + mkdir -p "${RAPIDS_DOCS_DIR}/dask-cudf/txt" + mv build/text/* "${RAPIDS_DOCS_DIR}/dask-cudf/txt" +fi popd rapids-upload-docs diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 8081d9de8b9..2794125b78a 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -93,6 +93,7 @@ dependencies: - sphinx-autobuild - sphinx-copybutton - sphinx-markdown-tables +- sphinx-remove-toctrees - sphinxcontrib-websupport - streamz - sysroot_linux-64==2.17 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 1cb8f376f82..91104c55961 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -91,6 +91,7 @@ dependencies: - sphinx-autobuild - sphinx-copybutton - sphinx-markdown-tables +- sphinx-remove-toctrees - sphinxcontrib-websupport - streamz - sysroot_linux-64==2.17 diff --git a/dependencies.yaml b/dependencies.yaml index e62fa86d4d4..09866869a8b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -472,6 +472,7 @@ dependencies: - sphinx-autobuild - sphinx-copybutton - sphinx-markdown-tables + - sphinx-remove-toctrees - sphinxcontrib-websupport notebooks: common: diff --git a/docs/cudf/source/conf.py b/docs/cudf/source/conf.py index 01a6c5316bd..5b04335f475 100644 --- a/docs/cudf/source/conf.py +++ b/docs/cudf/source/conf.py @@ -16,10 +16,12 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # +import filecmp import glob import os import re import sys +import tempfile import xml.etree.ElementTree as ET from docutils.nodes import Text @@ -62,6 +64,7 @@ class PseudoLexer(RegexLexer): "sphinx.ext.autodoc", "sphinx.ext.autosummary", "sphinx_copybutton", + "sphinx_remove_toctrees", "numpydoc", "IPython.sphinxext.ipython_console_highlighting", "IPython.sphinxext.ipython_directive", @@ -69,6 +72,8 @@ class PseudoLexer(RegexLexer): "myst_nb", ] +remove_from_toctrees = ["user_guide/api_docs/api/*"] + # Preprocess doxygen xml for compatibility with latest Breathe def clean_definitions(root): @@ -126,7 +131,13 @@ def clean_all_xml_files(path): for fn in glob.glob(os.path.join(path, "*.xml")): tree = ET.parse(fn) clean_definitions(tree.getroot()) - tree.write(fn) + with tempfile.NamedTemporaryFile() as tmp_fn: + tree.write(tmp_fn.name) + # Only write files that have actually changed. + if not filecmp.cmp(tmp_fn.name, fn): + tree.write(fn) + + # Breathe Configuration From 4444909b63b6854a9202f8093dfd7ae7833b0d1b Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 26 Jan 2024 08:27:50 +0530 Subject: [PATCH 130/384] FIx more miscellaneous pytests failures (#14895) This PR fixes multiple issues: Enables corr and cov for Datetime and Timedelta types. Properly disables all and any for StringColumn. Preserves groupby categorical index ordering. Catches FutureWarnings in pytests. --- python/cudf/cudf/core/column/datetime.py | 18 ++++++++++ python/cudf/cudf/core/column/string.py | 14 ++++++++ python/cudf/cudf/core/column/timedelta.py | 18 ++++++++++ python/cudf/cudf/core/dataframe.py | 4 +-- python/cudf/cudf/core/groupby/groupby.py | 9 ----- python/cudf/cudf/tests/test_dataframe.py | 43 ++++++++--------------- python/cudf/cudf/tests/test_groupby.py | 22 +++++++----- python/cudf/cudf/tests/test_joining.py | 4 +-- python/cudf/cudf/tests/test_stats.py | 38 ++++++++++---------- 9 files changed, 101 insertions(+), 69 deletions(-) diff --git a/python/cudf/cudf/core/column/datetime.py b/python/cudf/cudf/core/column/datetime.py index 6f7baebddd3..08a5103b409 100644 --- a/python/cudf/cudf/core/column/datetime.py +++ b/python/cudf/cudf/core/column/datetime.py @@ -532,6 +532,24 @@ def median(self, skipna: Optional[bool] = None) -> pd.Timestamp: unit=self.time_unit, ).as_unit(self.time_unit) + def cov(self, other: DatetimeColumn) -> float: + if not isinstance(other, DatetimeColumn): + raise TypeError( + f"cannot perform corr with types {self.dtype}, {other.dtype}" + ) + return self.as_numerical_column("int64").cov( + other.as_numerical_column("int64") + ) + + def corr(self, other: DatetimeColumn) -> float: + if not isinstance(other, DatetimeColumn): + raise TypeError( + f"cannot perform corr with types {self.dtype}, {other.dtype}" + ) + return self.as_numerical_column("int64").corr( + other.as_numerical_column("int64") + ) + def quantile( self, q: np.ndarray, diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index 3d222cb762e..b115e6cda48 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -5584,6 +5584,20 @@ def data(self): ] return self._data + def all(self, skipna: bool = True) -> bool: + # The skipna argument is only used for numerical columns. + # If all entries are null the result is True, including when the column + # is empty. + + raise NotImplementedError("`all` not implemented for `StringColumn`") + + def any(self, skipna: bool = True) -> bool: + # The skipna argument is only used for numerical columns. + # If all entries are null the result is True, including when the column + # is empty. + + raise NotImplementedError("`any` not implemented for `StringColumn`") + def data_array_view( self, *, mode="write" ) -> cuda.devicearray.DeviceNDArray: diff --git a/python/cudf/cudf/core/column/timedelta.py b/python/cudf/cudf/core/column/timedelta.py index b7209bbe7d0..2c12c77277c 100644 --- a/python/cudf/cudf/core/column/timedelta.py +++ b/python/cudf/cudf/core/column/timedelta.py @@ -404,6 +404,24 @@ def std( unit=self.time_unit, ).as_unit(self.time_unit) + def cov(self, other: TimeDeltaColumn) -> float: + if not isinstance(other, TimeDeltaColumn): + raise TypeError( + f"cannot perform corr with types {self.dtype}, {other.dtype}" + ) + return self.as_numerical_column("int64").cov( + other.as_numerical_column("int64") + ) + + def corr(self, other: TimeDeltaColumn) -> float: + if not isinstance(other, TimeDeltaColumn): + raise TypeError( + f"cannot perform corr with types {self.dtype}, {other.dtype}" + ) + return self.as_numerical_column("int64").corr( + other.as_numerical_column("int64") + ) + def components(self, index=None) -> "cudf.DataFrame": """ Return a Dataframe of the components of the Timedeltas. diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 5fa1956eaf1..c94b9040693 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -1038,7 +1038,6 @@ def _init_from_dict_like( empty_column = functools.partial( cudf.core.column.column_empty, row_count=(0 if index is None else len(index)), - dtype=None, masked=index is not None, ) @@ -6115,7 +6114,8 @@ def _reduce( return Series( index=self._data.to_pandas_index()[:0] if axis == 0 - else source.index + else source.index, + dtype="float64", ) if axis in {0, 2}: if axis == 2 and op in ("kurtosis", "kurt", "skew"): diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 6aba93855a7..3d0d7d9eba6 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -646,15 +646,6 @@ def agg(self, func): how="left", ) result = result.take(indices) - if isinstance(result._index, cudf.CategoricalIndex): - # Needs re-ordering the categories in the order - # they are after grouping. - result._index = cudf.Index( - result._index._column.reorder_categories( - result._index._column._get_decategorized_column() - ), - name=result._index.name, - ) if not self._as_index: result = result.reset_index() diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 026f0aa845d..69be352cf63 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -4173,15 +4173,7 @@ def test_dataframe_round_dict_decimal_validation(): [None, None], [[0, 5], [1, 6], [2, 7], [3, 8], [4, 9]], [[1, True], [2, False], [3, False]], - pytest.param( - [["a", True], ["b", False], ["c", False]], - marks=[ - pytest_xfail( - reason="NotImplementedError: all does not " - "support columns of object dtype." - ) - ], - ), + [["a", True], ["b", False], ["c", False]], ], ) def test_all(data): @@ -4192,6 +4184,9 @@ def test_all(data): if np.array(data).ndim <= 1: pdata = pd.Series(data=data, dtype=dtype).replace([None], False) gdata = cudf.Series.from_pandas(pdata) + got = gdata.all() + expected = pdata.all() + assert_eq(got, expected) else: pdata = pd.DataFrame(data, columns=["a", "b"], dtype=dtype).replace( [None], False @@ -4203,10 +4198,10 @@ def test_all(data): got = gdata.all(bool_only=True) expected = pdata.all(bool_only=True) assert_eq(got, expected) - - got = gdata.all() - expected = pdata.all() - assert_eq(got, expected) + else: + got = gdata.all() + expected = pdata.all() + assert_eq(got, expected) @pytest.mark.parametrize( @@ -4226,21 +4221,13 @@ def test_all(data): [None, None], [[0, 5], [1, 6], [2, 7], [3, 8], [4, 9]], [[1, True], [2, False], [3, False]], - pytest.param( - [["a", True], ["b", False], ["c", False]], - marks=[ - pytest_xfail( - reason="NotImplementedError: any does not " - "support columns of object dtype." - ) - ], - ), + [["a", True], ["b", False], ["c", False]], ], ) @pytest.mark.parametrize("axis", [0, 1]) def test_any(data, axis): # Provide a dtype when data is empty to avoid future pandas changes. - dtype = None if data else float + dtype = float if all(x is None for x in data) or len(data) < 1 else None if np.array(data).ndim <= 1: pdata = pd.Series(data=data, dtype=dtype) gdata = cudf.Series(data=data, dtype=dtype) @@ -4261,10 +4248,10 @@ def test_any(data, axis): got = gdata.any(bool_only=True) expected = pdata.any(bool_only=True) assert_eq(got, expected) - - got = gdata.any(axis=axis) - expected = pdata.any(axis=axis) - assert_eq(got, expected) + else: + got = gdata.any(axis=axis) + expected = pdata.any(axis=axis) + assert_eq(got, expected) @pytest.mark.parametrize("axis", [0, 1]) @@ -10197,7 +10184,7 @@ def test_empty_numeric_only(data): pdf = gdf.to_pandas() expected = pdf.prod(numeric_only=True) actual = gdf.prod(numeric_only=True) - assert_eq(expected, actual) + assert_eq(expected, actual, check_dtype=True) @pytest.fixture(params=[0, 10], ids=["empty", "10"]) diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index f594963dcda..e3dceeca1f3 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -565,7 +565,9 @@ def test_groupby_apply_jit_reductions_special_vals( func, dtype, dataset, groupby_jit_datasets, special_val ): dataset = groupby_jit_datasets[dataset] - with expect_warning_if(func in {"var", "std"} and not np.isnan(special_val), RuntimeWarning): + with expect_warning_if( + func in {"var", "std"} and not np.isnan(special_val), RuntimeWarning + ): groupby_apply_jit_reductions_special_vals_inner( func, dataset, dtype, special_val ) @@ -1409,7 +1411,7 @@ def test_groupby_multi_agg_hash_groupby(agg): @pytest.mark.parametrize( - "agg", ["min", "max", "idxmax", "idxmax", "sum", "prod", "count", "mean"] + "agg", ["min", "max", "idxmax", "idxmin", "sum", "prod", "count", "mean"] ) def test_groupby_nulls_basic(agg): check_dtype = agg not in _index_type_aggs @@ -1447,11 +1449,12 @@ def test_groupby_nulls_basic(agg): # TODO: fillna() used here since we don't follow # Pandas' null semantics. Should we change it? - assert_groupby_results_equal( - getattr(pdf.groupby("a"), agg)().fillna(0), - getattr(gdf.groupby("a"), agg)().fillna(0 if agg != "prod" else 1), - check_dtype=check_dtype, - ) + with expect_warning_if(agg in {"idxmax", "idxmin"}): + assert_groupby_results_equal( + getattr(pdf.groupby("a"), agg)().fillna(0), + getattr(gdf.groupby("a"), agg)().fillna(0 if agg != "prod" else 1), + check_dtype=check_dtype, + ) def test_groupby_nulls_in_index(): @@ -3702,8 +3705,9 @@ def test_categorical_grouping_pandas_compatibility(): with cudf.option_context("mode.pandas_compatible", True): actual = gdf.groupby("key", sort=False).sum() - expected = pdf.groupby("key", sort=False).sum() - + with pytest.warns(FutureWarning): + # observed param deprecation. + expected = pdf.groupby("key", sort=False).sum() assert_eq(actual, expected) diff --git a/python/cudf/cudf/tests/test_joining.py b/python/cudf/cudf/tests/test_joining.py index 8ce2adae15b..00b4a9b0e01 100644 --- a/python/cudf/cudf/tests/test_joining.py +++ b/python/cudf/cudf/tests/test_joining.py @@ -183,8 +183,8 @@ def test_dataframe_join_suffix(): assert list(expect.columns) == list(got.columns) assert_eq(expect.index.values, got.index.values) - got_sorted = got.sort_values(by=list(got.columns), axis=0) - expect_sorted = expect.sort_values(by=list(expect.columns), axis=0) + got_sorted = got.sort_values(by=["b_left", "c", "b_right"], axis=0) + expect_sorted = expect.sort_values(by=["b_left", "c", "b_right"], axis=0) for k in expect_sorted.columns: _check_series(expect_sorted[k].fillna(-1), got_sorted[k].fillna(-1)) diff --git a/python/cudf/cudf/tests/test_stats.py b/python/cudf/cudf/tests/test_stats.py index edd7da3d42c..6dbb23fbf04 100644 --- a/python/cudf/cudf/tests/test_stats.py +++ b/python/cudf/cudf/tests/test_stats.py @@ -581,28 +581,28 @@ def test_min_count_ops(data, ops, skipna, min_count): @pytest.mark.parametrize( - "gsr", + "data1", [ - cudf.Series([1, 2, 3, 4], dtype="datetime64[ns]"), - cudf.Series([1, 2, 3, 4], dtype="timedelta64[ns]"), + [1, 2, 3, 4], + [10, 1, 3, 5], ], ) -def test_cov_corr_invalid_dtypes(gsr): - psr = gsr.to_pandas() - - assert_exceptions_equal( - lfunc=psr.corr, - rfunc=gsr.corr, - lfunc_args_and_kwargs=([psr],), - rfunc_args_and_kwargs=([gsr],), - ) - - assert_exceptions_equal( - lfunc=psr.cov, - rfunc=gsr.cov, - lfunc_args_and_kwargs=([psr],), - rfunc_args_and_kwargs=([gsr],), - ) +@pytest.mark.parametrize( + "data2", + [ + [1, 2, 3, 4], + [10, 1, 3, 5], + ], +) +@pytest.mark.parametrize("dtype", ["datetime64[ns]", "timedelta64[ns]"]) +def test_cov_corr_datetime_timedelta(data1, data2, dtype): + gsr1 = cudf.Series(data1, dtype=dtype) + gsr2 = cudf.Series(data2, dtype=dtype) + psr1 = gsr1.to_pandas() + psr2 = gsr2.to_pandas() + + assert_eq(psr1.corr(psr2), gsr1.corr(gsr2)) + assert_eq(psr1.cov(psr2), gsr1.cov(gsr2)) @pytest.mark.parametrize( From 23d189beb1a6f4dc281f22f5c4ce7772d2848767 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 25 Jan 2024 17:09:34 -1000 Subject: [PATCH 131/384] Fix some pytests (#14894) np.product call I think will be redundant with the existing params, np.var call adjusted to what was tested before matmul failure existed upstream in pandas Snuck in a clean up files leftover by a parquet test (found these leftover when running the test suite locally) --- python/cudf/cudf/tests/test_array_function.py | 3 +-- python/cudf/cudf/tests/test_array_ufunc.py | 11 ++++++++++- python/cudf/cudf/tests/test_parquet.py | 18 ++++++++++-------- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/python/cudf/cudf/tests/test_array_function.py b/python/cudf/cudf/tests/test_array_function.py index 758a8cbb535..58658f8b3cc 100644 --- a/python/cudf/cudf/tests/test_array_function.py +++ b/python/cudf/cudf/tests/test_array_function.py @@ -65,11 +65,10 @@ def test_array_func_cudf_series(np_ar, func): [ lambda x: np.mean(x, axis=0), lambda x: np.sum(x, axis=0), - lambda x: np.var(x, ddof=1), + lambda x: np.var(x, ddof=1, axis=0), lambda x: np.dot(x, x.transpose()), lambda x: np.all(x), lambda x: np.any(x), - lambda x: np.product(x), lambda x: np.product(x, axis=0), lambda x: np.product(x, axis=1), ], diff --git a/python/cudf/cudf/tests/test_array_ufunc.py b/python/cudf/cudf/tests/test_array_ufunc.py index f5e999559b3..3e3f3aa5dfa 100644 --- a/python/cudf/cudf/tests/test_array_ufunc.py +++ b/python/cudf/cudf/tests/test_array_ufunc.py @@ -7,14 +7,16 @@ import cupy as cp import numpy as np +import pandas as pd import pytest +from packaging import version import cudf from cudf.core._compat import PANDAS_GE_150, PANDAS_GE_200, PANDAS_GE_210 from cudf.testing._utils import ( assert_eq, - set_random_null_mask_inplace, expect_warning_if, + set_random_null_mask_inplace, ) _UFUNCS = [ @@ -89,6 +91,13 @@ def test_ufunc_index(request, ufunc): reason=f"cupy has no support for '{fname}'", ) ) + request.applymarker( + pytest.mark.xfail( + condition=fname == "matmul" + and version.parse(pd.__version__) < version.parse("3.0"), + reason="Fixed by https://github.com/pandas-dev/pandas/pull/57079", + ) + ) N = 100 # Avoid zeros in either array to skip division by 0 errors. Also limit the diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index 73cbb924c65..69d3fe0b83f 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -21,7 +21,7 @@ from pyarrow import fs as pa_fs, parquet as pq import cudf -from cudf.core._compat import PANDAS_LT_153, PANDAS_GE_200 +from cudf.core._compat import PANDAS_GE_200, PANDAS_LT_153 from cudf.io.parquet import ( ParquetDatasetWriter, ParquetWriter, @@ -2683,29 +2683,31 @@ def test_parquet_writer_decimal(decimal_type, data): def test_parquet_writer_column_validation(): + cudf_parquet = BytesIO() + pandas_parquet = BytesIO() df = cudf.DataFrame({1: [1, 2, 3], "a": ["a", "b", "c"]}) pdf = df.to_pandas() with cudf.option_context("mode.pandas_compatible", True): with pytest.warns(UserWarning): - df.to_parquet("cudf.parquet") + df.to_parquet(cudf_parquet) if PANDAS_GE_200: with pytest.warns(UserWarning): - pdf.to_parquet("pandas.parquet") + pdf.to_parquet(pandas_parquet) assert_eq( - pd.read_parquet("cudf.parquet"), - cudf.read_parquet("pandas.parquet"), + pd.read_parquet(cudf_parquet), + cudf.read_parquet(pandas_parquet), ) assert_eq( - cudf.read_parquet("cudf.parquet"), - pd.read_parquet("pandas.parquet"), + cudf.read_parquet(cudf_parquet), + pd.read_parquet(pandas_parquet), ) with cudf.option_context("mode.pandas_compatible", False): with pytest.raises(ValueError): - df.to_parquet("cudf.parquet") + df.to_parquet(cudf_parquet) def test_parquet_writer_nulls_pandas_read(tmpdir, pdf): From 7df96e70289ee38a3a03ce7d70086edc9af62933 Mon Sep 17 00:00:00 2001 From: Ashwin Srinath <3190405+shwina@users.noreply.github.com> Date: Thu, 25 Jan 2024 23:24:09 -0500 Subject: [PATCH 132/384] Align datetimeindex slicing behaviour with Pandas 2.x (#14887) * Align with pandas slicing behaviour for non-monotonic datetime index * Not a TODO --------- Co-authored-by: Ashwin Srinath --- python/cudf/cudf/core/indexed_frame.py | 15 +++++++++++--- python/cudf/cudf/tests/test_indexing.py | 27 +++++++++++++++++++++---- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index fbbc606d7b8..cb7ff6a00d0 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -198,9 +198,18 @@ def _get_label_range_or_mask(index, start, stop, step): if start is not None and stop is not None: if start > stop: return slice(0, 0, None) - # TODO: Once Index binary ops are updated to support logical_and, - # can use that instead of using cupy. - boolean_mask = cp.logical_and((index >= start), (index <= stop)) + if (start in index) and (stop in index): + # when we have a non-monotonic datetime index, return + # values in the slice defined by index_of(start) and + # index_of(end) + start_loc = index.get_loc(start.to_datetime64()) + stop_loc = index.get_loc(stop.to_datetime64()) + 1 + return slice(start_loc, stop_loc) + else: + raise KeyError( + "Value based partial slicing on non-monotonic DatetimeIndexes " + "with non-existing keys is not allowed.", + ) elif start is not None: boolean_mask = index >= start else: diff --git a/python/cudf/cudf/tests/test_indexing.py b/python/cudf/cudf/tests/test_indexing.py index 8a84a84f681..1cdaa3c52a7 100644 --- a/python/cudf/cudf/tests/test_indexing.py +++ b/python/cudf/cudf/tests/test_indexing.py @@ -1278,15 +1278,15 @@ def test_iloc_categorical_index(index): @pytest.mark.parametrize( "sli", [ - slice("2001", "2020"), slice("2001", "2002"), slice("2002", "2001"), - slice(None, "2020"), slice("2001", None), ], ) @pytest.mark.parametrize("is_dataframe", [True, False]) def test_loc_datetime_index(sli, is_dataframe): + sli = slice(pd.to_datetime(sli.start), pd.to_datetime(sli.stop)) + if is_dataframe is True: pd_data = pd.DataFrame( {"a": [1, 2, 3]}, @@ -1299,13 +1299,32 @@ def test_loc_datetime_index(sli, is_dataframe): ) gd_data = cudf.from_pandas(pd_data) - expect = pd_data.loc[sli] got = gd_data.loc[sli] - assert_eq(expect, got) +@pytest.mark.parametrize( + "sli", + [ + slice("2001", "2020"), + slice(None, "2020"), + ], +) +def test_loc_datetime_index_slice_not_in(sli): + pd_data = pd.Series( + [1, 2, 3], + pd.Series(["2001", "2009", "2002"], dtype="datetime64[ns]"), + ) + gd_data = cudf.from_pandas(pd_data) + with pytest.raises(KeyError): + assert_eq(pd_data.loc[sli], gd_data.loc[sli]) + + with pytest.raises(KeyError): + sli = slice(pd.to_datetime(sli.start), pd.to_datetime(sli.stop)) + assert_eq(pd_data.loc[sli], gd_data.loc[sli]) + + @pytest.mark.parametrize( "gdf_kwargs", [ From 87a4d124f73a9f6283e708a876d78d4bdd2c162b Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Fri, 26 Jan 2024 14:11:14 +0000 Subject: [PATCH 133/384] Deprecations in replace --- python/cudf/cudf/core/indexed_frame.py | 34 ++++++++++++++++++++------ python/cudf/cudf/core/series.py | 5 ++-- python/cudf/cudf/tests/test_replace.py | 6 +++-- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 9d264822d14..ea85ee66f1b 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -598,11 +598,11 @@ def index(self, value): def replace( self, to_replace=None, - value=None, + value=no_default, inplace=False, limit=None, regex=False, - method=None, + method=no_default, ): """Replace values given in ``to_replace`` with ``value``. @@ -803,12 +803,30 @@ def replace( if regex: raise NotImplementedError("regex parameter is not implemented yet") - if method not in ("pad", None): - raise NotImplementedError( - "method parameter is not implemented yet" + if method is not no_default: + warnings.warn( + "The 'method' keyword in " + f"{type(self).__name__}.replace is deprecated and " + "will be removed in a future version.", + FutureWarning, ) + elif method not in ("pad", None, no_default): + raise NotImplementedError("method parameter is not implemented") - if not (to_replace is None and value is None): + if ( + value is no_default + and method is no_default + and not is_dict_like(to_replace) + and regex is False + ): + warnings.warn( + f"{type(self).__name__}.replace without 'value' and with " + "non-dict-like 'to_replace' is deprecated " + "and will raise in a future version. " + "Explicitly specify the new values instead.", + FutureWarning, + ) + if not (to_replace is None and value is no_default): copy_data = {} ( all_na_per_column, @@ -5320,7 +5338,7 @@ def _get_replacement_values_for_columns( "value argument must be scalar, list-like or Series" ) elif _is_series(to_replace): - if value is None: + if value is None or value is no_default: to_replace_columns = { col: as_column(to_replace.index) for col in columns_dtype_map } @@ -5351,7 +5369,7 @@ def _get_replacement_values_for_columns( "value" ) elif is_dict_like(to_replace): - if value is None: + if value is None or value is no_default: to_replace_columns = { col: list(to_replace.keys()) for col in columns_dtype_map } diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 5f03f368664..7ff529dbd05 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -594,7 +594,6 @@ def __init__( copy=False, nan_as_null=True, ): - index_from_data = None name_from_data = None if data is None: @@ -2317,8 +2316,8 @@ def argsort( return obj @_cudf_nvtx_annotate - def replace(self, to_replace=None, value=None, *args, **kwargs): - if is_dict_like(to_replace) and value is not None: + def replace(self, to_replace=None, value=no_default, *args, **kwargs): + if is_dict_like(to_replace) and value not in {None, no_default}: raise ValueError( "Series.replace cannot use dict-like to_replace and non-None " "value" diff --git a/python/cudf/cudf/tests/test_replace.py b/python/cudf/cudf/tests/test_replace.py index 94cee2dca68..ac2b2c6cd30 100644 --- a/python/cudf/cudf/tests/test_replace.py +++ b/python/cudf/cudf/tests/test_replace.py @@ -1068,8 +1068,10 @@ def test_replace_inplace(pframe, replace_args): assert_eq(gpu_frame, pandas_frame) assert_eq(gpu_copy, cpu_copy) - gpu_frame.replace(**replace_args) - pandas_frame.replace(**replace_args) + with expect_warning_if(len(replace_args) == 0): + gpu_frame.replace(**replace_args) + with expect_warning_if(len(replace_args) == 0): + pandas_frame.replace(**replace_args) assert_eq(gpu_frame, pandas_frame) assert_eq(gpu_copy, cpu_copy) From 7d3e72af69ea38f4150a5a2ff352a300f704fcd0 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 26 Jan 2024 23:28:39 +0530 Subject: [PATCH 134/384] Parquet Writer: Write `non-string` columns pandas-compatibility mode only (#14899) This PR enables writing of non-string columns in parquet writer only in pandas-compatibility mode. This PR: ``` = 8 failed, 102249 passed, 2090 skipped, 976 xfailed, 312 xpassed in 1363.59s (0:22:43) = ``` On `pandas_2.0_feature_branch`: ``` = 9 failed, 102247 passed, 2091 skipped, 976 xfailed, 312 xpassed in 1336.47s (0:22:16) = ``` Co-authored-by: Lawrence Mitchell --------- Co-authored-by: Lawrence Mitchell --- python/cudf/cudf/_lib/utils.pyx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/cudf/cudf/_lib/utils.pyx b/python/cudf/cudf/_lib/utils.pyx index 50a47b4f507..7ba717a0003 100644 --- a/python/cudf/cudf/_lib/utils.pyx +++ b/python/cudf/cudf/_lib/utils.pyx @@ -85,7 +85,12 @@ cpdef generate_pandas_metadata(table, index): # Columns for name, col in table._data.items(): - col_names.append(name) + if cudf.get_option("mode.pandas_compatible"): + # in pandas-compat mode, non-string column names are stringified. + col_names.append(str(name)) + else: + col_names.append(name) + if isinstance(col.dtype, cudf.CategoricalDtype): raise ValueError( "'category' column dtypes are currently not " From b61b39d0ebd33113d4070ceae24adf8c58a46ddf Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 26 Jan 2024 13:05:05 -0600 Subject: [PATCH 135/384] Use sets for argument checking. --- python/cudf/cudf/core/indexed_frame.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index ea85ee66f1b..6f80f6bb0bc 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -810,7 +810,7 @@ def replace( "will be removed in a future version.", FutureWarning, ) - elif method not in ("pad", None, no_default): + elif method not in {"pad", None, no_default}: raise NotImplementedError("method parameter is not implemented") if ( @@ -5338,7 +5338,7 @@ def _get_replacement_values_for_columns( "value argument must be scalar, list-like or Series" ) elif _is_series(to_replace): - if value is None or value is no_default: + if value in {None, no_default}: to_replace_columns = { col: as_column(to_replace.index) for col in columns_dtype_map } @@ -5369,7 +5369,7 @@ def _get_replacement_values_for_columns( "value" ) elif is_dict_like(to_replace): - if value is None or value is no_default: + if value in {None, no_default}: to_replace_columns = { col: list(to_replace.keys()) for col in columns_dtype_map } From 78eff481a6e8c96ae9076a79700f0873bbbd9fba Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 26 Jan 2024 20:06:24 +0000 Subject: [PATCH 136/384] Fix usage --- python/cudf/cudf/core/indexed_frame.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 6f80f6bb0bc..d7239dbcf2f 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -5338,7 +5338,7 @@ def _get_replacement_values_for_columns( "value argument must be scalar, list-like or Series" ) elif _is_series(to_replace): - if value in {None, no_default}: + if value is None or value is no_default: to_replace_columns = { col: as_column(to_replace.index) for col in columns_dtype_map } @@ -5369,7 +5369,7 @@ def _get_replacement_values_for_columns( "value" ) elif is_dict_like(to_replace): - if value in {None, no_default}: + if value is None or value is no_default: to_replace_columns = { col: list(to_replace.keys()) for col in columns_dtype_map } From b47f5ee7d76e2382906adc351caff91b13801843 Mon Sep 17 00:00:00 2001 From: Mark Harris <783069+harrism@users.noreply.github.com> Date: Sat, 27 Jan 2024 09:21:17 +1100 Subject: [PATCH 137/384] Remove supports_streams from cuDF custom memory resources. (#14857) Part of https://github.com/rapidsai/rmm/issues/1389. This removes now-optional and soon-to-be deprecated `supports_streams()` from cuDF's custom `device_memory_resource` implementations. Depends on https://github.com/rapidsai/rmm/pull/1437 Authors: - Mark Harris (https://github.com/harrism) - Bradley Dice (https://github.com/bdice) Approvers: - Nghia Truong (https://github.com/ttnghia) - Bradley Dice (https://github.com/bdice) - Yunsong Wang (https://github.com/PointKernel) - Michael Schellenberger Costa (https://github.com/miscco) URL: https://github.com/rapidsai/cudf/pull/14857 --- cpp/include/cudf_test/stream_checking_resource_adaptor.hpp | 7 ------- java/src/main/native/src/RmmJni.cpp | 4 ---- 2 files changed, 11 deletions(-) diff --git a/cpp/include/cudf_test/stream_checking_resource_adaptor.hpp b/cpp/include/cudf_test/stream_checking_resource_adaptor.hpp index 90a8c2ccc2f..d1841ff42a1 100644 --- a/cpp/include/cudf_test/stream_checking_resource_adaptor.hpp +++ b/cpp/include/cudf_test/stream_checking_resource_adaptor.hpp @@ -64,13 +64,6 @@ class stream_checking_resource_adaptor final : public rmm::mr::device_memory_res */ Upstream* get_upstream() const noexcept { return upstream_; } - /** - * @brief Checks whether the upstream resource supports streams. - * - * @return Whether or not the upstream resource supports streams - */ - bool supports_streams() const noexcept override { return upstream_->supports_streams(); } - private: /** * @brief Allocates memory of size at least `bytes` using the upstream diff --git a/java/src/main/native/src/RmmJni.cpp b/java/src/main/native/src/RmmJni.cpp index b92d9e4e891..81b8241bab0 100644 --- a/java/src/main/native/src/RmmJni.cpp +++ b/java/src/main/native/src/RmmJni.cpp @@ -96,8 +96,6 @@ class tracking_resource_adaptor final : public base_tracking_resource_adaptor { return scoped_max_total_allocated; } - bool supports_streams() const noexcept override { return resource->supports_streams(); } - private: Upstream *const resource; std::size_t const size_align; @@ -207,8 +205,6 @@ class java_event_handler_memory_resource : public device_memory_resource { device_memory_resource *get_wrapped_resource() { return resource; } - bool supports_streams() const noexcept override { return resource->supports_streams(); } - private: device_memory_resource *const resource; base_tracking_resource_adaptor *const tracker; From 5618d3da17b3d1d911151120e260e9c0dd5be6cf Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 26 Jan 2024 14:10:35 -1000 Subject: [PATCH 138/384] Remove pandas Index subclasses in cudf pandas (#14902) We won't have to proxy these types anymore since they are removed in pandas 2.0 Also removed references to the cudf Index subclasses that are removed in this branch Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Ashwin Srinath (https://github.com/shwina) URL: https://github.com/rapidsai/cudf/pull/14902 --- docs/cudf/source/conf.py | 6 -- python/cudf/cudf/core/index.py | 16 ++---- python/cudf/cudf/core/reshape.py | 4 +- python/cudf/cudf/pandas/_wrappers/pandas.py | 47 ---------------- python/cudf/cudf/tests/test_dataframe.py | 8 +-- python/cudf/cudf/tests/test_index.py | 21 +------ python/cudf/cudf/tests/test_sorting.py | 2 +- .../cudf_pandas_tests/test_cudf_pandas.py | 55 +------------------ 8 files changed, 17 insertions(+), 142 deletions(-) diff --git a/docs/cudf/source/conf.py b/docs/cudf/source/conf.py index 01a6c5316bd..e76bdb802e4 100644 --- a/docs/cudf/source/conf.py +++ b/docs/cudf/source/conf.py @@ -443,12 +443,6 @@ def on_missing_reference(app, env, node, contnode): _prefixed_domain_objects[f"{prefix}{name}"] = name reftarget = node.get("reftarget") - if reftarget == "cudf.core.index.GenericIndex": - # We don't exposed docs for `cudf.core.index.GenericIndex` - # hence we would want the docstring & mypy references to - # use `cudf.Index` - node["reftarget"] = "cudf.Index" - return contnode if "namespacecudf" in reftarget: node["reftarget"] = "cudf" return contnode diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index b506dfe6158..2bd4219997f 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -14,17 +14,17 @@ MutableMapping, Optional, Tuple, - Type, Union, cast, ) import cupy import numpy as np +import pandas as pd +from pandas._config import get_option from typing_extensions import Self import cudf -import pandas as pd from cudf import _lib as libcudf from cudf._lib.datetime import extract_quarter, is_leap_year from cudf._lib.filling import sequence @@ -66,12 +66,8 @@ is_mixed_with_object_dtype, numeric_normalize_types, ) -from cudf.utils.utils import ( - _warn_no_dask_cudf, - search_range, -) from cudf.utils.nvtx_annotation import _cudf_nvtx_annotate -from pandas._config import get_option +from cudf.utils.utils import _warn_no_dask_cudf, search_range class IndexMeta(type): @@ -1356,9 +1352,9 @@ def __repr__(self): # from the output due to the type-cast to # object dtype happening above. # Note : The replacing of single quotes has - # to happen only in case of non-StringIndex types, + # to happen only in case of non-Index[string] types, # as we want to preserve single quotes in case - # of StringIndex and it is valid to have them. + # of Index[string] and it is valid to have them. output = output.replace("'", "") else: output = repr(preprocess.to_pandas()) @@ -2947,7 +2943,7 @@ def as_index( result : subclass of Index - CategoricalIndex for Categorical input. - DatetimeIndex for Datetime input. - - GenericIndex for all other inputs. + - Index for all other inputs. Notes ----- diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index d8bb09c668a..05ab1edfaba 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -1,11 +1,11 @@ # Copyright (c) 2018-2024, NVIDIA CORPORATION. import itertools +import warnings from collections import abc from typing import Dict, Optional import cupy -import warnings import numpy as np import pandas as pd @@ -35,7 +35,7 @@ def _align_objs(objs, how="outer", sort=None): A list of reindexed and aligned objects ready for concatenation """ - # Check if multiindex then check if indexes match. GenericIndex + # Check if multiindex then check if indexes match. Index # returns ndarray tuple of bools requiring additional filter. # Then check for duplicate index value. i_objs = iter(objs) diff --git a/python/cudf/cudf/pandas/_wrappers/pandas.py b/python/cudf/cudf/pandas/_wrappers/pandas.py index afcfc13a9c4..137709925df 100644 --- a/python/cudf/cudf/pandas/_wrappers/pandas.py +++ b/python/cudf/cudf/pandas/_wrappers/pandas.py @@ -17,7 +17,6 @@ _FastSlowAttribute, _FunctionProxy, _Unusable, - get_final_type_map, make_final_proxy_type as _make_final_proxy_type, make_intermediate_proxy_type as _make_intermediate_proxy_type, register_proxy_func, @@ -203,19 +202,6 @@ def Index__new__(cls, *args, **kwargs): }, ) -get_final_type_map()[cudf.StringIndex] = Index -get_final_type_map()[cudf.Int8Index] = Index -get_final_type_map()[cudf.Int8Index] = Index -get_final_type_map()[cudf.Int16Index] = Index -get_final_type_map()[cudf.Int32Index] = Index -get_final_type_map()[cudf.UInt8Index] = Index -get_final_type_map()[cudf.UInt16Index] = Index -get_final_type_map()[cudf.UInt32Index] = Index -get_final_type_map()[cudf.UInt64Index] = Index -get_final_type_map()[cudf.Float32Index] = Index -get_final_type_map()[cudf.GenericIndex] = Index - - RangeIndex = make_final_proxy_type( "RangeIndex", cudf.RangeIndex, @@ -471,17 +457,6 @@ def Index__new__(cls, *args, **kwargs): additional_attributes={"__hash__": _FastSlowAttribute("__hash__")}, ) - -Int64Index = make_final_proxy_type( - "Int64Index", - cudf.Int64Index, - pd.core.indexes.numeric.Int64Index, - fast_to_slow=lambda fast: fast.to_pandas(), - slow_to_fast=cudf.from_pandas, - bases=(Index,), - additional_attributes={"__init__": _DELETE}, -) - UInt8Dtype = make_final_proxy_type( "UInt8Dtype", _Unusable, @@ -518,16 +493,6 @@ def Index__new__(cls, *args, **kwargs): additional_attributes={"__hash__": _FastSlowAttribute("__hash__")}, ) -UInt64Index = make_final_proxy_type( - "UInt64Index", - cudf.UInt64Index, - pd.core.indexes.numeric.UInt64Index, - fast_to_slow=lambda fast: fast.to_pandas(), - slow_to_fast=cudf.from_pandas, - bases=(Index,), - additional_attributes={"__init__": _DELETE}, -) - IntervalIndex = make_final_proxy_type( "IntervalIndex", cudf.IntervalIndex, @@ -593,16 +558,6 @@ def Index__new__(cls, *args, **kwargs): additional_attributes={"__hash__": _FastSlowAttribute("__hash__")}, ) -Float64Index = make_final_proxy_type( - "Float64Index", - cudf.Float64Index, - pd.core.indexes.numeric.Float64Index, - fast_to_slow=lambda fast: fast.to_pandas(), - slow_to_fast=cudf.from_pandas, - bases=(Index,), - additional_attributes={"__init__": _DELETE}, -) - SeriesGroupBy = make_intermediate_proxy_type( "SeriesGroupBy", cudf.core.groupby.groupby.SeriesGroupBy, @@ -1273,8 +1228,6 @@ def _df_query_method(self, *args, local_dict=None, global_dict=None, **kwargs): pd.core.indexes.datetimelike.DatetimeTimedeltaMixin, pd.core.indexes.datetimelike.DatetimeIndexOpsMixin, pd.core.indexes.extension.NDArrayBackedExtensionIndex, - pd.core.indexes.numeric.IntegerIndex, - pd.core.indexes.numeric.NumericIndex, pd.core.generic.NDFrame, pd.core.indexes.accessors.PeriodProperties, pd.core.indexes.accessors.Properties, diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index f7242941561..34dc7ebc68e 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -11,8 +11,8 @@ import string import textwrap import warnings -from contextlib import contextmanager from collections import OrderedDict, defaultdict, namedtuple +from contextlib import contextmanager from copy import copy import cupy @@ -24,6 +24,7 @@ from packaging import version import cudf +from cudf.api.extensions import no_default from cudf.core._compat import ( PANDAS_GE_134, PANDAS_GE_150, @@ -32,7 +33,6 @@ PANDAS_LT_140, PANDAS_LT_203, ) -from cudf.api.extensions import no_default from cudf.core.buffer.spill_manager import get_global_manager from cudf.core.column import column from cudf.errors import MixedTypeError @@ -5499,7 +5499,6 @@ def test_rowwise_ops_nullable_int_dtypes(op, expected): @pytest.mark.parametrize("skipna", [True, False]) @pytest.mark.parametrize("numeric_only", [True, False]) def test_rowwise_ops_datetime_dtypes(data, op, skipna, numeric_only): - gdf = cudf.DataFrame(data) pdf = gdf.to_pandas() @@ -5528,7 +5527,6 @@ def test_rowwise_ops_datetime_dtypes(data, op, skipna, numeric_only): # https://github.com/pandas-dev/pandas/issues/52524 assert_eq(got.astype("datetime64[ns]"), expected) else: - assert_eq(got, expected, check_dtype=False) @@ -10921,7 +10919,7 @@ def test_dataframe_contains(name, contains, other_names): assert (contains in pdf) == expectation assert (contains in gdf) == expectation elif pd.api.types.is_float_dtype(gdf.columns.dtype): - # In some cases, the columns are converted to a Float64Index based on + # In some cases, the columns are converted to a Index[float] based on # the other column names. That casts name values from None to np.nan. expectation = contains is np.nan and (name is None or name is np.nan) assert (contains in pdf) == expectation diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index e43416e323c..ca8ef83316d 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -1265,12 +1265,7 @@ def test_index_basic(data, dtype, name): @pytest.mark.parametrize("name", [1, "a", None]) @pytest.mark.parametrize("dtype", SIGNED_INTEGER_TYPES) def test_integer_index_apis(data, name, dtype): - if PANDAS_GE_200: - pindex = pd.Index(data, dtype=dtype, name=name) - else: - with pytest.warns(FutureWarning): - pindex = pd.Int64Index(data, dtype=dtype, name=name) - + pindex = pd.Index(data, dtype=dtype, name=name) gindex = cudf.Index(data, dtype=dtype, name=name) assert_eq(pindex, gindex) @@ -1281,12 +1276,7 @@ def test_integer_index_apis(data, name, dtype): @pytest.mark.parametrize("name", [1, "a", None]) @pytest.mark.parametrize("dtype", UNSIGNED_TYPES) def test_unsigned_integer_index_apis(data, name, dtype): - if PANDAS_GE_200: - pindex = pd.Index(data, dtype=dtype, name=name) - else: - with pytest.warns(FutureWarning): - pindex = pd.UInt64Index(data, dtype=dtype, name=name) - + pindex = pd.Index(data, dtype=dtype, name=name) gindex = cudf.Index(data, dtype=dtype, name=name) assert_eq(pindex, gindex) @@ -1297,12 +1287,7 @@ def test_unsigned_integer_index_apis(data, name, dtype): @pytest.mark.parametrize("name", [1, "a", None]) @pytest.mark.parametrize("dtype", FLOAT_TYPES) def test_float_index_apis(data, name, dtype): - if PANDAS_GE_200: - pindex = pd.Index(data, dtype=dtype, name=name) - else: - with pytest.warns(FutureWarning): - pindex = pd.Float64Index(data, dtype=dtype, name=name) - + pindex = pd.Index(data, dtype=dtype, name=name) gindex = cudf.Index(data, dtype=dtype, name=name) assert_eq(pindex, gindex) diff --git a/python/cudf/cudf/tests/test_sorting.py b/python/cudf/cudf/tests/test_sorting.py index 07f76f1103c..f30c14373bf 100644 --- a/python/cudf/cudf/tests/test_sorting.py +++ b/python/cudf/cudf/tests/test_sorting.py @@ -346,7 +346,7 @@ def _check_scatter_by_map(dfs, col): with pytest.warns(UserWarning): df.scatter_by_map("a", map_size=1, debug=True) # Bad map_size - # Test GenericIndex + # Test Index df2 = df.set_index("c") generic_result = df2.scatter_by_map("b", map_size, keep_index=keep) _check_scatter_by_map(generic_result, df2["b"]) diff --git a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py index df4bed0be0a..2425c323060 100644 --- a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py +++ b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py @@ -269,25 +269,6 @@ def test_rename_categories(): tm.assert_series_equal(psr, sr) -def test_rename_categories_inplace(): - psr = pd.Series([1, 2, 3], dtype="category") - sr = xpd.Series([1, 2, 3], dtype="category") - with pytest.warns(FutureWarning): - psr.cat.rename_categories({1: 5}, inplace=True) - sr.cat.rename_categories({1: 5}, inplace=True) - tm.assert_series_equal(psr, sr) - - -def test_rename_categories_inplace_after_copying_parent(): - s = xpd.Series([1, 2, 3], dtype="category") - # cudf does not define "rename_categories", - # so this copies `s` from device to host: - rename_categories = s.cat.rename_categories - _ = len(s) # trigger a copy of `s` from host to device: - with pytest.warns(FutureWarning): - rename_categories([5, 2, 3], inplace=True) - assert s.cat.categories.tolist() == [5, 2, 3] - def test_column_rename(dataframe): pdf, df = dataframe @@ -663,8 +644,7 @@ def test_rolling_win_type(): pdf = pd.DataFrame(range(5)) df = xpd.DataFrame(range(5)) result = df.rolling(2, win_type="boxcar").mean() - with pytest.warns(DeprecationWarning): - expected = pdf.rolling(2, win_type="boxcar").mean() + expected = pdf.rolling(2, win_type="boxcar").mean() tm.assert_equal(result, expected) @@ -1017,12 +997,6 @@ def __init__(self, myinput): xpd.PeriodIndex, xpd.MultiIndex, xpd.IntervalIndex, - xpd.UInt64Index, - xpd.Int64Index, - xpd.Float64Index, - xpd.core.indexes.numeric.UInt64Index, - xpd.core.indexes.numeric.Int64Index, - xpd.core.indexes.numeric.Float64Index, ], ) def test_index_subclass(index_type): @@ -1032,22 +1006,6 @@ def test_index_subclass(index_type): assert not issubclass(xpd.Index, index_type) -def test_index_internal_subclass(): - # test that proxy index types that are not related by inheritance - # still appear to be so if the underlying slow types are related - # by inheritance: - assert issubclass( - xpd.Int64Index, - xpd.core.indexes.numeric.NumericIndex, - ) == issubclass( - pd.Int64Index, - pd.core.indexes.numeric.NumericIndex, - ) - assert isinstance( - xpd.Index([1, 2, 3]), xpd.core.indexes.numeric.NumericIndex - ) == isinstance(pd.Index([1, 2, 3]), pd.core.indexes.numeric.NumericIndex) - - def test_np_array_of_timestamps(): expected = np.array([pd.Timestamp(1)]) + pd.tseries.offsets.MonthEnd() got = np.array([xpd.Timestamp(1)]) + xpd.tseries.offsets.MonthEnd() @@ -1080,7 +1038,7 @@ def test_np_array_of_timestamps(): # Other types xpd.tseries.offsets.BDay(5), xpd.Timestamp("2001-01-01"), - xpd.Timestamp("2001-01-01", freq="D"), + xpd.Timestamp("2001-01-01", tz="UTC"), xpd.Timedelta("1 days"), xpd.Timedelta(1, "D"), ], @@ -1214,15 +1172,6 @@ def test_read_sas_context(): assert isinstance(df, xpd.DataFrame) -@pytest.mark.parametrize( - "idx_obj", ["Float64Index", "Int64Index", "UInt64Index"] -) -def test_pandas_module_getattr_objects(idx_obj): - # Objects that are behind pandas.__getattr__ (version 1.5 specific) - idx = getattr(xpd, idx_obj)([1, 2, 3]) - assert isinstance(idx, xpd.Index) - - def test_concat_fast(): pytest.importorskip("cudf") From d8df8e469c9aa5668121c6e16636f1af3ec8c269 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Sun, 28 Jan 2024 11:09:39 -0500 Subject: [PATCH 139/384] Allow `any` and `all` only for all-`NA` and empty string columns (#14898) This PR allows any and all for all-NA string columns and string columns that have size 0. This is an essential workaround for time-being because any and all aren't natively supported for string types in libcudf and without these workarounds, multiple places in the reduction APIs will need if/elif checks which will make it harder to maintain. This PR: = 5 failed, 102252 passed, 2090 skipped, 976 xfailed, 312 xpassed in 1375.59s (0:22:55) = On pandas_2.0_feature_branch: = 9 failed, 102247 passed, 2091 skipped, 976 xfailed, 312 xpassed in 1336.47s (0:22:16) = --- python/cudf/cudf/core/column/string.py | 15 ++++++++------- python/cudf/cudf/tests/test_string.py | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index fa07b299ecf..2fdcf30606a 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -5594,16 +5594,17 @@ def data(self): return self._data def all(self, skipna: bool = True) -> bool: - # The skipna argument is only used for numerical columns. - # If all entries are null the result is True, including when the column - # is empty. - + if skipna and self.null_count == self.size: + return True + elif not skipna and self.has_nulls(): + raise TypeError("boolean value of NA is ambiguous") raise NotImplementedError("`all` not implemented for `StringColumn`") def any(self, skipna: bool = True) -> bool: - # The skipna argument is only used for numerical columns. - # If all entries are null the result is True, including when the column - # is empty. + if not skipna and self.has_nulls(): + raise TypeError("boolean value of NA is ambiguous") + elif skipna and self.null_count == self.size: + return False raise NotImplementedError("`any` not implemented for `StringColumn`") diff --git a/python/cudf/cudf/tests/test_string.py b/python/cudf/cudf/tests/test_string.py index 8eca1a56525..8c8a3cb2399 100644 --- a/python/cudf/cudf/tests/test_string.py +++ b/python/cudf/cudf/tests/test_string.py @@ -3484,3 +3484,21 @@ def test_str_iterate_error(): s = cudf.Series(["abc", "xyz"]) with pytest.raises(TypeError): iter(s.str) + + +def test_string_reduction_error(): + s = cudf.Series([None, None], dtype="str") + ps = s.to_pandas(nullable=True) + assert_exceptions_equal( + s.any, + ps.any, + lfunc_args_and_kwargs=([], {"skipna": False}), + rfunc_args_and_kwargs=([], {"skipna": False}), + ) + + assert_exceptions_equal( + s.all, + ps.all, + lfunc_args_and_kwargs=([], {"skipna": False}), + rfunc_args_and_kwargs=([], {"skipna": False}), + ) From 9fa9dc5222df5bf482b798206b70b02288bd23ca Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 29 Jan 2024 07:11:45 -0600 Subject: [PATCH 140/384] Prevent converting strings to arrow strings in `dask_cudf` pytests (#14914) dask is natively converting all object types to arrow[string] types if proper pyarrow dependency is detected. This is being done in assert_eq API. We will need a change in cudf and dask upstream to be able to support this kind of conversion. I'm coming up with a solution in 24.04 dev cycle, but in the interest of shipping pandas-2.x I'm feeling confident to disable this auto-conversion by setting the dataframe.convert-string dask config to False where necessary. --- .../dask_cudf/dask_cudf/io/tests/test_csv.py | 8 ++--- .../dask_cudf/dask_cudf/io/tests/test_json.py | 6 ++-- .../dask_cudf/io/tests/test_parquet.py | 2 +- .../dask_cudf/tests/test_accessor.py | 36 ++++++++++--------- python/dask_cudf/dask_cudf/tests/test_core.py | 26 ++++++++++---- .../dask_cudf/dask_cudf/tests/test_groupby.py | 3 +- .../dask_cudf/tests/test_reductions.py | 28 ++++++++------- 7 files changed, 65 insertions(+), 44 deletions(-) diff --git a/python/dask_cudf/dask_cudf/io/tests/test_csv.py b/python/dask_cudf/dask_cudf/io/tests/test_csv.py index 5f1aa98e888..987fcf6b4ae 100644 --- a/python/dask_cudf/dask_cudf/io/tests/test_csv.py +++ b/python/dask_cudf/dask_cudf/io/tests/test_csv.py @@ -226,11 +226,11 @@ def test_read_csv_skiprows_error(csv_begin_bad_lines): def test_read_csv_skipfooter(csv_end_bad_lines): # Repro from Issue#13552 + with dask.config.set({"dataframe.convert-string": False}): + ddf_cpu = dd.read_csv(csv_end_bad_lines, skipfooter=3).compute() + ddf_gpu = dask_cudf.read_csv(csv_end_bad_lines, skipfooter=3).compute() - ddf_cpu = dd.read_csv(csv_end_bad_lines, skipfooter=3).compute() - ddf_gpu = dask_cudf.read_csv(csv_end_bad_lines, skipfooter=3).compute() - - dd.assert_eq(ddf_cpu, ddf_gpu, check_dtype=False) + dd.assert_eq(ddf_cpu, ddf_gpu, check_dtype=False) def test_read_csv_skipfooter_error(csv_end_bad_lines): diff --git a/python/dask_cudf/dask_cudf/io/tests/test_json.py b/python/dask_cudf/dask_cudf/io/tests/test_json.py index fddbfb16e27..5e06832ed94 100644 --- a/python/dask_cudf/dask_cudf/io/tests/test_json.py +++ b/python/dask_cudf/dask_cudf/io/tests/test_json.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2023, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. import os @@ -80,7 +80,9 @@ def test_read_json_nested(tmp_path): } ) kwargs = dict(orient="records", lines=True) - with tmp_path / "data.json" as f: + with tmp_path / "data.json" as f, dask.config.set( + {"dataframe.convert-string": False} + ): df.to_json(f, **kwargs) # Ensure engine='cudf' is tested. actual = dask_cudf.read_json(f, engine="cudf", **kwargs) diff --git a/python/dask_cudf/dask_cudf/io/tests/test_parquet.py b/python/dask_cudf/dask_cudf/io/tests/test_parquet.py index 6c53193d7cd..583d4b07f6f 100644 --- a/python/dask_cudf/dask_cudf/io/tests/test_parquet.py +++ b/python/dask_cudf/dask_cudf/io/tests/test_parquet.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2023, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. import glob import math diff --git a/python/dask_cudf/dask_cudf/tests/test_accessor.py b/python/dask_cudf/dask_cudf/tests/test_accessor.py index 2f5dcb524a5..f6b8c34fef0 100644 --- a/python/dask_cudf/dask_cudf/tests/test_accessor.py +++ b/python/dask_cudf/dask_cudf/tests/test_accessor.py @@ -2,7 +2,9 @@ import numpy as np import pandas as pd +import dask import pytest + from pandas.testing import assert_series_equal from dask import dataframe as dd @@ -137,30 +139,30 @@ def test_categorical_basic(data): 4 a """ assert all(x == y for x, y in zip(string.split(), expect_str.split())) + with dask.config.set({"dataframe.convert-string": False}): + df = DataFrame() + df["a"] = ["xyz", "abc", "def"] * 10 - df = DataFrame() - df["a"] = ["xyz", "abc", "def"] * 10 - - pdf = df.to_pandas() - cddf = dgd.from_cudf(df, 1) - cddf["b"] = cddf["a"].astype("category") + pdf = df.to_pandas() + cddf = dgd.from_cudf(df, 1) + cddf["b"] = cddf["a"].astype("category") - ddf = dd.from_pandas(pdf, 1) - ddf["b"] = ddf["a"].astype("category") + ddf = dd.from_pandas(pdf, 1) + ddf["b"] = ddf["a"].astype("category") - assert_eq(ddf._meta_nonempty["b"], cddf._meta_nonempty["b"]) + assert_eq(ddf._meta_nonempty["b"], cddf._meta_nonempty["b"]) - with pytest.raises(NotImplementedError): - cddf["b"].cat.categories + with pytest.raises(NotImplementedError): + cddf["b"].cat.categories - with pytest.raises(NotImplementedError): - ddf["b"].cat.categories + with pytest.raises(NotImplementedError): + ddf["b"].cat.categories - cddf = cddf.categorize() - ddf = ddf.categorize() + cddf = cddf.categorize() + ddf = ddf.categorize() - assert_eq(ddf["b"].cat.categories, cddf["b"].cat.categories) - assert_eq(ddf["b"].cat.ordered, cddf["b"].cat.ordered) + assert_eq(ddf["b"].cat.categories, cddf["b"].cat.categories) + assert_eq(ddf["b"].cat.ordered, cddf["b"].cat.ordered) @pytest.mark.parametrize("data", [data_cat_1()]) diff --git a/python/dask_cudf/dask_cudf/tests/test_core.py b/python/dask_cudf/dask_cudf/tests/test_core.py index 4f77887033a..552d800e2dd 100644 --- a/python/dask_cudf/dask_cudf/tests/test_core.py +++ b/python/dask_cudf/dask_cudf/tests/test_core.py @@ -782,14 +782,16 @@ def test_dataframe_set_index(): df["str"] = list("abcdefghijklmnopqrstuvwxyz") pdf = df.to_pandas() - ddf = dgd.from_cudf(df, npartitions=4) - ddf = ddf.set_index("str") + with dask.config.set({"dataframe.convert-string": False}): + ddf = dgd.from_cudf(df, npartitions=4) + ddf = ddf.set_index("str") - pddf = dd.from_pandas(pdf, npartitions=4) - pddf = pddf.set_index("str") - from cudf.testing._utils import assert_eq + pddf = dd.from_pandas(pdf, npartitions=4) + pddf = pddf.set_index("str") + + from cudf.testing._utils import assert_eq - assert_eq(ddf.compute(), pddf.compute()) + assert_eq(ddf.compute(), pddf.compute()) def test_series_describe(): @@ -938,3 +940,15 @@ def test_categorical_dtype_round_trip(): actual = ds.compute() expected = pds.compute() assert actual.dtype.ordered == expected.dtype.ordered + + +def test_object_to_string_fail(request): + request.applymarker( + pytest.mark.xfail( + reason="https://github.com/rapidsai/cudf/issues/14915", + ) + ) + s = cudf.Series(["a", "b", "c"] * 10) + ds = dgd.from_cudf(s, npartitions=2) + pds = dd.from_pandas(s.to_pandas(), npartitions=2) + dd.assert_eq(ds.sort_values(), pds.sort_values()) diff --git a/python/dask_cudf/dask_cudf/tests/test_groupby.py b/python/dask_cudf/dask_cudf/tests/test_groupby.py index 0dc57d8df55..cef8bdacace 100644 --- a/python/dask_cudf/dask_cudf/tests/test_groupby.py +++ b/python/dask_cudf/dask_cudf/tests/test_groupby.py @@ -610,7 +610,8 @@ def test_groupby_agg_params(npartitions, split_every, split_out, as_index): if as_index: # Groupby columns became the index. # Sorting the index should not change anything. - dd.assert_eq(gf.index, gf.sort_index().index) + with dask.config.set({"dataframe.convert-string": False}): + dd.assert_eq(gf.index, gf.sort_index().index) else: # Groupby columns are did NOT become the index. # Sorting by these columns should not change anything. diff --git a/python/dask_cudf/dask_cudf/tests/test_reductions.py b/python/dask_cudf/dask_cudf/tests/test_reductions.py index 56d2b42efbc..e347e8be9e4 100644 --- a/python/dask_cudf/dask_cudf/tests/test_reductions.py +++ b/python/dask_cudf/dask_cudf/tests/test_reductions.py @@ -4,6 +4,7 @@ import pandas as pd import pytest +import dask from dask import dataframe as dd import cudf @@ -69,16 +70,17 @@ def test_rowwise_reductions(data, op): gddf = dgd.from_cudf(data, npartitions=10) pddf = gddf.to_dask_dataframe() - if op in ("var", "std"): - expected = getattr(pddf, op)(axis=1, numeric_only=True, ddof=0) - got = getattr(gddf, op)(axis=1, numeric_only=True, ddof=0) - else: - expected = getattr(pddf, op)(numeric_only=True, axis=1) - got = getattr(pddf, op)(numeric_only=True, axis=1) - - dd.assert_eq( - expected, - got, - check_exact=False, - check_dtype=op not in ("var", "std"), - ) + with dask.config.set({"dataframe.convert-string": False}): + if op in ("var", "std"): + expected = getattr(pddf, op)(axis=1, numeric_only=True, ddof=0) + got = getattr(gddf, op)(axis=1, numeric_only=True, ddof=0) + else: + expected = getattr(pddf, op)(numeric_only=True, axis=1) + got = getattr(pddf, op)(numeric_only=True, axis=1) + + dd.assert_eq( + expected, + got, + check_exact=False, + check_dtype=op not in ("var", "std"), + ) From 784fe95bf21b77e58102cee3bce8bea20ec262ee Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Mon, 29 Jan 2024 13:31:28 +0000 Subject: [PATCH 141/384] Enable full CI --- .github/workflows/pr.yaml | 181 +++++++++++++++++++------------------- 1 file changed, 90 insertions(+), 91 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 21138651f76..9c30161ab36 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -13,24 +13,24 @@ jobs: pr-builder: needs: - checks - #- conda-cpp-build - #- conda-cpp-checks - #- conda-cpp-tests - #- conda-python-build - #- conda-python-cudf-tests - #- conda-python-other-tests - #- conda-java-tests - #- conda-notebook-tests - #- docs-build + - conda-cpp-build + - conda-cpp-checks + - conda-cpp-tests + - conda-python-build + - conda-python-cudf-tests + - conda-python-other-tests + - conda-java-tests + - conda-notebook-tests + - docs-build - wheel-build-cudf - wheel-tests-cudf - wheel-build-dask-cudf - wheel-tests-dask-cudf - #- devcontainer + - devcontainer - unit-tests-cudf-pandas - pandas-tests - #- pandas-tests-diff - #- pandas-tests-diff-comment + - pandas-tests-diff + - pandas-tests-diff-comment secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.04 checks: @@ -38,77 +38,76 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.04 with: enable_check_generated_files: false - enable_check_style: false - #conda-cpp-build: - # needs: checks - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.04 - # with: - # build_type: pull-request - #conda-cpp-checks: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.04 - # with: - # build_type: pull-request - # enable_check_symbols: true - #conda-cpp-tests: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.04 - # with: - # build_type: pull-request - #conda-python-build: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.04 - # with: - # build_type: pull-request - #conda-python-cudf-tests: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 - # with: - # build_type: pull-request - # test_script: "ci/test_python_cudf.sh" - #conda-python-other-tests: - # # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 - # with: - # build_type: pull-request - # test_script: "ci/test_python_other.sh" - #conda-java-tests: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 - # with: - # build_type: pull-request - # node_type: "gpu-v100-latest-1" - # arch: "amd64" - # container_image: "rapidsai/ci-conda:latest" - # run_script: "ci/test_java.sh" - #conda-notebook-tests: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 - # with: - # build_type: pull-request - # node_type: "gpu-v100-latest-1" - # arch: "amd64" - # container_image: "rapidsai/ci-conda:latest" - # run_script: "ci/test_notebooks.sh" - #docs-build: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 - # with: - # build_type: pull-request - # node_type: "gpu-v100-latest-1" - # arch: "amd64" - # container_image: "rapidsai/ci-conda:latest" - # run_script: "ci/build_docs.sh" + conda-cpp-build: + needs: checks + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.04 + with: + build_type: pull-request + conda-cpp-checks: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.04 + with: + build_type: pull-request + enable_check_symbols: true + conda-cpp-tests: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.04 + with: + build_type: pull-request + conda-python-build: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.04 + with: + build_type: pull-request + conda-python-cudf-tests: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 + with: + build_type: pull-request + test_script: "ci/test_python_cudf.sh" + conda-python-other-tests: + # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 + with: + build_type: pull-request + test_script: "ci/test_python_other.sh" + conda-java-tests: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/test_java.sh" + conda-notebook-tests: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/test_notebooks.sh" + docs-build: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/build_docs.sh" wheel-build-cudf: needs: checks secrets: inherit @@ -140,14 +139,14 @@ jobs: matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) build_type: pull-request script: ci/test_wheel_dask_cudf.sh - #devcontainer: - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.04 - # with: - # build_command: | - # sccache -z; - # build-all -DBUILD_BENCHMARKS=ON --verbose; - # sccache -s; + devcontainer: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.04 + with: + build_command: | + sccache -z; + build-all -DBUILD_BENCHMARKS=ON --verbose; + sccache -s; unit-tests-cudf-pandas: needs: wheel-build-cudf secrets: inherit From 51e42c10fb8d2eab8d5e56502aaaae299af7d397 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Mon, 29 Jan 2024 13:34:13 +0000 Subject: [PATCH 142/384] Fix spacings --- .github/workflows/pr.yaml | 142 +++++++++++++++++++------------------- 1 file changed, 71 insertions(+), 71 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9c30161ab36..308f09b4b7f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -29,8 +29,8 @@ jobs: - devcontainer - unit-tests-cudf-pandas - pandas-tests - - pandas-tests-diff - - pandas-tests-diff-comment + #- pandas-tests-diff + #- pandas-tests-diff-comment secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.04 checks: @@ -39,75 +39,75 @@ jobs: with: enable_check_generated_files: false conda-cpp-build: - needs: checks - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.04 - with: - build_type: pull-request + needs: checks + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.04 + with: + build_type: pull-request conda-cpp-checks: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.04 - with: - build_type: pull-request - enable_check_symbols: true + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.04 + with: + build_type: pull-request + enable_check_symbols: true conda-cpp-tests: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.04 - with: - build_type: pull-request + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.04 + with: + build_type: pull-request conda-python-build: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.04 - with: - build_type: pull-request + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.04 + with: + build_type: pull-request conda-python-cudf-tests: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 - with: - build_type: pull-request - test_script: "ci/test_python_cudf.sh" + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 + with: + build_type: pull-request + test_script: "ci/test_python_cudf.sh" conda-python-other-tests: - # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 - with: - build_type: pull-request - test_script: "ci/test_python_other.sh" + # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 + with: + build_type: pull-request + test_script: "ci/test_python_other.sh" conda-java-tests: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/test_java.sh" + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/test_java.sh" conda-notebook-tests: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/test_notebooks.sh" + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/test_notebooks.sh" docs-build: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/build_docs.sh" + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/build_docs.sh" wheel-build-cudf: needs: checks secrets: inherit @@ -140,13 +140,13 @@ jobs: build_type: pull-request script: ci/test_wheel_dask_cudf.sh devcontainer: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.04 - with: - build_command: | - sccache -z; - build-all -DBUILD_BENCHMARKS=ON --verbose; - sccache -s; + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.04 + with: + build_command: | + sccache -z; + build-all -DBUILD_BENCHMARKS=ON --verbose; + sccache -s; unit-tests-cudf-pandas: needs: wheel-build-cudf secrets: inherit @@ -195,4 +195,4 @@ jobs: # owner: context.repo.owner, # repo: context.repo.repo, # body: `${summary_comment}\n\nHere is [a link to the full test summary](${summary_url}).\n` - # }) + # }) \ No newline at end of file From eae873e6b4f5f7234f7f8cee858d5d1adc788cc5 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 29 Jan 2024 07:36:06 -0600 Subject: [PATCH 143/384] Update pr.yaml --- .github/workflows/pr.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 308f09b4b7f..734c7643808 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -195,4 +195,5 @@ jobs: # owner: context.repo.owner, # repo: context.repo.repo, # body: `${summary_comment}\n\nHere is [a link to the full test summary](${summary_url}).\n` - # }) \ No newline at end of file + # }) + From d5db68e018d720094489325d5547a7ed82f22b0d Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Mon, 29 Jan 2024 08:47:31 -0500 Subject: [PATCH 144/384] Fix debug build by splitting row_operator_tests_utilities.cu (#14826) Splitting up `row_operator_tests_utilities.cu` works around a nvcc segfault when building with Debug. The segfault is found to occur on at least 11.8 and 12.0. ``` Building CUDA object tests/CMakeFiles/TABLE_TEST.dir/table/row_operator_tests_utilities.cu.o FAILED: tests/CMakeFiles/TABLE_TEST.dir/table/row_operator_tests_utilities.cu.o /usr/local/bin/nvcc -forward-unknown-to-host-compiler -DFMT_HEADER_ONLY=1 -DLIBCUDACXX_ENABLE_EXPERIMENTAL_MEMORY_RESOURCE -DSPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_INFO -DSPDLOG_FMT_EXTERNAL -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_CUDA -DTHRUST_HOST_SYSTEM=THRUST_HOST_SYSTEM_CPP -I/cudf/cpp -I/cudf/cpp/src -I/cudf/cpp/build/_deps/dlpack-src/include -I/cudf/cpp/build/_deps/jitify-src -I/cudf/cpp/include -I/cudf/cpp/build/include -I/cudf/cpp/build/_deps/cccl-src/thrust/thrust/cmake/../.. -I/cudf/cpp/build/_deps/cccl-src/libcudacxx/lib/cmake/libcudacxx/../../../include -I/cudf/cpp/build/_deps/cccl-src/cub/cub/cmake/../.. -isystem /conda/envs/rapids/include -isystem /usr/local/cuda/targets/x86_64-linux/include -Xcompiler=-fdiagnostics-color=always -g -std=c++17 "--generate-code=arch=compute_70,code=[sm_70]" -Xcompiler=-fPIE --expt-extended-lambda --expt-relaxed-constexpr -Werror=all-warnings -Xcompiler=-Wall,-Werror,-Wno-error=deprecated-declarations -Xfatbin=-compress-all -Xcompiler=-rdynamic -MD -MT tests/CMakeFiles/TABLE_TEST.dir/table/row_operator_tests_utilities.cu.o -MF tests/CMakeFiles/TABLE_TEST.dir/table/row_operator_tests_utilities.cu.o.d -x cu -c /cudf/cpp/tests/table/row_operator_tests_utilities.cu -o tests/CMakeFiles/TABLE_TEST.dir/table/row_operator_tests_utilities.cu.o Segmentation fault (core dumped) ninja: build stopped: subcommand failed. ``` This PR has been verified to workaround the error in both versions. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Karthikeyan (https://github.com/karthikeyann) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/14826 --- cpp/tests/CMakeLists.txt | 9 ++- .../table/row_operator_tests_utilities.cu | 51 +------------- .../table/row_operator_tests_utilities.hpp | 8 ++- .../table/row_operator_tests_utilities2.cu | 69 +++++++++++++++++++ 4 files changed, 84 insertions(+), 53 deletions(-) create mode 100644 cpp/tests/table/row_operator_tests_utilities2.cu diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index caa92d60151..8b0e625fecf 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -439,8 +439,13 @@ ConfigureTest( # ################################################################################################## # * table tests ----------------------------------------------------------------------------------- ConfigureTest( - TABLE_TEST table/table_tests.cpp table/table_view_tests.cu table/row_operators_tests.cpp - table/experimental_row_operator_tests.cu table/row_operator_tests_utilities.cu + TABLE_TEST + table/table_tests.cpp + table/table_view_tests.cu + table/row_operators_tests.cpp + table/experimental_row_operator_tests.cu + table/row_operator_tests_utilities.cu + table/row_operator_tests_utilities2.cu ) # ################################################################################################## diff --git a/cpp/tests/table/row_operator_tests_utilities.cu b/cpp/tests/table/row_operator_tests_utilities.cu index d1f918cc7af..cfffa1cdd54 100644 --- a/cpp/tests/table/row_operator_tests_utilities.cu +++ b/cpp/tests/table/row_operator_tests_utilities.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,60 +18,14 @@ #include #include -#include #include #include -#include #include #include #include -template -std::unique_ptr self_comparison(cudf::table_view input, - std::vector const& column_order, - PhysicalElementComparator comparator) -{ - rmm::cuda_stream_view stream{cudf::get_default_stream()}; - - auto const table_comparator = - cudf::experimental::row::lexicographic::self_comparator{input, column_order, {}, stream}; - - auto output = cudf::make_numeric_column( - cudf::data_type(cudf::type_id::BOOL8), input.num_rows(), cudf::mask_state::UNALLOCATED); - - if (cudf::detail::has_nested_columns(input)) { - thrust::transform(rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(input.num_rows()), - thrust::make_counting_iterator(0), - output->mutable_view().data(), - table_comparator.less(cudf::nullate::NO{}, comparator)); - } else { - thrust::transform(rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(input.num_rows()), - thrust::make_counting_iterator(0), - output->mutable_view().data(), - table_comparator.less(cudf::nullate::NO{}, comparator)); - } - return output; -} - -using physical_comparator_t = cudf::experimental::row::lexicographic::physical_element_comparator; -using sorting_comparator_t = - cudf::experimental::row::lexicographic::sorting_physical_element_comparator; - -template std::unique_ptr self_comparison( - cudf::table_view input, - std::vector const& column_order, - physical_comparator_t comparator); -template std::unique_ptr self_comparison( - cudf::table_view input, - std::vector const& column_order, - sorting_comparator_t comparator); - template std::unique_ptr two_table_comparison(cudf::table_view lhs, cudf::table_view rhs, @@ -199,9 +153,6 @@ std::unique_ptr two_table_equality(cudf::table_view lhs, return output; } -using physical_equality_t = cudf::experimental::row::equality::physical_equality_comparator; -using nan_equality_t = cudf::experimental::row::equality::nan_equal_physical_equality_comparator; - template std::unique_ptr two_table_equality( cudf::table_view lhs, cudf::table_view rhs, diff --git a/cpp/tests/table/row_operator_tests_utilities.hpp b/cpp/tests/table/row_operator_tests_utilities.hpp index b34bf65d176..023a54669b4 100644 --- a/cpp/tests/table/row_operator_tests_utilities.hpp +++ b/cpp/tests/table/row_operator_tests_utilities.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,12 @@ #include +using physical_comparator_t = cudf::experimental::row::lexicographic::physical_element_comparator; +using sorting_comparator_t = + cudf::experimental::row::lexicographic::sorting_physical_element_comparator; +using physical_equality_t = cudf::experimental::row::equality::physical_equality_comparator; +using nan_equality_t = cudf::experimental::row::equality::nan_equal_physical_equality_comparator; + template std::unique_ptr self_comparison(cudf::table_view input, std::vector const& column_order, diff --git a/cpp/tests/table/row_operator_tests_utilities2.cu b/cpp/tests/table/row_operator_tests_utilities2.cu new file mode 100644 index 00000000000..057d9ee1004 --- /dev/null +++ b/cpp/tests/table/row_operator_tests_utilities2.cu @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "row_operator_tests_utilities.hpp" + +#include +#include + +#include +#include + +#include +#include + +// Including this declaration/defintion in row_operator_tests_utilities.cu causes +// the nvcc compiler to segfault when built with the debug (-g) flag. + +template +std::unique_ptr self_comparison(cudf::table_view input, + std::vector const& column_order, + PhysicalElementComparator comparator) +{ + rmm::cuda_stream_view stream{cudf::get_default_stream()}; + + auto const table_comparator = + cudf::experimental::row::lexicographic::self_comparator{input, column_order, {}, stream}; + + auto output = cudf::make_numeric_column( + cudf::data_type(cudf::type_id::BOOL8), input.num_rows(), cudf::mask_state::UNALLOCATED); + + if (cudf::detail::has_nested_columns(input)) { + thrust::transform(rmm::exec_policy(stream), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(input.num_rows()), + thrust::make_counting_iterator(0), + output->mutable_view().data(), + table_comparator.less(cudf::nullate::NO{}, comparator)); + } else { + thrust::transform(rmm::exec_policy(stream), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(input.num_rows()), + thrust::make_counting_iterator(0), + output->mutable_view().data(), + table_comparator.less(cudf::nullate::NO{}, comparator)); + } + return output; +} + +template std::unique_ptr self_comparison( + cudf::table_view input, + std::vector const& column_order, + physical_comparator_t comparator); +template std::unique_ptr self_comparison( + cudf::table_view input, + std::vector const& column_order, + sorting_comparator_t comparator); From dbf08cb84026e185971f29b71e278765c3d55093 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 29 Jan 2024 08:48:51 -0600 Subject: [PATCH 145/384] Fix style issues in 2.0 feature branch (#14918) This PR fixes all style issues in pandas-2.0 feature branch --- .github/workflows/pr.yaml | 1 - python/cudf/cudf/_lib/column.pyx | 1 - python/cudf/cudf/api/types.py | 4 ++-- python/cudf/cudf/core/column/datetime.py | 1 - python/cudf/cudf/core/column/interval.py | 3 +-- python/cudf/cudf/core/column/timedelta.py | 3 +-- python/cudf/cudf/core/dataframe.py | 1 - python/cudf/cudf/core/dtypes.py | 14 +++++------ python/cudf/cudf/core/multiindex.py | 8 +++++-- python/cudf/cudf/testing/testing.py | 2 +- python/cudf/cudf/tests/test_api_types.py | 3 +-- python/cudf/cudf/tests/test_applymap.py | 2 +- python/cudf/cudf/tests/test_array_function.py | 3 ++- python/cudf/cudf/tests/test_categorical.py | 5 ---- .../cudf/cudf/tests/test_column_accessor.py | 2 +- python/cudf/cudf/tests/test_concat.py | 7 +++--- python/cudf/cudf/tests/test_csv.py | 2 +- python/cudf/cudf/tests/test_datasets.py | 2 +- python/cudf/cudf/tests/test_datetime.py | 24 ++++++------------- python/cudf/cudf/tests/test_dropna.py | 1 - python/cudf/cudf/tests/test_duplicates.py | 5 +--- python/cudf/cudf/tests/test_groupby.py | 2 +- python/cudf/cudf/tests/test_monotonic.py | 7 +----- python/cudf/cudf/tests/test_parquet.py | 1 - python/cudf/cudf/tests/test_resampling.py | 2 +- python/cudf/cudf/tests/test_rolling.py | 3 --- python/cudf/cudf/tests/test_series.py | 2 +- python/cudf/cudf/tests/test_stats.py | 5 +--- python/cudf/cudf/tests/test_timedelta.py | 2 +- .../cudf_pandas_tests/test_cudf_pandas.py | 1 - .../dask_cudf/tests/test_accessor.py | 3 +-- 31 files changed, 43 insertions(+), 79 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 734c7643808..14a74618413 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -196,4 +196,3 @@ jobs: # repo: context.repo.repo, # body: `${summary_comment}\n\nHere is [a link to the full test summary](${summary_url}).\n` # }) - diff --git a/python/cudf/cudf/_lib/column.pyx b/python/cudf/cudf/_lib/column.pyx index b97dc85ef8b..45aa1081b8d 100644 --- a/python/cudf/cudf/_lib/column.pyx +++ b/python/cudf/cudf/_lib/column.pyx @@ -12,7 +12,6 @@ import rmm import cudf import cudf._lib as libcudf from cudf._lib import pylibcudf -from cudf.api.types import _is_categorical_dtype, _is_datetime64tz_dtype from cudf.core.buffer import ( Buffer, ExposureTrackedBuffer, diff --git a/python/cudf/cudf/api/types.py b/python/cudf/cudf/api/types.py index 22fc3ea2c6f..f6f5e522cbd 100644 --- a/python/cudf/cudf/api/types.py +++ b/python/cudf/cudf/api/types.py @@ -12,9 +12,10 @@ import cupy as cp import numpy as np +import pandas as pd +from pandas.api import types as pd_types import cudf -import pandas as pd from cudf.core._compat import PANDAS_GE_150 from cudf.core.dtypes import ( # noqa: F401 _BaseDtype, @@ -30,7 +31,6 @@ is_list_dtype, is_struct_dtype, ) -from pandas.api import types as pd_types def is_numeric_dtype(obj): diff --git a/python/cudf/cudf/core/column/datetime.py b/python/cudf/cudf/core/column/datetime.py index 08a5103b409..a0c0b119ef7 100644 --- a/python/cudf/cudf/core/column/datetime.py +++ b/python/cudf/cudf/core/column/datetime.py @@ -28,7 +28,6 @@ is_scalar, is_timedelta64_dtype, ) - from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_220 from cudf.core.buffer import Buffer, cuda_array_interface_wrapper from cudf.core.column import ColumnBase, as_column, column, string diff --git a/python/cudf/cudf/core/column/interval.py b/python/cudf/cudf/core/column/interval.py index f73c222624a..f5d527ad201 100644 --- a/python/cudf/cudf/core/column/interval.py +++ b/python/cudf/cudf/core/column/interval.py @@ -1,11 +1,10 @@ # Copyright (c) 2018-2024, NVIDIA CORPORATION. from typing import Optional -import pyarrow as pa import pandas as pd +import pyarrow as pa import cudf - from cudf.api.types import _is_interval_dtype from cudf.core.column import StructColumn from cudf.core.dtypes import CategoricalDtype, IntervalDtype diff --git a/python/cudf/cudf/core/column/timedelta.py b/python/cudf/cudf/core/column/timedelta.py index 2c12c77277c..094ccb57a1c 100644 --- a/python/cudf/cudf/core/column/timedelta.py +++ b/python/cudf/cudf/core/column/timedelta.py @@ -14,12 +14,11 @@ from cudf import _lib as libcudf from cudf._typing import ColumnBinaryOperand, DatetimeLikeScalar, Dtype from cudf.api.types import is_scalar, is_timedelta64_dtype +from cudf.core._compat import PANDAS_GE_200 from cudf.core.buffer import Buffer, acquire_spill_lock from cudf.core.column import ColumnBase, column, string from cudf.utils.dtypes import np_to_pa_dtype from cudf.utils.utils import _all_bools_with_nulls -from cudf.core._compat import PANDAS_GE_200 - _dtype_to_format_conversion = { "timedelta64[ns]": "%D days %H:%M:%S", diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 86947fe6028..d0cbacfb7e8 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -6099,7 +6099,6 @@ def _reduce( numeric_only=False, **kwargs, ): - source = self if axis is None: diff --git a/python/cudf/cudf/core/dtypes.py b/python/cudf/cudf/core/dtypes.py index 0eb2e455544..734dd501d48 100644 --- a/python/cudf/cudf/core/dtypes.py +++ b/python/cudf/cudf/core/dtypes.py @@ -9,21 +9,21 @@ from typing import Any, Callable, Dict, List, Tuple, Type, Union import numpy as np +import pandas as pd import pyarrow as pa +from pandas.api import types as pd_types +from pandas.api.extensions import ExtensionDtype +from pandas.core.dtypes.dtypes import ( + CategoricalDtype as pd_CategoricalDtype, + CategoricalDtypeType as pd_CategoricalDtypeType, +) import cudf -import pandas as pd from cudf._typing import Dtype from cudf.core._compat import PANDAS_GE_150 from cudf.core.abc import Serializable from cudf.core.buffer import Buffer from cudf.utils.docutils import doc_apply -from pandas.api import types as pd_types -from pandas.api.extensions import ExtensionDtype -from pandas.core.dtypes.dtypes import ( - CategoricalDtype as pd_CategoricalDtype, - CategoricalDtypeType as pd_CategoricalDtypeType, -) if PANDAS_GE_150: from pandas.core.arrays.arrow.extension_types import ArrowIntervalType diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index e6ac114bfcb..081109e81bc 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -34,8 +34,8 @@ ) from cudf.core.join._join_helpers import _match_join_keys from cudf.utils.dtypes import is_column_like -from cudf.utils.utils import NotIterable, _external_only_api, _is_same_name from cudf.utils.nvtx_annotation import _cudf_nvtx_annotate +from cudf.utils.utils import NotIterable, _external_only_api, _is_same_name def _maybe_indices_to_slice(indices: cp.ndarray) -> Union[slice, cp.ndarray]: @@ -726,7 +726,11 @@ def _compute_validity_mask(self, index, row_tuple, max_length): [ frame, cudf.DataFrame( - {"idx": cudf.Series(column.as_column(range(len(frame))))} + { + "idx": cudf.Series( + column.as_column(range(len(frame))) + ) + } ), ], axis=1, diff --git a/python/cudf/cudf/testing/testing.py b/python/cudf/cudf/testing/testing.py index dc7d6b84d9b..fc253c5c197 100644 --- a/python/cudf/cudf/testing/testing.py +++ b/python/cudf/cudf/testing/testing.py @@ -4,9 +4,9 @@ import cupy as cp import numpy as np +import pandas as pd import cudf -import pandas as pd from cudf._lib.unary import is_nan from cudf.api.types import ( _is_categorical_dtype, diff --git a/python/cudf/cudf/tests/test_api_types.py b/python/cudf/cudf/tests/test_api_types.py index cc26815920c..7780f9853a2 100644 --- a/python/cudf/cudf/tests/test_api_types.py +++ b/python/cudf/cudf/tests/test_api_types.py @@ -6,9 +6,8 @@ from pandas.api import types as pd_types import cudf -from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210, PANDAS_GE_214 from cudf.api import types - +from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210, PANDAS_GE_214 from cudf.testing._utils import expect_warning_if diff --git a/python/cudf/cudf/tests/test_applymap.py b/python/cudf/cudf/tests/test_applymap.py index 523a7f424e8..38a34c206d7 100644 --- a/python/cudf/cudf/tests/test_applymap.py +++ b/python/cudf/cudf/tests/test_applymap.py @@ -3,8 +3,8 @@ import pytest from cudf import NA, DataFrame -from cudf.testing import _utils as utils from cudf.core._compat import PANDAS_GE_210 +from cudf.testing import _utils as utils @pytest.mark.parametrize( diff --git a/python/cudf/cudf/tests/test_array_function.py b/python/cudf/cudf/tests/test_array_function.py index b3f5267c3b2..090e8884991 100644 --- a/python/cudf/cudf/tests/test_array_function.py +++ b/python/cudf/cudf/tests/test_array_function.py @@ -1,4 +1,5 @@ -# Copyright (c) 2018-2023, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. + import numpy as np import pandas as pd import pytest diff --git a/python/cudf/cudf/tests/test_categorical.py b/python/cudf/cudf/tests/test_categorical.py index 0f7abab8104..ad32ebce01b 100644 --- a/python/cudf/cudf/tests/test_categorical.py +++ b/python/cudf/cudf/tests/test_categorical.py @@ -346,7 +346,6 @@ def test_categorical_set_categories_preserves_order(): def test_categorical_as_ordered(pd_str_cat): - pd_sr = pd.Series(pd_str_cat.copy().set_ordered(False)) cd_sr = cudf.Series(pd_str_cat.copy().set_ordered(False)) @@ -362,7 +361,6 @@ def test_categorical_as_ordered(pd_str_cat): def test_categorical_as_unordered(pd_str_cat): - pd_sr = pd.Series(pd_str_cat.copy().set_ordered(True)) cd_sr = cudf.Series(pd_str_cat.copy().set_ordered(True)) @@ -380,7 +378,6 @@ def test_categorical_as_unordered(pd_str_cat): @pytest.mark.parametrize("from_ordered", [True, False]) @pytest.mark.parametrize("to_ordered", [True, False]) def test_categorical_reorder_categories(pd_str_cat, from_ordered, to_ordered): - pd_sr = pd.Series(pd_str_cat.copy().set_ordered(from_ordered)) cd_sr = cudf.Series(pd_str_cat.copy().set_ordered(from_ordered)) @@ -401,7 +398,6 @@ def test_categorical_reorder_categories(pd_str_cat, from_ordered, to_ordered): def test_categorical_add_categories(pd_str_cat): - pd_sr = pd.Series(pd_str_cat.copy()) cd_sr = cudf.Series(pd_str_cat.copy()) @@ -419,7 +415,6 @@ def test_categorical_add_categories(pd_str_cat): def test_categorical_remove_categories(pd_str_cat): - pd_sr = pd.Series(pd_str_cat.copy()) cd_sr = cudf.Series(pd_str_cat.copy()) diff --git a/python/cudf/cudf/tests/test_column_accessor.py b/python/cudf/cudf/tests/test_column_accessor.py index e2a2b307856..bf764b02faa 100644 --- a/python/cudf/cudf/tests/test_column_accessor.py +++ b/python/cudf/cudf/tests/test_column_accessor.py @@ -5,9 +5,9 @@ import pytest import cudf +from cudf.core._compat import PANDAS_GE_200 from cudf.core.column_accessor import ColumnAccessor from cudf.testing._utils import assert_eq -from cudf.core._compat import PANDAS_GE_200 simple_test_data = [ {}, diff --git a/python/cudf/cudf/tests/test_concat.py b/python/cudf/cudf/tests/test_concat.py index 895e35614fa..ed8f32ed12b 100644 --- a/python/cudf/cudf/tests/test_concat.py +++ b/python/cudf/cudf/tests/test_concat.py @@ -1,17 +1,16 @@ # Copyright (c) 2018-2024, NVIDIA CORPORATION. +import warnings +from contextlib import contextmanager from decimal import Decimal -import warnings import numpy as np import pandas as pd import pytest -from contextlib import contextmanager - import cudf as gd from cudf.api.types import _is_categorical_dtype -from cudf.core._compat import PANDAS_GE_150, PANDAS_LT_140, PANDAS_GE_200 +from cudf.core._compat import PANDAS_GE_150, PANDAS_GE_200, PANDAS_LT_140 from cudf.core.dtypes import Decimal32Dtype, Decimal64Dtype, Decimal128Dtype from cudf.testing._utils import ( assert_eq, diff --git a/python/cudf/cudf/tests/test_csv.py b/python/cudf/cudf/tests/test_csv.py index 2fa4a313d6f..6de66bf1952 100644 --- a/python/cudf/cudf/tests/test_csv.py +++ b/python/cudf/cudf/tests/test_csv.py @@ -17,7 +17,7 @@ import cudf from cudf import read_csv -from cudf.core._compat import PANDAS_LT_140, PANDAS_GE_200 +from cudf.core._compat import PANDAS_GE_200, PANDAS_LT_140 from cudf.testing._utils import assert_eq, assert_exceptions_equal diff --git a/python/cudf/cudf/tests/test_datasets.py b/python/cudf/cudf/tests/test_datasets.py index fdab8cb5edf..320c221fcb2 100644 --- a/python/cudf/cudf/tests/test_datasets.py +++ b/python/cudf/cudf/tests/test_datasets.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2022, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. import numpy as np diff --git a/python/cudf/cudf/tests/test_datetime.py b/python/cudf/cudf/tests/test_datetime.py index 14a732deea4..d83b46250d0 100644 --- a/python/cudf/cudf/tests/test_datetime.py +++ b/python/cudf/cudf/tests/test_datetime.py @@ -2,6 +2,7 @@ import datetime import operator +import warnings import cupy as cp import numpy as np @@ -10,15 +11,14 @@ import pytest import cudf -import warnings import cudf.testing.dataset_generator as dataset_generator from cudf import DataFrame, Series from cudf.core._compat import ( - PANDAS_GE_150, - PANDAS_LT_140, PANDAS_EQ_200, + PANDAS_GE_150, PANDAS_GE_200, PANDAS_GE_210, + PANDAS_LT_140, ) from cudf.core.index import DatetimeIndex from cudf.testing._utils import ( @@ -623,22 +623,13 @@ def test_datetime_dataframe(): @pytest.mark.parametrize("dayfirst", [True, False]) def test_cudf_to_datetime(data, dayfirst): pd_data = data - is_string_data = False if isinstance(pd_data, (pd.Series, pd.DataFrame, pd.Index)): gd_data = cudf.from_pandas(pd_data) - is_string_data = ( - gd_data.ndim == 1 - and not gd_data.empty - and gd_data.dtype.kind == "O" - ) else: if type(pd_data).__module__ == np.__name__: gd_data = cp.array(pd_data) else: gd_data = pd_data - is_string_data = isinstance(gd_data, list) and isinstance( - next(iter(gd_data), None), str - ) expected = pd.to_datetime(pd_data, dayfirst=dayfirst) actual = cudf.to_datetime(gd_data, dayfirst=dayfirst) @@ -696,7 +687,6 @@ def test_to_datetime_errors(data): def test_to_datetime_not_implemented(): - with pytest.raises(NotImplementedError): cudf.to_datetime([], exact=False) @@ -817,7 +807,6 @@ def test_to_datetime_different_formats_notimplemented(): def test_datetime_can_cast_safely(): - sr = cudf.Series( ["1679-01-01", "2000-01-31", "2261-01-01"], dtype="datetime64[ms]" ) @@ -938,7 +927,6 @@ def test_str_to_datetime_error(): @pytest.mark.parametrize("data_dtype", DATETIME_TYPES) @pytest.mark.parametrize("other_dtype", DATETIME_TYPES) def test_datetime_subtract(data, other, data_dtype, other_dtype): - gsr = cudf.Series(data, dtype=data_dtype) psr = gsr.to_pandas() @@ -1580,7 +1568,8 @@ def test_date_range_start_end_freq(request, start, end, freq): request.applymarker( pytest.mark.xfail( condition=( - not PANDAS_GE_200 and isinstance(freq, dict) + not PANDAS_GE_200 + and isinstance(freq, dict) and freq.get("hours", None) == 10 and freq.get("days", None) == 57 and freq.get("nanoseconds", None) == 3 @@ -1634,7 +1623,8 @@ def test_date_range_start_freq_periods(request, start, freq, periods): request.applymarker( pytest.mark.xfail( condition=( - not PANDAS_GE_200 and isinstance(freq, dict) + not PANDAS_GE_200 + and isinstance(freq, dict) and freq.get("hours", None) == 10 and freq.get("days", None) == 57 and freq.get("nanoseconds", None) == 3 diff --git a/python/cudf/cudf/tests/test_dropna.py b/python/cudf/cudf/tests/test_dropna.py index 4b665cb6f0a..f1acd7b4320 100644 --- a/python/cudf/cudf/tests/test_dropna.py +++ b/python/cudf/cudf/tests/test_dropna.py @@ -21,7 +21,6 @@ @pytest.mark.parametrize("nulls", ["one", "some", "all", "none"]) @pytest.mark.parametrize("inplace", [True, False]) def test_dropna_series(data, nulls, inplace): - psr = pd.Series(data) if len(data) > 0: diff --git a/python/cudf/cudf/tests/test_duplicates.py b/python/cudf/cudf/tests/test_duplicates.py index b01130d5fa1..447b2b3c4f5 100644 --- a/python/cudf/cudf/tests/test_duplicates.py +++ b/python/cudf/cudf/tests/test_duplicates.py @@ -9,10 +9,7 @@ import cudf from cudf import concat -from cudf.testing._utils import ( - assert_eq, - assert_exceptions_equal, -) +from cudf.testing._utils import assert_eq, assert_exceptions_equal # TODO: PANDAS 1.0 support # Revisit drop_duplicates() tests to update parameters like ignore_index. diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index 77a25e2dbae..0c71d74f89f 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -22,9 +22,9 @@ from cudf.api.extensions import no_default from cudf.core._compat import ( PANDAS_GE_150, - PANDAS_LT_140, PANDAS_GE_200, PANDAS_GE_210, + PANDAS_LT_140, ) from cudf.core.udf._ops import arith_ops, comparison_ops, unary_ops from cudf.core.udf.groupby_typing import SUPPORTED_GROUPBY_NUMPY_TYPES diff --git a/python/cudf/cudf/tests/test_monotonic.py b/python/cudf/cudf/tests/test_monotonic.py index c84088c1cd3..53919a95115 100644 --- a/python/cudf/cudf/tests/test_monotonic.py +++ b/python/cudf/cudf/tests/test_monotonic.py @@ -10,11 +10,7 @@ import cudf from cudf import Index, MultiIndex, Series -from cudf.core.index import ( - CategoricalIndex, - DatetimeIndex, - RangeIndex, -) +from cudf.core.index import CategoricalIndex, DatetimeIndex, RangeIndex from cudf.testing._utils import assert_eq @@ -46,7 +42,6 @@ def test_range_index(testrange): ], ) def test_generic_index(testlist): - index = Index(testlist) index_pd = pd.Index(testlist) diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index 69d3fe0b83f..105c31cc71f 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -1622,7 +1622,6 @@ def test_parquet_writer_int96_timestamps(tmpdir, pdf, gdf): def test_multifile_parquet_folder(tmpdir): - test_pdf1 = make_pdf(nrows=10, nvalids=10 // 2, dtype="float64") test_pdf2 = make_pdf(nrows=20, dtype="float64") expect = pd.concat([test_pdf1, test_pdf2]) diff --git a/python/cudf/cudf/tests/test_resampling.py b/python/cudf/cudf/tests/test_resampling.py index 7cc4b465873..6281d54aa60 100644 --- a/python/cudf/cudf/tests/test_resampling.py +++ b/python/cudf/cudf/tests/test_resampling.py @@ -5,8 +5,8 @@ import pytest import cudf -from cudf.testing._utils import assert_eq from cudf.core._compat import PANDAS_GE_200 +from cudf.testing._utils import assert_eq def assert_resample_results_equal(lhs, rhs, **kwargs): diff --git a/python/cudf/cudf/tests/test_rolling.py b/python/cudf/cudf/tests/test_rolling.py index 7d3e19c002b..22dcf5dfa7e 100644 --- a/python/cudf/cudf/tests/test_rolling.py +++ b/python/cudf/cudf/tests/test_rolling.py @@ -151,7 +151,6 @@ def test_rolling_with_offset(agg): @pytest.mark.parametrize("seed", [100, 2000]) @pytest.mark.parametrize("window_size", [2, 10, 100]) def test_rolling_var_std_large(agg, ddof, center, seed, window_size): - iupper_bound = math.sqrt(np.iinfo(np.int64).max / window_size) ilower_bound = -math.sqrt(abs(np.iinfo(np.int64).min) / window_size) @@ -312,7 +311,6 @@ def test_rolling_getitem_window(): ) @pytest.mark.parametrize("center", [True, False]) def test_rollling_series_numba_udf_basic(data, index, center): - psr = pd.Series(data, index=index) gsr = cudf.from_pandas(psr) @@ -349,7 +347,6 @@ def some_func(A): ) @pytest.mark.parametrize("center", [True, False]) def test_rolling_dataframe_numba_udf_basic(data, center): - pdf = pd.DataFrame(data) gdf = cudf.from_pandas(pdf) diff --git a/python/cudf/cudf/tests/test_series.py b/python/cudf/cudf/tests/test_series.py index 2e39345f63a..b7be3878412 100644 --- a/python/cudf/cudf/tests/test_series.py +++ b/python/cudf/cudf/tests/test_series.py @@ -14,6 +14,7 @@ import pytest import cudf +from cudf.api.extensions import no_default from cudf.core._compat import PANDAS_LT_140 from cudf.errors import MixedTypeError from cudf.testing._utils import ( @@ -25,7 +26,6 @@ expect_warning_if, gen_rand, ) -from cudf.api.extensions import no_default def _series_na_data(): diff --git a/python/cudf/cudf/tests/test_stats.py b/python/cudf/cudf/tests/test_stats.py index 6dbb23fbf04..b35dd28c4ec 100644 --- a/python/cudf/cudf/tests/test_stats.py +++ b/python/cudf/cudf/tests/test_stats.py @@ -9,13 +9,13 @@ import cudf from cudf.api.extensions import no_default +from cudf.core._compat import PANDAS_GE_210 from cudf.datasets import randomdata from cudf.testing._utils import ( assert_eq, assert_exceptions_equal, expect_warning_if, ) -from cudf.core._compat import PANDAS_GE_210 params_dtypes = [np.int32, np.uint32, np.float32, np.float64] methods = ["min", "max", "sum", "mean", "var", "std"] @@ -182,7 +182,6 @@ def test_exact_quantiles_int(int_method): def test_approx_quantiles(): - arr = np.asarray([6.8, 0.15, 3.4, 4.17, 2.13, 1.11, -1.01, 0.8, 5.7]) quant_values = [0.0, 0.25, 0.33, 0.5, 1.0] @@ -222,7 +221,6 @@ def test_approx_quantiles_int(): ], ) def test_misc_quantiles(data, q): - pdf_series = pd.Series(data, dtype="float64" if len(data) == 0 else None) gdf_series = cudf.from_pandas(pdf_series) @@ -503,7 +501,6 @@ def test_corr1d(data1, data2, method): @pytest.mark.parametrize("method", ["spearman", "pearson"]) def test_df_corr(method): - gdf = randomdata(100, {str(x): float for x in range(50)}) pdf = gdf.to_pandas() got = gdf.corr(method) diff --git a/python/cudf/cudf/tests/test_timedelta.py b/python/cudf/cudf/tests/test_timedelta.py index 850c56b7614..980a8c0df2e 100644 --- a/python/cudf/cudf/tests/test_timedelta.py +++ b/python/cudf/cudf/tests/test_timedelta.py @@ -9,9 +9,9 @@ import pytest import cudf +from cudf.core._compat import PANDAS_GE_200 from cudf.testing import _utils as utils from cudf.testing._utils import assert_eq, assert_exceptions_equal -from cudf.core._compat import PANDAS_GE_200 _TIMEDELTA_DATA = [ [1000000, 200000, 3000000], diff --git a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py index 2425c323060..e36e1a68114 100644 --- a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py +++ b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py @@ -269,7 +269,6 @@ def test_rename_categories(): tm.assert_series_equal(psr, sr) - def test_column_rename(dataframe): pdf, df = dataframe pdf.columns = ["x", "y"] diff --git a/python/dask_cudf/dask_cudf/tests/test_accessor.py b/python/dask_cudf/dask_cudf/tests/test_accessor.py index f6b8c34fef0..a6a457d98a4 100644 --- a/python/dask_cudf/dask_cudf/tests/test_accessor.py +++ b/python/dask_cudf/dask_cudf/tests/test_accessor.py @@ -2,11 +2,10 @@ import numpy as np import pandas as pd -import dask import pytest - from pandas.testing import assert_series_equal +import dask from dask import dataframe as dd from cudf import DataFrame, Series, date_range From e74fe0a1a0f3e40e4514fca510429f5e2e33fa76 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 29 Jan 2024 06:58:34 -0800 Subject: [PATCH 146/384] Remove gated xfails (#14905) This removes xpassing tests from the test output. --- python/cudf/cudf/tests/test_index.py | 40 +++--------------------- python/cudf/cudf/tests/test_timedelta.py | 8 +---- 2 files changed, 5 insertions(+), 43 deletions(-) diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index ca8ef83316d..4dfbcf138c3 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -2274,45 +2274,13 @@ def test_range_index_concat(objs): [ (pd.RangeIndex(0, 10), pd.RangeIndex(3, 7)), (pd.RangeIndex(0, 10), pd.RangeIndex(10, 20)), - pytest.param( - pd.RangeIndex(0, 10, 2), - pd.RangeIndex(1, 5, 3), - marks=pytest.mark.xfail( - condition=PANDAS_GE_200, - reason="https://github.com/pandas-dev/pandas/issues/53490", - strict=False, - ), - ), - pytest.param( - pd.RangeIndex(1, 5, 3), - pd.RangeIndex(0, 10, 2), - marks=pytest.mark.xfail( - condition=PANDAS_GE_200, - reason="https://github.com/pandas-dev/pandas/issues/53490", - strict=False, - ), - ), - pytest.param( - pd.RangeIndex(1, 10, 3), - pd.RangeIndex(1, 5, 2), - marks=pytest.mark.xfail( - condition=PANDAS_GE_200, - reason="https://github.com/pandas-dev/pandas/issues/53490", - strict=False, - ), - ), + (pd.RangeIndex(0, 10, 2), pd.RangeIndex(1, 5, 3)), + (pd.RangeIndex(1, 5, 3), pd.RangeIndex(0, 10, 2)), + (pd.RangeIndex(1, 10, 3), pd.RangeIndex(1, 5, 2)), (pd.RangeIndex(1, 5, 2), pd.RangeIndex(1, 10, 3)), (pd.RangeIndex(1, 100, 3), pd.RangeIndex(1, 50, 3)), (pd.RangeIndex(1, 100, 3), pd.RangeIndex(1, 50, 6)), - pytest.param( - pd.RangeIndex(1, 100, 6), - pd.RangeIndex(1, 50, 3), - marks=pytest.mark.xfail( - condition=PANDAS_GE_200, - reason="https://github.com/pandas-dev/pandas/issues/53490", - strict=False, - ), - ), + (pd.RangeIndex(1, 100, 6), pd.RangeIndex(1, 50, 3)), (pd.RangeIndex(0, 10, name="a"), pd.RangeIndex(90, 100, name="b")), (pd.Index([0, 1, 2, 30], name="a"), pd.Index([90, 100])), (pd.Index([0, 1, 2, 30], name="a"), [90, 100]), diff --git a/python/cudf/cudf/tests/test_timedelta.py b/python/cudf/cudf/tests/test_timedelta.py index 980a8c0df2e..7cae2f3a30f 100644 --- a/python/cudf/cudf/tests/test_timedelta.py +++ b/python/cudf/cudf/tests/test_timedelta.py @@ -538,13 +538,7 @@ def test_timedelta_series_mod_with_scalar_zero(reverse): datetime.timedelta(seconds=768), datetime.timedelta(microseconds=7), np.timedelta64(4, "s"), - pytest.param( - np.timedelta64("nat", "s"), - marks=pytest.mark.xfail( - strict=False, - reason="https://github.com/pandas-dev/pandas/issues/52295", - ), - ), + np.timedelta64("nat", "s"), np.timedelta64(1, "s"), np.timedelta64(1, "ms"), np.timedelta64(1, "us"), From f69ae1d110ce6389ccef115fe5ca49d36066b8ca Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 29 Jan 2024 09:22:37 -0600 Subject: [PATCH 147/384] Add `Groupby.indices` property and deprecate `obj` in `get_group` (#14912) This PR: Introduces Groupby.indices property. Deprecates obj in Groupby.get_group --- python/cudf/cudf/core/groupby/groupby.py | 34 ++++++++++++++++++++++++ python/cudf/cudf/tests/test_groupby.py | 13 +++++++-- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index b73d5532100..b3577444f6b 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -344,6 +344,33 @@ def groups(self): zip(group_names.to_pandas(), grouped_index._split(offsets[1:-1])) ) + @cached_property + def indices(self): + """ + Dict {group name -> group indices}. + + Examples + -------- + >>> import cudf + >>> data = [[10, 20, 30], [10, 30, 40], [40, 50, 30]] + >>> df = cudf.DataFrame(data, columns=["a", "b", "c"]) + >>> df + a b c + 0 10 20 30 + 1 10 30 40 + 2 40 50 30 + >>> df.groupby(by=["a"]).indices + {10: array([0, 1]), 40: array([2])} + """ + group_names, offsets, _, grouped_values = self._grouped() + + return dict( + zip( + group_names.to_pandas(), + np.split(grouped_values.index.values, offsets[1:-1]), + ) + ) + @_cudf_nvtx_annotate def get_group(self, name, obj=None): """ @@ -379,6 +406,13 @@ def get_group(self, name, obj=None): """ if obj is None: obj = self.obj + else: + warnings.warn( + "obj is deprecated and will be removed in a future version. " + "Use ``df.iloc[gb.indices.get(name)]`` " + "instead of ``gb.get_group(name, obj=df)``.", + FutureWarning, + ) return obj.loc[self.groups[name].drop_duplicates()] diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index 0c71d74f89f..526aa9f503a 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -3130,11 +3130,20 @@ def test_groupby_get_group(pdf, group, name, obj): else: gobj = obj - expected = pdf.groupby(group).get_group(name=name, obj=obj) - actual = gdf.groupby(group).get_group(name=name, obj=gobj) + pgb = pdf.groupby(group) + ggb = gdf.groupby(group) + with expect_warning_if(obj is not None): + expected = pgb.get_group(name=name, obj=obj) + with expect_warning_if(obj is not None): + actual = ggb.get_group(name=name, obj=gobj) assert_groupby_results_equal(expected, actual) + expected = pdf.iloc[pgb.indices.get(name)] + actual = gdf.iloc[ggb.indices.get(name)] + + assert_eq(expected, actual) + @pytest.mark.parametrize( "by", From 7cd3834987c7bd635c4758b939544518cb3d1236 Mon Sep 17 00:00:00 2001 From: Suraj Aralihalli Date: Mon, 29 Jan 2024 10:34:35 -0800 Subject: [PATCH 148/384] revert sum agg (#14907) This pull request reverses the modifications made to the sum/product aggregation target type, ensuring it always produces int64. The changes implemented by PR [14679](https://github.com/rapidsai/cudf/pull/14679) which led to degraded performance when the aggregation column had an unsigned type, are reverted. Additional details can be found in the issue [14886](https://github.com/rapidsai/cudf/issues/14886). Authors: - Suraj Aralihalli (https://github.com/SurajAralihalli) Approvers: - David Wendt (https://github.com/davidwendt) - Nghia Truong (https://github.com/ttnghia) - Karthikeyan (https://github.com/karthikeyann) --- cpp/include/cudf/detail/aggregation/aggregation.hpp | 4 ++-- cpp/tests/groupby/sum_tests.cpp | 11 ++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/cpp/include/cudf/detail/aggregation/aggregation.hpp b/cpp/include/cudf/detail/aggregation/aggregation.hpp index c35d56b4c13..a8f164646a5 100644 --- a/cpp/include/cudf/detail/aggregation/aggregation.hpp +++ b/cpp/include/cudf/detail/aggregation/aggregation.hpp @@ -1234,12 +1234,12 @@ constexpr bool is_sum_product_agg(aggregation::Kind k) (k == aggregation::SUM_OF_SQUARES); } -// Summing/Multiplying integers of any type, always use uint64_t for unsigned and int64_t for signed +// Summing/Multiplying integers of any type, always use int64_t accumulator template struct target_type_impl && is_sum_product_agg(k)>> { - using type = std::conditional_t, uint64_t, int64_t>; + using type = int64_t; }; // Summing fixed_point numbers diff --git a/cpp/tests/groupby/sum_tests.cpp b/cpp/tests/groupby/sum_tests.cpp index abf25eb0aa9..03cc3fab568 100644 --- a/cpp/tests/groupby/sum_tests.cpp +++ b/cpp/tests/groupby/sum_tests.cpp @@ -28,10 +28,10 @@ using namespace cudf::test::iterators; template struct groupby_sum_test : public cudf::test::BaseFixture {}; -using K = int32_t; -using supported_types = cudf::test::Concat< - cudf::test::Types, - cudf::test::DurationTypes>; +using K = int32_t; +using supported_types = + cudf::test::Concat, + cudf::test::DurationTypes>; TYPED_TEST_SUITE(groupby_sum_test, supported_types); @@ -40,9 +40,6 @@ TYPED_TEST(groupby_sum_test, basic) using V = TypeParam; using R = cudf::detail::target_type_t; - static_assert(std::is_signed_v == std::is_signed_v, - "Both Result type and Source type must have same signedness"); - cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; cudf::test::fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; From 5cc021af0ef934ddf3f5f66cee2d8dd2490ba623 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Mon, 29 Jan 2024 13:51:26 -0500 Subject: [PATCH 149/384] Use offsetalator in cudf::strings::copy_slice (#14844) Replace hardcoded offset types with offsetalator in `cudf::strings::detail::copy_slice`. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Mark Harris (https://github.com/harrism) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/14844 --- cpp/src/strings/copying/copying.cu | 35 ++++++++++++++++-------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/cpp/src/strings/copying/copying.cu b/cpp/src/strings/copying/copying.cu index 4f37d3864ac..013028d6df3 100644 --- a/cpp/src/strings/copying/copying.cu +++ b/cpp/src/strings/copying/copying.cu @@ -16,9 +16,10 @@ #include #include -#include #include +#include #include +#include #include #include @@ -33,47 +34,49 @@ namespace cudf { namespace strings { namespace detail { -std::unique_ptr copy_slice(strings_column_view const& strings, +std::unique_ptr copy_slice(strings_column_view const& input, size_type start, size_type end, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { - if (strings.is_empty()) return make_empty_column(type_id::STRING); - if (end < 0 || end > strings.size()) end = strings.size(); + if (input.is_empty()) { return make_empty_column(type_id::STRING); } CUDF_EXPECTS(((start >= 0) && (start < end)), "Invalid start parameter value."); auto const strings_count = end - start; - auto const offsets_offset = start + strings.offset(); + auto const offsets_offset = start + input.offset(); // slice the offsets child column auto offsets_column = std::make_unique( cudf::detail::slice( - strings.offsets(), {offsets_offset, offsets_offset + strings_count + 1}, stream) + input.offsets(), {offsets_offset, offsets_offset + strings_count + 1}, stream) .front(), stream, mr); auto const chars_offset = - offsets_offset == 0 ? 0 : cudf::detail::get_value(offsets_column->view(), 0, stream); + offsets_offset == 0 ? 0L : get_offset_value(offsets_column->view(), 0, stream); if (chars_offset > 0) { // adjust the individual offset values only if needed - auto d_offsets = offsets_column->mutable_view(); + auto d_offsets = + cudf::detail::offsetalator_factory::make_output_iterator(offsets_column->mutable_view()); + auto input_offsets = + cudf::detail::offsetalator_factory::make_input_iterator(input.offsets(), offsets_offset); thrust::transform(rmm::exec_policy(stream), - d_offsets.begin(), - d_offsets.end(), - d_offsets.begin(), - cuda::proclaim_return_type( + input_offsets, + input_offsets + offsets_column->size(), + d_offsets, + cuda::proclaim_return_type( [chars_offset] __device__(auto offset) { return offset - chars_offset; })); } // slice the chars child column - auto const data_size = static_cast( - cudf::detail::get_value(offsets_column->view(), strings_count, stream)); + auto const data_size = + static_cast(get_offset_value(offsets_column->view(), strings_count, stream)); auto chars_buffer = - rmm::device_buffer{strings.chars_begin(stream) + chars_offset, data_size, stream, mr}; + rmm::device_buffer{input.chars_begin(stream) + chars_offset, data_size, stream, mr}; // slice the null mask auto null_mask = cudf::detail::copy_bitmask( - strings.null_mask(), offsets_offset, offsets_offset + strings_count, stream, mr); + input.null_mask(), offsets_offset, offsets_offset + strings_count, stream, mr); auto null_count = cudf::detail::null_count( static_cast(null_mask.data()), 0, strings_count, stream); From fc2b9771f17644243817a339e218360aa97a1a79 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 29 Jan 2024 13:29:46 -0600 Subject: [PATCH 150/384] Pin pytest to <8 (#14920) --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-120_arch-x86_64.yaml | 2 +- dependencies.yaml | 2 +- python/cudf/pyproject.toml | 2 +- python/cudf_kafka/pyproject.toml | 2 +- python/custreamz/pyproject.toml | 2 +- python/dask_cudf/pyproject.toml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 47b377013ce..956c685f7de 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -73,11 +73,11 @@ dependencies: - ptxcompiler - pyarrow==14.0.1.* - pydata-sphinx-theme!=0.14.2 -- pytest - pytest-benchmark - pytest-cases>=3.8.2 - pytest-cov - pytest-xdist +- pytest<8 - python-confluent-kafka>=1.9.0,<1.10.0a0 - python-snappy>=0.6.0 - python>=3.9,<3.11 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 4cf1d5427f4..cd2c70577f9 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -71,11 +71,11 @@ dependencies: - pyarrow==14.0.1.* - pydata-sphinx-theme!=0.14.2 - pynvjitlink -- pytest - pytest-benchmark - pytest-cases>=3.8.2 - pytest-cov - pytest-xdist +- pytest<8 - python-confluent-kafka>=1.9.0,<1.10.0a0 - python-snappy>=0.6.0 - python>=3.9,<3.11 diff --git a/dependencies.yaml b/dependencies.yaml index 90a04b2f876..9a1d11af02d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -612,7 +612,7 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - pytest + - pytest<8 - pytest-cov - pytest-xdist test_python_cudf: diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index c7b66abea27..ce30230398f 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -57,11 +57,11 @@ test = [ "fastavro>=0.22.9", "hypothesis", "msgpack", - "pytest", "pytest-benchmark", "pytest-cases>=3.8.2", "pytest-cov", "pytest-xdist", + "pytest<8", "python-snappy>=0.6.0", "scipy", "tokenizers==0.13.1", diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index 062a0224c1f..d6574c32873 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -27,9 +27,9 @@ dependencies = [ [project.optional-dependencies] test = [ - "pytest", "pytest-cov", "pytest-xdist", + "pytest<8", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] diff --git a/python/custreamz/pyproject.toml b/python/custreamz/pyproject.toml index 3e6c74ab570..03ec079a890 100644 --- a/python/custreamz/pyproject.toml +++ b/python/custreamz/pyproject.toml @@ -36,9 +36,9 @@ classifiers = [ [project.optional-dependencies] test = [ - "pytest", "pytest-cov", "pytest-xdist", + "pytest<8", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index 33065da6e8d..c3185bcb793 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -43,9 +43,9 @@ cudf = "dask_cudf.backends:CudfBackendEntrypoint" test = [ "dask-cuda==24.2.*", "numba>=0.57", - "pytest", "pytest-cov", "pytest-xdist", + "pytest<8", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] From fc790ab0a4650188f975ca139313f011d6427a4d Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 29 Jan 2024 13:41:20 -0600 Subject: [PATCH 151/384] Change pandas version range (#14919) This PR pins pandas version range from 2.0 through 2.1.4 --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-120_arch-x86_64.yaml | 2 +- conda/recipes/cudf/meta.yaml | 2 +- dependencies.yaml | 2 +- python/cudf/pyproject.toml | 2 +- python/dask_cudf/pyproject.toml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 18575ba861b..e749d223bea 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -65,7 +65,7 @@ dependencies: - nvcomp==3.0.5 - nvtx>=0.2.1 - packaging -- pandas==2.1.4 +- pandas>=2.0,<2.1.5dev0 - pandoc - pip - pre-commit diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index abbbb7c2758..80ca746fb38 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -63,7 +63,7 @@ dependencies: - nvcomp==3.0.5 - nvtx>=0.2.1 - packaging -- pandas==2.1.4 +- pandas>=2.0,<2.1.5dev0 - pandoc - pip - pre-commit diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 73cb59bd97a..0dffdc10421 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -76,7 +76,7 @@ requirements: - {{ pin_compatible('protobuf', min_pin='x.x', max_pin='x') }} - python - typing_extensions >=4.0.0 - - pandas >=1.3,<1.6.0dev0 + - pandas >=2.0,<2.1.5dev0 - cupy >=12.0.0 # TODO: Pin to numba<0.58 until #14160 is resolved - numba >=0.57,<0.58 diff --git a/dependencies.yaml b/dependencies.yaml index 96bcf66f99b..cb2102910b5 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -501,7 +501,7 @@ dependencies: packages: - fsspec>=0.6.0 - *numpy - - pandas==2.1.4 + - pandas>=2.0,<2.1.5dev0 run_cudf: common: - output_types: [conda, requirements, pyproject] diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 4e6b8d984b1..81fe0bec325 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ "numpy>=1.21", "nvtx>=0.2.1", "packaging", - "pandas==2.1.4", + "pandas>=2.0,<2.1.5dev0", "protobuf>=4.21,<5", "ptxcompiler", "pyarrow>=14.0.1,<15.0.0a0", diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index 5f0596a1d6a..52ff31af7ba 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "cupy-cuda11x>=12.0.0", "fsspec>=0.6.0", "numpy>=1.21", - "pandas==2.1.4", + "pandas>=2.0,<2.1.5dev0", "rapids-dask-dependency==24.4.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ From 5abe6b545198d87fe822a8fbc53ecd5271c9056b Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 29 Jan 2024 15:10:20 -0600 Subject: [PATCH 152/384] Fix custreamz pytests to test on float64 types (#14925) This PR passes types to empty dataframe construction because reductions were being performed on float64 types and now empty column default type is object. --- python/custreamz/custreamz/tests/test_dataframes.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/custreamz/custreamz/tests/test_dataframes.py b/python/custreamz/custreamz/tests/test_dataframes.py index 1a1fc84ef89..7ce398c7617 100644 --- a/python/custreamz/custreamz/tests/test_dataframes.py +++ b/python/custreamz/custreamz/tests/test_dataframes.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. """ Tests for Streamz Dataframes (SDFs) built on top of cuDF DataFrames. @@ -863,7 +863,7 @@ def test_rolling_aggs_with_start_state(stream): def test_window_aggs_with_start_state(stream): - example = cudf.DataFrame({"name": [], "amount": []}) + example = cudf.DataFrame({"name": [], "amount": []}, dtype="float64") sdf = DataFrame(stream, example=example) output0 = ( sdf.window(2, with_state=True, start=None) @@ -881,7 +881,7 @@ def test_window_aggs_with_start_state(stream): assert output0[-1][1] == 450 stream = Stream() - example = cudf.DataFrame({"name": [], "amount": []}) + example = cudf.DataFrame({"name": [], "amount": []}, dtype="float64") sdf = DataFrame(stream, example=example) output1 = ( sdf.window(2, with_state=True, start=output0[-1][0]) @@ -895,7 +895,7 @@ def test_window_aggs_with_start_state(stream): def test_windowed_groupby_aggs_with_start_state(stream): - example = cudf.DataFrame({"name": [], "amount": []}) + example = cudf.DataFrame({"name": [], "amount": []}, dtype="float64") sdf = DataFrame(stream, example=example) output0 = ( sdf.window(5, with_state=True, start=None) @@ -915,7 +915,7 @@ def test_windowed_groupby_aggs_with_start_state(stream): stream.emit(df) stream = Stream() - example = cudf.DataFrame({"name": [], "amount": []}) + example = cudf.DataFrame({"name": [], "amount": []}, dtype="float64") sdf = DataFrame(stream, example=example) output1 = ( sdf.window(5, with_state=True, start=output0[-1][0]) From eb957d97b07fdd5cac88348f9568db96ff9baeb4 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 29 Jan 2024 15:40:51 -0800 Subject: [PATCH 153/384] Revert unnecessary copyright changes --- python/cudf/CMakeLists.txt | 2 +- python/cudf/cmake/Modules/LinkPyarrowHeaders.cmake | 2 +- python/cudf/cmake/Modules/WheelHelpers.cmake | 2 +- python/cudf/cudf/_lib/CMakeLists.txt | 2 +- python/cudf/cudf/_lib/copying.pyx | 2 +- python/cudf/cudf/_lib/cpp/copying.pxd | 2 +- python/cudf/cudf/_lib/cpp/stream_compaction.pxd | 2 +- python/cudf/cudf/_lib/io/CMakeLists.txt | 2 +- python/cudf/cudf/_lib/nvtext/CMakeLists.txt | 2 +- python/cudf/cudf/_lib/pylibcudf/copying.pxd | 2 +- python/cudf/cudf/_lib/pylibcudf/scalar.pxd | 2 +- python/cudf/cudf/_lib/pylibcudf/table.pxd | 2 +- python/cudf/cudf/_lib/scalar.pxd | 2 +- python/cudf/cudf/_lib/strings/CMakeLists.txt | 2 +- python/cudf/cudf/_lib/strings/convert/CMakeLists.txt | 2 +- python/cudf/cudf/_lib/strings/split/CMakeLists.txt | 2 +- python/cudf/cudf/core/_internals/where.py | 2 +- python/cudf/cudf/core/column/struct.py | 2 +- python/cudf/cudf/core/udf/groupby_typing.py | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/python/cudf/CMakeLists.txt b/python/cudf/CMakeLists.txt index 481d6194a03..77771afe0e6 100644 --- a/python/cudf/CMakeLists.txt +++ b/python/cudf/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at diff --git a/python/cudf/cmake/Modules/LinkPyarrowHeaders.cmake b/python/cudf/cmake/Modules/LinkPyarrowHeaders.cmake index 93695dd44dc..6b543433a5d 100644 --- a/python/cudf/cmake/Modules/LinkPyarrowHeaders.cmake +++ b/python/cudf/cmake/Modules/LinkPyarrowHeaders.cmake @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at diff --git a/python/cudf/cmake/Modules/WheelHelpers.cmake b/python/cudf/cmake/Modules/WheelHelpers.cmake index 3abe98a0647..278d6751c15 100644 --- a/python/cudf/cmake/Modules/WheelHelpers.cmake +++ b/python/cudf/cmake/Modules/WheelHelpers.cmake @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at diff --git a/python/cudf/cudf/_lib/CMakeLists.txt b/python/cudf/cudf/_lib/CMakeLists.txt index 8697f731dfb..b67c26f779f 100644 --- a/python/cudf/cudf/_lib/CMakeLists.txt +++ b/python/cudf/cudf/_lib/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at diff --git a/python/cudf/cudf/_lib/copying.pyx b/python/cudf/cudf/_lib/copying.pyx index b707fa2a6fc..8eb0500617f 100644 --- a/python/cudf/cudf/_lib/copying.pyx +++ b/python/cudf/cudf/_lib/copying.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2024, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. import pickle diff --git a/python/cudf/cudf/_lib/cpp/copying.pxd b/python/cudf/cudf/_lib/cpp/copying.pxd index 6b6b4c87c1d..f3e5c0aec72 100644 --- a/python/cudf/cudf/_lib/cpp/copying.pxd +++ b/python/cudf/cudf/_lib/cpp/copying.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2024, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. from libc.stdint cimport int32_t, int64_t, uint8_t from libcpp cimport bool diff --git a/python/cudf/cudf/_lib/cpp/stream_compaction.pxd b/python/cudf/cudf/_lib/cpp/stream_compaction.pxd index 803b06bc8ae..aef2f639d76 100644 --- a/python/cudf/cudf/_lib/cpp/stream_compaction.pxd +++ b/python/cudf/cudf/_lib/cpp/stream_compaction.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2024, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. from libcpp cimport bool from libcpp.memory cimport unique_ptr diff --git a/python/cudf/cudf/_lib/io/CMakeLists.txt b/python/cudf/cudf/_lib/io/CMakeLists.txt index 34c5e8a6e5f..2408fa1c12f 100644 --- a/python/cudf/cudf/_lib/io/CMakeLists.txt +++ b/python/cudf/cudf/_lib/io/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at diff --git a/python/cudf/cudf/_lib/nvtext/CMakeLists.txt b/python/cudf/cudf/_lib/nvtext/CMakeLists.txt index f099adf7e0a..55301789812 100644 --- a/python/cudf/cudf/_lib/nvtext/CMakeLists.txt +++ b/python/cudf/cudf/_lib/nvtext/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at diff --git a/python/cudf/cudf/_lib/pylibcudf/copying.pxd b/python/cudf/cudf/_lib/pylibcudf/copying.pxd index ed29a3b3be3..21785a9b108 100644 --- a/python/cudf/cudf/_lib/pylibcudf/copying.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/copying.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2023, NVIDIA CORPORATION. from libcpp cimport bool as cbool diff --git a/python/cudf/cudf/_lib/pylibcudf/scalar.pxd b/python/cudf/cudf/_lib/pylibcudf/scalar.pxd index b95f8233f4d..4c47de5c0c6 100644 --- a/python/cudf/cudf/_lib/pylibcudf/scalar.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/scalar.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2023, NVIDIA CORPORATION. from libcpp cimport bool from libcpp.memory cimport unique_ptr diff --git a/python/cudf/cudf/_lib/pylibcudf/table.pxd b/python/cudf/cudf/_lib/pylibcudf/table.pxd index 2d7fb7d7149..de4ffd73be3 100644 --- a/python/cudf/cudf/_lib/pylibcudf/table.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/table.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2023, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr from pyarrow cimport lib as pa diff --git a/python/cudf/cudf/_lib/scalar.pxd b/python/cudf/cudf/_lib/scalar.pxd index eaf67aa659f..b5c5a8a64a3 100644 --- a/python/cudf/cudf/_lib/scalar.pxd +++ b/python/cudf/cudf/_lib/scalar.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2024, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. from libcpp cimport bool from libcpp.memory cimport unique_ptr diff --git a/python/cudf/cudf/_lib/strings/CMakeLists.txt b/python/cudf/cudf/_lib/strings/CMakeLists.txt index dbb2f9d1734..081b84db79c 100644 --- a/python/cudf/cudf/_lib/strings/CMakeLists.txt +++ b/python/cudf/cudf/_lib/strings/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at diff --git a/python/cudf/cudf/_lib/strings/convert/CMakeLists.txt b/python/cudf/cudf/_lib/strings/convert/CMakeLists.txt index 05bf6109ad3..ebd7a793bf4 100644 --- a/python/cudf/cudf/_lib/strings/convert/CMakeLists.txt +++ b/python/cudf/cudf/_lib/strings/convert/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at diff --git a/python/cudf/cudf/_lib/strings/split/CMakeLists.txt b/python/cudf/cudf/_lib/strings/split/CMakeLists.txt index 930e42d44ef..105e73788fe 100644 --- a/python/cudf/cudf/_lib/strings/split/CMakeLists.txt +++ b/python/cudf/cudf/_lib/strings/split/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at diff --git a/python/cudf/cudf/core/_internals/where.py b/python/cudf/cudf/core/_internals/where.py index 2f7827521a0..ef6b10f66c1 100644 --- a/python/cudf/cudf/core/_internals/where.py +++ b/python/cudf/cudf/core/_internals/where.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2024, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. import warnings from typing import Tuple, Union diff --git a/python/cudf/cudf/core/column/struct.py b/python/cudf/cudf/core/column/struct.py index 95e776c8720..6cfa8db0d96 100644 --- a/python/cudf/cudf/core/column/struct.py +++ b/python/cudf/cudf/core/column/struct.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2024, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. from __future__ import annotations from functools import cached_property diff --git a/python/cudf/cudf/core/udf/groupby_typing.py b/python/cudf/cudf/core/udf/groupby_typing.py index 70864320fd8..72088493074 100644 --- a/python/cudf/cudf/core/udf/groupby_typing.py +++ b/python/cudf/cudf/core/udf/groupby_typing.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2024, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. from typing import Any, Dict import numba From 7f7e237dbbdc107a46b00924574b1d349807a47a Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 30 Jan 2024 00:25:48 +0000 Subject: [PATCH 154/384] Undo a few incorrect copyright fixes --- python/cudf/cmake/Modules/LinkPyarrowHeaders.cmake | 2 +- python/cudf/cudf/_lib/pylibcudf/copying.pxd | 2 +- python/cudf/cudf/_lib/pylibcudf/scalar.pxd | 2 +- python/cudf/cudf/_lib/pylibcudf/table.pxd | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/cudf/cmake/Modules/LinkPyarrowHeaders.cmake b/python/cudf/cmake/Modules/LinkPyarrowHeaders.cmake index 6b543433a5d..d432f9fe1f5 100644 --- a/python/cudf/cmake/Modules/LinkPyarrowHeaders.cmake +++ b/python/cudf/cmake/Modules/LinkPyarrowHeaders.cmake @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2023-2023, NVIDIA CORPORATION. +# Copyright (c) 2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at diff --git a/python/cudf/cudf/_lib/pylibcudf/copying.pxd b/python/cudf/cudf/_lib/pylibcudf/copying.pxd index 21785a9b108..3567df9ac9c 100644 --- a/python/cudf/cudf/_lib/pylibcudf/copying.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/copying.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2023, NVIDIA CORPORATION. +# Copyright (c) 2023, NVIDIA CORPORATION. from libcpp cimport bool as cbool diff --git a/python/cudf/cudf/_lib/pylibcudf/scalar.pxd b/python/cudf/cudf/_lib/pylibcudf/scalar.pxd index 4c47de5c0c6..0edc934ca22 100644 --- a/python/cudf/cudf/_lib/pylibcudf/scalar.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/scalar.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2023, NVIDIA CORPORATION. +# Copyright (c) 2023, NVIDIA CORPORATION. from libcpp cimport bool from libcpp.memory cimport unique_ptr diff --git a/python/cudf/cudf/_lib/pylibcudf/table.pxd b/python/cudf/cudf/_lib/pylibcudf/table.pxd index de4ffd73be3..6fe06f00491 100644 --- a/python/cudf/cudf/_lib/pylibcudf/table.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/table.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2023, NVIDIA CORPORATION. +# Copyright (c) 2023, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr from pyarrow cimport lib as pa From 57bbe94e995b9a0365276e4cb26853dce219e22a Mon Sep 17 00:00:00 2001 From: Yunsong Wang Date: Mon, 29 Jan 2024 17:00:15 -0800 Subject: [PATCH 155/384] Add detail `cuco_allocator` (#14877) Surpass #14827 Related to https://github.com/rapidsai/cudf/issues/11176 This PR adds a new `cudf::detail::cuco_allocator` to deprecate and replace the old `default_allocator` in the global namespace. Following the comments in https://github.com/rapidsai/cudf/pull/14827#pullrequestreview-1839545432, the new `cudf::detail::cuco_allocator` class is moved to `detail/cuco_helpers.cuh`. Free functions in `hashing/detail/helper_functions.cuh` are left in the global namespace without changes due to the verbose nested namespace expression. Authors: - Yunsong Wang (https://github.com/PointKernel) Approvers: - Nghia Truong (https://github.com/ttnghia) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/14877 --- cpp/include/cudf/detail/cuco_helpers.hpp | 46 ++++++++++++++ .../cudf/detail/hash_reduce_by_row.cuh | 8 +-- cpp/include/cudf/detail/join.hpp | 11 ++-- .../cudf/hashing/detail/hash_allocator.cuh | 62 ------------------- cpp/src/hash/concurrent_unordered_map.cuh | 6 +- cpp/src/io/json/json_tree.cu | 11 ++-- cpp/src/join/hash_join.cu | 5 +- cpp/src/join/join_common_utils.hpp | 13 ++-- cpp/src/join/mixed_join.cu | 25 ++++---- cpp/src/join/mixed_join_semi.cu | 7 ++- cpp/src/reductions/histogram.cu | 13 ++-- cpp/src/search/contains_table.cu | 4 +- cpp/src/stream_compaction/distinct.cu | 4 +- cpp/src/stream_compaction/distinct_count.cu | 16 ++--- .../stream_compaction_common.hpp | 10 +-- cpp/src/text/bpe/byte_pair_encoding.cuh | 11 ++-- cpp/src/text/bpe/load_merge_pairs.cu | 33 +++++----- cpp/src/text/vocabulary_tokenize.cu | 12 ++-- 18 files changed, 131 insertions(+), 166 deletions(-) create mode 100644 cpp/include/cudf/detail/cuco_helpers.hpp delete mode 100644 cpp/include/cudf/hashing/detail/hash_allocator.cuh diff --git a/cpp/include/cudf/detail/cuco_helpers.hpp b/cpp/include/cudf/detail/cuco_helpers.hpp new file mode 100644 index 00000000000..5f3c31479de --- /dev/null +++ b/cpp/include/cudf/detail/cuco_helpers.hpp @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +namespace cudf::detail { + +/** + * @brief Stream-ordered allocator adaptor used for cuco data structures + * + * The stream-ordered `rmm::mr::polymorphic_allocator` cannot be used in `cuco` directly since the + * later expects a standard C++ `Allocator` interface. This allocator helper provides a simple way + * to handle cuco memory allocation/deallocation with the given `stream` and the rmm default memory + * resource. + */ +class cuco_allocator + : public rmm::mr::stream_allocator_adaptor> { + /// Default stream-ordered allocator type + using default_allocator = rmm::mr::polymorphic_allocator; + /// The base allocator adaptor type + using base_type = rmm::mr::stream_allocator_adaptor; + + public: + /** + * @brief Constructs the allocator adaptor with the given `stream` + */ + cuco_allocator(rmm::cuda_stream_view stream) : base_type{default_allocator{}, stream} {} +}; + +} // namespace cudf::detail diff --git a/cpp/include/cudf/detail/hash_reduce_by_row.cuh b/cpp/include/cudf/detail/hash_reduce_by_row.cuh index f63d1922950..006cb5142c9 100644 --- a/cpp/include/cudf/detail/hash_reduce_by_row.cuh +++ b/cpp/include/cudf/detail/hash_reduce_by_row.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include #include #include @@ -22,7 +22,6 @@ #include #include #include -#include #include #include @@ -32,9 +31,8 @@ namespace cudf::detail { -using hash_table_allocator_type = rmm::mr::stream_allocator_adaptor>; using hash_map_type = - cuco::static_map; + cuco::static_map; /** * @brief The base struct for customized reduction functor to perform reduce-by-key with keys are diff --git a/cpp/include/cudf/detail/join.hpp b/cpp/include/cudf/detail/join.hpp index b69632c83ca..ad6269dae30 100644 --- a/cpp/include/cudf/detail/join.hpp +++ b/cpp/include/cudf/detail/join.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,6 @@ #include #include #include -#include #include @@ -33,9 +32,6 @@ #include // Forward declaration -template -class default_allocator; - namespace cudf::experimental::row::equality { class preprocessed_table; } @@ -43,6 +39,9 @@ class preprocessed_table; namespace cudf { namespace detail { +// Forward declaration +class cuco_allocator; + constexpr int DEFAULT_JOIN_CG_SIZE = 2; enum class join_kind { INNER_JOIN, LEFT_JOIN, FULL_JOIN, LEFT_SEMI_JOIN, LEFT_ANTI_JOIN }; @@ -62,7 +61,7 @@ struct hash_join { cuco::static_multimap>, + cudf::detail::cuco_allocator, cuco::double_hashing>; hash_join() = delete; diff --git a/cpp/include/cudf/hashing/detail/hash_allocator.cuh b/cpp/include/cudf/hashing/detail/hash_allocator.cuh deleted file mode 100644 index 64a2a852ae4..00000000000 --- a/cpp/include/cudf/hashing/detail/hash_allocator.cuh +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2017-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#include - -#include -#include -#include -#include - -template -struct default_allocator { - using value_type = T; - rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource(); - - default_allocator() = default; - - template - constexpr default_allocator(default_allocator const&) noexcept - { - } - - T* allocate(std::size_t n, rmm::cuda_stream_view stream = cudf::get_default_stream()) const - { - return static_cast(mr->allocate(n * sizeof(T), stream)); - } - - void deallocate(T* p, - std::size_t n, - rmm::cuda_stream_view stream = cudf::get_default_stream()) const - { - mr->deallocate(p, n * sizeof(T), stream); - } -}; - -template -bool operator==(default_allocator const&, default_allocator const&) -{ - return true; -} -template -bool operator!=(default_allocator const&, default_allocator const&) -{ - return false; -} diff --git a/cpp/src/hash/concurrent_unordered_map.cuh b/cpp/src/hash/concurrent_unordered_map.cuh index d773c2763df..adc87c2400e 100644 --- a/cpp/src/hash/concurrent_unordered_map.cuh +++ b/cpp/src/hash/concurrent_unordered_map.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2023, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2017-2024, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,12 +20,12 @@ #include #include -#include #include #include #include #include +#include #include @@ -117,7 +117,7 @@ template , typename Equality = equal_to, - typename Allocator = default_allocator>> + typename Allocator = rmm::mr::polymorphic_allocator>> class concurrent_unordered_map { public: using size_type = size_t; diff --git a/cpp/src/io/json/json_tree.cu b/cpp/src/io/json/json_tree.cu index 275907c19c9..db9daf28c06 100644 --- a/cpp/src/io/json/json_tree.cu +++ b/cpp/src/io/json/json_tree.cu @@ -17,12 +17,12 @@ #include "nested_json.hpp" #include +#include #include #include #include #include #include -#include #include #include #include @@ -31,7 +31,6 @@ #include #include #include -#include #include @@ -511,7 +510,6 @@ rmm::device_uvector hash_node_type_with_field_name(device_span>; auto const num_nodes = d_tree.node_categories.size(); auto const num_fields = thrust::count(rmm::exec_policy(stream), @@ -555,7 +553,7 @@ rmm::device_uvector hash_node_type_with_field_name(device_span{d_hasher}, - hash_table_allocator_type{default_allocator{}, stream}, + cudf::detail::cuco_allocator{stream}, stream.value()}; key_set.insert_if_async(iter, iter + num_nodes, @@ -735,15 +733,14 @@ std::pair, rmm::device_uvector> hash_n }; constexpr size_type empty_node_index_sentinel = -1; - using hash_table_allocator_type = rmm::mr::stream_allocator_adaptor>; - using hasher_type = decltype(d_hashed_cache); + using hasher_type = decltype(d_hashed_cache); auto key_set = cuco::experimental::static_set{ cuco::experimental::extent{compute_hash_table_size(num_nodes)}, cuco::empty_key{empty_node_index_sentinel}, d_equal, cuco::experimental::linear_probing<1, hasher_type>{d_hashed_cache}, - hash_table_allocator_type{default_allocator{}, stream}, + cudf::detail::cuco_allocator{stream}, stream.value()}; // insert and convert node ids to unique set ids diff --git a/cpp/src/join/hash_join.cu b/cpp/src/join/hash_join.cu index 453257ab228..17616818a58 100644 --- a/cpp/src/join/hash_join.cu +++ b/cpp/src/join/hash_join.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -368,7 +369,7 @@ hash_join::hash_join(cudf::table_view const& build, cuco::empty_key{std::numeric_limits::max()}, cuco::empty_value{cudf::detail::JoinNoneValue}, stream.value(), - detail::hash_table_allocator_type{default_allocator{}, stream}}, + cudf::detail::cuco_allocator{stream}}, _build{build}, _preprocessed_build{ cudf::experimental::row::equality::preprocessed_table::create(_build, stream)} diff --git a/cpp/src/join/join_common_utils.hpp b/cpp/src/join/join_common_utils.hpp index e96505e5ed6..b88a4fdef58 100644 --- a/cpp/src/join/join_common_utils.hpp +++ b/cpp/src/join/join_common_utils.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,16 +15,13 @@ */ #pragma once +#include #include #include -#include -#include #include #include #include -#include - #include #include @@ -43,8 +40,6 @@ using pair_type = cuco::pair; using hash_type = cuco::murmurhash3_32; -using hash_table_allocator_type = rmm::mr::stream_allocator_adaptor>; - using multimap_type = cudf::hash_join::impl_type::map_type; // Multimap type used for mixed joins. TODO: This is a temporary alias used @@ -53,11 +48,11 @@ using multimap_type = cudf::hash_join::impl_type::map_type; using mixed_multimap_type = cuco::static_multimap>; using semi_map_type = cuco:: - static_map; + static_map; using row_hash_legacy = cudf::row_hasher; diff --git a/cpp/src/join/mixed_join.cu b/cpp/src/join/mixed_join.cu index 3d902bf93b2..6223114fcd0 100644 --- a/cpp/src/join/mixed_join.cu +++ b/cpp/src/join/mixed_join.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -127,12 +128,11 @@ mixed_join( auto build_view = table_device_view::create(build, stream); // Don't use multimap_type because we want a CG size of 1. - mixed_multimap_type hash_table{ - compute_hash_table_size(build.num_rows()), - cuco::empty_key{std::numeric_limits::max()}, - cuco::empty_value{cudf::detail::JoinNoneValue}, - stream.value(), - detail::hash_table_allocator_type{default_allocator{}, stream}}; + mixed_multimap_type hash_table{compute_hash_table_size(build.num_rows()), + cuco::empty_key{std::numeric_limits::max()}, + cuco::empty_value{cudf::detail::JoinNoneValue}, + stream.value(), + cudf::detail::cuco_allocator{stream}}; // TODO: To add support for nested columns we will need to flatten in many // places. However, this probably isn't worth adding any time soon since we @@ -393,12 +393,11 @@ compute_mixed_join_output_size(table_view const& left_equality, auto build_view = table_device_view::create(build, stream); // Don't use multimap_type because we want a CG size of 1. - mixed_multimap_type hash_table{ - compute_hash_table_size(build.num_rows()), - cuco::empty_key{std::numeric_limits::max()}, - cuco::empty_value{cudf::detail::JoinNoneValue}, - stream.value(), - detail::hash_table_allocator_type{default_allocator{}, stream}}; + mixed_multimap_type hash_table{compute_hash_table_size(build.num_rows()), + cuco::empty_key{std::numeric_limits::max()}, + cuco::empty_value{cudf::detail::JoinNoneValue}, + stream.value(), + cudf::detail::cuco_allocator{stream}}; // TODO: To add support for nested columns we will need to flatten in many // places. However, this probably isn't worth adding any time soon since we diff --git a/cpp/src/join/mixed_join_semi.cu b/cpp/src/join/mixed_join_semi.cu index f619ed0d558..edf6c32eadf 100644 --- a/cpp/src/join/mixed_join_semi.cu +++ b/cpp/src/join/mixed_join_semi.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -172,7 +173,7 @@ std::unique_ptr> mixed_join_semi( semi_map_type hash_table{compute_hash_table_size(build.num_rows()), cuco::empty_key{std::numeric_limits::max()}, cuco::empty_value{cudf::detail::JoinNoneValue}, - detail::hash_table_allocator_type{default_allocator{}, stream}, + cudf::detail::cuco_allocator{stream}, stream.value()}; // Create hash table containing all keys found in right table @@ -433,7 +434,7 @@ compute_mixed_join_output_size_semi(table_view const& left_equality, semi_map_type hash_table{compute_hash_table_size(build.num_rows()), cuco::empty_key{std::numeric_limits::max()}, cuco::empty_value{cudf::detail::JoinNoneValue}, - detail::hash_table_allocator_type{default_allocator{}, stream}, + cudf::detail::cuco_allocator{stream}, stream.value()}; // Create hash table containing all keys found in right table diff --git a/cpp/src/reductions/histogram.cu b/cpp/src/reductions/histogram.cu index 218e2e57420..42ef266a684 100644 --- a/cpp/src/reductions/histogram.cu +++ b/cpp/src/reductions/histogram.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -163,12 +163,11 @@ compute_row_frequencies(table_view const& input, "Nested types are not yet supported in histogram aggregation.", std::invalid_argument); - auto map = cudf::detail::hash_map_type{ - compute_hash_table_size(input.num_rows()), - cuco::empty_key{-1}, - cuco::empty_value{std::numeric_limits::min()}, - cudf::detail::hash_table_allocator_type{default_allocator{}, stream}, - stream.value()}; + auto map = cudf::detail::hash_map_type{compute_hash_table_size(input.num_rows()), + cuco::empty_key{-1}, + cuco::empty_value{std::numeric_limits::min()}, + cudf::detail::cuco_allocator{stream}, + stream.value()}; auto const preprocessed_input = cudf::experimental::row::hash::preprocessed_table::create(input, stream); diff --git a/cpp/src/search/contains_table.cu b/cpp/src/search/contains_table.cu index b8ece03c4a0..ce069abcb78 100644 --- a/cpp/src/search/contains_table.cu +++ b/cpp/src/search/contains_table.cu @@ -16,7 +16,9 @@ #include +#include #include +#include #include #include #include @@ -231,7 +233,7 @@ rmm::device_uvector contains(table_view const& haystack, cuco::empty_key{lhs_index_type{-1}}, d_equal, probing_scheme, - detail::hash_table_allocator_type{default_allocator{}, stream}, + cudf::detail::cuco_allocator{stream}, stream.value()}; if (haystack_has_nulls && compare_nulls == null_equality::UNEQUAL) { diff --git a/cpp/src/stream_compaction/distinct.cu b/cpp/src/stream_compaction/distinct.cu index b867df1565a..e73bab1345e 100644 --- a/cpp/src/stream_compaction/distinct.cu +++ b/cpp/src/stream_compaction/distinct.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -54,7 +54,7 @@ rmm::device_uvector distinct_indices(table_view const& input, auto map = hash_map_type{compute_hash_table_size(input.num_rows()), cuco::empty_key{-1}, cuco::empty_value{std::numeric_limits::min()}, - detail::hash_table_allocator_type{default_allocator{}, stream}, + cudf::detail::cuco_allocator{stream}, stream.value()}; auto const preprocessed_input = diff --git a/cpp/src/stream_compaction/distinct_count.cu b/cpp/src/stream_compaction/distinct_count.cu index ac4811ad279..507bad777eb 100644 --- a/cpp/src/stream_compaction/distinct_count.cu +++ b/cpp/src/stream_compaction/distinct_count.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -141,13 +141,13 @@ cudf::size_type distinct_count(table_view const& keys, auto const comparator_helper = [&](auto const row_equal) { using hasher_type = decltype(hash_key); - auto key_set = cuco::experimental::static_set{ - cuco::experimental::extent{compute_hash_table_size(num_rows)}, - cuco::empty_key{-1}, - row_equal, - cuco::experimental::linear_probing<1, hasher_type>{hash_key}, - detail::hash_table_allocator_type{default_allocator{}, stream}, - stream.value()}; + auto key_set = + cuco::experimental::static_set{cuco::experimental::extent{compute_hash_table_size(num_rows)}, + cuco::empty_key{-1}, + row_equal, + cuco::experimental::linear_probing<1, hasher_type>{hash_key}, + cudf::detail::cuco_allocator{stream}, + stream.value()}; auto const iter = thrust::counting_iterator(0); // when nulls are equal, we skip hashing any row that has a null diff --git a/cpp/src/stream_compaction/stream_compaction_common.hpp b/cpp/src/stream_compaction/stream_compaction_common.hpp index 18c531e3e69..ceb62d1d059 100644 --- a/cpp/src/stream_compaction/stream_compaction_common.hpp +++ b/cpp/src/stream_compaction/stream_compaction_common.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,13 +15,11 @@ */ #pragma once -#include +#include #include #include #include -#include - #include #include @@ -29,10 +27,8 @@ namespace cudf { namespace detail { -using hash_table_allocator_type = rmm::mr::stream_allocator_adaptor>; - using hash_map_type = - cuco::static_map; + cuco::static_map; } // namespace detail } // namespace cudf diff --git a/cpp/src/text/bpe/byte_pair_encoding.cuh b/cpp/src/text/bpe/byte_pair_encoding.cuh index 2a170317909..1a3f8eadea0 100644 --- a/cpp/src/text/bpe/byte_pair_encoding.cuh +++ b/cpp/src/text/bpe/byte_pair_encoding.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,14 +20,13 @@ #include #include -#include +#include #include #include #include #include #include -#include #include @@ -46,8 +45,6 @@ using string_hasher_type = cudf::hashing::detail::MurmurHash3_x86_32; -using hash_table_allocator_type = rmm::mr::stream_allocator_adaptor>; - /** * @brief Hasher function used for building and using the cuco static-map * @@ -109,7 +106,7 @@ using merge_pairs_map_type = cuco::experimental::static_map; + cudf::detail::cuco_allocator>; /** * @brief Hasher function used for building and using the cuco static-map @@ -166,7 +163,7 @@ using mp_table_map_type = cuco::experimental::static_map; + cudf::detail::cuco_allocator>; } // namespace detail diff --git a/cpp/src/text/bpe/load_merge_pairs.cu b/cpp/src/text/bpe/load_merge_pairs.cu index c07d929e98a..3b630886b3e 100644 --- a/cpp/src/text/bpe/load_merge_pairs.cu +++ b/cpp/src/text/bpe/load_merge_pairs.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,14 +42,14 @@ namespace { std::unique_ptr initialize_merge_pairs_map( cudf::column_device_view const& input, rmm::cuda_stream_view stream) { - auto merge_pairs_map = std::make_unique( - static_cast(input.size()), - cuco::empty_key{-1}, - cuco::empty_value{-1}, - bpe_equal{input}, - bpe_probe_scheme{bpe_hasher{input}}, - hash_table_allocator_type{default_allocator{}, stream}, - stream.value()); + auto merge_pairs_map = + std::make_unique(static_cast(input.size()), + cuco::empty_key{-1}, + cuco::empty_value{-1}, + bpe_equal{input}, + bpe_probe_scheme{bpe_hasher{input}}, + cudf::detail::cuco_allocator{stream}, + stream.value()); auto iter = cudf::detail::make_counting_transform_iterator( 0, @@ -64,14 +64,13 @@ std::unique_ptr initialize_merge_pairs_map( std::unique_ptr initialize_mp_table_map( cudf::column_device_view const& input, rmm::cuda_stream_view stream) { - auto mp_table_map = std::make_unique( - static_cast(input.size()), - cuco::empty_key{-1}, - cuco::empty_value{-1}, - mp_equal{input}, - mp_probe_scheme{mp_hasher{input}}, - hash_table_allocator_type{default_allocator{}, stream}, - stream.value()); + auto mp_table_map = std::make_unique(static_cast(input.size()), + cuco::empty_key{-1}, + cuco::empty_value{-1}, + mp_equal{input}, + mp_probe_scheme{mp_hasher{input}}, + cudf::detail::cuco_allocator{stream}, + stream.value()); auto iter = cudf::detail::make_counting_transform_iterator( 0, diff --git a/cpp/src/text/vocabulary_tokenize.cu b/cpp/src/text/vocabulary_tokenize.cu index a9e8d4d9a24..80f275dba7d 100644 --- a/cpp/src/text/vocabulary_tokenize.cu +++ b/cpp/src/text/vocabulary_tokenize.cu @@ -21,13 +21,13 @@ #include #include #include +#include #include #include #include #include #include #include -#include #include #include #include @@ -35,7 +35,6 @@ #include #include -#include #include @@ -93,15 +92,14 @@ struct vocab_equal { } }; -using hash_table_allocator_type = rmm::mr::stream_allocator_adaptor>; -using probe_scheme = cuco::experimental::linear_probing<1, vocab_hasher>; -using vocabulary_map_type = cuco::experimental::static_map; +using vocabulary_map_type = cuco::experimental::static_map, cuda::thread_scope_device, vocab_equal, probe_scheme, - hash_table_allocator_type>; + cudf::detail::cuco_allocator>; } // namespace } // namespace detail @@ -150,7 +148,7 @@ tokenize_vocabulary::tokenize_vocabulary(cudf::strings_column_view const& input, cuco::empty_value{-1}, detail::vocab_equal{*d_vocabulary}, detail::probe_scheme{detail::vocab_hasher{*d_vocabulary}}, - detail::hash_table_allocator_type{default_allocator{}, stream}, + cudf::detail::cuco_allocator{stream}, stream.value()); // the row index is the token id (value for each key in the map) From c6353358541e0401402a32a7384ed8c57c7aae7a Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 29 Jan 2024 15:38:53 -1000 Subject: [PATCH 156/384] Remove pandas 1.3, 1.4 checks (#14927) Removes pandas 1.3, 1.4 checks in unit tests Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/14927 --- python/cudf/cudf/core/_compat.py | 3 --- python/cudf/cudf/tests/test_concat.py | 6 +----- python/cudf/cudf/tests/test_csv.py | 6 +----- python/cudf/cudf/tests/test_dataframe.py | 23 ++--------------------- python/cudf/cudf/tests/test_datetime.py | 10 +--------- python/cudf/cudf/tests/test_groupby.py | 11 +---------- python/cudf/cudf/tests/test_index.py | 8 ++------ python/cudf/cudf/tests/test_replace.py | 9 ++------- python/cudf/cudf/tests/test_series.py | 11 +---------- 9 files changed, 11 insertions(+), 76 deletions(-) diff --git a/python/cudf/cudf/core/_compat.py b/python/cudf/cudf/core/_compat.py index fb267bdf7df..b602dfdf23c 100644 --- a/python/cudf/cudf/core/_compat.py +++ b/python/cudf/cudf/core/_compat.py @@ -4,9 +4,6 @@ from packaging import version PANDAS_VERSION = version.parse(pd.__version__) -PANDAS_GE_133 = PANDAS_VERSION >= version.parse("1.3.3") -PANDAS_GE_134 = PANDAS_VERSION >= version.parse("1.3.4") -PANDAS_LT_140 = PANDAS_VERSION < version.parse("1.4.0") PANDAS_GE_150 = PANDAS_VERSION >= version.parse("1.5.0") PANDAS_LT_153 = PANDAS_VERSION < version.parse("1.5.3") PANDAS_EQ_200 = PANDAS_VERSION == version.parse("2.0.0") diff --git a/python/cudf/cudf/tests/test_concat.py b/python/cudf/cudf/tests/test_concat.py index ed8f32ed12b..9078d54c193 100644 --- a/python/cudf/cudf/tests/test_concat.py +++ b/python/cudf/cudf/tests/test_concat.py @@ -10,7 +10,7 @@ import cudf as gd from cudf.api.types import _is_categorical_dtype -from cudf.core._compat import PANDAS_GE_150, PANDAS_GE_200, PANDAS_LT_140 +from cudf.core._compat import PANDAS_GE_150, PANDAS_GE_200 from cudf.core.dtypes import Decimal32Dtype, Decimal64Dtype, Decimal128Dtype from cudf.testing._utils import ( assert_eq, @@ -932,10 +932,6 @@ def test_concat_join_one_df(ignore_index, sort, join, axis): @pytest.mark.parametrize("sort", [True, False]) @pytest.mark.parametrize("join", ["inner", "outer"]) @pytest.mark.parametrize("axis", [0, 1]) -@pytest.mark.xfail( - condition=PANDAS_LT_140, - reason="https://github.com/pandas-dev/pandas/issues/43584", -) def test_concat_join_no_overlapping_columns( pdf1, pdf2, ignore_index, sort, join, axis ): diff --git a/python/cudf/cudf/tests/test_csv.py b/python/cudf/cudf/tests/test_csv.py index 6de66bf1952..8171f3a1872 100644 --- a/python/cudf/cudf/tests/test_csv.py +++ b/python/cudf/cudf/tests/test_csv.py @@ -17,7 +17,7 @@ import cudf from cudf import read_csv -from cudf.core._compat import PANDAS_GE_200, PANDAS_LT_140 +from cudf.core._compat import PANDAS_GE_200 from cudf.testing._utils import assert_eq, assert_exceptions_equal @@ -1368,10 +1368,6 @@ def test_csv_reader_column_names(names): assert list(df) == list(names) -@pytest.mark.xfail( - condition=PANDAS_LT_140, - reason="https://github.com/rapidsai/cudf/issues/10618", -) def test_csv_reader_repeated_column_name(): buffer = """A,A,A.1,A,A.2,A,A.4,A,A 1,2,3.1,4,a.2,a,a.4,a,a diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 34dc7ebc68e..fa130a99c72 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -26,11 +26,9 @@ import cudf from cudf.api.extensions import no_default from cudf.core._compat import ( - PANDAS_GE_134, PANDAS_GE_150, PANDAS_GE_200, PANDAS_GE_210, - PANDAS_LT_140, PANDAS_LT_203, ) from cudf.core.buffer.spill_manager import get_global_manager @@ -3589,15 +3587,7 @@ def test_dataframe_empty_sort_index(): [2, 0, 1], ] ), - pytest.param( - pd.RangeIndex(2, -1, -1), - marks=[ - pytest_xfail( - condition=PANDAS_LT_140, - reason="https://github.com/pandas-dev/pandas/issues/43591", - ) - ], - ), + pd.RangeIndex(2, -1, -1), ], ) @pytest.mark.parametrize("axis", [0, 1, "index", "columns"]) @@ -9584,16 +9574,7 @@ def test_explode(data, labels, ignore_index, p_index, label_to_explode): pdf = pd.DataFrame(data, index=p_index, columns=labels) gdf = cudf.from_pandas(pdf) - if PANDAS_GE_134: - expect = pdf.explode(label_to_explode, ignore_index) - else: - # https://github.com/pandas-dev/pandas/issues/43314 - if isinstance(label_to_explode, int): - pdlabel_to_explode = [label_to_explode] - else: - pdlabel_to_explode = label_to_explode - expect = pdf.explode(pdlabel_to_explode, ignore_index) - + expect = pdf.explode(label_to_explode, ignore_index) got = gdf.explode(label_to_explode, ignore_index) assert_eq(expect, got, check_dtype=False) diff --git a/python/cudf/cudf/tests/test_datetime.py b/python/cudf/cudf/tests/test_datetime.py index d83b46250d0..60b0d787278 100644 --- a/python/cudf/cudf/tests/test_datetime.py +++ b/python/cudf/cudf/tests/test_datetime.py @@ -18,7 +18,6 @@ PANDAS_GE_150, PANDAS_GE_200, PANDAS_GE_210, - PANDAS_LT_140, ) from cudf.core.index import DatetimeIndex from cudf.testing._utils import ( @@ -1500,14 +1499,7 @@ def test_is_month_start(data, dtype): date_range_test_periods = [1, 10, 100] date_range_test_freq = [ {"months": 3, "years": 1}, - pytest.param( - {"hours": 10, "days": 57, "nanoseconds": 3}, - marks=pytest.mark.xfail( - condition=PANDAS_LT_140, - reason="Pandas ignoring nanoseconds component. " - "https://github.com/pandas-dev/pandas/issues/44393", - ), - ), + {"hours": 10, "days": 57, "nanoseconds": 3}, "83D", "17h", "-680T", diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index 526aa9f503a..dcfc9d801a4 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -20,12 +20,7 @@ import cudf from cudf import DataFrame, Series from cudf.api.extensions import no_default -from cudf.core._compat import ( - PANDAS_GE_150, - PANDAS_GE_200, - PANDAS_GE_210, - PANDAS_LT_140, -) +from cudf.core._compat import PANDAS_GE_150, PANDAS_GE_200, PANDAS_GE_210 from cudf.core.udf._ops import arith_ops, comparison_ops, unary_ops from cudf.core.udf.groupby_typing import SUPPORTED_GROUPBY_NUMPY_TYPES from cudf.core.udf.utils import UDFError, precompiled @@ -909,10 +904,6 @@ def test_groupby_2keys_agg(nelem, func): # https://github.com/pandas-dev/pandas/issues/40685 is resolved. # "func", ["min", "max", "idxmin", "idxmax", "count", "sum"], ) -@pytest.mark.xfail( - condition=PANDAS_LT_140, - reason="https://github.com/pandas-dev/pandas/issues/43209", -) def test_groupby_agg_decimal(num_groups, nelem_per_group, func): # The number of digits after the decimal to use. decimal_digits = 2 diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index 4dfbcf138c3..5cc1c93deff 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -15,7 +15,7 @@ import cudf from cudf.api.extensions import no_default from cudf.api.types import is_bool_dtype -from cudf.core._compat import PANDAS_GE_133, PANDAS_GE_200 +from cudf.core._compat import PANDAS_GE_200 from cudf.core.index import ( CategoricalIndex, DatetimeIndex, @@ -514,15 +514,11 @@ def test_empty_df_head_tail_index(n): None, ), (pd.Index(range(5)), pd.Index(range(4)) > 0, None, ValueError), - pytest.param( + ( pd.Index(range(5)), pd.Index(range(5)) > 1, 10, None, - marks=pytest.mark.xfail( - condition=not PANDAS_GE_133, - reason="https://github.com/pandas-dev/pandas/issues/43240", - ), ), ( pd.Index(np.arange(10)), diff --git a/python/cudf/cudf/tests/test_replace.py b/python/cudf/cudf/tests/test_replace.py index ac2b2c6cd30..3050ce75d12 100644 --- a/python/cudf/cudf/tests/test_replace.py +++ b/python/cudf/cudf/tests/test_replace.py @@ -8,12 +8,7 @@ import pytest import cudf -from cudf.core._compat import ( - PANDAS_GE_134, - PANDAS_GE_150, - PANDAS_GE_200, - PANDAS_GE_210, -) +from cudf.core._compat import PANDAS_GE_150, PANDAS_GE_200, PANDAS_GE_210 from cudf.core.dtypes import Decimal32Dtype, Decimal64Dtype, Decimal128Dtype from cudf.testing._utils import ( INTEGER_TYPES, @@ -1016,7 +1011,7 @@ def test_numeric_series_replace_dtype(series_dtype, replacement): pd.Series(["one", "two", "three"], dtype="category"), {"to_replace": "one", "value": "two", "inplace": True}, marks=pytest.mark.xfail( - condition=(not PANDAS_GE_134) or (PANDAS_GE_200), + condition=PANDAS_GE_200, reason="https://github.com/pandas-dev/pandas/issues/43232" "https://github.com/pandas-dev/pandas/issues/53358", ), diff --git a/python/cudf/cudf/tests/test_series.py b/python/cudf/cudf/tests/test_series.py index b7be3878412..14006f90b45 100644 --- a/python/cudf/cudf/tests/test_series.py +++ b/python/cudf/cudf/tests/test_series.py @@ -15,7 +15,6 @@ import cudf from cudf.api.extensions import no_default -from cudf.core._compat import PANDAS_LT_140 from cudf.errors import MixedTypeError from cudf.testing._utils import ( NUMERIC_TYPES, @@ -1318,15 +1317,7 @@ def test_series_raises_float16(data): pd.RangeIndex(0, 3, 1), [3.0, 1.0, np.nan], ["a", "z", None], - pytest.param( - pd.RangeIndex(4, -1, -2), - marks=[ - pytest.mark.xfail( - condition=PANDAS_LT_140, - reason="https://github.com/pandas-dev/pandas/issues/43591", - ) - ], - ), + pd.RangeIndex(4, -1, -2), ], ) @pytest.mark.parametrize("axis", [0, "index"]) From adcd7e9c757fdda3e25d7c469d062a7402a4ad95 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 30 Jan 2024 00:41:21 -0600 Subject: [PATCH 157/384] Apply suggestions from code review Co-authored-by: Bradley Dice Co-authored-by: Vyas Ramasubramani --- python/cudf/cudf/api/types.py | 3 +-- python/cudf/cudf/core/column/datetime.py | 6 ++---- python/cudf/cudf/core/column/string.py | 6 +++--- python/cudf/cudf/core/column/timedelta.py | 2 +- python/cudf/cudf/core/dataframe.py | 4 ++-- python/cudf/cudf/core/dtypes.py | 2 +- python/cudf/cudf/core/frame.py | 2 +- python/cudf/cudf/core/groupby/groupby.py | 6 +++--- python/cudf/cudf/core/series.py | 4 ++-- python/cudf/cudf/core/single_column_frame.py | 4 ++-- python/cudf/cudf/tests/test_binops.py | 4 ++-- python/cudf/cudf/tests/test_dataframe.py | 2 +- 12 files changed, 21 insertions(+), 24 deletions(-) diff --git a/python/cudf/cudf/api/types.py b/python/cudf/cudf/api/types.py index f6f5e522cbd..6a9e5933e12 100644 --- a/python/cudf/cudf/api/types.py +++ b/python/cudf/cudf/api/types.py @@ -540,8 +540,7 @@ def _is_pandas_nullable_extension_dtype(dtype_to_check) -> bool: is_named_tuple = pd_types.is_named_tuple is_iterator = pd_types.is_iterator is_bool = pd_types.is_bool -is_categorical = pd_types.is_categorical_dtype -# TODO +is_categorical_dtype = pd_types.is_categorical_dtype is_complex = pd_types.is_complex is_float = pd_types.is_float is_hashable = pd_types.is_hashable diff --git a/python/cudf/cudf/core/column/datetime.py b/python/cudf/cudf/core/column/datetime.py index a0c0b119ef7..6682bbb333b 100644 --- a/python/cudf/cudf/core/column/datetime.py +++ b/python/cudf/cudf/core/column/datetime.py @@ -113,14 +113,12 @@ def infer_format(element: str, **kwargs) -> str: raise NotImplementedError( "cuDF does not yet support timezone-aware datetimes" ) - if ".%f" in fmt: + if ".%f" not in fmt: # For context read: # https://github.com/pandas-dev/pandas/issues/52418 # We cannot rely on format containing only %f # c++/libcudf expects .%3f, .%6f, .%9f # Logic below handles those cases well. - pass - else: return fmt element_parts = element.split(".") @@ -534,7 +532,7 @@ def median(self, skipna: Optional[bool] = None) -> pd.Timestamp: def cov(self, other: DatetimeColumn) -> float: if not isinstance(other, DatetimeColumn): raise TypeError( - f"cannot perform corr with types {self.dtype}, {other.dtype}" + f"cannot perform cov with types {self.dtype}, {other.dtype}" ) return self.as_numerical_column("int64").cov( other.as_numerical_column("int64") diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index 2fdcf30606a..2373f94ee97 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -697,7 +697,7 @@ def contains( >>> data = ['Mouse', 'dog', 'house and parrot', '23.0', np.NaN] >>> idx = cudf.Index(data) >>> idx - Index(['Mouse' 'dog' 'house and parrot' '23.0' None], dtype='object') + Index(['Mouse', 'dog', 'house and parrot', '23.0', None], dtype='object') >>> idx.str.contains('23', regex=False) Index([False, False, False, True, ], dtype='bool') @@ -2805,7 +2805,7 @@ def partition(self, sep: str = " ", expand: bool = True) -> SeriesOrIndex: >>> idx = cudf.Index(['X 123', 'Y 999']) >>> idx - Index(['X 123' 'Y 999'], dtype='object') + Index(['X 123', 'Y 999'], dtype='object') Which will create a MultiIndex: @@ -2878,7 +2878,7 @@ def rpartition(self, sep: str = " ", expand: bool = True) -> SeriesOrIndex: >>> idx = cudf.Index(['X 123', 'Y 999']) >>> idx - Index(['X 123' 'Y 999'], dtype='object') + Index(['X 123', 'Y 999'], dtype='object') Which will create a MultiIndex: diff --git a/python/cudf/cudf/core/column/timedelta.py b/python/cudf/cudf/core/column/timedelta.py index 094ccb57a1c..edf05fbb264 100644 --- a/python/cudf/cudf/core/column/timedelta.py +++ b/python/cudf/cudf/core/column/timedelta.py @@ -406,7 +406,7 @@ def std( def cov(self, other: TimeDeltaColumn) -> float: if not isinstance(other, TimeDeltaColumn): raise TypeError( - f"cannot perform corr with types {self.dtype}, {other.dtype}" + f"cannot perform cov with types {self.dtype}, {other.dtype}" ) return self.as_numerical_column("int64").cov( other.as_numerical_column("int64") diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index d0cbacfb7e8..23f153e14fa 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -7267,14 +7267,14 @@ def pct_change( fill_method : str, default 'ffill' How to handle NAs before computing percent changes. - .. deprecated:: 23.12 + .. deprecated:: 24.04 All options of `fill_method` are deprecated except `fill_method=None`. limit : int, optional The number of consecutive NAs to fill before stopping. Not yet implemented. - .. deprecated:: 23.12 + .. deprecated:: 24.04 `limit` is deprecated. freq : str, optional Increment to use from time series API. diff --git a/python/cudf/cudf/core/dtypes.py b/python/cudf/cudf/core/dtypes.py index 734dd501d48..11e64faecf9 100644 --- a/python/cudf/cudf/core/dtypes.py +++ b/python/cudf/cudf/core/dtypes.py @@ -1026,7 +1026,7 @@ def _is_categorical_dtype(obj): def is_categorical_dtype(obj): """Check whether an array-like or dtype is of the Categorical dtype. - .. deprecated:: 23.12 + .. deprecated:: 24.04 Use isinstance(dtype, cudf.CategoricalDtype) instead Parameters diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index d061045fb2d..a1c5cf40024 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -748,7 +748,7 @@ def fillna( non-null value. `bfill` propagates backward with the next non-null value. Cannot be used with ``value``. - .. deprecated:: 23.12 + .. deprecated:: 24.04 `method` is deprecated. Returns diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index b3577444f6b..ba802c47479 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -290,7 +290,7 @@ def dtypes(self): """ Return the dtypes in this group. - .. deprecated:: 23.08 + .. deprecated:: 24.04 Use `.dtypes` on base object instead. Returns @@ -2343,14 +2343,14 @@ def pct_change( fill_method : str, default 'ffill' How to handle NAs before computing percent changes. - .. deprecated:: 23.12 + .. deprecated:: 24.04 All options of `fill_method` are deprecated except `fill_method=None`. limit : int, optional The number of consecutive NAs to fill before stopping. Not yet implemented. - .. deprecated:: 23.12 + .. deprecated:: 24.04 `limit` is deprecated. freq : str, optional Increment to use from time series API. diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 7ff529dbd05..649b0688992 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -3582,14 +3582,14 @@ def pct_change( fill_method : str, default 'ffill' How to handle NAs before computing percent changes. - .. deprecated:: 23.12 + .. deprecated:: 24.04 All options of `fill_method` are deprecated except `fill_method=None`. limit : int, optional The number of consecutive NAs to fill before stopping. Not yet implemented. - .. deprecated:: 23.12 + .. deprecated:: 24.04 `limit` is deprecated. freq : str, optional Increment to use from time series API. diff --git a/python/cudf/cudf/core/single_column_frame.py b/python/cudf/cudf/core/single_column_frame.py index 65fbc968c12..97779522b8b 100644 --- a/python/cudf/cudf/core/single_column_frame.py +++ b/python/cudf/cudf/core/single_column_frame.py @@ -159,7 +159,7 @@ def from_arrow(cls, array): >>> import cudf >>> import pyarrow as pa >>> cudf.Index.from_arrow(pa.array(["a", "b", None])) - Index(['a' 'b' None], dtype='object') + Index(['a', 'b', None], dtype='object') >>> cudf.Series.from_arrow(pa.array(["a", "b", None])) 0 a 1 b @@ -273,7 +273,7 @@ def factorize(self, sort=False, use_na_sentinel=True): >>> codes array([0, 0, 1], dtype=int8) >>> uniques - Index(['a' 'c'], dtype='object') + Index(['a', 'c'], dtype='object') """ return cudf.core.algorithms.factorize( self, diff --git a/python/cudf/cudf/tests/test_binops.py b/python/cudf/cudf/tests/test_binops.py index 3d920f225d3..2c5d46f2ca2 100644 --- a/python/cudf/cudf/tests/test_binops.py +++ b/python/cudf/cudf/tests/test_binops.py @@ -663,11 +663,11 @@ def test_different_shapes_and_columns_with_unaligned_indices(binop): # Test with a RangeIndex pdf1 = pd.DataFrame({"x": [4, 3, 2, 1], "y": [7, 3, 8, 6]}) - # Test with a Index + # Test with an Index pdf2 = pd.DataFrame( {"x": [1, 2, 3, 7], "y": [4, 5, 6, 7]}, index=[0, 1, 3, 4] ) - # Test with a Index in a different order + # Test with an Index in a different order pdf3 = pd.DataFrame( {"x": [4, 5, 6, 7], "y": [1, 2, 3, 7], "z": [0, 5, 3, 7]}, index=[0, 3, 5, 3], diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index fa130a99c72..a483657a334 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -10900,7 +10900,7 @@ def test_dataframe_contains(name, contains, other_names): assert (contains in pdf) == expectation assert (contains in gdf) == expectation elif pd.api.types.is_float_dtype(gdf.columns.dtype): - # In some cases, the columns are converted to a Index[float] based on + # In some cases, the columns are converted to an Index[float] based on # the other column names. That casts name values from None to np.nan. expectation = contains is np.nan and (name is None or name is np.nan) assert (contains in pdf) == expectation From 86a4068cc61494b56c66b9384a66206d46b06e06 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 29 Jan 2024 20:42:38 -1000 Subject: [PATCH 158/384] Allow hash_array to be findable in pandas 2.0; add workaround for test_resample for cudf.pandas (#14908) Fixes new failure in test_hash_array. Open to feedback on a better approach. The main issue is that some public methods are defined under __getattr__ with no __dir__ to find them (which we rely on for module population) --- python/cudf/cudf/pandas/_wrappers/pandas.py | 17 +++++++++++++++++ .../cudf/cudf_pandas_tests/test_cudf_pandas.py | 3 +++ 2 files changed, 20 insertions(+) diff --git a/python/cudf/cudf/pandas/_wrappers/pandas.py b/python/cudf/cudf/pandas/_wrappers/pandas.py index 137709925df..b7c8e92e8db 100644 --- a/python/cudf/cudf/pandas/_wrappers/pandas.py +++ b/python/cudf/cudf/pandas/_wrappers/pandas.py @@ -2,6 +2,7 @@ # All rights reserved. # SPDX-License-Identifier: Apache-2.0 import copyreg +import importlib import pickle import sys @@ -47,6 +48,22 @@ cudf.set_option("mode.pandas_compatible", True) +def _pandas_util_dir(): + # In pandas 2.0, pandas.util contains public APIs under + # __getattr__ but no __dir__ to find them + # https://github.com/pandas-dev/pandas/blob/2.2.x/pandas/util/__init__.py + return list(importlib.import_module("pandas.util").__dict__.keys()) + [ + "hash_array", + "hash_pandas_object", + "Appender", + "Substitution", + "cache_readonly", + ] + + +pd.util.__dir__ = _pandas_util_dir + + def make_final_proxy_type( name, fast_type, diff --git a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py index e36e1a68114..546f8df95f3 100644 --- a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py +++ b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py @@ -909,6 +909,9 @@ def test_resample(): ) expected = ser.resample("D").max() result = xser.resample("D").max() + # TODO: See if as_unit can be avoided + expected.index = expected.index.as_unit("s") + result.index = result.index.as_unit("s") tm.assert_series_equal(result, expected) From 92b6472cd3d8ce2bdbee3e8d9dbae187ec227c31 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 29 Jan 2024 20:44:46 -1000 Subject: [PATCH 159/384] Remove pandas 1.5 checks (#14928) Remove pandas 1.5 checks mostly in unit tests. --- python/cudf/cudf/api/types.py | 4 +- python/cudf/cudf/core/_compat.py | 2 - python/cudf/cudf/core/column/column.py | 8 +-- python/cudf/cudf/core/dtypes.py | 7 +-- python/cudf/cudf/core/multiindex.py | 17 +----- python/cudf/cudf/core/window/rolling.py | 23 +++----- python/cudf/cudf/tests/test_array_ufunc.py | 11 +--- python/cudf/cudf/tests/test_binops.py | 45 ++++----------- python/cudf/cudf/tests/test_concat.py | 62 ++++----------------- python/cudf/cudf/tests/test_dataframe.py | 53 ++---------------- python/cudf/cudf/tests/test_datetime.py | 16 +----- python/cudf/cudf/tests/test_df_protocol.py | 5 -- python/cudf/cudf/tests/test_dtypes.py | 9 +-- python/cudf/cudf/tests/test_groupby.py | 25 +-------- python/cudf/cudf/tests/test_numerical.py | 15 +---- python/cudf/cudf/tests/test_parquet.py | 64 ++++++++++------------ python/cudf/cudf/tests/test_replace.py | 20 +++---- python/cudf/cudf/tests/test_rolling.py | 22 +------- python/cudf/cudf/tests/test_setitem.py | 31 +++-------- python/cudf/cudf/tests/test_string.py | 9 +-- 20 files changed, 98 insertions(+), 350 deletions(-) diff --git a/python/cudf/cudf/api/types.py b/python/cudf/cudf/api/types.py index 6a9e5933e12..6a9eb68d6f5 100644 --- a/python/cudf/cudf/api/types.py +++ b/python/cudf/cudf/api/types.py @@ -16,7 +16,6 @@ from pandas.api import types as pd_types import cudf -from cudf.core._compat import PANDAS_GE_150 from cudf.core.dtypes import ( # noqa: F401 _BaseDtype, _is_categorical_dtype, @@ -497,8 +496,9 @@ def _is_pandas_nullable_extension_dtype(dtype_to_check) -> bool: pd.Float64Dtype, pd.BooleanDtype, pd.StringDtype, + pd.ArrowDtype, ), - ) or (PANDAS_GE_150 and isinstance(dtype_to_check, pd.ArrowDtype)): + ): return True elif isinstance(dtype_to_check, pd.CategoricalDtype): return _is_pandas_nullable_extension_dtype( diff --git a/python/cudf/cudf/core/_compat.py b/python/cudf/cudf/core/_compat.py index b602dfdf23c..f15e85b7a88 100644 --- a/python/cudf/cudf/core/_compat.py +++ b/python/cudf/cudf/core/_compat.py @@ -4,8 +4,6 @@ from packaging import version PANDAS_VERSION = version.parse(pd.__version__) -PANDAS_GE_150 = PANDAS_VERSION >= version.parse("1.5.0") -PANDAS_LT_153 = PANDAS_VERSION < version.parse("1.5.3") PANDAS_EQ_200 = PANDAS_VERSION == version.parse("2.0.0") PANDAS_GE_200 = PANDAS_VERSION >= version.parse("2.0.0") PANDAS_GE_201 = PANDAS_VERSION >= version.parse("2.0.1") diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 569e8e30dd2..ad56cabb48e 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -28,6 +28,7 @@ import pyarrow as pa import pyarrow.compute as pc from numba import cuda +from pandas.core.arrays.arrow.extension_types import ArrowIntervalType from typing_extensions import Self import rmm @@ -66,7 +67,7 @@ is_scalar, is_string_dtype, ) -from cudf.core._compat import PANDAS_GE_150, PANDAS_GE_210 +from cudf.core._compat import PANDAS_GE_210 from cudf.core.abc import Serializable from cudf.core.buffer import ( Buffer, @@ -97,11 +98,6 @@ ) from cudf.utils.utils import _array_ufunc, mask_dtype -if PANDAS_GE_150: - from pandas.core.arrays.arrow.extension_types import ArrowIntervalType -else: - from pandas.core.arrays._arrow_utils import ArrowIntervalType - if PANDAS_GE_210: NumpyExtensionArray = pd.arrays.NumpyExtensionArray else: diff --git a/python/cudf/cudf/core/dtypes.py b/python/cudf/cudf/core/dtypes.py index 11e64faecf9..f05758d6993 100644 --- a/python/cudf/cudf/core/dtypes.py +++ b/python/cudf/cudf/core/dtypes.py @@ -13,6 +13,7 @@ import pyarrow as pa from pandas.api import types as pd_types from pandas.api.extensions import ExtensionDtype +from pandas.core.arrays.arrow.extension_types import ArrowIntervalType from pandas.core.dtypes.dtypes import ( CategoricalDtype as pd_CategoricalDtype, CategoricalDtypeType as pd_CategoricalDtypeType, @@ -20,16 +21,10 @@ import cudf from cudf._typing import Dtype -from cudf.core._compat import PANDAS_GE_150 from cudf.core.abc import Serializable from cudf.core.buffer import Buffer from cudf.utils.docutils import doc_apply -if PANDAS_GE_150: - from pandas.core.arrays.arrow.extension_types import ArrowIntervalType -else: - from pandas.core.arrays._arrow_utils import ArrowIntervalType - def dtype(arbitrary): """ diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 081109e81bc..a747ca8eea0 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -24,7 +24,6 @@ from cudf.api.extensions import no_default from cudf.api.types import is_integer, is_list_like, is_object_dtype from cudf.core import column -from cudf.core._compat import PANDAS_GE_150 from cudf.core.frame import Frame from cudf.core.index import ( BaseIndex, @@ -469,21 +468,7 @@ def __repr__(self): ) ) - if not PANDAS_GE_150: - # Need this whole `if` block, - # this is a workaround for the following issue: - # https://github.com/pandas-dev/pandas/issues/39984 - preprocess_pdf = pd.DataFrame( - { - name: col.to_pandas(nullable=(col.dtype.kind != "f")) - for name, col in preprocess._data.items() - } - ) - - preprocess_pdf.columns = preprocess.names - preprocess = pd.MultiIndex.from_frame(preprocess_pdf) - else: - preprocess = preprocess.to_pandas(nullable=True) + preprocess = preprocess.to_pandas(nullable=True) preprocess.values[:] = tuples_list else: preprocess = preprocess.to_pandas(nullable=True) diff --git a/python/cudf/cudf/core/window/rolling.py b/python/cudf/cudf/core/window/rolling.py index f4322aefceb..890e4ecc2f0 100644 --- a/python/cudf/cudf/core/window/rolling.py +++ b/python/cudf/cudf/core/window/rolling.py @@ -10,7 +10,6 @@ from cudf import _lib as libcudf from cudf.api.types import is_integer, is_number from cudf.core import column -from cudf.core._compat import PANDAS_GE_150 from cudf.core.buffer import acquire_spill_lock from cudf.core.column.column import as_column from cudf.core.mixins import Reducible @@ -217,21 +216,13 @@ def _apply_agg_column(self, source_column, agg_name): following_window = None window = self.window elif isinstance(self.window, BaseIndexer): - if PANDAS_GE_150: - start, end = self.window.get_window_bounds( - num_values=len(self.obj), - min_periods=self.min_periods, - center=self.center, - closed=None, - step=None, - ) - else: - start, end = self.window.get_window_bounds( - num_values=len(self.obj), - min_periods=self.min_periods, - center=self.center, - closed=None, - ) + start, end = self.window.get_window_bounds( + num_values=len(self.obj), + min_periods=self.min_periods, + center=self.center, + closed=None, + step=None, + ) start = as_column(start, dtype="int32") end = as_column(end, dtype="int32") diff --git a/python/cudf/cudf/tests/test_array_ufunc.py b/python/cudf/cudf/tests/test_array_ufunc.py index 3e3f3aa5dfa..d6b944ebeac 100644 --- a/python/cudf/cudf/tests/test_array_ufunc.py +++ b/python/cudf/cudf/tests/test_array_ufunc.py @@ -12,7 +12,7 @@ from packaging import version import cudf -from cudf.core._compat import PANDAS_GE_150, PANDAS_GE_200, PANDAS_GE_210 +from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210 from cudf.testing._utils import ( assert_eq, expect_warning_if, @@ -76,15 +76,6 @@ def _hide_ufunc_warnings(ufunc): def test_ufunc_index(request, ufunc): # Note: This test assumes that all ufuncs are unary or binary. fname = ufunc.__name__ - request.applymarker( - pytest.mark.xfail( - condition=( - fname in {"bitwise_and", "bitwise_or", "bitwise_xor"} - and not PANDAS_GE_150 - ), - reason="https://github.com/pandas-dev/pandas/issues/46769", - ) - ) request.applymarker( pytest.mark.xfail( condition=not hasattr(cp, fname), diff --git a/python/cudf/cudf/tests/test_binops.py b/python/cudf/cudf/tests/test_binops.py index 2c5d46f2ca2..3ebefa6e071 100644 --- a/python/cudf/cudf/tests/test_binops.py +++ b/python/cudf/cudf/tests/test_binops.py @@ -13,7 +13,6 @@ import cudf from cudf import Series -from cudf.core._compat import PANDAS_GE_150 from cudf.core.buffer.spill_manager import get_global_manager from cudf.core.index import as_index from cudf.testing import _utils as utils @@ -1706,13 +1705,7 @@ def test_scalar_null_binops(op, dtype_l, dtype_r): "minutes", "seconds", "microseconds", - pytest.param( - "nanoseconds", - marks=pytest_xfail( - condition=not PANDAS_GE_150, - reason="https://github.com/pandas-dev/pandas/issues/36589", - ), - ), + "nanoseconds", ], ) @pytest.mark.parametrize( @@ -1758,29 +1751,17 @@ def test_datetime_dateoffset_binaryop( {"months": 2, "years": 5}, {"microseconds": 1, "seconds": 1}, {"months": 2, "years": 5, "seconds": 923, "microseconds": 481}, - pytest.param( - {"milliseconds": 4}, - marks=pytest.mark.xfail( - condition=not PANDAS_GE_150, - reason="Pandas gets the wrong answer for milliseconds", - ), - ), - pytest.param( - {"milliseconds": 4, "years": 2}, - marks=pytest_xfail( - reason="https://github.com/pandas-dev/pandas/issues/49897" - ), - ), - pytest.param( - {"nanoseconds": 12}, - marks=pytest.mark.xfail( - condition=not PANDAS_GE_150, - reason="Pandas gets the wrong answer for nanoseconds", - ), - ), + {"milliseconds": 4}, + {"milliseconds": 4, "years": 2}, {"nanoseconds": 12}, ], ) +@pytest.mark.filterwarnings( + "ignore:Non-vectorized DateOffset:pandas.errors.PerformanceWarning" +) +@pytest.mark.filterwarnings( + "ignore:Discarding nonzero nanoseconds:UserWarning" +) @pytest.mark.parametrize("op", [operator.add, operator.sub]) def test_datetime_dateoffset_binaryop_multiple(date_col, kwargs, op): gsr = cudf.Series(date_col, dtype="datetime64[ns]") @@ -1816,13 +1797,7 @@ def test_datetime_dateoffset_binaryop_multiple(date_col, kwargs, op): "minutes", "seconds", "microseconds", - pytest.param( - "nanoseconds", - marks=pytest_xfail( - condition=not PANDAS_GE_150, - reason="https://github.com/pandas-dev/pandas/issues/36589", - ), - ), + "nanoseconds", ], ) @pytest.mark.parametrize( diff --git a/python/cudf/cudf/tests/test_concat.py b/python/cudf/cudf/tests/test_concat.py index 9078d54c193..4b0e46bf286 100644 --- a/python/cudf/cudf/tests/test_concat.py +++ b/python/cudf/cudf/tests/test_concat.py @@ -10,7 +10,7 @@ import cudf as gd from cudf.api.types import _is_categorical_dtype -from cudf.core._compat import PANDAS_GE_150, PANDAS_GE_200 +from cudf.core._compat import PANDAS_GE_200 from cudf.core.dtypes import Decimal32Dtype, Decimal64Dtype, Decimal128Dtype from cudf.testing._utils import ( assert_eq, @@ -828,13 +828,7 @@ def test_concat_join_axis_1(objs, ignore_index, sort, join, axis): axis=axis, ) - if PANDAS_GE_150: - assert_eq(expected, actual, check_index_type=True) - else: - # special handling of check_index_type below - # required because: - # https://github.com/pandas-dev/pandas/issues/47501 - assert_eq(expected, actual, check_index_type=not (axis == 1 and sort)) + assert_eq(expected, actual, check_index_type=True) @pytest.mark.parametrize("ignore_index", [True, False]) @@ -902,13 +896,7 @@ def test_concat_join_one_df(ignore_index, sort, join, axis): [gdf1], sort=sort, join=join, ignore_index=ignore_index, axis=axis ) - if PANDAS_GE_150: - assert_eq(expected, actual, check_index_type=True) - else: - # special handling of check_index_type below - # required because: - # https://github.com/pandas-dev/pandas/issues/47501 - assert_eq(expected, actual, check_index_type=not (axis == 1 and sort)) + assert_eq(expected, actual, check_index_type=True) @pytest.mark.parametrize( @@ -953,13 +941,7 @@ def test_concat_join_no_overlapping_columns( axis=axis, ) - if PANDAS_GE_150: - assert_eq(expected, actual, check_index_type=True) - else: - # special handling of check_index_type below - # required because: - # https://github.com/pandas-dev/pandas/issues/47501 - assert_eq(expected, actual, check_index_type=not (axis == 1 and sort)) + assert_eq(expected, actual, check_index_type=True) @pytest.mark.parametrize("ignore_index", [False, True]) @@ -1113,7 +1095,7 @@ def test_concat_join_no_overlapping_columns_empty_df_basic( assert_eq( expected, actual, - check_index_type=PANDAS_GE_150, + check_index_type=True, check_column_type=not PANDAS_GE_200, ) @@ -1149,21 +1131,11 @@ def test_concat_join_series(ignore_index, sort, join, axis): axis=axis, ) - if PANDAS_GE_150: - assert_eq( - expected, - actual, - check_index_type=True, - ) - else: - # special handling of check_index_type required below: - # https://github.com/pandas-dev/pandas/issues/46675 - # https://github.com/pandas-dev/pandas/issues/47501 - assert_eq( - expected, - actual, - check_index_type=(axis == 0), - ) + assert_eq( + expected, + actual, + check_index_type=True, + ) @pytest.mark.parametrize( @@ -1323,19 +1295,7 @@ def test_concat_join_empty_dataframes( ) @pytest.mark.parametrize("ignore_index", [True, False]) @pytest.mark.parametrize("sort", [True, False]) -@pytest.mark.parametrize( - "join", - [ - "inner", - pytest.param( - "outer", - marks=pytest.mark.xfail( - condition=not PANDAS_GE_150, - reason="https://github.com/pandas-dev/pandas/issues/37937", - ), - ), - ], -) +@pytest.mark.parametrize("join", ["inner", "outer"]) @pytest.mark.parametrize("axis", [1]) def test_concat_join_empty_dataframes_axis_1( df, other, ignore_index, axis, join, sort diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index a483657a334..a0f6c4c3cfc 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -25,12 +25,7 @@ import cudf from cudf.api.extensions import no_default -from cudf.core._compat import ( - PANDAS_GE_150, - PANDAS_GE_200, - PANDAS_GE_210, - PANDAS_LT_203, -) +from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210, PANDAS_LT_203 from cudf.core.buffer.spill_manager import get_global_manager from cudf.core.column import column from cudf.errors import MixedTypeError @@ -345,27 +340,9 @@ def test_concat_index(a, b): {"a": [1, None, None], "b": [3, np.nan, np.nan]}, {1: ["a", "b", "c"], 2: ["q", "w", "u"]}, {1: ["a", np.nan, "c"], 2: ["q", None, "u"]}, - pytest.param( - {}, - marks=pytest_xfail( - condition=not PANDAS_GE_150, - reason="https://github.com/rapidsai/cudf/issues/11080", - ), - ), - pytest.param( - {1: [], 2: [], 3: []}, - marks=pytest_xfail( - condition=not PANDAS_GE_150, - reason="https://github.com/rapidsai/cudf/issues/11080", - ), - ), - pytest.param( - [1, 2, 3], - marks=pytest_xfail( - condition=not PANDAS_GE_150, - reason="https://github.com/rapidsai/cudf/issues/11080", - ), - ), + {}, + {1: [], 2: [], 3: []}, + [1, 2, 3], ], ) def test_axes(data): @@ -1882,18 +1859,7 @@ def test_nonmatching_index_setitem(nrows): assert_eq(gdf["c"].to_pandas(), gdf_series.to_pandas()) -@pytest.mark.parametrize( - "dtype", - [ - "int", - pytest.param( - "int64[pyarrow]", - marks=pytest.mark.skipif( - not PANDAS_GE_150, reason="pyarrow support only in >=1.5" - ), - ), - ], -) +@pytest.mark.parametrize("dtype", ["int", "int64[pyarrow]"]) def test_from_pandas(dtype): df = pd.DataFrame({"x": [1, 2, 3]}, index=[4.0, 5.0, 6.0], dtype=dtype) df.columns.name = "custom_column_name" @@ -7710,14 +7676,7 @@ def test_dataframe_concat_dataframe(df, other, sort, ignore_index): "other", [ pd.Series([10, 11, 23, 234, 13]), - pytest.param( - pd.Series([10, 11, 23, 234, 13], index=[11, 12, 13, 44, 33]), - marks=pytest.mark.xfail( - condition=not PANDAS_GE_150, - reason="pandas bug: " - "https://github.com/pandas-dev/pandas/issues/35092", - ), - ), + pd.Series([10, 11, 23, 234, 13], index=[11, 12, 13, 44, 33]), {1: 1}, {0: 10, 1: 100, 2: 102}, ], diff --git a/python/cudf/cudf/tests/test_datetime.py b/python/cudf/cudf/tests/test_datetime.py index 60b0d787278..62733625485 100644 --- a/python/cudf/cudf/tests/test_datetime.py +++ b/python/cudf/cudf/tests/test_datetime.py @@ -13,12 +13,7 @@ import cudf import cudf.testing.dataset_generator as dataset_generator from cudf import DataFrame, Series -from cudf.core._compat import ( - PANDAS_EQ_200, - PANDAS_GE_150, - PANDAS_GE_200, - PANDAS_GE_210, -) +from cudf.core._compat import PANDAS_EQ_200, PANDAS_GE_200, PANDAS_GE_210 from cudf.core.index import DatetimeIndex from cudf.testing._utils import ( DATETIME_TYPES, @@ -1504,14 +1499,7 @@ def test_is_month_start(data, dtype): "17h", "-680T", "110546s", - pytest.param( - "110546789L", - marks=pytest.mark.xfail( - condition=not PANDAS_GE_150, - reason="Pandas DateOffset ignores milliseconds. " - "https://github.com/pandas-dev/pandas/issues/43371", - ), - ), + "110546789L", "110546789248U", ] diff --git a/python/cudf/cudf/tests/test_df_protocol.py b/python/cudf/cudf/tests/test_df_protocol.py index bffbade14d8..a22b678ebe6 100644 --- a/python/cudf/cudf/tests/test_df_protocol.py +++ b/python/cudf/cudf/tests/test_df_protocol.py @@ -7,7 +7,6 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_150 from cudf.core.buffer import as_buffer from cudf.core.column import as_column, build_column from cudf.core.df_protocol import ( @@ -278,9 +277,5 @@ def test_NA_mixed_dtype(): assert_df_unique_dtype_cols(data_mixed) -@pytest.mark.skipif( - not PANDAS_GE_150, - reason="Pandas versions < 1.5.0 do not support interchange protocol", -) def test_from_cpu_df(pandas_df): cudf.from_dataframe(pandas_df, allow_copy=True) diff --git a/python/cudf/cudf/tests/test_dtypes.py b/python/cudf/cudf/tests/test_dtypes.py index 6e24099f1a8..0efd8d9781c 100644 --- a/python/cudf/cudf/tests/test_dtypes.py +++ b/python/cudf/cudf/tests/test_dtypes.py @@ -1,12 +1,12 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. import numpy as np import pandas as pd import pyarrow as pa import pytest +from pandas.core.arrays.arrow.extension_types import ArrowIntervalType import cudf -from cudf.core._compat import PANDAS_GE_150 from cudf.core.column import ColumnBase from cudf.core.dtypes import ( CategoricalDtype, @@ -20,11 +20,6 @@ from cudf.testing._utils import assert_eq from cudf.utils.dtypes import np_to_pa_dtype -if PANDAS_GE_150: - from pandas.core.arrays.arrow.extension_types import ArrowIntervalType -else: - from pandas.core.arrays._arrow_utils import ArrowIntervalType - def test_cdt_basic(): psr = pd.Series(["a", "b", "a", "c"], dtype="category") diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index dcfc9d801a4..a0b86d735cc 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -20,7 +20,7 @@ import cudf from cudf import DataFrame, Series from cudf.api.extensions import no_default -from cudf.core._compat import PANDAS_GE_150, PANDAS_GE_200, PANDAS_GE_210 +from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210 from cudf.core.udf._ops import arith_ops, comparison_ops, unary_ops from cudf.core.udf.groupby_typing import SUPPORTED_GROUPBY_NUMPY_TYPES from cudf.core.udf.utils import UDFError, precompiled @@ -1190,13 +1190,7 @@ def test_advanced_groupby_levels(): @pytest.mark.parametrize( "func", [ - pytest.param( - lambda df: df.groupby(["x", "y", "z"]).sum(), - marks=pytest.mark.xfail( - condition=not PANDAS_GE_150, - reason="https://github.com/pandas-dev/pandas/issues/32464", - ), - ), + lambda df: df.groupby(["x", "y", "z"]).sum(), lambda df: df.groupby(["x", "y"]).sum(), lambda df: df.groupby(["x", "y"]).agg("sum"), lambda df: df.groupby(["y"]).sum(), @@ -3294,20 +3288,7 @@ def test_groupby_pct_change_empty_columns(): assert_eq(expected, actual) -@pytest.mark.parametrize( - "group_keys", - [ - None, - pytest.param( - True, - marks=pytest.mark.xfail( - condition=not PANDAS_GE_150, - reason="https://github.com/pandas-dev/pandas/pull/34998", - ), - ), - False, - ], -) +@pytest.mark.parametrize("group_keys", [None, True, False]) @pytest.mark.parametrize("by", ["A", ["A", "B"]]) def test_groupby_group_keys(group_keys, by): gdf = cudf.DataFrame( diff --git a/python/cudf/cudf/tests/test_numerical.py b/python/cudf/cudf/tests/test_numerical.py index fee5cc0ad21..2139e7b9860 100644 --- a/python/cudf/cudf/tests/test_numerical.py +++ b/python/cudf/cudf/tests/test_numerical.py @@ -5,7 +5,6 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_150 from cudf.testing._utils import NUMERIC_TYPES, assert_eq from cudf.utils.dtypes import np_dtypes_to_pandas_dtypes @@ -267,12 +266,7 @@ def test_to_numeric_downcast_large_float_pd_bug(data, downcast): expected = pd.to_numeric(ps, downcast=downcast) got = cudf.to_numeric(gs, downcast=downcast) - if PANDAS_GE_150: - assert_eq(expected, got) - else: - # Pandas bug: https://github.com/pandas-dev/pandas/issues/19729 - with pytest.raises(AssertionError, match="Series are different"): - assert_eq(expected, got) + assert_eq(expected, got) @pytest.mark.parametrize( @@ -350,12 +344,7 @@ def test_to_numeric_downcast_string_large_float(data, downcast): expected = pd.to_numeric(ps, downcast=downcast) got = cudf.to_numeric(gs, downcast=downcast) - if PANDAS_GE_150: - assert_eq(expected, got) - else: - # Pandas bug: https://github.com/pandas-dev/pandas/issues/19729 - with pytest.raises(AssertionError, match="Series are different"): - assert_eq(expected, got) + assert_eq(expected, got) else: expected = pd.Series([np.inf, -np.inf]) with pytest.warns( diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index 105c31cc71f..b4e24bd1617 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -21,7 +21,7 @@ from pyarrow import fs as pa_fs, parquet as pq import cudf -from cudf.core._compat import PANDAS_GE_200, PANDAS_LT_153 +from cudf.core._compat import PANDAS_GE_200 from cudf.io.parquet import ( ParquetDatasetWriter, ParquetWriter, @@ -2612,43 +2612,37 @@ def test_parquet_writer_list_statistics(tmpdir): ] }, # Struct of Lists - pytest.param( - { - "Real estate records": [ - None, - { - "Status": "NRI", - "Ownerships": { - "land_unit": [None, 2, None], - "flats": [[1, 2, 3], [], [4, 5], [], [0, 6, 0]], - }, - }, - { - "Status": None, - "Ownerships": { - "land_unit": [4, 5], - "flats": [[7, 8], []], - }, + { + "Real estate records": [ + None, + { + "Status": "NRI", + "Ownerships": { + "land_unit": [None, 2, None], + "flats": [[1, 2, 3], [], [4, 5], [], [0, 6, 0]], }, - { - "Status": "RI", - "Ownerships": {"land_unit": None, "flats": [[]]}, + }, + { + "Status": None, + "Ownerships": { + "land_unit": [4, 5], + "flats": [[7, 8], []], }, - {"Status": "RI", "Ownerships": None}, - { - "Status": None, - "Ownerships": { - "land_unit": [7, 8, 9], - "flats": [[], [], []], - }, + }, + { + "Status": "RI", + "Ownerships": {"land_unit": None, "flats": [[]]}, + }, + {"Status": "RI", "Ownerships": None}, + { + "Status": None, + "Ownerships": { + "land_unit": [7, 8, 9], + "flats": [[], [], []], }, - ] - }, - marks=pytest.mark.xfail( - condition=PANDAS_LT_153, - reason="pandas assertion fixed in pandas 1.5.3", - ), - ), + }, + ] + }, ], ) def test_parquet_writer_nested(tmpdir, data): diff --git a/python/cudf/cudf/tests/test_replace.py b/python/cudf/cudf/tests/test_replace.py index 3050ce75d12..6db1c97b9fd 100644 --- a/python/cudf/cudf/tests/test_replace.py +++ b/python/cudf/cudf/tests/test_replace.py @@ -8,7 +8,7 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_150, PANDAS_GE_200, PANDAS_GE_210 +from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210 from cudf.core.dtypes import Decimal32Dtype, Decimal64Dtype, Decimal128Dtype from cudf.testing._utils import ( INTEGER_TYPES, @@ -167,18 +167,12 @@ def test_series_replace_with_nulls(): "c": ["abc", "def", ".", None, None], } ), - pytest.param( - cudf.DataFrame( - { - "a": ["one", "two", None, "three"], - "b": ["one", None, "two", "three"], - }, - dtype="category", - ), - marks=pytest.mark.xfail( - condition=not PANDAS_GE_150, - reason="https://github.com/pandas-dev/pandas/issues/46672", - ), + cudf.DataFrame( + { + "a": ["one", "two", None, "three"], + "b": ["one", None, "two", "three"], + }, + dtype="category", ), cudf.DataFrame( { diff --git a/python/cudf/cudf/tests/test_rolling.py b/python/cudf/cudf/tests/test_rolling.py index 22dcf5dfa7e..9c3c9d1082c 100644 --- a/python/cudf/cudf/tests/test_rolling.py +++ b/python/cudf/cudf/tests/test_rolling.py @@ -8,7 +8,7 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_150, PANDAS_GE_200 +from cudf.core._compat import PANDAS_GE_200 from cudf.testing._utils import assert_eq from cudf.testing.dataset_generator import rand_dataframe @@ -480,7 +480,7 @@ def test_rolling_custom_index_support(): from pandas.api.indexers import BaseIndexer class CustomIndexer(BaseIndexer): - def custom_get_window_bounds( + def get_window_bounds( self, num_values, min_periods, center, closed, step=None ): start = np.empty(num_values, dtype=np.int64) @@ -496,24 +496,6 @@ def custom_get_window_bounds( return start, end - if PANDAS_GE_150: - - def get_window_bounds( - self, num_values, min_periods, center, closed, step - ): - return self.custom_get_window_bounds( - num_values, min_periods, center, closed, step - ) - - else: - - def get_window_bounds( - self, num_values, min_periods, center, closed - ): - return self.custom_get_window_bounds( - num_values, min_periods, center, closed - ) - use_expanding = [True, False, True, False, True] indexer = CustomIndexer(window_size=1, use_expanding=use_expanding) diff --git a/python/cudf/cudf/tests/test_setitem.py b/python/cudf/cudf/tests/test_setitem.py index e8d7fdadbff..de0826d61e9 100644 --- a/python/cudf/cudf/tests/test_setitem.py +++ b/python/cudf/cudf/tests/test_setitem.py @@ -5,7 +5,7 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_150, PANDAS_GE_210 +from cudf.core._compat import PANDAS_GE_210 from cudf.testing._utils import ( assert_eq, assert_exceptions_equal, @@ -230,22 +230,12 @@ def test_categorical_setitem_invalid(): ps = pd.Series([1, 2, 3], dtype="category") gs = cudf.Series([1, 2, 3], dtype="category") - if PANDAS_GE_150: - assert_exceptions_equal( - lfunc=ps.__setitem__, - rfunc=gs.__setitem__, - lfunc_args_and_kwargs=([0, 5], {}), - rfunc_args_and_kwargs=([0, 5], {}), - ) - else: - # Following workaround is needed because: - # https://github.com/pandas-dev/pandas/issues/46646 - with pytest.raises( - ValueError, - match="Cannot setitem on a Categorical with a new category, set " - "the categories first", - ): - gs[0] = 5 + assert_exceptions_equal( + lfunc=ps.__setitem__, + rfunc=gs.__setitem__, + lfunc_args_and_kwargs=([0, 5], {}), + rfunc_args_and_kwargs=([0, 5], {}), + ) def test_series_slice_setitem_list(): @@ -318,11 +308,8 @@ def test_series_setitem_upcasting(dtype, indices): sr[indices] = new_value with expect_warning_if(dtype != np.float64): cr[indices] = new_value - if PANDAS_GE_150: - assert_eq(sr, cr) - else: - # pandas bug, incorrectly fails to upcast from float32 to float64 - assert_eq(sr.values, cr.values) + assert_eq(sr, cr) + if dtype == np.float64: # no-op type cast should not modify backing column assert col_ref == cr._column diff --git a/python/cudf/cudf/tests/test_string.py b/python/cudf/cudf/tests/test_string.py index 8c8a3cb2399..b2bf687ba06 100644 --- a/python/cudf/cudf/tests/test_string.py +++ b/python/cudf/cudf/tests/test_string.py @@ -15,7 +15,6 @@ import cudf from cudf import concat -from cudf.core._compat import PANDAS_GE_150 from cudf.core.column.string import StringColumn from cudf.core.index import Index, as_index from cudf.testing._utils import ( @@ -1721,13 +1720,7 @@ def test_strings_filling_tests(data, width, fillchar): ["A,,B", "1,,5", "3,00,0"], ["Linda van der Berg", "George Pitt-Rivers"], ["³", "⅕", ""], - pytest.param( - ["hello", "there", "world", "+1234", "-1234", None, "accént", ""], - marks=pytest.mark.xfail( - condition=not PANDAS_GE_150, - reason="https://github.com/pandas-dev/pandas/issues/20868", - ), - ), + ["hello", "there", "world", "+1234", "-1234", None, "accént", ""], [" ", "\t\r\n ", ""], ["1. Ant. ", "2. Bee!\n", "3. Cat?\t", None], ], From 132978fe85e7f700078a1eb1f0a4264ff404274f Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 30 Jan 2024 08:19:38 +0000 Subject: [PATCH 160/384] Address all remaining reviews --- .pre-commit-config.yaml | 2 ++ python/cudf/cudf/api/types.py | 4 +++- python/cudf/cudf/core/_compat.py | 1 + python/cudf/cudf/core/column_accessor.py | 28 ++++------------------ python/cudf/cudf/core/dataframe.py | 12 +++++++--- python/cudf/cudf/core/dtypes.py | 2 ++ python/cudf/cudf/core/groupby/groupby.py | 7 ++++++ python/cudf/cudf/core/index.py | 5 +++- python/cudf/cudf/core/indexed_frame.py | 6 +++++ python/cudf/cudf/core/reshape.py | 4 ++++ python/cudf/cudf/core/series.py | 13 ++++++++++ python/cudf/cudf/tests/test_array_ufunc.py | 7 ++---- python/cudf/cudf/tests/test_timedelta.py | 5 +--- python/cudf/cudf/utils/ioutils.py | 10 ++++++++ 14 files changed, 68 insertions(+), 38 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ad8e2f6c5ee..ccda2596031 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -91,6 +91,8 @@ repos: entry: '(category=|\s)DeprecationWarning[,)]' language: pygrep types_or: [python, cython] + # We need to exclude just the following file because few APIs still need + # DeprecationWarning: https://github.com/pandas-dev/pandas/issues/54970 exclude: | (?x)^( ^python/cudf/cudf/core/dtypes.py diff --git a/python/cudf/cudf/api/types.py b/python/cudf/cudf/api/types.py index 6a9eb68d6f5..a422eb82231 100644 --- a/python/cudf/cudf/api/types.py +++ b/python/cudf/cudf/api/types.py @@ -16,6 +16,7 @@ from pandas.api import types as pd_types import cudf +from cudf.core._compat import PANDAS_LT_300 from cudf.core.dtypes import ( # noqa: F401 _BaseDtype, _is_categorical_dtype, @@ -467,11 +468,13 @@ def is_any_real_numeric_dtype(arr_or_dtype) -> bool: def _is_datetime64tz_dtype(obj): with warnings.catch_warnings(): warnings.simplefilter("ignore") + assert PANDAS_LT_300, "Need to drop after pandas-3.0 support is added." return _wrap_pandas_is_dtype_api(pd_types.is_datetime64tz_dtype)(obj) def is_datetime64tz_dtype(obj): # Do not remove until pandas 3.0 support is added. + assert PANDAS_LT_300, "Need to drop after pandas-3.0 support is added." warnings.warn( "is_datetime64tz_dtype is deprecated and will be removed in a future " "version.", @@ -540,7 +543,6 @@ def _is_pandas_nullable_extension_dtype(dtype_to_check) -> bool: is_named_tuple = pd_types.is_named_tuple is_iterator = pd_types.is_iterator is_bool = pd_types.is_bool -is_categorical_dtype = pd_types.is_categorical_dtype is_complex = pd_types.is_complex is_float = pd_types.is_float is_hashable = pd_types.is_hashable diff --git a/python/cudf/cudf/core/_compat.py b/python/cudf/cudf/core/_compat.py index f15e85b7a88..5aa685560c8 100644 --- a/python/cudf/cudf/core/_compat.py +++ b/python/cudf/cudf/core/_compat.py @@ -11,3 +11,4 @@ PANDAS_GE_214 = PANDAS_VERSION >= version.parse("2.1.4") PANDAS_GE_220 = PANDAS_VERSION >= version.parse("2.2.0") PANDAS_LT_203 = PANDAS_VERSION < version.parse("2.0.3") +PANDAS_LT_300 = PANDAS_VERSION < version.parse("3.0.0") diff --git a/python/cudf/cudf/core/column_accessor.py b/python/cudf/cudf/core/column_accessor.py index 93abaae6120..33085bede78 100644 --- a/python/cudf/cudf/core/column_accessor.py +++ b/python/cudf/cudf/core/column_accessor.py @@ -3,7 +3,6 @@ from __future__ import annotations import itertools -import warnings from collections import abc from functools import cached_property, reduce from typing import ( @@ -23,7 +22,6 @@ import cudf from cudf.core import column -from cudf.core._compat import PANDAS_GE_200 if TYPE_CHECKING: from cudf._typing import Dtype @@ -237,28 +235,10 @@ def _clear_cache(self): def to_pandas_index(self) -> pd.Index: """Convert the keys of the ColumnAccessor to a Pandas Index object.""" if self.multiindex and len(self.level_names) > 0: - if PANDAS_GE_200: - result = pd.MultiIndex.from_tuples( - self.names, - names=self.level_names, - ) - else: - # Using `from_frame()` instead of `from_tuples` - # prevents coercion of values to a different type - # (e.g., ''->NaT) - with warnings.catch_warnings(): - # Specifying `dtype="object"` here and passing that to - # `from_frame` is deprecated in pandas, but we cannot - # remove that without also losing compatibility with other - # current pandas behaviors like the NaT inference above. - warnings.simplefilter("ignore") - result = pd.MultiIndex.from_frame( - pd.DataFrame( - self.names, - columns=self.level_names, - dtype="object", - ), - ) + result = pd.MultiIndex.from_tuples( + self.names, + names=self.level_names, + ) else: # Determine if we can return a RangeIndex if self.rangeindex: diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 23f153e14fa..1b0f83c5d70 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -58,7 +58,7 @@ is_string_dtype, ) from cudf.core import column, df_protocol, indexing_utils, reshape -from cudf.core._compat import PANDAS_GE_200 +from cudf.core._compat import PANDAS_GE_200, PANDAS_LT_300 from cudf.core.abc import Serializable from cudf.core.column import ( CategoricalColumn, @@ -4589,6 +4589,7 @@ def applymap( Transformed DataFrame. """ # Do not remove until pandas 3.0 support is added. + assert PANDAS_LT_300, "Need to drop after pandas-3.0 support is added." warnings.warn( "DataFrame.applymap has been deprecated. Use DataFrame.map " "instead.", @@ -6102,8 +6103,6 @@ def _reduce( source = self if axis is None: - # if op in {"any", "all"}: - # axis = 2 if op in {"sum", "product", "std", "var"}: # Do not remove until pandas 2.0 support is added. warnings.warn( @@ -6140,6 +6139,7 @@ def _reduce( if axis == 2 and op in ("kurtosis", "kurt", "skew"): # TODO: concat + op can probably be done in the general case # for axis == 2. + # https://github.com/rapidsai/cudf/issues/14930 return getattr(concat_columns(source._data.columns), op)( **kwargs ) @@ -6323,6 +6323,9 @@ def mode(self, axis=0, numeric_only=False, dropna=True): return DataFrame() with warnings.catch_warnings(): + assert ( + PANDAS_LT_300 + ), "Need to drop after pandas-3.0 support is added." warnings.simplefilter("ignore", FutureWarning) df = cudf.concat(mode_results, axis=1) @@ -7303,6 +7306,9 @@ def pct_change( if fill_method not in (no_default, None) or limit is not no_default: # Do not remove until pandas 3.0 support is added. + assert ( + PANDAS_LT_300 + ), "Need to drop after pandas-3.0 support is added." warnings.warn( "The 'fill_method' and 'limit' keywords in " f"{type(self).__name__}.pct_change are deprecated and will be " diff --git a/python/cudf/cudf/core/dtypes.py b/python/cudf/cudf/core/dtypes.py index f05758d6993..17d6d42618a 100644 --- a/python/cudf/cudf/core/dtypes.py +++ b/python/cudf/cudf/core/dtypes.py @@ -21,6 +21,7 @@ import cudf from cudf._typing import Dtype +from cudf.core._compat import PANDAS_LT_300 from cudf.core.abc import Serializable from cudf.core.buffer import Buffer from cudf.utils.docutils import doc_apply @@ -1035,6 +1036,7 @@ def is_categorical_dtype(obj): Whether or not the array-like or dtype is of a categorical dtype. """ # Do not remove until pandas 3.0 support is added. + assert PANDAS_LT_300, "Need to drop after pandas-3.0 support is added." warnings.warn( "is_categorical_dtype is deprecated and will be removed in a future " "version. Use isinstance(dtype, cudf.CategoricalDtype) instead", diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index ba802c47479..1f08abdc7fc 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -23,6 +23,7 @@ from cudf._typing import AggType, DataFrameOrSeries, MultiColumnAggType from cudf.api.extensions import no_default from cudf.api.types import is_bool_dtype, is_float_dtype, is_list_like +from cudf.core._compat import PANDAS_LT_300 from cudf.core.abc import Serializable from cudf.core.column.column import ColumnBase, as_column from cudf.core.column_accessor import ColumnAccessor @@ -2244,6 +2245,9 @@ def fillna( if method not in {"ffill", "bfill"}: raise ValueError("Method can only be of 'ffill', 'bfill'.") # Do not remove until pandas 3.0 support is added. + assert ( + PANDAS_LT_300 + ), "Need to drop after pandas-3.0 support is added." warnings.warn( f"{type(self).__name__}.fillna with 'method' is " "deprecated and will raise in a future version. " @@ -2374,6 +2378,9 @@ def pct_change( if fill_method not in (no_default, None) or limit is not no_default: # Do not remove until pandas 3.0 support is added. + assert ( + PANDAS_LT_300 + ), "Need to drop after pandas-3.0 support is added." warnings.warn( "The 'fill_method' keyword being not None and the 'limit' " f"keywords in {type(self).__name__}.pct_change are " diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 2bd4219997f..fa9e49baaa2 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -40,7 +40,7 @@ is_signed_integer_dtype, ) from cudf.core._base_index import BaseIndex -from cudf.core._compat import PANDAS_GE_200 +from cudf.core._compat import PANDAS_GE_200, PANDAS_LT_300 from cudf.core.column import ( CategoricalColumn, ColumnBase, @@ -1107,6 +1107,9 @@ def _concat(cls, objs): non_empties = [index for index in objs if len(index)] if len(objs) != len(non_empties): # Do not remove until pandas-3.0 support is added. + assert ( + PANDAS_LT_300 + ), "Need to drop after pandas-3.0 support is added." warnings.warn( "The behavior of array concatenation with empty entries is " "deprecated. In a future version, this will no longer exclude " diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index d7239dbcf2f..8c3276d7703 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -48,6 +48,7 @@ is_scalar, ) from cudf.core._base_index import BaseIndex +from cudf.core._compat import PANDAS_LT_300 from cudf.core.buffer import acquire_spill_lock from cudf.core.column import ColumnBase, as_column, full from cudf.core.column_accessor import ColumnAccessor @@ -2160,6 +2161,9 @@ def fillna( ): # noqa: D102 if method is not None: # Do not remove until pandas 3.0 support is added. + assert ( + PANDAS_LT_300 + ), "Need to drop after pandas-3.0 support is added." warnings.warn( f"{type(self).__name__}.fillna with 'method' is " "deprecated and will raise in a future version. " @@ -3389,6 +3393,7 @@ def first(self, offset): 2018-04-11 2 """ # Do not remove until pandas 3.0 support is added. + assert PANDAS_LT_300, "Need to drop after pandas-3.0 support is added." warnings.warn( "first is deprecated and will be removed in a future version. " "Please create a mask and filter using `.loc` instead", @@ -3441,6 +3446,7 @@ def last(self, offset): 2018-04-15 4 """ # Do not remove until pandas 3.0 support is added. + assert PANDAS_LT_300, "Need to drop after pandas-3.0 support is added." warnings.warn( "last is deprecated and will be removed in a future version. " "Please create a mask and filter using `.loc` instead", diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index 05ab1edfaba..2ea538d66a1 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -14,6 +14,7 @@ from cudf._lib.types import size_type_dtype from cudf._typing import Dtype from cudf.api.extensions import no_default +from cudf.core._compat import PANDAS_LT_300 from cudf.core.column import ColumnBase, as_column, column_empty_like from cudf.core.column.categorical import CategoricalColumn from cudf.utils.dtypes import min_unsigned_type @@ -324,6 +325,9 @@ def concat(objs, axis=0, join="outer", ignore_index=False, sort=None): any_empty = any(obj.empty for obj in objs) if any_empty: # Do not remove until pandas-3.0 support is added. + assert ( + PANDAS_LT_300 + ), "Need to drop after pandas-3.0 support is added." warnings.warn( "The behavior of array concatenation with empty entries is " "deprecated. In a future version, this will no longer exclude " diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 649b0688992..77ed7644f69 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -48,6 +48,7 @@ is_string_dtype, ) from cudf.core import indexing_utils +from cudf.core._compat import PANDAS_LT_300 from cudf.core.abc import Serializable from cudf.core.buffer import acquire_spill_lock from cudf.core.column import ( @@ -278,6 +279,9 @@ def __setitem__(self, key, value): value = value.astype(to_dtype) if to_dtype != self._frame._column.dtype: # Do not remove until pandas-3.0 support is added. + assert ( + PANDAS_LT_300 + ), "Need to drop after pandas-3.0 support is added." warnings.warn( f"Setting an item of incompatible dtype is deprecated " "and will raise in a future error of pandas. " @@ -388,10 +392,16 @@ def _loc_to_iloc(self, arg): arg.dtype ): # Do not remove until pandas 3.0 support is added. + assert ( + PANDAS_LT_300 + ), "Need to drop after pandas-3.0 support is added." warnings.warn(warn_msg, FutureWarning) return arg.value elif is_integer(arg): # Do not remove until pandas 3.0 support is added. + assert ( + PANDAS_LT_300 + ), "Need to drop after pandas-3.0 support is added." warnings.warn(warn_msg, FutureWarning) return arg try: @@ -3617,6 +3627,9 @@ def pct_change( ) if fill_method not in (no_default, None) or limit is not no_default: # Do not remove until pandas 3.0 support is added. + assert ( + PANDAS_LT_300 + ), "Need to drop after pandas-3.0 support is added." warnings.warn( "The 'fill_method' and 'limit' keywords in " f"{type(self).__name__}.pct_change are deprecated and will be " diff --git a/python/cudf/cudf/tests/test_array_ufunc.py b/python/cudf/cudf/tests/test_array_ufunc.py index d6b944ebeac..3ba0403d67c 100644 --- a/python/cudf/cudf/tests/test_array_ufunc.py +++ b/python/cudf/cudf/tests/test_array_ufunc.py @@ -7,12 +7,10 @@ import cupy as cp import numpy as np -import pandas as pd import pytest -from packaging import version import cudf -from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210 +from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210, PANDAS_LT_300 from cudf.testing._utils import ( assert_eq, expect_warning_if, @@ -84,8 +82,7 @@ def test_ufunc_index(request, ufunc): ) request.applymarker( pytest.mark.xfail( - condition=fname == "matmul" - and version.parse(pd.__version__) < version.parse("3.0"), + condition=fname == "matmul" and PANDAS_LT_300, reason="Fixed by https://github.com/pandas-dev/pandas/pull/57079", ) ) diff --git a/python/cudf/cudf/tests/test_timedelta.py b/python/cudf/cudf/tests/test_timedelta.py index 7cae2f3a30f..18fe1700e25 100644 --- a/python/cudf/cudf/tests/test_timedelta.py +++ b/python/cudf/cudf/tests/test_timedelta.py @@ -696,10 +696,7 @@ def test_timedelta_dt_components(data, dtype): @pytest.mark.parametrize( "data", - _TIMEDELTA_DATA_NON_OVERFLOW, - # TODO-PANDAS-2.0: Replace above with `_TIMEDELTA_DATA` - # after the following issue is fixed: - # https://github.com/pandas-dev/pandas/issues/52386 + _TIMEDELTA_DATA, ) @pytest.mark.parametrize("dtype", utils.TIMEDELTA_TYPES) def test_timedelta_dt_properties(data, dtype): diff --git a/python/cudf/cudf/utils/ioutils.py b/python/cudf/cudf/utils/ioutils.py index 1c5bde89800..feb02bac60d 100644 --- a/python/cudf/cudf/utils/ioutils.py +++ b/python/cudf/cudf/utils/ioutils.py @@ -15,6 +15,7 @@ from pyarrow import PythonFile as ArrowPythonFile from pyarrow.lib import NativeFile +from cudf.core._compat import PANDAS_LT_300 from cudf.utils.docutils import docfmt_partial try: @@ -1683,6 +1684,9 @@ def get_reader_filepath_or_buffer( if fs is None: if warn_on_raw_text_input: # Do not remove until pandas 3.0 support is added. + assert ( + PANDAS_LT_300 + ), "Need to drop after pandas-3.0 support is added." warnings.warn( f"Passing literal {warn_meta[0]} to {warn_meta[1]} is " "deprecated and will be removed in a future version. " @@ -1704,6 +1708,9 @@ def get_reader_filepath_or_buffer( ) elif warn_on_raw_text_input: # Do not remove until pandas 3.0 support is added. + assert ( + PANDAS_LT_300 + ), "Need to drop after pandas-3.0 support is added." warnings.warn( f"Passing literal {warn_meta[0]} to {warn_meta[1]} is " "deprecated and will be removed in a future version. " @@ -1713,6 +1720,9 @@ def get_reader_filepath_or_buffer( ) elif warn_on_raw_text_input: # Do not remove until pandas 3.0 support is added. + assert ( + PANDAS_LT_300 + ), "Need to drop after pandas-3.0 support is added." warnings.warn( f"Passing literal {warn_meta[0]} to {warn_meta[1]} is " "deprecated and will be removed in a future version. " From 30f873d86cbb0f7c9536acd5530ae2b7f9d7b68e Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 30 Jan 2024 09:11:13 +0000 Subject: [PATCH 161/384] Address all dask_cudf reviews --- python/dask_cudf/dask_cudf/io/tests/test_csv.py | 12 ++++++++++++ python/dask_cudf/dask_cudf/tests/test_core.py | 12 ------------ python/dask_cudf/dask_cudf/tests/test_groupby.py | 3 +-- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/python/dask_cudf/dask_cudf/io/tests/test_csv.py b/python/dask_cudf/dask_cudf/io/tests/test_csv.py index 987fcf6b4ae..a35a9f1be48 100644 --- a/python/dask_cudf/dask_cudf/io/tests/test_csv.py +++ b/python/dask_cudf/dask_cudf/io/tests/test_csv.py @@ -233,6 +233,18 @@ def test_read_csv_skipfooter(csv_end_bad_lines): dd.assert_eq(ddf_cpu, ddf_gpu, check_dtype=False) +def test_read_csv_skipfooter_arrow_string_fail(request, csv_end_bad_lines): + request.applymarker( + pytest.mark.xfail( + reason="https://github.com/rapidsai/cudf/issues/14915", + ) + ) + ddf_cpu = dd.read_csv(csv_end_bad_lines, skipfooter=3).compute() + ddf_gpu = dask_cudf.read_csv(csv_end_bad_lines, skipfooter=3).compute() + + dd.assert_eq(ddf_cpu, ddf_gpu, check_dtype=False) + + def test_read_csv_skipfooter_error(csv_end_bad_lines): with pytest.raises(ValueError): dask_cudf.read_csv( diff --git a/python/dask_cudf/dask_cudf/tests/test_core.py b/python/dask_cudf/dask_cudf/tests/test_core.py index 552d800e2dd..afe2a050695 100644 --- a/python/dask_cudf/dask_cudf/tests/test_core.py +++ b/python/dask_cudf/dask_cudf/tests/test_core.py @@ -940,15 +940,3 @@ def test_categorical_dtype_round_trip(): actual = ds.compute() expected = pds.compute() assert actual.dtype.ordered == expected.dtype.ordered - - -def test_object_to_string_fail(request): - request.applymarker( - pytest.mark.xfail( - reason="https://github.com/rapidsai/cudf/issues/14915", - ) - ) - s = cudf.Series(["a", "b", "c"] * 10) - ds = dgd.from_cudf(s, npartitions=2) - pds = dd.from_pandas(s.to_pandas(), npartitions=2) - dd.assert_eq(ds.sort_values(), pds.sort_values()) diff --git a/python/dask_cudf/dask_cudf/tests/test_groupby.py b/python/dask_cudf/dask_cudf/tests/test_groupby.py index cef8bdacace..c8cc6e65fa5 100644 --- a/python/dask_cudf/dask_cudf/tests/test_groupby.py +++ b/python/dask_cudf/dask_cudf/tests/test_groupby.py @@ -610,8 +610,7 @@ def test_groupby_agg_params(npartitions, split_every, split_out, as_index): if as_index: # Groupby columns became the index. # Sorting the index should not change anything. - with dask.config.set({"dataframe.convert-string": False}): - dd.assert_eq(gf.index, gf.sort_index().index) + dd.assert_eq(gf.index.to_frame(), gf.sort_index().index.to_frame()) else: # Groupby columns are did NOT become the index. # Sorting by these columns should not change anything. From 2b05b59720ea0d25566b844a6d5cfd0afef74ab3 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 30 Jan 2024 04:23:33 -0600 Subject: [PATCH 162/384] Fix custreamz pytests to test on float64 types (#14934) This PR passes types to empty dataframe construction because reductions were being performed on float64 types and now empty column default type is object. From 2e307535554664180fc06de4805dbe0a297bbdaf Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 30 Jan 2024 12:40:36 +0000 Subject: [PATCH 163/384] Remaining custreamz test fix --- python/custreamz/custreamz/tests/test_dataframes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/custreamz/custreamz/tests/test_dataframes.py b/python/custreamz/custreamz/tests/test_dataframes.py index 7ce398c7617..779560a394a 100644 --- a/python/custreamz/custreamz/tests/test_dataframes.py +++ b/python/custreamz/custreamz/tests/test_dataframes.py @@ -749,7 +749,7 @@ def on_old(self, state, new): def test_groupby_aggregate_with_start_state(stream): - example = cudf.DataFrame({"name": [], "amount": []}) + example = cudf.DataFrame({"name": [], "amount": []}, dtype="float64") sdf = DataFrame(stream, example=example).groupby(["name"]) output0 = sdf.amount.sum(start=None).stream.gather().sink_to_list() output1 = ( @@ -817,7 +817,7 @@ def test_reductions_with_start_state(stream): def test_rolling_aggs_with_start_state(stream): - example = cudf.DataFrame({"name": [], "amount": []}) + example = cudf.DataFrame({"name": [], "amount": []}, dtype="float64") sdf = DataFrame(stream, example=example) output0 = ( sdf.rolling(2, with_state=True, start=()) From 1937252684b2589781e6a13075ce9458b649d40e Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 30 Jan 2024 12:55:29 +0000 Subject: [PATCH 164/384] Remove missing docstrings --- docs/cudf/source/user_guide/api_docs/groupby.rst | 2 -- docs/cudf/source/user_guide/api_docs/series.rst | 1 - 2 files changed, 3 deletions(-) diff --git a/docs/cudf/source/user_guide/api_docs/groupby.rst b/docs/cudf/source/user_guide/api_docs/groupby.rst index 26dd9bb354b..80811efa33f 100644 --- a/docs/cudf/source/user_guide/api_docs/groupby.rst +++ b/docs/cudf/source/user_guide/api_docs/groupby.rst @@ -80,7 +80,6 @@ application to columns of a specific data type. .. autosummary:: :toctree: api/ - DataFrameGroupBy.backfill DataFrameGroupBy.bfill DataFrameGroupBy.count DataFrameGroupBy.cumcount @@ -94,7 +93,6 @@ application to columns of a specific data type. DataFrameGroupBy.idxmax DataFrameGroupBy.idxmin DataFrameGroupBy.nunique - DataFrameGroupBy.pad DataFrameGroupBy.quantile DataFrameGroupBy.shift DataFrameGroupBy.size diff --git a/docs/cudf/source/user_guide/api_docs/series.rst b/docs/cudf/source/user_guide/api_docs/series.rst index 4672db04eb3..28931d567b4 100644 --- a/docs/cudf/source/user_guide/api_docs/series.rst +++ b/docs/cudf/source/user_guide/api_docs/series.rst @@ -158,7 +158,6 @@ Computations / descriptive stats Series.unique Series.nunique Series.is_unique - Series.is_monotonic Series.is_monotonic_increasing Series.is_monotonic_decreasing Series.value_counts From 6d07cc2d0cbcf7913c5a4bf3a4d20ea82dcef8e4 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 30 Jan 2024 14:19:21 +0000 Subject: [PATCH 165/384] Fix another custreamz test --- python/custreamz/custreamz/tests/test_dataframes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/custreamz/custreamz/tests/test_dataframes.py b/python/custreamz/custreamz/tests/test_dataframes.py index 779560a394a..bae4b051cae 100644 --- a/python/custreamz/custreamz/tests/test_dataframes.py +++ b/python/custreamz/custreamz/tests/test_dataframes.py @@ -771,7 +771,7 @@ def test_groupby_aggregate_with_start_state(stream): assert assert_eq(output1[0][1].reset_index(), out_df1) assert assert_eq(output2[0].reset_index(), out_df2) - example = cudf.DataFrame({"name": [], "amount": []}) + example = cudf.DataFrame({"name": [], "amount": []}, dtype="float64") sdf = DataFrame(stream, example=example).groupby(["name"]) output3 = sdf.amount.sum(start=output0[0]).stream.gather().sink_to_list() output4 = ( From 71d87d53632ff03a7fa92901c8d066ffeab3847a Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 30 Jan 2024 08:20:41 -0800 Subject: [PATCH 166/384] Add back reftarget change for cudf.Index --- docs/cudf/source/conf.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/cudf/source/conf.py b/docs/cudf/source/conf.py index 035ee586822..0100c331e72 100644 --- a/docs/cudf/source/conf.py +++ b/docs/cudf/source/conf.py @@ -454,6 +454,12 @@ def on_missing_reference(app, env, node, contnode): _prefixed_domain_objects[f"{prefix}{name}"] = name reftarget = node.get("reftarget") + if reftarget == "cudf.core.index.Index": + # We don't exposed docs for `cudf.core.index.Index` + # hence we would want the docstring & mypy references to + # use `cudf.Index` + node["reftarget"] = "cudf.Index" + return contnode if "namespacecudf" in reftarget: node["reftarget"] = "cudf" return contnode From 3438af0e3aa2ae7b6b16bd8e5a0ff0141dd633c7 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 30 Jan 2024 17:33:44 +0000 Subject: [PATCH 167/384] Revert "Add back reftarget change for cudf.Index" This reverts commit 71d87d53632ff03a7fa92901c8d066ffeab3847a. --- docs/cudf/source/conf.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/cudf/source/conf.py b/docs/cudf/source/conf.py index 0100c331e72..035ee586822 100644 --- a/docs/cudf/source/conf.py +++ b/docs/cudf/source/conf.py @@ -454,12 +454,6 @@ def on_missing_reference(app, env, node, contnode): _prefixed_domain_objects[f"{prefix}{name}"] = name reftarget = node.get("reftarget") - if reftarget == "cudf.core.index.Index": - # We don't exposed docs for `cudf.core.index.Index` - # hence we would want the docstring & mypy references to - # use `cudf.Index` - node["reftarget"] = "cudf.Index" - return contnode if "namespacecudf" in reftarget: node["reftarget"] = "cudf" return contnode From ffa473e4f6b7515cd78e9cd5f9bcb8537e32ae62 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 30 Jan 2024 18:11:19 +0000 Subject: [PATCH 168/384] Move abs to IndexedFrame --- python/cudf/cudf/core/frame.py | 26 -------------------------- python/cudf/cudf/core/indexed_frame.py | 26 ++++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index a1c5cf40024..9342e9439c3 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -1549,32 +1549,6 @@ def _get_sorted_inds( stable=True, ) - @_cudf_nvtx_annotate - def abs(self): - """ - Return a Series/DataFrame with absolute numeric value of each element. - - This function only applies to elements that are all numeric. - - Returns - ------- - DataFrame/Series - Absolute value of each element. - - Examples - -------- - Absolute numeric values in a Series - - >>> s = cudf.Series([-1.10, 2, -3.33, 4]) - >>> s.abs() - 0 1.10 - 1 2.00 - 2 3.33 - 3 4.00 - dtype: float64 - """ - return self._unaryop("abs") - @_cudf_nvtx_annotate def _is_sorted(self, ascending=None, null_position=None): """ diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 8c3276d7703..52fc5b3808e 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -987,6 +987,32 @@ def clip(self, lower=None, upper=None, inplace=False, axis=1): output._copy_type_metadata(self, include_index=False) return self._mimic_inplace(output, inplace=inplace) + @_cudf_nvtx_annotate + def abs(self): + """ + Return a Series/DataFrame with absolute numeric value of each element. + + This function only applies to elements that are all numeric. + + Returns + ------- + DataFrame/Series + Absolute value of each element. + + Examples + -------- + Absolute numeric values in a Series + + >>> s = cudf.Series([-1.10, 2, -3.33, 4]) + >>> s.abs() + 0 1.10 + 1 2.00 + 2 3.33 + 3 4.00 + dtype: float64 + """ + return self._unaryop("abs") + def _copy_type_metadata( self, other: Self, From abcd15d1bcfaed8fda2180fae4ecc3b5dd325f8c Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 30 Jan 2024 18:15:59 +0000 Subject: [PATCH 169/384] Move head and tail to IndexedFrame --- python/cudf/cudf/core/frame.py | 227 +------------------------ python/cudf/cudf/core/indexed_frame.py | 225 ++++++++++++++++++++++++ 2 files changed, 226 insertions(+), 226 deletions(-) diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index 9342e9439c3..ae4f6180eec 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -46,7 +46,7 @@ from cudf.core.window import Rolling from cudf.utils import ioutils from cudf.utils.docutils import copy_docstring -from cudf.utils.dtypes import can_convert_to_column, find_common_type +from cudf.utils.dtypes import find_common_type from cudf.utils.nvtx_annotation import _cudf_nvtx_annotate from cudf.utils.utils import _array_ufunc, _warn_no_dask_cudf @@ -1745,121 +1745,6 @@ def _apply_cupy_ufunc_to_operands( data[i][name] = as_column(out).set_mask(mask) return data - @_cudf_nvtx_annotate - def dot(self, other, reflect=False): - """ - Get dot product of frame and other, (binary operator `dot`). - - Among flexible wrappers (`add`, `sub`, `mul`, `div`, `mod`, `pow`, - `dot`) to arithmetic operators: `+`, `-`, `*`, `/`, `//`, `%`, `**`, - `@`. - - Parameters - ---------- - other : Sequence, Series, or DataFrame - Any multiple element data structure, or list-like object. - reflect : bool, default False - If ``True``, swap the order of the operands. See - https://docs.python.org/3/reference/datamodel.html#object.__ror__ - for more information on when this is necessary. - - Returns - ------- - scalar, Series, or DataFrame - The result of the operation. - - Examples - -------- - >>> import cudf - >>> df = cudf.DataFrame([[1, 2, 3, 4], - ... [5, 6, 7, 8]]) - >>> df @ df.T - 0 1 - 0 30 70 - 1 70 174 - >>> s = cudf.Series([1, 1, 1, 1]) - >>> df @ s - 0 10 - 1 26 - dtype: int64 - >>> [1, 2, 3, 4] @ s - 10 - """ - # TODO: This function does not currently support nulls. - lhs = self.values - result_index = None - result_cols = None - if isinstance(self, cudf.Series) and isinstance( - other, (cudf.Series, cudf.DataFrame) - ): - common = self.index.union(other.index) - if len(common) > len(self.index) or len(common) > len(other.index): - raise ValueError("matrices are not aligned") - - lhs = self.reindex(index=common, copy=False).values - rhs = other.reindex(index=common, copy=False).values - if isinstance(other, cudf.DataFrame): - result_index = other._data.to_pandas_index() - elif isinstance(self, cudf.DataFrame) and isinstance( - other, (cudf.Series, cudf.DataFrame) - ): - common = self._data.to_pandas_index().union( - other.index.to_pandas() - ) - if len(common) > len(self._data.names) or len(common) > len( - other.index - ): - raise ValueError("matrices are not aligned") - - lhs = self.reindex(columns=common, copy=False) - result_index = lhs.index - - rhs = other.reindex(index=common, copy=False).values - lhs = lhs.values - if isinstance(other, cudf.DataFrame): - result_cols = other._data.to_pandas_index() - - elif isinstance( - other, (cupy.ndarray, np.ndarray) - ) or can_convert_to_column(other): - rhs = cupy.asarray(other) - else: - # TODO: This should raise an exception, not return NotImplemented, - # but __matmul__ relies on the current behavior. We should either - # move this implementation to __matmul__ and call it from here - # (checking for NotImplemented and raising NotImplementedError if - # that's what's returned), or __matmul__ should catch a - # NotImplementedError from here and return NotImplemented. The - # latter feels cleaner (putting the implementation in this method - # rather than in the operator) but will be slower in the (highly - # unlikely) case that we're multiplying a cudf object with another - # type of object that somehow supports this behavior. - return NotImplemented - if reflect: - lhs, rhs = rhs, lhs - - result = lhs.dot(rhs) - if len(result.shape) == 1: - return cudf.Series( - result, - index=self.index if result_index is None else result_index, - ) - if len(result.shape) == 2: - return cudf.DataFrame( - result, - index=self.index if result_index is None else result_index, - columns=result_cols, - ) - return result.item() - - @_cudf_nvtx_annotate - def __matmul__(self, other): - return self.dot(other) - - @_cudf_nvtx_annotate - def __rmatmul__(self, other): - return self.dot(other, reflect=True) - # Unary logical operators @_cudf_nvtx_annotate def __neg__(self): @@ -2629,116 +2514,6 @@ def __deepcopy__(self, memo): def __copy__(self): return self.copy(deep=False) - @_cudf_nvtx_annotate - def head(self, n=5): - """ - Return the first `n` rows. - This function returns the first `n` rows for the object based - on position. It is useful for quickly testing if your object - has the right type of data in it. - For negative values of `n`, this function returns all rows except - the last `n` rows, equivalent to ``df[:-n]``. - - Parameters - ---------- - n : int, default 5 - Number of rows to select. - - Returns - ------- - DataFrame or Series - The first `n` rows of the caller object. - - Examples - -------- - **Series** - - >>> ser = cudf.Series(['alligator', 'bee', 'falcon', - ... 'lion', 'monkey', 'parrot', 'shark', 'whale', 'zebra']) - >>> ser - 0 alligator - 1 bee - 2 falcon - 3 lion - 4 monkey - 5 parrot - 6 shark - 7 whale - 8 zebra - dtype: object - - Viewing the first 5 lines - - >>> ser.head() - 0 alligator - 1 bee - 2 falcon - 3 lion - 4 monkey - dtype: object - - Viewing the first `n` lines (three in this case) - - >>> ser.head(3) - 0 alligator - 1 bee - 2 falcon - dtype: object - - For negative values of `n` - - >>> ser.head(-3) - 0 alligator - 1 bee - 2 falcon - 3 lion - 4 monkey - 5 parrot - dtype: object - - **DataFrame** - - >>> df = cudf.DataFrame() - >>> df['key'] = [0, 1, 2, 3, 4] - >>> df['val'] = [float(i + 10) for i in range(5)] # insert column - >>> df.head(2) - key val - 0 0 10.0 - 1 1 11.0 - """ - return self.iloc[:n] - - @_cudf_nvtx_annotate - def tail(self, n=5): - """ - Returns the last n rows as a new DataFrame or Series - - Examples - -------- - **DataFrame** - - >>> import cudf - >>> df = cudf.DataFrame() - >>> df['key'] = [0, 1, 2, 3, 4] - >>> df['val'] = [float(i + 10) for i in range(5)] # insert column - >>> df.tail(2) - key val - 3 3 13.0 - 4 4 14.0 - - **Series** - - >>> import cudf - >>> ser = cudf.Series([4, 3, 2, 1, 0]) - >>> ser.tail(2) - 3 1 - 4 0 - """ - if n == 0: - return self.iloc[0:0] - - return self.iloc[-n:] - @_cudf_nvtx_annotate @copy_docstring(Rolling) def rolling( diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 52fc5b3808e..efa75772053 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -1013,6 +1013,231 @@ def abs(self): """ return self._unaryop("abs") + @_cudf_nvtx_annotate + def dot(self, other, reflect=False): + """ + Get dot product of frame and other, (binary operator `dot`). + + Among flexible wrappers (`add`, `sub`, `mul`, `div`, `mod`, `pow`, + `dot`) to arithmetic operators: `+`, `-`, `*`, `/`, `//`, `%`, `**`, + `@`. + + Parameters + ---------- + other : Sequence, Series, or DataFrame + Any multiple element data structure, or list-like object. + reflect : bool, default False + If ``True``, swap the order of the operands. See + https://docs.python.org/3/reference/datamodel.html#object.__ror__ + for more information on when this is necessary. + + Returns + ------- + scalar, Series, or DataFrame + The result of the operation. + + Examples + -------- + >>> import cudf + >>> df = cudf.DataFrame([[1, 2, 3, 4], + ... [5, 6, 7, 8]]) + >>> df @ df.T + 0 1 + 0 30 70 + 1 70 174 + >>> s = cudf.Series([1, 1, 1, 1]) + >>> df @ s + 0 10 + 1 26 + dtype: int64 + >>> [1, 2, 3, 4] @ s + 10 + """ + # TODO: This function does not currently support nulls. + lhs = self.values + result_index = None + result_cols = None + if isinstance(self, cudf.Series) and isinstance( + other, (cudf.Series, cudf.DataFrame) + ): + common = self.index.union(other.index) + if len(common) > len(self.index) or len(common) > len(other.index): + raise ValueError("matrices are not aligned") + + lhs = self.reindex(index=common, copy=False).values + rhs = other.reindex(index=common, copy=False).values + if isinstance(other, cudf.DataFrame): + result_index = other._data.to_pandas_index() + elif isinstance(self, cudf.DataFrame) and isinstance( + other, (cudf.Series, cudf.DataFrame) + ): + common = self._data.to_pandas_index().union( + other.index.to_pandas() + ) + if len(common) > len(self._data.names) or len(common) > len( + other.index + ): + raise ValueError("matrices are not aligned") + + lhs = self.reindex(columns=common, copy=False) + result_index = lhs.index + + rhs = other.reindex(index=common, copy=False).values + lhs = lhs.values + if isinstance(other, cudf.DataFrame): + result_cols = other._data.to_pandas_index() + + elif isinstance( + other, (cp.ndarray, np.ndarray) + ) or cudf.utils.dtypes.can_convert_to_column(other): + rhs = cp.asarray(other) + else: + # TODO: This should raise an exception, not return NotImplemented, + # but __matmul__ relies on the current behavior. We should either + # move this implementation to __matmul__ and call it from here + # (checking for NotImplemented and raising NotImplementedError if + # that's what's returned), or __matmul__ should catch a + # NotImplementedError from here and return NotImplemented. The + # latter feels cleaner (putting the implementation in this method + # rather than in the operator) but will be slower in the (highly + # unlikely) case that we're multiplying a cudf object with another + # type of object that somehow supports this behavior. + return NotImplemented + if reflect: + lhs, rhs = rhs, lhs + + result = lhs.dot(rhs) + if len(result.shape) == 1: + return cudf.Series( + result, + index=self.index if result_index is None else result_index, + ) + if len(result.shape) == 2: + return cudf.DataFrame( + result, + index=self.index if result_index is None else result_index, + columns=result_cols, + ) + return result.item() + + @_cudf_nvtx_annotate + def __matmul__(self, other): + return self.dot(other) + + @_cudf_nvtx_annotate + def __rmatmul__(self, other): + return self.dot(other, reflect=True) + + @_cudf_nvtx_annotate + def head(self, n=5): + """ + Return the first `n` rows. + This function returns the first `n` rows for the object based + on position. It is useful for quickly testing if your object + has the right type of data in it. + For negative values of `n`, this function returns all rows except + the last `n` rows, equivalent to ``df[:-n]``. + + Parameters + ---------- + n : int, default 5 + Number of rows to select. + + Returns + ------- + DataFrame or Series + The first `n` rows of the caller object. + + Examples + -------- + **Series** + + >>> ser = cudf.Series(['alligator', 'bee', 'falcon', + ... 'lion', 'monkey', 'parrot', 'shark', 'whale', 'zebra']) + >>> ser + 0 alligator + 1 bee + 2 falcon + 3 lion + 4 monkey + 5 parrot + 6 shark + 7 whale + 8 zebra + dtype: object + + Viewing the first 5 lines + + >>> ser.head() + 0 alligator + 1 bee + 2 falcon + 3 lion + 4 monkey + dtype: object + + Viewing the first `n` lines (three in this case) + + >>> ser.head(3) + 0 alligator + 1 bee + 2 falcon + dtype: object + + For negative values of `n` + + >>> ser.head(-3) + 0 alligator + 1 bee + 2 falcon + 3 lion + 4 monkey + 5 parrot + dtype: object + + **DataFrame** + + >>> df = cudf.DataFrame() + >>> df['key'] = [0, 1, 2, 3, 4] + >>> df['val'] = [float(i + 10) for i in range(5)] # insert column + >>> df.head(2) + key val + 0 0 10.0 + 1 1 11.0 + """ + return self.iloc[:n] + + @_cudf_nvtx_annotate + def tail(self, n=5): + """ + Returns the last n rows as a new DataFrame or Series + + Examples + -------- + **DataFrame** + + >>> import cudf + >>> df = cudf.DataFrame() + >>> df['key'] = [0, 1, 2, 3, 4] + >>> df['val'] = [float(i + 10) for i in range(5)] # insert column + >>> df.tail(2) + key val + 3 3 13.0 + 4 4 14.0 + + **Series** + + >>> import cudf + >>> ser = cudf.Series([4, 3, 2, 1, 0]) + >>> ser.tail(2) + 3 1 + 4 0 + """ + if n == 0: + return self.iloc[0:0] + + return self.iloc[-n:] + def _copy_type_metadata( self, other: Self, From 50d287f97bec818a6027b55727ab1d8e538fd4ce Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 30 Jan 2024 18:17:53 +0000 Subject: [PATCH 170/384] Move isnull (alias) to IndexedFrame --- python/cudf/cudf/core/frame.py | 3 --- python/cudf/cudf/core/indexed_frame.py | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index ae4f6180eec..5d33fbf70c3 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -1234,9 +1234,6 @@ def isna(self): data_columns = (col.isnull() for col in self._columns) return self._from_data_like_self(zip(self._column_names, data_columns)) - # Alias for isna - isnull = isna - @_cudf_nvtx_annotate def notna(self): """ diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index efa75772053..6443391bfe1 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -1238,6 +1238,9 @@ def tail(self, n=5): return self.iloc[-n:] + # Alias for isna + isnull = Frame.isna + def _copy_type_metadata( self, other: Self, From 0013faa416fe3032dc6f2242204ef282bc8036f9 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 30 Jan 2024 18:18:38 +0000 Subject: [PATCH 171/384] Move kurtosis and skew to IndexedFrame --- python/cudf/cudf/core/frame.py | 119 ------------------------- python/cudf/cudf/core/indexed_frame.py | 119 +++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 119 deletions(-) diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index 5d33fbf70c3..3a7545c93c0 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -2161,125 +2161,6 @@ def var( **kwargs, ) - @_cudf_nvtx_annotate - def kurtosis(self, axis=0, skipna=True, numeric_only=False, **kwargs): - """ - Return Fisher's unbiased kurtosis of a sample. - - Kurtosis obtained using Fisher's definition of - kurtosis (kurtosis of normal == 0.0). Normalized by N-1. - - Parameters - ---------- - axis: {index (0), columns(1)} - Axis for the function to be applied on. - skipna: bool, default True - Exclude NA/null values when computing the result. - numeric_only : bool, default False - If True, includes only float, int, boolean columns. - If False, will raise error in-case there are - non-numeric columns. - - Returns - ------- - Series or scalar - - Examples - -------- - **Series** - - >>> import cudf - >>> series = cudf.Series([1, 2, 3, 4]) - >>> series.kurtosis() - -1.1999999999999904 - - **DataFrame** - - >>> import cudf - >>> df = cudf.DataFrame({'a': [1, 2, 3, 4], 'b': [7, 8, 9, 10]}) - >>> df.kurt() - a -1.2 - b -1.2 - dtype: float64 - - .. pandas-compat:: - **DataFrame.kurtosis** - - Parameters currently not supported are `level` and `numeric_only` - """ - if axis not in (0, "index", None, no_default): - raise NotImplementedError("Only axis=0 is currently supported.") - - return self._reduce( - "kurtosis", - axis=axis, - skipna=skipna, - numeric_only=numeric_only, - **kwargs, - ) - - # Alias for kurtosis. - kurt = kurtosis - - @_cudf_nvtx_annotate - def skew(self, axis=0, skipna=True, numeric_only=False, **kwargs): - """ - Return unbiased Fisher-Pearson skew of a sample. - - Parameters - ---------- - skipna: bool, default True - Exclude NA/null values when computing the result. - numeric_only : bool, default False - If True, includes only float, int, boolean columns. - If False, will raise error in-case there are - non-numeric columns. - - Returns - ------- - Series - - Examples - -------- - **Series** - - >>> import cudf - >>> series = cudf.Series([1, 2, 3, 4, 5, 6, 6]) - >>> series - 0 1 - 1 2 - 2 3 - 3 4 - 4 5 - 5 6 - 6 6 - dtype: int64 - - **DataFrame** - - >>> import cudf - >>> df = cudf.DataFrame({'a': [3, 2, 3, 4], 'b': [7, 8, 10, 10]}) - >>> df.skew() - a 0.00000 - b -0.37037 - dtype: float64 - - .. pandas-compat:: - **DataFrame.skew, Series.skew, Frame.skew** - - The `axis` parameter is not currently supported. - """ - if axis not in (0, "index", None, no_default): - raise NotImplementedError("Only axis=0 is currently supported.") - - return self._reduce( - "skew", - axis=axis, - skipna=skipna, - numeric_only=numeric_only, - **kwargs, - ) - @_cudf_nvtx_annotate def all(self, axis=0, skipna=True, **kwargs): """ diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 6443391bfe1..d63921c2c68 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -1241,6 +1241,125 @@ def tail(self, n=5): # Alias for isna isnull = Frame.isna + @_cudf_nvtx_annotate + def kurtosis(self, axis=0, skipna=True, numeric_only=False, **kwargs): + """ + Return Fisher's unbiased kurtosis of a sample. + + Kurtosis obtained using Fisher's definition of + kurtosis (kurtosis of normal == 0.0). Normalized by N-1. + + Parameters + ---------- + axis: {index (0), columns(1)} + Axis for the function to be applied on. + skipna: bool, default True + Exclude NA/null values when computing the result. + numeric_only : bool, default False + If True, includes only float, int, boolean columns. + If False, will raise error in-case there are + non-numeric columns. + + Returns + ------- + Series or scalar + + Examples + -------- + **Series** + + >>> import cudf + >>> series = cudf.Series([1, 2, 3, 4]) + >>> series.kurtosis() + -1.1999999999999904 + + **DataFrame** + + >>> import cudf + >>> df = cudf.DataFrame({'a': [1, 2, 3, 4], 'b': [7, 8, 9, 10]}) + >>> df.kurt() + a -1.2 + b -1.2 + dtype: float64 + + .. pandas-compat:: + **DataFrame.kurtosis** + + Parameters currently not supported are `level` and `numeric_only` + """ + if axis not in (0, "index", None, no_default): + raise NotImplementedError("Only axis=0 is currently supported.") + + return self._reduce( + "kurtosis", + axis=axis, + skipna=skipna, + numeric_only=numeric_only, + **kwargs, + ) + + # Alias for kurtosis. + kurt = kurtosis + + @_cudf_nvtx_annotate + def skew(self, axis=0, skipna=True, numeric_only=False, **kwargs): + """ + Return unbiased Fisher-Pearson skew of a sample. + + Parameters + ---------- + skipna: bool, default True + Exclude NA/null values when computing the result. + numeric_only : bool, default False + If True, includes only float, int, boolean columns. + If False, will raise error in-case there are + non-numeric columns. + + Returns + ------- + Series + + Examples + -------- + **Series** + + >>> import cudf + >>> series = cudf.Series([1, 2, 3, 4, 5, 6, 6]) + >>> series + 0 1 + 1 2 + 2 3 + 3 4 + 4 5 + 5 6 + 6 6 + dtype: int64 + + **DataFrame** + + >>> import cudf + >>> df = cudf.DataFrame({'a': [3, 2, 3, 4], 'b': [7, 8, 10, 10]}) + >>> df.skew() + a 0.00000 + b -0.37037 + dtype: float64 + + .. pandas-compat:: + **DataFrame.skew, Series.skew, Frame.skew** + + The `axis` parameter is not currently supported. + """ + if axis not in (0, "index", None, no_default): + raise NotImplementedError("Only axis=0 is currently supported.") + + return self._reduce( + "skew", + axis=axis, + skipna=skipna, + numeric_only=numeric_only, + **kwargs, + ) + def _copy_type_metadata( self, other: Self, From 11ab9e818798a6f3d8d5def1beeb287e80cf63e8 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 30 Jan 2024 18:23:24 +0000 Subject: [PATCH 172/384] Move mask to IndexedFrame --- python/cudf/cudf/core/frame.py | 62 -------------------------- python/cudf/cudf/core/indexed_frame.py | 62 ++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 62 deletions(-) diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index 3a7545c93c0..6a1f6b76302 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -615,68 +615,6 @@ def where(self, cond, other=None, inplace: bool = False) -> Optional[Self]: """ raise NotImplementedError - @_cudf_nvtx_annotate - def mask(self, cond, other=None, inplace: bool = False) -> Optional[Self]: - """ - Replace values where the condition is True. - - Parameters - ---------- - cond : bool Series/DataFrame, array-like - Where cond is False, keep the original value. - Where True, replace with corresponding value from other. - Callables are not supported. - other: scalar, list of scalars, Series/DataFrame - Entries where cond is True are replaced with - corresponding value from other. Callables are not - supported. Default is None. - - DataFrame expects only Scalar or array like with scalars or - dataframe with same dimension as self. - - Series expects only scalar or series like with same length - inplace : bool, default False - Whether to perform the operation in place on the data. - - Returns - ------- - Same type as caller - - Examples - -------- - >>> import cudf - >>> df = cudf.DataFrame({"A":[1, 4, 5], "B":[3, 5, 8]}) - >>> df.mask(df % 2 == 0, [-1, -1]) - A B - 0 1 3 - 1 -1 5 - 2 5 -1 - - >>> ser = cudf.Series([4, 3, 2, 1, 0]) - >>> ser.mask(ser > 2, 10) - 0 10 - 1 10 - 2 2 - 3 1 - 4 0 - dtype: int64 - >>> ser.mask(ser > 2) - 0 - 1 - 2 2 - 3 1 - 4 0 - dtype: int64 - """ - - if not hasattr(cond, "__invert__"): - # We Invert `cond` below and call `where`, so - # making sure the object supports - # `~`(inversion) operator or `__invert__` method - cond = cupy.asarray(cond) - - return self.where(cond=~cond, other=other, inplace=inplace) - @_cudf_nvtx_annotate def pipe(self, func, *args, **kwargs): """ diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index d63921c2c68..c08571a4752 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -1360,6 +1360,68 @@ def skew(self, axis=0, skipna=True, numeric_only=False, **kwargs): **kwargs, ) + @_cudf_nvtx_annotate + def mask(self, cond, other=None, inplace: bool = False) -> Optional[Self]: + """ + Replace values where the condition is True. + + Parameters + ---------- + cond : bool Series/DataFrame, array-like + Where cond is False, keep the original value. + Where True, replace with corresponding value from other. + Callables are not supported. + other: scalar, list of scalars, Series/DataFrame + Entries where cond is True are replaced with + corresponding value from other. Callables are not + supported. Default is None. + + DataFrame expects only Scalar or array like with scalars or + dataframe with same dimension as self. + + Series expects only scalar or series like with same length + inplace : bool, default False + Whether to perform the operation in place on the data. + + Returns + ------- + Same type as caller + + Examples + -------- + >>> import cudf + >>> df = cudf.DataFrame({"A":[1, 4, 5], "B":[3, 5, 8]}) + >>> df.mask(df % 2 == 0, [-1, -1]) + A B + 0 1 3 + 1 -1 5 + 2 5 -1 + + >>> ser = cudf.Series([4, 3, 2, 1, 0]) + >>> ser.mask(ser > 2, 10) + 0 10 + 1 10 + 2 2 + 3 1 + 4 0 + dtype: int64 + >>> ser.mask(ser > 2) + 0 + 1 + 2 2 + 3 1 + 4 0 + dtype: int64 + """ + + if not hasattr(cond, "__invert__"): + # We Invert `cond` below and call `where`, so + # making sure the object supports + # `~`(inversion) operator or `__invert__` method + cond = cp.asarray(cond) + + return self.where(cond=~cond, other=other, inplace=inplace) + def _copy_type_metadata( self, other: Self, From 2563b90f7e8b0a099b5acc6770d6fba2f33216a9 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 30 Jan 2024 18:25:42 +0000 Subject: [PATCH 173/384] Move various reductions to IndexedFrame --- python/cudf/cudf/core/frame.py | 340 ------------------------- python/cudf/cudf/core/indexed_frame.py | 339 ++++++++++++++++++++++++ 2 files changed, 339 insertions(+), 340 deletions(-) diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index 6a1f6b76302..5d2d054d20c 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -31,7 +31,6 @@ import cudf from cudf import _lib as libcudf from cudf._typing import Dtype -from cudf.api.extensions import no_default from cudf.api.types import is_bool_dtype, is_dtype_equal, is_scalar from cudf.core.buffer import acquire_spill_lock from cudf.core.column import ( @@ -1815,290 +1814,6 @@ def max( **kwargs, ) - @_cudf_nvtx_annotate - def sum( - self, - axis=no_default, - skipna=True, - dtype=None, - numeric_only=False, - min_count=0, - **kwargs, - ): - """ - Return sum of the values in the DataFrame. - - Parameters - ---------- - axis: {index (0), columns(1)} - Axis for the function to be applied on. - skipna: bool, default True - Exclude NA/null values when computing the result. - dtype: data type - Data type to cast the result to. - numeric_only : bool, default False - If True, includes only float, int, boolean columns. - If False, will raise error in-case there are - non-numeric columns. - min_count: int, default 0 - The required number of valid values to perform the operation. - If fewer than min_count non-NA values are present the result - will be NA. - - The default being 0. This means the sum of an all-NA or empty - Series is 0, and the product of an all-NA or empty Series is 1. - - Returns - ------- - Series - - Examples - -------- - >>> import cudf - >>> df = cudf.DataFrame({'a': [1, 2, 3, 4], 'b': [7, 8, 9, 10]}) - >>> df.sum() - a 10 - b 34 - dtype: int64 - - .. pandas-compat:: - **DataFrame.sum, Series.sum** - - Parameters currently not supported are `level`, `numeric_only`. - """ - return self._reduce( - "sum", - axis=axis, - skipna=skipna, - dtype=dtype, - numeric_only=numeric_only, - min_count=min_count, - **kwargs, - ) - - @_cudf_nvtx_annotate - def product( - self, - axis=no_default, - skipna=True, - dtype=None, - numeric_only=False, - min_count=0, - **kwargs, - ): - """ - Return product of the values in the DataFrame. - - Parameters - ---------- - axis: {index (0), columns(1)} - Axis for the function to be applied on. - skipna: bool, default True - Exclude NA/null values when computing the result. - dtype: data type - Data type to cast the result to. - numeric_only : bool, default False - If True, includes only float, int, boolean columns. - If False, will raise error in-case there are - non-numeric columns. - min_count: int, default 0 - The required number of valid values to perform the operation. - If fewer than min_count non-NA values are present the result - will be NA. - - The default being 0. This means the sum of an all-NA or empty - Series is 0, and the product of an all-NA or empty Series is 1. - - Returns - ------- - Series - - Examples - -------- - >>> import cudf - >>> df = cudf.DataFrame({'a': [1, 2, 3, 4], 'b': [7, 8, 9, 10]}) - >>> df.product() - a 24 - b 5040 - dtype: int64 - - .. pandas-compat:: - **DataFrame.product, Series.product** - - Parameters currently not supported are level`, `numeric_only`. - """ - - return self._reduce( - # cuDF columns use "product" as the op name, but cupy uses "prod" - # and we need cupy if axis == 1. - "prod" if axis in {1, "columns"} else "product", - axis=axis, - skipna=skipna, - dtype=dtype, - numeric_only=numeric_only, - min_count=min_count, - **kwargs, - ) - - # Alias for pandas compatibility. - prod = product - - @_cudf_nvtx_annotate - def mean(self, axis=0, skipna=True, numeric_only=False, **kwargs): - """ - Return the mean of the values for the requested axis. - - Parameters - ---------- - axis : {0 or 'index', 1 or 'columns'} - Axis for the function to be applied on. - skipna : bool, default True - Exclude NA/null values when computing the result. - numeric_only : bool, default False - If True, includes only float, int, boolean columns. - If False, will raise error in-case there are - non-numeric columns. - **kwargs - Additional keyword arguments to be passed to the function. - - Returns - ------- - mean : Series or DataFrame (if level specified) - - Examples - -------- - >>> import cudf - >>> df = cudf.DataFrame({'a': [1, 2, 3, 4], 'b': [7, 8, 9, 10]}) - >>> df.mean() - a 2.5 - b 8.5 - dtype: float64 - """ - return self._reduce( - "mean", - axis=axis, - skipna=skipna, - numeric_only=numeric_only, - **kwargs, - ) - - @_cudf_nvtx_annotate - def std( - self, - axis=no_default, - skipna=True, - ddof=1, - numeric_only=False, - **kwargs, - ): - """ - Return sample standard deviation of the DataFrame. - - Normalized by N-1 by default. This can be changed using - the `ddof` argument - - Parameters - ---------- - axis: {index (0), columns(1)} - Axis for the function to be applied on. - skipna: bool, default True - Exclude NA/null values. If an entire row/column is NA, the result - will be NA. - ddof: int, default 1 - Delta Degrees of Freedom. The divisor used in calculations - is N - ddof, where N represents the number of elements. - numeric_only : bool, default False - If True, includes only float, int, boolean columns. - If False, will raise error in-case there are - non-numeric columns. - - Returns - ------- - Series - - Examples - -------- - >>> import cudf - >>> df = cudf.DataFrame({'a': [1, 2, 3, 4], 'b': [7, 8, 9, 10]}) - >>> df.std() - a 1.290994 - b 1.290994 - dtype: float64 - - .. pandas-compat:: - **DataFrame.std, Series.std** - - Parameters currently not supported are `level` and - `numeric_only` - """ - - return self._reduce( - "std", - axis=axis, - skipna=skipna, - ddof=ddof, - numeric_only=numeric_only, - **kwargs, - ) - - @_cudf_nvtx_annotate - def var( - self, - axis=no_default, - skipna=True, - ddof=1, - numeric_only=False, - **kwargs, - ): - """ - Return unbiased variance of the DataFrame. - - Normalized by N-1 by default. This can be changed using the - ddof argument. - - Parameters - ---------- - axis: {index (0), columns(1)} - Axis for the function to be applied on. - skipna: bool, default True - Exclude NA/null values. If an entire row/column is NA, the result - will be NA. - ddof: int, default 1 - Delta Degrees of Freedom. The divisor used in calculations is - N - ddof, where N represents the number of elements. - numeric_only : bool, default False - If True, includes only float, int, boolean columns. - If False, will raise error in-case there are - non-numeric columns. - - Returns - ------- - scalar - - Examples - -------- - >>> import cudf - >>> df = cudf.DataFrame({'a': [1, 2, 3, 4], 'b': [7, 8, 9, 10]}) - >>> df.var() - a 1.666667 - b 1.666667 - dtype: float64 - - .. pandas-compat:: - **DataFrame.var, Series.var** - - Parameters currently not supported are `level` and - `numeric_only` - """ - return self._reduce( - "var", - axis=axis, - skipna=skipna, - ddof=ddof, - numeric_only=numeric_only, - **kwargs, - ) - @_cudf_nvtx_annotate def all(self, axis=0, skipna=True, **kwargs): """ @@ -2217,61 +1932,6 @@ def any(self, axis=0, skipna=True, **kwargs): **kwargs, ) - def median( - self, axis=None, skipna=True, level=None, numeric_only=None, **kwargs - ): - """ - Return the median of the values for the requested axis. - - Parameters - ---------- - axis : {index (0), columns (1)} - Axis for the function to be applied on. For Series this - parameter is unused and defaults to 0. - skipna : bool, default True - Exclude NA/null values when computing the result. - numeric_only : bool, default False - If True, includes only float, int, boolean columns. - If False, will raise error in-case there are - non-numeric columns. - - Returns - ------- - scalar - - Examples - -------- - >>> import cudf - >>> ser = cudf.Series([10, 25, 3, 25, 24, 6]) - >>> ser - 0 10 - 1 25 - 2 3 - 3 25 - 4 24 - 5 6 - dtype: int64 - >>> ser.median() - 17.0 - - .. pandas-compat:: - **DataFrame.median, Series.median** - - Parameters currently not supported are `level` and `numeric_only`. - - .. pandas-compat:: - **DataFrame.median, Series.median** - - Parameters currently not supported are `level` and `numeric_only`. - """ - return self._reduce( - "median", - axis=axis, - skipna=skipna, - numeric_only=numeric_only, - **kwargs, - ) - @_cudf_nvtx_annotate @ioutils.doc_to_json() def to_json(self, path_or_buf=None, *args, **kwargs): diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index c08571a4752..82e355ddfd0 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -1241,6 +1241,345 @@ def tail(self, n=5): # Alias for isna isnull = Frame.isna + @_cudf_nvtx_annotate + def sum( + self, + axis=no_default, + skipna=True, + dtype=None, + numeric_only=False, + min_count=0, + **kwargs, + ): + """ + Return sum of the values in the DataFrame. + + Parameters + ---------- + axis: {index (0), columns(1)} + Axis for the function to be applied on. + skipna: bool, default True + Exclude NA/null values when computing the result. + dtype: data type + Data type to cast the result to. + numeric_only : bool, default False + If True, includes only float, int, boolean columns. + If False, will raise error in-case there are + non-numeric columns. + min_count: int, default 0 + The required number of valid values to perform the operation. + If fewer than min_count non-NA values are present the result + will be NA. + + The default being 0. This means the sum of an all-NA or empty + Series is 0, and the product of an all-NA or empty Series is 1. + + Returns + ------- + Series + + Examples + -------- + >>> import cudf + >>> df = cudf.DataFrame({'a': [1, 2, 3, 4], 'b': [7, 8, 9, 10]}) + >>> df.sum() + a 10 + b 34 + dtype: int64 + + .. pandas-compat:: + **DataFrame.sum, Series.sum** + + Parameters currently not supported are `level`, `numeric_only`. + """ + return self._reduce( + "sum", + axis=axis, + skipna=skipna, + dtype=dtype, + numeric_only=numeric_only, + min_count=min_count, + **kwargs, + ) + + @_cudf_nvtx_annotate + def product( + self, + axis=no_default, + skipna=True, + dtype=None, + numeric_only=False, + min_count=0, + **kwargs, + ): + """ + Return product of the values in the DataFrame. + + Parameters + ---------- + axis: {index (0), columns(1)} + Axis for the function to be applied on. + skipna: bool, default True + Exclude NA/null values when computing the result. + dtype: data type + Data type to cast the result to. + numeric_only : bool, default False + If True, includes only float, int, boolean columns. + If False, will raise error in-case there are + non-numeric columns. + min_count: int, default 0 + The required number of valid values to perform the operation. + If fewer than min_count non-NA values are present the result + will be NA. + + The default being 0. This means the sum of an all-NA or empty + Series is 0, and the product of an all-NA or empty Series is 1. + + Returns + ------- + Series + + Examples + -------- + >>> import cudf + >>> df = cudf.DataFrame({'a': [1, 2, 3, 4], 'b': [7, 8, 9, 10]}) + >>> df.product() + a 24 + b 5040 + dtype: int64 + + .. pandas-compat:: + **DataFrame.product, Series.product** + + Parameters currently not supported are level`, `numeric_only`. + """ + + return self._reduce( + # cuDF columns use "product" as the op name, but cupy uses "prod" + # and we need cupy if axis == 1. + "prod" if axis in {1, "columns"} else "product", + axis=axis, + skipna=skipna, + dtype=dtype, + numeric_only=numeric_only, + min_count=min_count, + **kwargs, + ) + + # Alias for pandas compatibility. + prod = product + + @_cudf_nvtx_annotate + def mean(self, axis=0, skipna=True, numeric_only=False, **kwargs): + """ + Return the mean of the values for the requested axis. + + Parameters + ---------- + axis : {0 or 'index', 1 or 'columns'} + Axis for the function to be applied on. + skipna : bool, default True + Exclude NA/null values when computing the result. + numeric_only : bool, default False + If True, includes only float, int, boolean columns. + If False, will raise error in-case there are + non-numeric columns. + **kwargs + Additional keyword arguments to be passed to the function. + + Returns + ------- + mean : Series or DataFrame (if level specified) + + Examples + -------- + >>> import cudf + >>> df = cudf.DataFrame({'a': [1, 2, 3, 4], 'b': [7, 8, 9, 10]}) + >>> df.mean() + a 2.5 + b 8.5 + dtype: float64 + """ + return self._reduce( + "mean", + axis=axis, + skipna=skipna, + numeric_only=numeric_only, + **kwargs, + ) + + def median( + self, axis=None, skipna=True, level=None, numeric_only=None, **kwargs + ): + """ + Return the median of the values for the requested axis. + + Parameters + ---------- + axis : {index (0), columns (1)} + Axis for the function to be applied on. For Series this + parameter is unused and defaults to 0. + skipna : bool, default True + Exclude NA/null values when computing the result. + numeric_only : bool, default False + If True, includes only float, int, boolean columns. + If False, will raise error in-case there are + non-numeric columns. + + Returns + ------- + scalar + + Examples + -------- + >>> import cudf + >>> ser = cudf.Series([10, 25, 3, 25, 24, 6]) + >>> ser + 0 10 + 1 25 + 2 3 + 3 25 + 4 24 + 5 6 + dtype: int64 + >>> ser.median() + 17.0 + + .. pandas-compat:: + **DataFrame.median, Series.median** + + Parameters currently not supported are `level` and `numeric_only`. + + .. pandas-compat:: + **DataFrame.median, Series.median** + + Parameters currently not supported are `level` and `numeric_only`. + """ + return self._reduce( + "median", + axis=axis, + skipna=skipna, + numeric_only=numeric_only, + **kwargs, + ) + + @_cudf_nvtx_annotate + def std( + self, + axis=no_default, + skipna=True, + ddof=1, + numeric_only=False, + **kwargs, + ): + """ + Return sample standard deviation of the DataFrame. + + Normalized by N-1 by default. This can be changed using + the `ddof` argument + + Parameters + ---------- + axis: {index (0), columns(1)} + Axis for the function to be applied on. + skipna: bool, default True + Exclude NA/null values. If an entire row/column is NA, the result + will be NA. + ddof: int, default 1 + Delta Degrees of Freedom. The divisor used in calculations + is N - ddof, where N represents the number of elements. + numeric_only : bool, default False + If True, includes only float, int, boolean columns. + If False, will raise error in-case there are + non-numeric columns. + + Returns + ------- + Series + + Examples + -------- + >>> import cudf + >>> df = cudf.DataFrame({'a': [1, 2, 3, 4], 'b': [7, 8, 9, 10]}) + >>> df.std() + a 1.290994 + b 1.290994 + dtype: float64 + + .. pandas-compat:: + **DataFrame.std, Series.std** + + Parameters currently not supported are `level` and + `numeric_only` + """ + + return self._reduce( + "std", + axis=axis, + skipna=skipna, + ddof=ddof, + numeric_only=numeric_only, + **kwargs, + ) + + @_cudf_nvtx_annotate + def var( + self, + axis=no_default, + skipna=True, + ddof=1, + numeric_only=False, + **kwargs, + ): + """ + Return unbiased variance of the DataFrame. + + Normalized by N-1 by default. This can be changed using the + ddof argument. + + Parameters + ---------- + axis: {index (0), columns(1)} + Axis for the function to be applied on. + skipna: bool, default True + Exclude NA/null values. If an entire row/column is NA, the result + will be NA. + ddof: int, default 1 + Delta Degrees of Freedom. The divisor used in calculations is + N - ddof, where N represents the number of elements. + numeric_only : bool, default False + If True, includes only float, int, boolean columns. + If False, will raise error in-case there are + non-numeric columns. + + Returns + ------- + scalar + + Examples + -------- + >>> import cudf + >>> df = cudf.DataFrame({'a': [1, 2, 3, 4], 'b': [7, 8, 9, 10]}) + >>> df.var() + a 1.666667 + b 1.666667 + dtype: float64 + + .. pandas-compat:: + **DataFrame.var, Series.var** + + Parameters currently not supported are `level` and + `numeric_only` + """ + return self._reduce( + "var", + axis=axis, + skipna=skipna, + ddof=ddof, + numeric_only=numeric_only, + **kwargs, + ) + @_cudf_nvtx_annotate def kurtosis(self, axis=0, skipna=True, numeric_only=False, **kwargs): """ From 9716f52ac51c2a23aa7eb4585d0c47f1db0a378c Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 30 Jan 2024 18:27:25 +0000 Subject: [PATCH 174/384] Move nans_to_nulls to IndexedFrame --- python/cudf/cudf/core/frame.py | 54 -------------------------- python/cudf/cudf/core/indexed_frame.py | 54 ++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 54 deletions(-) diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index 5d2d054d20c..b06afa5da0b 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -2004,60 +2004,6 @@ def rolling( win_type=win_type, ) - @_cudf_nvtx_annotate - def nans_to_nulls(self): - """ - Convert nans (if any) to nulls - - Returns - ------- - DataFrame or Series - - Examples - -------- - **Series** - - >>> import cudf, numpy as np - >>> series = cudf.Series([1, 2, np.nan, None, 10], nan_as_null=False) - >>> series - 0 1.0 - 1 2.0 - 2 NaN - 3 - 4 10.0 - dtype: float64 - >>> series.nans_to_nulls() - 0 1.0 - 1 2.0 - 2 - 3 - 4 10.0 - dtype: float64 - - **DataFrame** - - >>> df = cudf.DataFrame() - >>> df['a'] = cudf.Series([1, None, np.nan], nan_as_null=False) - >>> df['b'] = cudf.Series([None, 3.14, np.nan], nan_as_null=False) - >>> df - a b - 0 1.0 - 1 3.14 - 2 NaN NaN - >>> df.nans_to_nulls() - a b - 0 1.0 - 1 3.14 - 2 - """ - result_data = {} - for name, col in self._data.items(): - try: - result_data[name] = col.nans_to_nulls() - except AttributeError: - result_data[name] = col.copy() - return self._from_data_like_self(result_data) - @_cudf_nvtx_annotate def __invert__(self): """Bitwise invert (~) for integral dtypes, logical NOT for bools.""" diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 82e355ddfd0..5d634418655 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -1761,6 +1761,60 @@ def mask(self, cond, other=None, inplace: bool = False) -> Optional[Self]: return self.where(cond=~cond, other=other, inplace=inplace) + @_cudf_nvtx_annotate + def nans_to_nulls(self): + """ + Convert nans (if any) to nulls + + Returns + ------- + DataFrame or Series + + Examples + -------- + **Series** + + >>> import cudf, numpy as np + >>> series = cudf.Series([1, 2, np.nan, None, 10], nan_as_null=False) + >>> series + 0 1.0 + 1 2.0 + 2 NaN + 3 + 4 10.0 + dtype: float64 + >>> series.nans_to_nulls() + 0 1.0 + 1 2.0 + 2 + 3 + 4 10.0 + dtype: float64 + + **DataFrame** + + >>> df = cudf.DataFrame() + >>> df['a'] = cudf.Series([1, None, np.nan], nan_as_null=False) + >>> df['b'] = cudf.Series([None, 3.14, np.nan], nan_as_null=False) + >>> df + a b + 0 1.0 + 1 3.14 + 2 NaN NaN + >>> df.nans_to_nulls() + a b + 0 1.0 + 1 3.14 + 2 + """ + result_data = {} + for name, col in self._data.items(): + try: + result_data[name] = col.nans_to_nulls() + except AttributeError: + result_data[name] = col.copy() + return self._from_data_like_self(result_data) + def _copy_type_metadata( self, other: Self, From fdf31e382833133d91d151fc868a78d4ad5d9ff4 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 30 Jan 2024 18:28:36 +0000 Subject: [PATCH 175/384] Move rolling to IndexedFrame --- python/cudf/cudf/core/frame.py | 16 ---------------- python/cudf/cudf/core/indexed_frame.py | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index b06afa5da0b..5d4bea580bb 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -42,9 +42,7 @@ ) from cudf.core.column_accessor import ColumnAccessor from cudf.core.mixins import BinaryOperand, Scannable -from cudf.core.window import Rolling from cudf.utils import ioutils -from cudf.utils.docutils import copy_docstring from cudf.utils.dtypes import find_common_type from cudf.utils.nvtx_annotation import _cudf_nvtx_annotate from cudf.utils.utils import _array_ufunc, _warn_no_dask_cudf @@ -1990,20 +1988,6 @@ def __deepcopy__(self, memo): def __copy__(self): return self.copy(deep=False) - @_cudf_nvtx_annotate - @copy_docstring(Rolling) - def rolling( - self, window, min_periods=None, center=False, axis=0, win_type=None - ): - return Rolling( - self, - window, - min_periods=min_periods, - center=center, - axis=axis, - win_type=win_type, - ) - @_cudf_nvtx_annotate def __invert__(self): """Bitwise invert (~) for integral dtypes, logical NOT for bools.""" diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 5d634418655..10e6493ce7e 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -66,8 +66,10 @@ _post_process_output_col, _return_arr_from_dtype, ) +from cudf.core.window import Rolling from cudf.utils import docutils from cudf.utils._numba import _CUDFNumbaConfig +from cudf.utils.docutils import copy_docstring from cudf.utils.nvtx_annotation import _cudf_nvtx_annotate from cudf.utils.utils import _warn_no_dask_cudf @@ -1761,6 +1763,20 @@ def mask(self, cond, other=None, inplace: bool = False) -> Optional[Self]: return self.where(cond=~cond, other=other, inplace=inplace) + @_cudf_nvtx_annotate + @copy_docstring(Rolling) + def rolling( + self, window, min_periods=None, center=False, axis=0, win_type=None + ): + return Rolling( + self, + window, + min_periods=min_periods, + center=center, + axis=axis, + win_type=win_type, + ) + @_cudf_nvtx_annotate def nans_to_nulls(self): """ From 0bcdb2de6fb8c355e53c96fb166fe779a3b28807 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 30 Jan 2024 18:29:13 +0000 Subject: [PATCH 176/384] Move notnull (alias) to IndexedFrame --- python/cudf/cudf/core/frame.py | 3 --- python/cudf/cudf/core/indexed_frame.py | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index 5d4bea580bb..5618647b7f7 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -1245,9 +1245,6 @@ def notna(self): data_columns = (col.notnull() for col in self._columns) return self._from_data_like_self(zip(self._column_names, data_columns)) - # Alias for notna - notnull = notna - @_cudf_nvtx_annotate def searchsorted( self, diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 10e6493ce7e..3688a65ff82 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -1243,6 +1243,9 @@ def tail(self, n=5): # Alias for isna isnull = Frame.isna + # Alias for notna + notnull = Frame.notna + @_cudf_nvtx_annotate def sum( self, From ea7ebfbf28a2d7ad19488622b20c6013bade7016 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 30 Jan 2024 18:30:47 +0000 Subject: [PATCH 177/384] Move pipe to IndexedFrame --- python/cudf/cudf/core/frame.py | 47 -------------------------- python/cudf/cudf/core/indexed_frame.py | 47 ++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 47 deletions(-) diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index 5618647b7f7..b230bac3706 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -612,53 +612,6 @@ def where(self, cond, other=None, inplace: bool = False) -> Optional[Self]: """ raise NotImplementedError - @_cudf_nvtx_annotate - def pipe(self, func, *args, **kwargs): - """ - Apply ``func(self, *args, **kwargs)``. - - Parameters - ---------- - func : function - Function to apply to the Series/DataFrame/Index. - ``args``, and ``kwargs`` are passed into ``func``. - Alternatively a ``(callable, data_keyword)`` tuple where - ``data_keyword`` is a string indicating the keyword of - ``callable`` that expects the Series/DataFrame/Index. - args : iterable, optional - Positional arguments passed into ``func``. - kwargs : mapping, optional - A dictionary of keyword arguments passed into ``func``. - - Returns - ------- - object : the return type of ``func``. - - Examples - -------- - Use ``.pipe`` when chaining together functions that expect - Series, DataFrames or GroupBy objects. Instead of writing - - >>> func(g(h(df), arg1=a), arg2=b, arg3=c) - - You can write - - >>> (df.pipe(h) - ... .pipe(g, arg1=a) - ... .pipe(func, arg2=b, arg3=c) - ... ) - - If you have a function that takes the data as (say) the second - argument, pass a tuple indicating which keyword expects the - data. For example, suppose ``f`` takes its data as ``arg2``: - - >>> (df.pipe(h) - ... .pipe(g, arg1=a) - ... .pipe((func, 'arg2'), arg1=a, arg3=c) - ... ) - """ - return cudf.core.common.pipe(self, func, *args, **kwargs) - @_cudf_nvtx_annotate def fillna( self, diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 3688a65ff82..807445af2a9 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -1246,6 +1246,53 @@ def tail(self, n=5): # Alias for notna notnull = Frame.notna + @_cudf_nvtx_annotate + def pipe(self, func, *args, **kwargs): + """ + Apply ``func(self, *args, **kwargs)``. + + Parameters + ---------- + func : function + Function to apply to the Series/DataFrame/Index. + ``args``, and ``kwargs`` are passed into ``func``. + Alternatively a ``(callable, data_keyword)`` tuple where + ``data_keyword`` is a string indicating the keyword of + ``callable`` that expects the Series/DataFrame/Index. + args : iterable, optional + Positional arguments passed into ``func``. + kwargs : mapping, optional + A dictionary of keyword arguments passed into ``func``. + + Returns + ------- + object : the return type of ``func``. + + Examples + -------- + Use ``.pipe`` when chaining together functions that expect + Series, DataFrames or GroupBy objects. Instead of writing + + >>> func(g(h(df), arg1=a), arg2=b, arg3=c) + + You can write + + >>> (df.pipe(h) + ... .pipe(g, arg1=a) + ... .pipe(func, arg2=b, arg3=c) + ... ) + + If you have a function that takes the data as (say) the second + argument, pass a tuple indicating which keyword expects the + data. For example, suppose ``f`` takes its data as ``arg2``: + + >>> (df.pipe(h) + ... .pipe(g, arg1=a) + ... .pipe((func, 'arg2'), arg1=a, arg3=c) + ... ) + """ + return cudf.core.common.pipe(self, func, *args, **kwargs) + @_cudf_nvtx_annotate def sum( self, From 7b0bcde0c4147b280ff12bfe478942bf863c907f Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 30 Jan 2024 18:32:16 +0000 Subject: [PATCH 178/384] Move conversion functions --- python/cudf/cudf/core/frame.py | 39 ------------------------ python/cudf/cudf/core/indexed_frame.py | 41 +++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index b230bac3706..9164c35c00b 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -1880,22 +1880,6 @@ def any(self, axis=0, skipna=True, **kwargs): **kwargs, ) - @_cudf_nvtx_annotate - @ioutils.doc_to_json() - def to_json(self, path_or_buf=None, *args, **kwargs): - """{docstring}""" - - return cudf.io.json.to_json( - self, path_or_buf=path_or_buf, *args, **kwargs - ) - - @_cudf_nvtx_annotate - @ioutils.doc_to_hdf() - def to_hdf(self, path_or_buf, key, *args, **kwargs): - """{docstring}""" - - cudf.io.hdf.to_hdf(path_or_buf, key, self, *args, **kwargs) - @_cudf_nvtx_annotate @ioutils.doc_to_dlpack() def to_dlpack(self): @@ -1903,29 +1887,6 @@ def to_dlpack(self): return cudf.io.dlpack.to_dlpack(self) - @_cudf_nvtx_annotate - def to_string(self): - r""" - Convert to string - - cuDF uses Pandas internals for efficient string formatting. - Set formatting options using pandas string formatting options and - cuDF objects will print identically to Pandas objects. - - cuDF supports `null/None` as a value in any column type, which - is transparently supported during this output process. - - Examples - -------- - >>> import cudf - >>> df = cudf.DataFrame() - >>> df['key'] = [0, 1, 2] - >>> df['val'] = [float(i + 10) for i in range(3)] - >>> df.to_string() - ' key val\n0 0 10.0\n1 1 11.0\n2 2 12.0' - """ - return repr(self) - @_cudf_nvtx_annotate def __str__(self): return self.to_string() diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 807445af2a9..df2fd881432 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -67,7 +67,7 @@ _return_arr_from_dtype, ) from cudf.core.window import Rolling -from cudf.utils import docutils +from cudf.utils import docutils, ioutils from cudf.utils._numba import _CUDFNumbaConfig from cudf.utils.docutils import copy_docstring from cudf.utils.nvtx_annotation import _cudf_nvtx_annotate @@ -507,6 +507,45 @@ def empty(self): """ return self.size == 0 + @_cudf_nvtx_annotate + @ioutils.doc_to_json() + def to_json(self, path_or_buf=None, *args, **kwargs): + """{docstring}""" + + return cudf.io.json.to_json( + self, path_or_buf=path_or_buf, *args, **kwargs + ) + + @_cudf_nvtx_annotate + @ioutils.doc_to_hdf() + def to_hdf(self, path_or_buf, key, *args, **kwargs): + """{docstring}""" + + cudf.io.hdf.to_hdf(path_or_buf, key, self, *args, **kwargs) + + @_cudf_nvtx_annotate + def to_string(self): + r""" + Convert to string + + cuDF uses Pandas internals for efficient string formatting. + Set formatting options using pandas string formatting options and + cuDF objects will print identically to Pandas objects. + + cuDF supports `null/None` as a value in any column type, which + is transparently supported during this output process. + + Examples + -------- + >>> import cudf + >>> df = cudf.DataFrame() + >>> df['key'] = [0, 1, 2] + >>> df['val'] = [float(i + 10) for i in range(3)] + >>> df.to_string() + ' key val\n0 0 10.0\n1 1 11.0\n2 2 12.0' + """ + return repr(self) + def copy(self, deep: bool = True) -> Self: """Make a copy of this object's indices and data. From 28548f6c3b7b4aabecc7a325cb86f550f8026377 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 30 Jan 2024 18:32:58 +0000 Subject: [PATCH 179/384] Add missing methods to the docs --- docs/cudf/source/user_guide/api_docs/index_objects.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/cudf/source/user_guide/api_docs/index_objects.rst b/docs/cudf/source/user_guide/api_docs/index_objects.rst index ff190da86bf..e669b95198c 100644 --- a/docs/cudf/source/user_guide/api_docs/index_objects.rst +++ b/docs/cudf/source/user_guide/api_docs/index_objects.rst @@ -41,6 +41,7 @@ Modifying and computations .. autosummary:: :toctree: api/ + Index.all Index.any Index.copy Index.drop_duplicates @@ -60,6 +61,7 @@ Modifying and computations Index.where Index.take Index.unique + Index.nunique Compatibility with MultiIndex ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 59af57d3f5a776b636ec7d66b9d6079e314f07e9 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 30 Jan 2024 18:45:47 +0000 Subject: [PATCH 180/384] Add isnull and notnull to index docs --- docs/cudf/source/user_guide/api_docs/index_objects.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/cudf/source/user_guide/api_docs/index_objects.rst b/docs/cudf/source/user_guide/api_docs/index_objects.rst index e669b95198c..9c84f206010 100644 --- a/docs/cudf/source/user_guide/api_docs/index_objects.rst +++ b/docs/cudf/source/user_guide/api_docs/index_objects.rst @@ -78,7 +78,9 @@ Missing values Index.fillna Index.dropna Index.isna + Index.isnull Index.notna + Index.notnull Memory usage ~~~~~~~~~~~~ From c6f5392a0a8f0ad5687716944b697ebc9965cc82 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 30 Jan 2024 18:47:50 +0000 Subject: [PATCH 181/384] Revert "Move isnull (alias) to IndexedFrame" This reverts commit 50d287f97bec818a6027b55727ab1d8e538fd4ce. --- python/cudf/cudf/core/frame.py | 3 +++ python/cudf/cudf/core/indexed_frame.py | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index 9164c35c00b..abbb730fb71 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -1122,6 +1122,9 @@ def isna(self): data_columns = (col.isnull() for col in self._columns) return self._from_data_like_self(zip(self._column_names, data_columns)) + # Alias for isna + isnull = isna + @_cudf_nvtx_annotate def notna(self): """ diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index df2fd881432..c12d55d6873 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -1279,9 +1279,6 @@ def tail(self, n=5): return self.iloc[-n:] - # Alias for isna - isnull = Frame.isna - # Alias for notna notnull = Frame.notna From 63015385cf7cde97dead6b1ce973faef78d25865 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 30 Jan 2024 18:48:30 +0000 Subject: [PATCH 182/384] Revert "Move notnull (alias) to IndexedFrame" This reverts commit 0bcdb2de6fb8c355e53c96fb166fe779a3b28807. --- python/cudf/cudf/core/frame.py | 3 +++ python/cudf/cudf/core/indexed_frame.py | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index abbb730fb71..996b1a80c79 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -1201,6 +1201,9 @@ def notna(self): data_columns = (col.notnull() for col in self._columns) return self._from_data_like_self(zip(self._column_names, data_columns)) + # Alias for notna + notnull = notna + @_cudf_nvtx_annotate def searchsorted( self, diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index c12d55d6873..6d53198611e 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -1279,9 +1279,6 @@ def tail(self, n=5): return self.iloc[-n:] - # Alias for notna - notnull = Frame.notna - @_cudf_nvtx_annotate def pipe(self, func, *args, **kwargs): """ From a95bc6a0fabece2ce10d493c0c7acca743a09f29 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 30 Jan 2024 19:13:33 +0000 Subject: [PATCH 183/384] Make sure str works even if to_string does not --- python/cudf/cudf/core/frame.py | 2 +- python/cudf/cudf/core/indexed_frame.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index 996b1a80c79..96b62e185b3 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -1895,7 +1895,7 @@ def to_dlpack(self): @_cudf_nvtx_annotate def __str__(self): - return self.to_string() + return repr(self) @_cudf_nvtx_annotate def __deepcopy__(self, memo): diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 6d53198611e..9c35dba7cfd 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -544,7 +544,7 @@ def to_string(self): >>> df.to_string() ' key val\n0 0 10.0\n1 1 11.0\n2 2 12.0' """ - return repr(self) + return str(self) def copy(self, deep: bool = True) -> Self: """Make a copy of this object's indices and data. From 4f0563d563bda4608ce93e2436a9d224abf4f073 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 30 Jan 2024 19:13:56 +0000 Subject: [PATCH 184/384] Remove tests of now unsupported reductions --- python/cudf/cudf/tests/test_array_function.py | 4 ---- python/cudf/cudf/tests/test_index.py | 3 --- 2 files changed, 7 deletions(-) diff --git a/python/cudf/cudf/tests/test_array_function.py b/python/cudf/cudf/tests/test_array_function.py index 090e8884991..58939f0ddd9 100644 --- a/python/cudf/cudf/tests/test_array_function.py +++ b/python/cudf/cudf/tests/test_array_function.py @@ -104,11 +104,7 @@ def test_array_func_missing_cudf_dataframe(pd_df, func): @pytest.mark.parametrize( "func", [ - lambda x: np.mean(x), - lambda x: np.sum(x), - lambda x: np.var(x, ddof=1), lambda x: np.unique(x), - lambda x: np.dot(x, x), ], ) def test_array_func_cudf_index(np_ar, func): diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index 5cc1c93deff..996b651b9fe 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -132,11 +132,8 @@ def test_index_comparision(): [ lambda x: x.min(), lambda x: x.max(), - lambda x: x.sum(), - lambda x: x.mean(), lambda x: x.any(), lambda x: x.all(), - lambda x: x.prod(), ], ) def test_reductions(func): From 07e98723864fb719eb2682b4aa0abab54770d67a Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 30 Jan 2024 19:24:31 +0000 Subject: [PATCH 185/384] Address feedback --- python/cudf/cudf/core/indexed_frame.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 9c35dba7cfd..15277ff5586 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -1287,11 +1287,11 @@ def pipe(self, func, *args, **kwargs): Parameters ---------- func : function - Function to apply to the Series/DataFrame/Index. + Function to apply to the Series/DataFrame. ``args``, and ``kwargs`` are passed into ``func``. Alternatively a ``(callable, data_keyword)`` tuple where ``data_keyword`` is a string indicating the keyword of - ``callable`` that expects the Series/DataFrame/Index. + ``callable`` that expects the Series/DataFrame. args : iterable, optional Positional arguments passed into ``func``. kwargs : mapping, optional From 6ed75ffddcd678a5dbcfd5f0e2dccf98531b4282 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Tue, 30 Jan 2024 14:04:52 -0800 Subject: [PATCH 186/384] Fix V2 Parquet page alignment for use with zStandard compression (#14841) Fixes #14781 This PR makes changes to the Parquet writer to ensure that data to be compressed is properly aligned. Changes have also been made to the `EncPage` struct to make it easier to keep fields in that struct aligned, and also to reduce confusing re-use of fields. In particular, the `max_data_size` field can be any of a) the maximum possible size for the page data, b) the actual size of page data after encoding, c) the actual size of compressed page data. The latter two now have their own fields, `data_size` and `comp_data_size`. Authors: - Ed Seidl (https://github.com/etseidl) - Mike Wilson (https://github.com/hyperbolic2346) Approvers: - Mike Wilson (https://github.com/hyperbolic2346) - Vukasin Milovanovic (https://github.com/vuule) URL: https://github.com/rapidsai/cudf/pull/14841 --- cpp/src/io/parquet/page_enc.cu | 129 +++++++++++++++++---------- cpp/src/io/parquet/parquet_gpu.hpp | 43 +++++---- cpp/src/io/parquet/writer_impl.cu | 7 +- cpp/tests/io/parquet_v2_test.cpp | 5 +- cpp/tests/io/parquet_writer_test.cpp | 17 ---- 5 files changed, 115 insertions(+), 86 deletions(-) diff --git a/cpp/src/io/parquet/page_enc.cu b/cpp/src/io/parquet/page_enc.cu index 3cc4fda695f..2f351edd2b9 100644 --- a/cpp/src/io/parquet/page_enc.cu +++ b/cpp/src/io/parquet/page_enc.cu @@ -388,6 +388,27 @@ __device__ uint8_t const* delta_encode(page_enc_state_s<0>* s, uint64_t* buffer, return packer.flush(); } +/** + * @brief Sets `s->cur` to point to the start of encoded page data. + * + * For V1 headers, this will be immediately after the repetition and definition level data. For V2, + * it will be at the next properly aligned location after the level data. The padding in V2 is + * needed for compressors that require aligned input. + */ +template +inline void __device__ set_page_data_start(state_type* s) +{ + s->cur = s->page.page_data + s->page.max_hdr_size; + switch (s->page.page_type) { + case PageType::DATA_PAGE: + s->cur += s->page.level_bytes(); + if (s->col.num_def_level_bits() != 0) { s->cur += RLE_LENGTH_FIELD_LEN; } + if (s->col.num_rep_level_bits() != 0) { s->cur += RLE_LENGTH_FIELD_LEN; } + break; + case PageType::DATA_PAGE_V2: s->cur += s->page.max_lvl_size; break; + } +} + } // anonymous namespace // blockDim {512,1,1} @@ -594,8 +615,13 @@ CUDF_KERNEL void __launch_bounds__(128) page_g.chunk = &chunks[blockIdx.y][blockIdx.x]; page_g.chunk_id = blockIdx.y * num_columns + blockIdx.x; page_g.hdr_size = 0; + page_g.def_lvl_bytes = 0; + page_g.rep_lvl_bytes = 0; + page_g.max_lvl_size = 0; + page_g.comp_data_size = 0; page_g.max_hdr_size = MAX_V1_HDR_SIZE; page_g.max_data_size = ck_g.uniq_data_size; + page_g.data_size = ck_g.uniq_data_size; page_g.start_row = cur_row; page_g.num_rows = ck_g.num_dict_entries; page_g.num_leaf_values = ck_g.num_dict_entries; @@ -689,12 +715,17 @@ CUDF_KERNEL void __launch_bounds__(128) page_size = 1 + max_RLE_page_size(ck_g.dict_rle_bits, values_in_page); } if (!t) { - page_g.num_fragments = fragments_in_chunk - page_start; - page_g.chunk = &chunks[blockIdx.y][blockIdx.x]; - page_g.chunk_id = blockIdx.y * num_columns + blockIdx.x; - page_g.page_type = data_page_type; - page_g.hdr_size = 0; - page_g.max_hdr_size = max_data_page_hdr_size; // Max size excluding statistics + page_g.num_fragments = fragments_in_chunk - page_start; + page_g.chunk = &chunks[blockIdx.y][blockIdx.x]; + page_g.chunk_id = blockIdx.y * num_columns + blockIdx.x; + page_g.page_type = data_page_type; + page_g.hdr_size = 0; + page_g.def_lvl_bytes = 0; + page_g.rep_lvl_bytes = 0; + page_g.max_lvl_size = 0; + page_g.data_size = 0; + page_g.comp_data_size = 0; + page_g.max_hdr_size = max_data_page_hdr_size; // Max size excluding statistics if (ck_g.stats) { uint32_t stats_hdr_len = 16; if (col_g.stats_dtype == dtype_string || col_g.stats_dtype == dtype_byte_array) { @@ -716,13 +747,19 @@ CUDF_KERNEL void __launch_bounds__(128) page_g.num_valid = num_valid; auto const def_level_size = max_RLE_page_size(col_g.num_def_level_bits(), values_in_page); auto const rep_level_size = max_RLE_page_size(col_g.num_rep_level_bits(), values_in_page); + if (write_v2_headers) { + page_g.max_lvl_size = + util::round_up_unsafe(def_level_size + rep_level_size, page_align); + } // get a different bound if using delta encoding if (is_use_delta) { auto const delta_len = delta_data_len(physical_type, type_id, page_g.num_leaf_values, page_size); page_size = max(page_size, delta_len); } - auto const max_data_size = page_size + def_level_size + rep_level_size + rle_pad; + auto const max_data_size = + page_size + rle_pad + + (write_v2_headers ? page_g.max_lvl_size : def_level_size + rep_level_size); // page size must fit in 32-bit signed integer if (max_data_size > std::numeric_limits::max()) { CUDF_UNREACHABLE("page size exceeds maximum for i32"); @@ -739,7 +776,9 @@ CUDF_KERNEL void __launch_bounds__(128) page_offset += util::round_up_unsafe(page_g.max_hdr_size + page_g.max_data_size, page_align); if (not comp_page_sizes.empty()) { - comp_page_offset += page_g.max_hdr_size + comp_page_sizes[ck_g.first_page + num_pages]; + // V2 does not include level data in compressed size estimate + comp_page_offset += page_g.max_hdr_size + page_g.max_lvl_size + + comp_page_sizes[ck_g.first_page + num_pages]; } page_headers_size += page_g.max_hdr_size; max_page_data_size = max(max_page_data_size, page_g.max_data_size); @@ -774,8 +813,10 @@ CUDF_KERNEL void __launch_bounds__(128) } pages[ck_g.first_page + num_pages] = page_g; } + // page_sizes should be the number of bytes to be compressed, so don't include level + // data for V2. if (not page_sizes.empty()) { - page_sizes[ck_g.first_page + num_pages] = page_g.max_data_size; + page_sizes[ck_g.first_page + num_pages] = page_g.max_data_size - page_g.max_lvl_size; } if (page_grstats) { page_grstats[ck_g.first_page + num_pages] = pagestats_g; } } @@ -1429,10 +1470,6 @@ __device__ void finish_page_encode(state_buf* s, return thrust::reduce(thrust::seq, hist_start, hist_end, 0U); }; - // V2 does not compress rep and def level data - size_t const skip_comp_size = - write_v2_headers ? s->page.def_lvl_bytes + s->page.rep_lvl_bytes : 0; - // this will be true if max_rep > 0 (i.e. there are lists) if (s->page.rep_histogram != nullptr) { // for repetition we get hist[0] from num_rows, and can derive hist[max_rep_level] @@ -1489,10 +1526,17 @@ __device__ void finish_page_encode(state_buf* s, // FIXME(ets): this needs to do error propagation back to the host CUDF_UNREACHABLE("detected possible page data corruption"); } - s->page.max_data_size = actual_data_size; + if (s->page.is_v2()) { + auto const d_base = base + s->page.max_lvl_size; + s->page.data_size = static_cast(end_ptr - d_base) + s->page.level_bytes(); + } else { + s->page.data_size = actual_data_size; + } if (not comp_in.empty()) { - comp_in[blockIdx.x] = {base + skip_comp_size, actual_data_size - skip_comp_size}; - comp_out[blockIdx.x] = {s->page.compressed_data + s->page.max_hdr_size + skip_comp_size, + auto const c_base = base + s->page.max_lvl_size; + auto const bytes_to_compress = static_cast(end_ptr - c_base); + comp_in[blockIdx.x] = {c_base, bytes_to_compress}; + comp_out[blockIdx.x] = {s->page.compressed_data + s->page.max_hdr_size + s->page.max_lvl_size, 0}; // size is unused } pages[blockIdx.x] = s->page; @@ -1503,10 +1547,10 @@ __device__ void finish_page_encode(state_buf* s, } // copy uncompressed bytes over - if (skip_comp_size != 0 && not comp_in.empty()) { + if (s->page.is_v2() and not comp_in.empty()) { uint8_t* const src = s->page.page_data + s->page.max_hdr_size; uint8_t* const dst = s->page.compressed_data + s->page.max_hdr_size; - for (int i = t; i < skip_comp_size; i += block_size) { + for (int i = t; i < s->page.level_bytes(); i += block_size) { dst[i] = src[i]; } } @@ -1536,13 +1580,7 @@ CUDF_KERNEL void __launch_bounds__(block_size, 8) s->col = *s->ck.col_desc; s->rle_len_pos = nullptr; // get s->cur back to where it was at the end of encoding the rep and def level data - s->cur = - s->page.page_data + s->page.max_hdr_size + s->page.def_lvl_bytes + s->page.rep_lvl_bytes; - // if V1 data page, need space for the RLE length fields - if (s->page.page_type == PageType::DATA_PAGE) { - if (s->col.num_def_level_bits() != 0) { s->cur += RLE_LENGTH_FIELD_LEN; } - if (s->col.num_rep_level_bits() != 0) { s->cur += RLE_LENGTH_FIELD_LEN; } - } + set_page_data_start(s); } __syncthreads(); @@ -1771,13 +1809,7 @@ CUDF_KERNEL void __launch_bounds__(block_size, 8) s->col = *s->ck.col_desc; s->rle_len_pos = nullptr; // get s->cur back to where it was at the end of encoding the rep and def level data - s->cur = - s->page.page_data + s->page.max_hdr_size + s->page.def_lvl_bytes + s->page.rep_lvl_bytes; - // if V1 data page, need space for the RLE length fields - if (s->page.page_type == PageType::DATA_PAGE) { - if (s->col.num_def_level_bits() != 0) { s->cur += RLE_LENGTH_FIELD_LEN; } - if (s->col.num_rep_level_bits() != 0) { s->cur += RLE_LENGTH_FIELD_LEN; } - } + set_page_data_start(s); } __syncthreads(); @@ -1908,8 +1940,7 @@ CUDF_KERNEL void __launch_bounds__(block_size, 8) s->col = *s->ck.col_desc; s->rle_len_pos = nullptr; // get s->cur back to where it was at the end of encoding the rep and def level data - s->cur = - s->page.page_data + s->page.max_hdr_size + s->page.def_lvl_bytes + s->page.rep_lvl_bytes; + set_page_data_start(s); } __syncthreads(); @@ -2017,8 +2048,7 @@ CUDF_KERNEL void __launch_bounds__(block_size, 8) s->col = *s->ck.col_desc; s->rle_len_pos = nullptr; // get s->cur back to where it was at the end of encoding the rep and def level data - s->cur = - s->page.page_data + s->page.max_hdr_size + s->page.def_lvl_bytes + s->page.rep_lvl_bytes; + set_page_data_start(s); } __syncthreads(); @@ -2142,11 +2172,10 @@ CUDF_KERNEL void __launch_bounds__(decide_compression_block_size) auto const num_pages = ck_g[warp_id].num_pages; for (auto page_id = lane_id; page_id < num_pages; page_id += cudf::detail::warp_size) { auto const& curr_page = ck_g[warp_id].pages[page_id]; - auto const page_data_size = curr_page.max_data_size; - auto const is_v2 = curr_page.page_type == PageType::DATA_PAGE_V2; - auto const lvl_bytes = is_v2 ? curr_page.def_lvl_bytes + curr_page.rep_lvl_bytes : 0; + auto const page_data_size = curr_page.data_size; uncompressed_data_size += page_data_size; if (auto comp_res = curr_page.comp_res; comp_res != nullptr) { + auto const lvl_bytes = curr_page.is_v2() ? curr_page.level_bytes() : 0; compressed_data_size += comp_res->bytes_written + lvl_bytes; if (comp_res->status != compression_status::SUCCESS) { atomicOr(&compression_error[warp_id], 1); @@ -2614,14 +2643,13 @@ CUDF_KERNEL void __launch_bounds__(128) EncodeStatistics(hdr_start, &chunk_stats[page_g.chunk_id], col_g.stats_dtype, scratch); page_g.chunk->ck_stat_size = static_cast(hdr_end - hdr_start); } - uncompressed_page_size = page_g.max_data_size; + uncompressed_page_size = page_g.data_size; if (ck_g.is_compressed) { - auto const is_v2 = page_g.page_type == PageType::DATA_PAGE_V2; - auto const lvl_bytes = is_v2 ? page_g.def_lvl_bytes + page_g.rep_lvl_bytes : 0; + auto const lvl_bytes = page_g.is_v2() ? page_g.level_bytes() : 0; hdr_start = page_g.compressed_data; compressed_page_size = static_cast(comp_results[blockIdx.x].bytes_written) + lvl_bytes; - page_g.max_data_size = compressed_page_size; + page_g.comp_data_size = compressed_page_size; } else { hdr_start = page_g.page_data; compressed_page_size = uncompressed_page_size; @@ -2708,19 +2736,26 @@ CUDF_KERNEL void __launch_bounds__(1024) if (t == 0) { page_g = first_page[page]; } __syncthreads(); - src = (ck_g.is_compressed) ? page_g.compressed_data : page_g.page_data; + src = ck_g.is_compressed ? page_g.compressed_data : page_g.page_data; // Copy page header hdr_len = page_g.hdr_size; memcpy_block<1024, true>(dst, src, hdr_len, t); src += page_g.max_hdr_size; dst += hdr_len; - // Copy page data uncompressed_size += hdr_len; - data_len = page_g.max_data_size; + data_len = ck_g.is_compressed ? page_g.comp_data_size : page_g.data_size; + // Copy page data. For V2, the level data and page data are disjoint. + if (page_g.is_v2()) { + auto const lvl_len = page_g.level_bytes(); + memcpy_block<1024, true>(dst, src, lvl_len, t); + src += page_g.max_lvl_size; + dst += lvl_len; + data_len -= lvl_len; + } memcpy_block<1024, true>(dst, src, data_len, t); dst += data_len; __syncthreads(); - if (!t && page == 0 && ck_g.use_dictionary) { ck_g.dictionary_size = hdr_len + data_len; } + if (t == 0 && page == 0 && ck_g.use_dictionary) { ck_g.dictionary_size = hdr_len + data_len; } } if (t == 0) { chunks[blockIdx.x].bfr_size = uncompressed_size; diff --git a/cpp/src/io/parquet/parquet_gpu.hpp b/cpp/src/io/parquet/parquet_gpu.hpp index d58c7f95389..b215cd7a20b 100644 --- a/cpp/src/io/parquet/parquet_gpu.hpp +++ b/cpp/src/io/parquet/parquet_gpu.hpp @@ -560,30 +560,41 @@ struct EncColumnChunk { * @brief Struct describing an encoder data page */ struct EncPage { - uint8_t* page_data; //!< Ptr to uncompressed page - uint8_t* compressed_data; //!< Ptr to compressed page - uint16_t num_fragments; //!< Number of fragments in page - PageType page_type; //!< Page type - Encoding encoding; //!< Encoding used for page data - EncColumnChunk* chunk; //!< Chunk that this page belongs to + // all pointers at the top to keep things properly aligned + uint8_t* page_data; //!< Ptr to uncompressed page + uint8_t* compressed_data; //!< Ptr to compressed page + EncColumnChunk* chunk; //!< Chunk that this page belongs to + compression_result* comp_res; //!< Ptr to compression result + uint32_t* def_histogram; //!< Histogram of counts for each definition level + uint32_t* rep_histogram; //!< Histogram of counts for each repetition level + // put this here in case it's ever made 64-bit + encode_kernel_mask kernel_mask; //!< Mask used to control which encoding kernels to run + // the rest can be 4 byte aligned uint32_t chunk_id; //!< Index in chunk array - uint32_t hdr_size; //!< Size of page header + uint32_t hdr_size; //!< Actual size of encoded page header uint32_t max_hdr_size; //!< Maximum size of page header - uint32_t max_data_size; //!< Maximum size of coded page data (excluding header) + uint32_t max_data_size; //!< Maximum size of encoded page data (excluding header) + uint32_t data_size; //!< Actual size of encoded page data (includes level data) + uint32_t comp_data_size; //!< Actual size of compressed page data uint32_t start_row; //!< First row of page uint32_t num_rows; //!< Rows in page uint32_t num_leaf_values; //!< Values in page. Different from num_rows in case of nested types uint32_t num_values; //!< Number of def/rep level values in page. Includes null/empty elements in //!< non-leaf levels - uint32_t def_lvl_bytes; //!< Number of bytes of encoded definition level data (V2 only) - uint32_t rep_lvl_bytes; //!< Number of bytes of encoded repetition level data (V2 only) - compression_result* comp_res; //!< Ptr to compression result - uint32_t num_nulls; //!< Number of null values (V2 only) (down here for alignment) - encode_kernel_mask kernel_mask; //!< Mask used to control which encoding kernels to run - uint32_t* def_histogram; //!< Histogram of counts for each definition level - uint32_t* rep_histogram; //!< Histogram of counts for each repetition level - uint32_t var_bytes_size; //!< Number of variable length bytes in the page (byte arrays only) + uint32_t def_lvl_bytes; //!< Number of bytes of encoded definition level data + uint32_t rep_lvl_bytes; //!< Number of bytes of encoded repetition level data + uint32_t max_lvl_size; //!< Maximum size of level data (V2 only, 0 for V1) + uint32_t num_nulls; //!< Number of null values uint32_t num_valid; //!< Number of valid leaf values + uint32_t var_bytes_size; //!< Number of variable length bytes in the page (byte arrays only) + // enums and smaller stuff down here + PageType page_type; //!< Page type + Encoding encoding; //!< Encoding used for page data + uint16_t num_fragments; //!< Number of fragments in page + + constexpr bool is_v2() const { return page_type == PageType::DATA_PAGE_V2; } + + constexpr auto level_bytes() const { return def_lvl_bytes + rep_lvl_bytes; } }; /** diff --git a/cpp/src/io/parquet/writer_impl.cu b/cpp/src/io/parquet/writer_impl.cu index 93b225dca1b..0303439fb27 100644 --- a/cpp/src/io/parquet/writer_impl.cu +++ b/cpp/src/io/parquet/writer_impl.cu @@ -2220,10 +2220,6 @@ writer::impl::~impl() { close(); } void writer::impl::init_state() { - // See issue #14781. Can remove this check once that is fixed. - CUDF_EXPECTS(not(_write_v2_headers and _compression == Compression::ZSTD), - "V2 page headers cannot be used with ZSTD compression"); - _current_chunk_offset.resize(_out_sink.size()); // Write file header file_header_s fhdr; @@ -2405,7 +2401,8 @@ void writer::impl::write_parquet_data_to_sink( // skip dict pages if (enc_page.page_type == PageType::DICTIONARY_PAGE) { continue; } - int32_t this_page_size = enc_page.hdr_size + enc_page.max_data_size; + int32_t const this_page_size = + enc_page.hdr_size + (ck.is_compressed ? enc_page.comp_data_size : enc_page.data_size); // first_row_idx is relative to start of row group PageLocation loc{curr_pg_offset, this_page_size, enc_page.start_row - ck.start_row}; if (is_byte_arr) { var_bytes.push_back(enc_page.var_bytes_size); } diff --git a/cpp/tests/io/parquet_v2_test.cpp b/cpp/tests/io/parquet_v2_test.cpp index 1a373ed92ae..25d58a96512 100644 --- a/cpp/tests/io/parquet_v2_test.cpp +++ b/cpp/tests/io/parquet_v2_test.cpp @@ -35,7 +35,7 @@ INSTANTIATE_TEST_SUITE_P(ParquetV2ReadWriteTest, TEST_P(ParquetV2Test, MultiColumn) { - constexpr auto num_rows = 50000; + constexpr auto num_rows = 50'000; auto const is_v2 = GetParam(); // auto col0_data = random_values(num_rows); @@ -84,6 +84,7 @@ TEST_P(ParquetV2Test, MultiColumn) cudf::io::parquet_writer_options out_opts = cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, expected) .write_v2_headers(is_v2) + .compression(cudf::io::compression_type::ZSTD) .metadata(expected_metadata); cudf::io::write_parquet(out_opts); @@ -156,6 +157,7 @@ TEST_P(ParquetV2Test, MultiColumnWithNulls) cudf::io::parquet_writer_options out_opts = cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, expected) .write_v2_headers(is_v2) + .compression(cudf::io::compression_type::ZSTD) .metadata(expected_metadata); cudf::io::write_parquet(out_opts); @@ -197,6 +199,7 @@ TEST_P(ParquetV2Test, Strings) cudf::io::parquet_writer_options out_opts = cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, expected) .write_v2_headers(is_v2) + .compression(cudf::io::compression_type::ZSTD) .metadata(expected_metadata); cudf::io::write_parquet(out_opts); diff --git a/cpp/tests/io/parquet_writer_test.cpp b/cpp/tests/io/parquet_writer_test.cpp index 2df34c7928b..34061cb7bf8 100644 --- a/cpp/tests/io/parquet_writer_test.cpp +++ b/cpp/tests/io/parquet_writer_test.cpp @@ -1064,7 +1064,6 @@ TEST_F(ParquetWriterTest, DictionaryAdaptiveTest) auto const expected = table_view{{col0, col1}}; auto const filepath = temp_env->get_temp_filepath("DictionaryAdaptiveTest.parquet"); - // no compression so we can easily read page data cudf::io::parquet_writer_options out_opts = cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, expected) .compression(cudf::io::compression_type::ZSTD) @@ -1116,7 +1115,6 @@ TEST_F(ParquetWriterTest, DictionaryAlwaysTest) auto const expected = table_view{{col0, col1}}; auto const filepath = temp_env->get_temp_filepath("DictionaryAlwaysTest.parquet"); - // no compression so we can easily read page data cudf::io::parquet_writer_options out_opts = cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, expected) .compression(cudf::io::compression_type::ZSTD) @@ -1428,21 +1426,6 @@ TEST_F(ParquetWriterTest, RowGroupMetadata) static_cast(num_rows * sizeof(column_type))); } -// See #14772. -// zStandard compression cannot currently be used with V2 page headers due to buffer -// alignment issues. -// TODO: Remove this test when #14781 is closed. -TEST_F(ParquetWriterTest, ZstdWithV2Header) -{ - auto const expected = table_view{}; - - cudf::io::parquet_writer_options const out_opts = - cudf::io::parquet_writer_options::builder(cudf::io::sink_info{"14772.pq"}, expected) - .compression(cudf::io::compression_type::ZSTD) - .write_v2_headers(true); - EXPECT_THROW(cudf::io::write_parquet(out_opts), cudf::logic_error); -} - ///////////////////////////////////////////////////////////// // custom mem mapped data sink that supports device writes template From bb59715162218c0c638f5c368e6871ca15168838 Mon Sep 17 00:00:00 2001 From: "Richard (Rick) Zamora" Date: Wed, 31 Jan 2024 14:52:37 -0600 Subject: [PATCH 187/384] Fix dask token normalization (#14829) This PR fixes cudf's `__dask_tokenization__` definitions so that they will produce data that can be deterministically tokenized when a `MultiIndex` is present. I ran into this problem in dask-expr for an index with datetime data (a case reflected by the new test). Authors: - Richard (Rick) Zamora (https://github.com/rjzamora) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) - Charles Blackmon-Luca (https://github.com/charlesbluca) URL: https://github.com/rapidsai/cudf/pull/14829 --- python/cudf/cudf/core/frame.py | 6 ++++-- python/cudf/cudf/core/indexed_frame.py | 8 +++++--- python/dask_cudf/dask_cudf/tests/test_dispatch.py | 14 +++++++++++++- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index 96b62e185b3..79005193b4e 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -1954,10 +1954,12 @@ def _repeat( @_cudf_nvtx_annotate @_warn_no_dask_cudf def __dask_tokenize__(self): + from dask.base import normalize_token + return [ type(self), - self._dtypes, - self.to_pandas(), + normalize_token(self._dtypes), + normalize_token(self.to_pandas()), ] diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 15277ff5586..0a0cefde9cd 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -6176,11 +6176,13 @@ def convert_dtypes( @_warn_no_dask_cudf def __dask_tokenize__(self): + from dask.base import normalize_token + return [ type(self), - self._dtypes, - self.index, - self.hash_values().values_host, + normalize_token(self._dtypes), + normalize_token(self.index), + normalize_token(self.hash_values().values_host), ] diff --git a/python/dask_cudf/dask_cudf/tests/test_dispatch.py b/python/dask_cudf/dask_cudf/tests/test_dispatch.py index c64e25fd437..76703206726 100644 --- a/python/dask_cudf/dask_cudf/tests/test_dispatch.py +++ b/python/dask_cudf/dask_cudf/tests/test_dispatch.py @@ -1,4 +1,6 @@ -# Copyright (c) 2021-2023, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. + +from datetime import datetime import numpy as np import pandas as pd @@ -82,6 +84,16 @@ def test_deterministic_tokenize(index): assert tokenize(df2) == tokenize(df2) +def test_deterministic_tokenize_multiindex(): + dt = datetime.strptime("1995-03-15", "%Y-%m-%d") + index = cudf.MultiIndex( + levels=[[1, 2], [dt]], + codes=[[0, 1], [0, 0]], + ) + df = cudf.DataFrame(index=index) + assert tokenize(df) == tokenize(df) + + @pytest.mark.parametrize("preserve_index", [True, False]) def test_pyarrow_schema_dispatch(preserve_index): from dask.dataframe.dispatch import ( From 767dde16e413f34cac16cb0b96b7eca18d71b7e9 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 31 Jan 2024 19:47:07 -1000 Subject: [PATCH 188/384] Use more public pandas APIs (#14929) As noted what's public in https://pandas.pydata.org/docs/reference/index.html Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/14929 --- python/cudf/cudf/_lib/groupby.pyx | 2 +- python/cudf/cudf/core/column/column.py | 2 +- python/cudf/cudf/core/dataframe.py | 18 ++++++------- python/cudf/cudf/core/dtypes.py | 12 +++------ python/cudf/cudf/core/index.py | 3 +-- python/cudf/cudf/core/multiindex.py | 3 +-- python/cudf/cudf/core/series.py | 14 +++++----- python/cudf/cudf/core/tools/datetimes.py | 26 ++++++++++++++++++- python/cudf/cudf/io/hdf.py | 4 +-- python/cudf/cudf/io/json.py | 3 +-- .../cudf/pandas/scripts/run-pandas-tests.sh | 5 +--- python/cudf/cudf/tests/test_datetime.py | 4 +-- python/dask_cudf/dask_cudf/backends.py | 4 +-- 13 files changed, 55 insertions(+), 45 deletions(-) diff --git a/python/cudf/cudf/_lib/groupby.pyx b/python/cudf/cudf/_lib/groupby.pyx index 8848649736b..db4c5e6173a 100644 --- a/python/cudf/cudf/_lib/groupby.pyx +++ b/python/cudf/cudf/_lib/groupby.pyx @@ -1,7 +1,7 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from functools import singledispatch -from pandas.core.groupby.groupby import DataError +from pandas.errors import DataError from cudf.api.types import is_string_dtype from cudf.core.buffer import acquire_spill_lock diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index ad56cabb48e..9143c7f5e9e 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -2189,7 +2189,7 @@ def as_column( elif ( arbitrary.size != 0 and arb_dtype.kind in ("O") - and isinstance(arbitrary[0], pd._libs.interval.Interval) + and isinstance(arbitrary[0], pd.Interval) ): # changing from pd array to series,possible arrow bug interval_series = pd.Series(arbitrary) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 1b0f83c5d70..727d5135297 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -34,8 +34,6 @@ import pandas as pd import pyarrow as pa from nvtx import annotate -from pandas._config import get_option -from pandas.core.dtypes.common import is_float, is_integer from pandas.io.formats import console from pandas.io.formats.printing import pprint_thing from typing_extensions import Self, assert_never @@ -1817,12 +1815,12 @@ def _clean_renderable_dataframe(self, output): dimensions (rows x columns) """ - max_rows = get_option("display.max_rows") - min_rows = get_option("display.min_rows") - max_cols = get_option("display.max_columns") - max_colwidth = get_option("display.max_colwidth") - show_dimensions = get_option("display.show_dimensions") - if get_option("display.expand_frame_repr"): + max_rows = pd.get_option("display.max_rows") + min_rows = pd.get_option("display.min_rows") + max_cols = pd.get_option("display.max_columns") + max_colwidth = pd.get_option("display.max_colwidth") + show_dimensions = pd.get_option("display.show_dimensions") + if pd.get_option("display.expand_frame_repr"): width, _ = console.get_console_size() else: width = None @@ -3318,8 +3316,8 @@ def diff(self, periods=1, axis=0): Diff currently only supports numeric dtype columns. """ - if not is_integer(periods): - if not (is_float(periods) and periods.is_integer()): + if not isinstance(periods, int): + if not (isinstance(periods, float) and periods.is_integer()): raise ValueError("periods must be an integer") periods = int(periods) diff --git a/python/cudf/cudf/core/dtypes.py b/python/cudf/cudf/core/dtypes.py index 17d6d42618a..7892f8065d0 100644 --- a/python/cudf/cudf/core/dtypes.py +++ b/python/cudf/cudf/core/dtypes.py @@ -14,10 +14,6 @@ from pandas.api import types as pd_types from pandas.api.extensions import ExtensionDtype from pandas.core.arrays.arrow.extension_types import ArrowIntervalType -from pandas.core.dtypes.dtypes import ( - CategoricalDtype as pd_CategoricalDtype, - CategoricalDtypeType as pd_CategoricalDtypeType, -) import cudf from cudf._typing import Dtype @@ -971,7 +967,7 @@ def _is_categorical_dtype(obj): if isinstance( obj, ( - pd_CategoricalDtype, + pd.CategoricalDtype, cudf.CategoricalDtype, cudf.core.index.CategoricalIndex, cudf.core.column.CategoricalColumn, @@ -988,8 +984,8 @@ def _is_categorical_dtype(obj): obj is t for t in ( cudf.CategoricalDtype, - pd_CategoricalDtype, - pd_CategoricalDtypeType, + pd.CategoricalDtype, + pd.CategoricalDtype.type, ) ): return True @@ -1010,7 +1006,7 @@ def _is_categorical_dtype(obj): ): return _is_categorical_dtype(obj.dtype) if hasattr(obj, "type"): - if obj.type is pd_CategoricalDtypeType: + if obj.type is pd.CategoricalDtype.type: return True # TODO: A lot of the above checks are probably redundant and should be # farmed out to this function here instead. diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index fa9e49baaa2..c8eedae200b 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -21,7 +21,6 @@ import cupy import numpy as np import pandas as pd -from pandas._config import get_option from typing_extensions import Self import cudf @@ -1306,7 +1305,7 @@ def get_loc(self, key): @_cudf_nvtx_annotate def __repr__(self): - max_seq_items = get_option("max_seq_items") or len(self) + max_seq_items = pd.get_option("max_seq_items") or len(self) mr = 0 if 2 * max_seq_items < len(self): mr = max_seq_items + 1 diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index a747ca8eea0..a3f7be7b266 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -15,7 +15,6 @@ import cupy as cp import numpy as np import pandas as pd -from pandas._config import get_option import cudf import cudf._lib as libcudf @@ -428,7 +427,7 @@ def copy( @_cudf_nvtx_annotate def __repr__(self): - max_seq_items = get_option("display.max_seq_items") or len(self) + max_seq_items = pd.get_option("display.max_seq_items") or len(self) if len(self) > max_seq_items: n = int(max_seq_items / 2) + 1 diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 77ed7644f69..3f51ecdf7dc 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -23,8 +23,6 @@ import cupy import numpy as np import pandas as pd -from pandas._config import get_option -from pandas.core.dtypes.common import is_float from typing_extensions import Self, assert_never import cudf @@ -1405,8 +1403,8 @@ def __repr__(self): _, height = get_terminal_size() max_rows = ( height - if get_option("display.max_rows") == 0 - else get_option("display.max_rows") + if pd.get_option("display.max_rows") == 0 + else pd.get_option("display.max_rows") ) if max_rows not in (0, None) and len(self) > max_rows: top = self.head(int(max_rows / 2 + 1)) @@ -1451,10 +1449,10 @@ def __repr__(self): ): min_rows = ( height - if get_option("display.min_rows") == 0 - else get_option("display.min_rows") + if pd.get_option("display.min_rows") == 0 + else pd.get_option("display.min_rows") ) - show_dimensions = get_option("display.show_dimensions") + show_dimensions = pd.get_option("display.show_dimensions") if preprocess._column.categories.dtype.kind == "f": pd_series = ( preprocess.astype("str") @@ -3392,7 +3390,7 @@ def diff(self, periods=1): dtype: int64 """ if not is_integer(periods): - if not (is_float(periods) and periods.is_integer()): + if not (isinstance(periods, float) and periods.is_integer()): raise ValueError("periods must be an integer") periods = int(periods) diff --git a/python/cudf/cudf/core/tools/datetimes.py b/python/cudf/cudf/core/tools/datetimes.py index 80a79e60ea9..faa7407daaf 100644 --- a/python/cudf/cudf/core/tools/datetimes.py +++ b/python/cudf/cudf/core/tools/datetimes.py @@ -9,7 +9,6 @@ import numpy as np import pandas as pd import pandas.tseries.offsets as pd_offset -from pandas.core.tools.datetimes import _unit_map from typing_extensions import Self import cudf @@ -21,6 +20,31 @@ from cudf.core import column from cudf.core.index import as_index +# https://github.com/pandas-dev/pandas/blob/2.2.x/pandas/core/tools/datetimes.py#L1112 +_unit_map = { + "year": "year", + "years": "year", + "month": "month", + "months": "month", + "day": "day", + "days": "day", + "hour": "h", + "hours": "h", + "minute": "m", + "minutes": "m", + "second": "s", + "seconds": "s", + "ms": "ms", + "millisecond": "ms", + "milliseconds": "ms", + "us": "us", + "microsecond": "us", + "microseconds": "us", + "ns": "ns", + "nanosecond": "ns", + "nanoseconds": "ns", +} + _unit_dtype_map = { "ns": "datetime64[ns]", "us": "datetime64[us]", diff --git a/python/cudf/cudf/io/hdf.py b/python/cudf/cudf/io/hdf.py index 8cf8c01c1df..78e7df649cb 100644 --- a/python/cudf/cudf/io/hdf.py +++ b/python/cudf/cudf/io/hdf.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. import warnings @@ -27,4 +27,4 @@ def to_hdf(path_or_buf, key, value, *args, **kwargs): "be GPU accelerated in the future" ) pd_value = value.to_pandas() - pd.io.pytables.to_hdf(path_or_buf, key, pd_value, *args, **kwargs) + pd_value.to_hdf(path_or_buf, key, *args, **kwargs) diff --git a/python/cudf/cudf/io/json.py b/python/cudf/cudf/io/json.py index 4f16263dd05..b2f3fd09146 100644 --- a/python/cudf/cudf/io/json.py +++ b/python/cudf/cudf/io/json.py @@ -247,9 +247,8 @@ def to_json( pd_value = pd.DataFrame(pd_data) else: pd_value = maybe_return_nullable_pd_obj(cudf_val) - return pd.io.json.to_json( + return pd_value.to_json( path_or_buf, - pd_value, orient=orient, storage_options=storage_options, *args, diff --git a/python/cudf/cudf/pandas/scripts/run-pandas-tests.sh b/python/cudf/cudf/pandas/scripts/run-pandas-tests.sh index c4dfe427c93..4fe152cc493 100755 --- a/python/cudf/cudf/pandas/scripts/run-pandas-tests.sh +++ b/python/cudf/cudf/pandas/scripts/run-pandas-tests.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. # All rights reserved. # SPDX-License-Identifier: Apache-2.0 @@ -50,9 +50,6 @@ xfail_strict = true filterwarnings = [ "error:Sparse:FutureWarning", "error:The SparseArray:FutureWarning", - # Deprecation gives warning on import during pytest collection - "ignore:pandas.core.index is deprecated:FutureWarning:importlib", - "ignore:pandas.util.testing is deprecated:FutureWarning:importlib", # Will be fixed in numba 0.56: https://github.com/numba/numba/issues/7758 "ignore:`np.MachAr` is deprecated:DeprecationWarning:numba", ] diff --git a/python/cudf/cudf/tests/test_datetime.py b/python/cudf/cudf/tests/test_datetime.py index 62733625485..24d8aa052e8 100644 --- a/python/cudf/cudf/tests/test_datetime.py +++ b/python/cudf/cudf/tests/test_datetime.py @@ -1705,14 +1705,14 @@ def test_date_range_raise_overflow(): start = np.datetime64(np.iinfo("int64").max, "ns") periods = 2 freq = cudf.DateOffset(nanoseconds=1) - with pytest.raises(pd._libs.tslibs.np_datetime.OutOfBoundsDatetime): + with pytest.raises(pd.errors.OutOfBoundsDatetime): cudf.date_range(start=start, periods=periods, freq=freq) # Non-fixed offset start = np.datetime64(np.iinfo("int64").max, "ns") periods = 2 freq = cudf.DateOffset(months=1) - with pytest.raises(pd._libs.tslibs.np_datetime.OutOfBoundsDatetime): + with pytest.raises(pd.errors.OutOfBoundsDatetime): # Extending beyond the max value will trigger a warning when pandas # does an internal conversion to a Python built-in datetime.datetime # object, which only supports down to microsecond resolution. diff --git a/python/dask_cudf/dask_cudf/backends.py b/python/dask_cudf/dask_cudf/backends.py index 026ab1d304a..454cce76ff2 100644 --- a/python/dask_cudf/dask_cudf/backends.py +++ b/python/dask_cudf/dask_cudf/backends.py @@ -41,7 +41,7 @@ from dask.utils import Dispatch, is_arraylike import cudf -from cudf.api.types import _is_datetime64tz_dtype, is_string_dtype +from cudf.api.types import is_string_dtype from cudf.utils.nvtx_annotation import _dask_cudf_nvtx_annotate from .core import DataFrame, Index, Series @@ -126,7 +126,7 @@ def _get_non_empty_data(s): data = cudf.core.column.as_column(data, dtype=s.dtype) elif is_string_dtype(s.dtype): data = pa.array(["cat", "dog"]) - elif _is_datetime64tz_dtype(s.dtype): + elif isinstance(s.dtype, pd.DatetimeTZDtype): from cudf.utils.dtypes import get_time_unit data = cudf.date_range("2001-01-01", periods=2, freq=get_time_unit(s)) From 6b3462bfbe796950dc3838e16e58d8bb2c0c9690 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 1 Feb 2024 07:04:26 -1000 Subject: [PATCH 189/384] Replace legacy cudf and dask_cudf imports as (d)gd (#14944) Discussed offline, replacing this legacy import style without aliasing Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Richard (Rick) Zamora (https://github.com/rjzamora) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/14944 --- python/cudf/cudf/api/extensions/accessor.py | 20 +- python/cudf/cudf/core/indexed_frame.py | 2 +- python/cudf/cudf/datasets.py | 10 +- python/cudf/cudf/tests/test_concat.py | 410 +++++++++--------- .../cudf/cudf/tests/test_custom_accessor.py | 26 +- python/cudf/cudf/tests/test_datasets.py | 14 +- python/dask_cudf/dask_cudf/sorting.py | 10 +- .../dask_cudf/tests/test_accessor.py | 64 +-- python/dask_cudf/dask_cudf/tests/test_core.py | 58 +-- .../dask_cudf/tests/test_delayed_io.py | 58 +-- python/dask_cudf/dask_cudf/tests/test_join.py | 26 +- .../dask_cudf/tests/test_reductions.py | 6 +- 12 files changed, 357 insertions(+), 347 deletions(-) diff --git a/python/cudf/cudf/api/extensions/accessor.py b/python/cudf/cudf/api/extensions/accessor.py index 311b33a5ab8..e4988c1fa68 100644 --- a/python/cudf/cudf/api/extensions/accessor.py +++ b/python/cudf/cudf/api/extensions/accessor.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. import warnings @@ -37,8 +37,8 @@ _dataframe_example = """ In your library code: - >>> import cudf as gd - >>> @gd.api.extensions.register_dataframe_accessor("point") + >>> import cudf + >>> @cudf.api.extensions.register_dataframe_accessor("point") ... class PointsAccessor: ... def __init__(self, obj): ... self._validate(obj) @@ -57,7 +57,7 @@ Then in user code: - >>> df = gd.DataFrame({'x': [1,2,3,4,5,6], 'y':[7,6,5,4,3,2]}) + >>> df = cudf.DataFrame({'x': [1,2,3,4,5,6], 'y':[7,6,5,4,3,2]}) >>> df.point.bounding_box (1, 2, 6, 7) @@ -66,8 +66,8 @@ _index_example = """ In your library code: - >>> import cudf as gd - >>> @gd.api.extensions.register_index_accessor("odd") + >>> import cudf + >>> @cudf.api.extensions.register_index_accessor("odd") ... class OddRowAccessor: ... def __init__(self, obj): ... self._obj = obj @@ -76,7 +76,7 @@ Then in user code: - >>> gs = gd.Index(list(range(0, 50))) + >>> gs = cudf.Index(list(range(0, 50))) >>> gs.odd[1] 1 >>> gs.odd[2] @@ -89,8 +89,8 @@ _series_example = """ In your library code: - >>> import cudf as gd - >>> @gd.api.extensions.register_series_accessor("odd") + >>> import cudf + >>> @cudf.api.extensions.register_series_accessor("odd") ... class OddRowAccessor: ... def __init__(self, obj): ... self._obj = obj @@ -99,7 +99,7 @@ Then in user code: - >>> gs = gd.Series(list(range(0, 50))) + >>> gs = cudf.Series(list(range(0, 50))) >>> gs.odd[1] 1 >>> gs.odd[2] diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 0a0cefde9cd..659e323c57d 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -4442,7 +4442,7 @@ def sample( Examples -------- - >>> import cudf as cudf + >>> import cudf >>> df = cudf.DataFrame({"a":{1, 2, 3, 4, 5}}) >>> df.sample(3) a diff --git a/python/cudf/cudf/datasets.py b/python/cudf/cudf/datasets.py index c6091ab60fc..7b183d5f1a3 100644 --- a/python/cudf/cudf/datasets.py +++ b/python/cudf/cudf/datasets.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. import numpy as np import pandas as pd @@ -42,8 +42,8 @@ def timeseries( Examples -------- - >>> import cudf as gd - >>> gdf = gd.datasets.timeseries() + >>> import cudf + >>> gdf = cudf.datasets.timeseries() >>> gdf.head() # doctest: +SKIP timestamp id name x y 2000-01-01 00:00:00 967 Jerry -0.031348 -0.040633 @@ -97,8 +97,8 @@ def randomdata(nrows=10, dtypes=None, seed=None): Examples -------- - >>> import cudf as gd - >>> gdf = gd.datasets.randomdata() + >>> import cudf + >>> gdf = cudf.datasets.randomdata() >>> cdf.head() # doctest: +SKIP id x y 0 1014 0.28361267466770146 -0.44274170661264334 diff --git a/python/cudf/cudf/tests/test_concat.py b/python/cudf/cudf/tests/test_concat.py index 4b0e46bf286..01c37005271 100644 --- a/python/cudf/cudf/tests/test_concat.py +++ b/python/cudf/cudf/tests/test_concat.py @@ -8,7 +8,7 @@ import pandas as pd import pytest -import cudf as gd +import cudf from cudf.api.types import _is_categorical_dtype from cudf.core._compat import PANDAS_GE_200 from cudf.core.dtypes import Decimal32Dtype, Decimal64Dtype, Decimal128Dtype @@ -59,8 +59,8 @@ def make_frames(index=None, nulls="none"): mask = mask[:5] df.loc[mask, "y"] = np.nan df2.loc[mask, "y"] = np.nan - gdf = gd.DataFrame.from_pandas(df) - gdf2 = gd.DataFrame.from_pandas(df2) + gdf = cudf.DataFrame.from_pandas(df) + gdf2 = cudf.DataFrame.from_pandas(df2) if index: df = df.set_index(index) df2 = df2.set_index(index) @@ -83,7 +83,7 @@ def test_concat_dataframe(index, nulls, axis): # DataFrame with _hide_concat_empty_dtype_warning(): - res = gd.concat([gdf, gdf2, gdf, gdf_empty1], axis=axis).to_pandas() + res = cudf.concat([gdf, gdf2, gdf, gdf_empty1], axis=axis).to_pandas() sol = pd.concat([df, df2, df, df_empty1], axis=axis) assert_eq( res, @@ -95,7 +95,7 @@ def test_concat_dataframe(index, nulls, axis): # Series for c in [i for i in ("x", "y", "z") if i != index]: - res = gd.concat([gdf[c], gdf2[c], gdf[c]], axis=axis).to_pandas() + res = cudf.concat([gdf[c], gdf2[c], gdf[c]], axis=axis).to_pandas() sol = pd.concat([df[c], df2[c], df[c]], axis=axis) assert_eq( res, @@ -106,7 +106,7 @@ def test_concat_dataframe(index, nulls, axis): ) # Index - res = gd.concat([gdf.index, gdf2.index], axis=axis).to_pandas() + res = cudf.concat([gdf.index, gdf2.index], axis=axis).to_pandas() sol = df.index.append(df2.index) assert_eq(res, sol, check_names=False, check_categorical=False) @@ -120,9 +120,9 @@ def test_concat_all_nulls(values): pb = pd.Series([None]) ps = pd.concat([pa, pb]) - ga = gd.Series(values) - gb = gd.Series([None]) - gs = gd.concat([ga, gb]) + ga = cudf.Series(values) + gb = cudf.Series([None]) + gs = cudf.concat([ga, gb]) assert_eq( ps, @@ -139,7 +139,7 @@ def test_concat_errors(): # No objs assert_exceptions_equal( lfunc=pd.concat, - rfunc=gd.concat, + rfunc=cudf.concat, lfunc_args_and_kwargs=([], {"objs": []}), rfunc_args_and_kwargs=([], {"objs": []}), ) @@ -147,7 +147,7 @@ def test_concat_errors(): # All None assert_exceptions_equal( lfunc=pd.concat, - rfunc=gd.concat, + rfunc=cudf.concat, lfunc_args_and_kwargs=([], {"objs": [None, None]}), rfunc_args_and_kwargs=([], {"objs": [None, None]}), ) @@ -155,7 +155,7 @@ def test_concat_errors(): # Mismatched types assert_exceptions_equal( lfunc=pd.concat, - rfunc=gd.concat, + rfunc=cudf.concat, lfunc_args_and_kwargs=([], {"objs": [df, df.index, df.x]}), rfunc_args_and_kwargs=([], {"objs": [gdf, gdf.index, gdf.x]}), ) @@ -163,7 +163,7 @@ def test_concat_errors(): # Unknown type assert_exceptions_equal( lfunc=pd.concat, - rfunc=gd.concat, + rfunc=cudf.concat, lfunc_args_and_kwargs=([], {"objs": ["bar", "foo"]}), rfunc_args_and_kwargs=([], {"objs": ["bar", "foo"]}), ) @@ -174,12 +174,12 @@ def test_concat_errors(): gdf4 = gdf2.set_index("z") with pytest.raises(ValueError, match="All columns must be the same type"): - gd.concat([gdf3, gdf4]) + cudf.concat([gdf3, gdf4]) # Bad axis value assert_exceptions_equal( lfunc=pd.concat, - rfunc=gd.concat, + rfunc=cudf.concat, lfunc_args_and_kwargs=( [], {"objs": [gdf.to_pandas(), gdf2.to_pandas()], "axis": "bad_value"}, @@ -193,7 +193,7 @@ def test_concat_misordered_columns(): gdf2 = gdf2[["z", "x", "y"]] df2 = df2[["z", "x", "y"]] - res = gd.concat([gdf, gdf2]).to_pandas() + res = cudf.concat([gdf, gdf2]).to_pandas() sol = pd.concat([df, df2], sort=False) assert_eq( @@ -211,17 +211,17 @@ def test_concat_columns(axis): pdf2 = pd.DataFrame( np.random.randint(10, size=(5, 4)), columns=[4, 5, 6, 7] ) - gdf1 = gd.from_pandas(pdf1) - gdf2 = gd.from_pandas(pdf2) + gdf1 = cudf.from_pandas(pdf1) + gdf2 = cudf.from_pandas(pdf2) expect = pd.concat([pdf1, pdf2], axis=axis) - got = gd.concat([gdf1, gdf2], axis=axis) + got = cudf.concat([gdf1, gdf2], axis=axis) assert_eq(expect, got, check_index_type=True) def test_concat_multiindex_dataframe(): - gdf = gd.DataFrame( + gdf = cudf.DataFrame( { "w": np.arange(4), "x": np.arange(4), @@ -233,22 +233,22 @@ def test_concat_multiindex_dataframe(): pdg = gdg.to_pandas() pdg1 = pdg.iloc[:, :1] pdg2 = pdg.iloc[:, 1:] - gdg1 = gd.from_pandas(pdg1) - gdg2 = gd.from_pandas(pdg2) + gdg1 = cudf.from_pandas(pdg1) + gdg2 = cudf.from_pandas(pdg2) assert_eq( - gd.concat([gdg1, gdg2]).astype("float64"), + cudf.concat([gdg1, gdg2]).astype("float64"), pd.concat([pdg1, pdg2]), check_index_type=True, ) assert_eq( - gd.concat([gdg1, gdg2], axis=1), + cudf.concat([gdg1, gdg2], axis=1), pd.concat([pdg1, pdg2], axis=1), check_index_type=True, ) def test_concat_multiindex_series(): - gdf = gd.DataFrame( + gdf = cudf.DataFrame( { "w": np.arange(4), "x": np.arange(4), @@ -260,16 +260,20 @@ def test_concat_multiindex_series(): pdg = gdg.to_pandas() pdg1 = pdg["y"] pdg2 = pdg["z"] - gdg1 = gd.from_pandas(pdg1) - gdg2 = gd.from_pandas(pdg2) + gdg1 = cudf.from_pandas(pdg1) + gdg2 = cudf.from_pandas(pdg2) assert_eq( - gd.concat([gdg1, gdg2]), pd.concat([pdg1, pdg2]), check_index_type=True + cudf.concat([gdg1, gdg2]), + pd.concat([pdg1, pdg2]), + check_index_type=True, + ) + assert_eq( + cudf.concat([gdg1, gdg2], axis=1), pd.concat([pdg1, pdg2], axis=1) ) - assert_eq(gd.concat([gdg1, gdg2], axis=1), pd.concat([pdg1, pdg2], axis=1)) def test_concat_multiindex_dataframe_and_series(): - gdf = gd.DataFrame( + gdf = cudf.DataFrame( { "w": np.arange(4), "x": np.arange(4), @@ -282,17 +286,17 @@ def test_concat_multiindex_dataframe_and_series(): pdg1 = pdg[["y", "z"]] pdg2 = pdg["z"] pdg2.name = "a" - gdg1 = gd.from_pandas(pdg1) - gdg2 = gd.from_pandas(pdg2) + gdg1 = cudf.from_pandas(pdg1) + gdg2 = cudf.from_pandas(pdg2) assert_eq( - gd.concat([gdg1, gdg2], axis=1), + cudf.concat([gdg1, gdg2], axis=1), pd.concat([pdg1, pdg2], axis=1), check_index_type=True, ) def test_concat_multiindex_series_and_dataframe(): - gdf = gd.DataFrame( + gdf = cudf.DataFrame( { "w": np.arange(4), "x": np.arange(4), @@ -305,10 +309,10 @@ def test_concat_multiindex_series_and_dataframe(): pdg1 = pdg["z"] pdg2 = pdg[["y", "z"]] pdg1.name = "a" - gdg1 = gd.from_pandas(pdg1) - gdg2 = gd.from_pandas(pdg2) + gdg1 = cudf.from_pandas(pdg1) + gdg2 = cudf.from_pandas(pdg2) assert_eq( - gd.concat([gdg1, gdg2], axis=1), + cudf.concat([gdg1, gdg2], axis=1), pd.concat([pdg1, pdg2], axis=1), check_index_type=True, ) @@ -318,27 +322,27 @@ def test_concat_multiindex_series_and_dataframe(): def test_concat_string_index_name(myindex): # GH-Issue #3420 data = {"a": [123, 456], "b": ["s1", "s2"]} - df1 = gd.DataFrame(data).set_index(myindex) + df1 = cudf.DataFrame(data).set_index(myindex) df2 = df1.copy() - df3 = gd.concat([df1, df2]) + df3 = cudf.concat([df1, df2]) assert df3.index.name == myindex def test_pandas_concat_compatibility_axis1(): - d1 = gd.datasets.randomdata( + d1 = cudf.datasets.randomdata( 3, dtypes={"a": float, "ind": float} ).set_index("ind") - d2 = gd.datasets.randomdata( + d2 = cudf.datasets.randomdata( 3, dtypes={"b": float, "ind": float} ).set_index("ind") - d3 = gd.datasets.randomdata( + d3 = cudf.datasets.randomdata( 3, dtypes={"c": float, "ind": float} ).set_index("ind") - d4 = gd.datasets.randomdata( + d4 = cudf.datasets.randomdata( 3, dtypes={"d": float, "ind": float} ).set_index("ind") - d5 = gd.datasets.randomdata( + d5 = cudf.datasets.randomdata( 3, dtypes={"e": float, "ind": float} ).set_index("ind") @@ -349,7 +353,7 @@ def test_pandas_concat_compatibility_axis1(): pd5 = d5.to_pandas() expect = pd.concat([pd1, pd2, pd3, pd4, pd5], axis=1) - got = gd.concat([d1, d2, d3, d4, d5], axis=1) + got = cudf.concat([d1, d2, d3, d4, d5], axis=1) assert_eq( got.sort_index(), @@ -368,28 +372,28 @@ def test_pandas_concat_compatibility_axis1(): ], ) def test_pandas_concat_compatibility_axis1_overlap(index, names, data): - s1 = gd.Series(data[0], index=[0, 1, 2]) - s2 = gd.Series(data[1], index=index) + s1 = cudf.Series(data[0], index=[0, 1, 2]) + s2 = cudf.Series(data[1], index=index) if names: s1.name = names[0] s2.name = names[1] ps1 = s1.to_pandas() ps2 = s2.to_pandas() - got = gd.concat([s1, s2], axis=1) + got = cudf.concat([s1, s2], axis=1) expect = pd.concat([ps1, ps2], axis=1) assert_eq(got, expect, check_index_type=True) def test_pandas_concat_compatibility_axis1_eq_index(): - s1 = gd.Series(["a", "b", "c"], index=[0, 1, 2]) - s2 = gd.Series(["a", "b", "c"], index=[1, 1, 1]) + s1 = cudf.Series(["a", "b", "c"], index=[0, 1, 2]) + s2 = cudf.Series(["a", "b", "c"], index=[1, 1, 1]) ps1 = s1.to_pandas() ps2 = s2.to_pandas() with expect_warning_if(not PANDAS_GE_200): assert_exceptions_equal( lfunc=pd.concat, - rfunc=gd.concat, + rfunc=cudf.concat, lfunc_args_and_kwargs=([], {"objs": [ps1, ps2], "axis": 1}), rfunc_args_and_kwargs=([], {"objs": [s1, s2], "axis": 1}), ) @@ -399,14 +403,14 @@ def test_pandas_concat_compatibility_axis1_eq_index(): def test_pandas_concat_compatibility_axis1_single_column(name): # Pandas renames series name `None` to 0 # and preserves anything else - s = gd.Series([1, 2, 3], name=name) - got = gd.concat([s], axis=1) + s = cudf.Series([1, 2, 3], name=name) + got = cudf.concat([s], axis=1) expected = pd.concat([s.to_pandas()], axis=1) assert_eq(expected, got) def test_concat_duplicate_columns(): - cdf = gd.DataFrame( + cdf = cudf.DataFrame( { "id4": 4 * list(range(6)), "id5": 4 * list(reversed(range(6))), @@ -416,30 +420,34 @@ def test_concat_duplicate_columns(): cdf_std = cdf.groupby(["id4", "id5"])[["v3"]].std() cdf_med = cdf.groupby(["id4", "id5"])[["v3"]].quantile(q=0.5) with pytest.raises(NotImplementedError): - gd.concat([cdf_med, cdf_std], axis=1) + cudf.concat([cdf_med, cdf_std], axis=1) def test_concat_mixed_input(): pdf1 = pd.DataFrame({"a": [10, 20, 30]}) pdf2 = pd.DataFrame({"a": [11, 22, 33]}) - gdf1 = gd.from_pandas(pdf1) - gdf2 = gd.from_pandas(pdf2) + gdf1 = cudf.from_pandas(pdf1) + gdf2 = cudf.from_pandas(pdf2) assert_eq( pd.concat([pdf1, None, pdf2, None]), - gd.concat([gdf1, None, gdf2, None]), + cudf.concat([gdf1, None, gdf2, None]), check_index_type=True, ) assert_eq( - pd.concat([pdf1, None]), gd.concat([gdf1, None]), check_index_type=True + pd.concat([pdf1, None]), + cudf.concat([gdf1, None]), + check_index_type=True, ) assert_eq( - pd.concat([None, pdf2]), gd.concat([None, gdf2]), check_index_type=True + pd.concat([None, pdf2]), + cudf.concat([None, gdf2]), + check_index_type=True, ) assert_eq( pd.concat([None, pdf2, pdf1]), - gd.concat([None, gdf2, gdf1]), + cudf.concat([None, gdf2, gdf1]), check_index_type=True, ) @@ -491,11 +499,11 @@ def test_concat_mixed_input(): ) def test_concat_series_dataframe_input(objs): pd_objs = objs - gd_objs = [gd.from_pandas(obj) for obj in objs] + gd_objs = [cudf.from_pandas(obj) for obj in objs] with _hide_concat_empty_dtype_warning(): expected = pd.concat(pd_objs) - actual = gd.concat(gd_objs) + actual = cudf.concat(gd_objs) assert_eq( expected.fillna(-1), @@ -537,10 +545,10 @@ def test_concat_series_dataframe_input(objs): ) def test_concat_series_dataframe_input_str(objs): pd_objs = objs - gd_objs = [gd.from_pandas(obj) for obj in objs] + gd_objs = [cudf.from_pandas(obj) for obj in objs] expected = pd.concat(pd_objs) - actual = gd.concat(gd_objs) + actual = cudf.concat(gd_objs) assert_eq(expected, actual, check_dtype=False, check_index_type=False) @@ -593,11 +601,11 @@ def test_concat_series_dataframe_input_str(objs): def test_concat_empty_dataframes(df, other, ignore_index): other_pd = [df] + other - gdf = gd.from_pandas(df) - other_gd = [gdf] + [gd.from_pandas(o) for o in other] + gdf = cudf.from_pandas(df) + other_gd = [gdf] + [cudf.from_pandas(o) for o in other] expected = pd.concat(other_pd, ignore_index=ignore_index) - actual = gd.concat(other_gd, ignore_index=ignore_index) + actual = cudf.concat(other_gd, ignore_index=ignore_index) if expected.shape != df.shape: for key, col in actual[actual.columns].items(): if _is_categorical_dtype(col.dtype): @@ -636,11 +644,11 @@ def test_concat_empty_dataframes(df, other, ignore_index): ], ) def test_concat_empty_and_nonempty_series(ignore_index, data, axis): - s1 = gd.Series() - s2 = gd.Series(data[0]) + s1 = cudf.Series() + s2 = cudf.Series(data[0]) ps1 = s1.to_pandas() ps2 = s2.to_pandas() - got = gd.concat([s1, s2], axis=axis, ignore_index=ignore_index) + got = cudf.concat([s1, s2], axis=axis, ignore_index=ignore_index) expect = pd.concat([ps1, ps2], axis=axis, ignore_index=ignore_index) assert_eq(got, expect, check_index_type=True) @@ -649,11 +657,11 @@ def test_concat_empty_and_nonempty_series(ignore_index, data, axis): @pytest.mark.parametrize("ignore_index", [True, False]) @pytest.mark.parametrize("axis", [0, "index"]) def test_concat_two_empty_series(ignore_index, axis): - s1 = gd.Series() - s2 = gd.Series() + s1 = cudf.Series() + s2 = cudf.Series() ps1 = s1.to_pandas() ps2 = s2.to_pandas() - got = gd.concat([s1, s2], axis=axis, ignore_index=ignore_index) + got = cudf.concat([s1, s2], axis=axis, ignore_index=ignore_index) expect = pd.concat([ps1, ps2], axis=axis, ignore_index=ignore_index) assert_eq(got, expect, check_index_type=True) @@ -663,12 +671,12 @@ def test_concat_two_empty_series(ignore_index, axis): "df1,df2", [ ( - gd.DataFrame({"k1": [0, 1], "k2": [2, 3], "v1": [4, 5]}), - gd.DataFrame({"k1": [1, 0], "k2": [3, 2], "v2": [6, 7]}), + cudf.DataFrame({"k1": [0, 1], "k2": [2, 3], "v1": [4, 5]}), + cudf.DataFrame({"k1": [1, 0], "k2": [3, 2], "v2": [6, 7]}), ), ( - gd.DataFrame({"k1": [0, 1], "k2": [2, 3], "v1": [4, 5]}), - gd.DataFrame({"k1": [0, 1], "k2": [3, 2], "v2": [6, 7]}), + cudf.DataFrame({"k1": [0, 1], "k2": [2, 3], "v1": [4, 5]}), + cudf.DataFrame({"k1": [0, 1], "k2": [3, 2], "v2": [6, 7]}), ), ], ) @@ -682,7 +690,7 @@ def test_concat_dataframe_with_multiindex(df1, df2): pdf1 = gdf1.to_pandas() pdf2 = gdf2.to_pandas() - actual = gd.concat([gdf1, gdf2], axis=1) + actual = cudf.concat([gdf1, gdf2], axis=1) expected = pd.concat([pdf1, pdf2], axis=1) # Will need to sort_index before comparing as @@ -743,13 +751,13 @@ def test_concat_dataframe_with_multiindex(df1, df2): @pytest.mark.parametrize("join", ["inner", "outer"]) @pytest.mark.parametrize("axis", [0]) def test_concat_join(objs, ignore_index, sort, join, axis): - gpu_objs = [gd.from_pandas(o) for o in objs] + gpu_objs = [cudf.from_pandas(o) for o in objs] assert_eq( pd.concat( objs, sort=sort, join=join, ignore_index=ignore_index, axis=axis ), - gd.concat( + cudf.concat( gpu_objs, sort=sort, join=join, @@ -778,7 +786,7 @@ def test_concat_join(objs, ignore_index, sort, join, axis): ], ) def test_concat_join_axis_1_dup_error(objs): - gpu_objs = [gd.from_pandas(o) for o in objs] + gpu_objs = [cudf.from_pandas(o) for o in objs] # we do not support duplicate columns with pytest.raises(NotImplementedError): assert_eq( @@ -786,7 +794,7 @@ def test_concat_join_axis_1_dup_error(objs): objs, axis=1, ), - gd.concat( + cudf.concat( gpu_objs, axis=1, ), @@ -816,11 +824,11 @@ def test_concat_join_axis_1_dup_error(objs): @pytest.mark.parametrize("axis", [1]) def test_concat_join_axis_1(objs, ignore_index, sort, join, axis): # no duplicate columns - gpu_objs = [gd.from_pandas(o) for o in objs] + gpu_objs = [cudf.from_pandas(o) for o in objs] expected = pd.concat( objs, sort=sort, join=join, ignore_index=ignore_index, axis=axis ) - actual = gd.concat( + actual = cudf.concat( gpu_objs, sort=sort, join=join, @@ -850,10 +858,10 @@ def test_concat_join_many_df_and_empty_df(ignore_index, sort, join, axis): pdf3 = pd.DataFrame({"j": [1, 2], "k": [1, 2], "s": [1, 2], "t": [1, 2]}) pdf_empty1 = pd.DataFrame() - gdf1 = gd.from_pandas(pdf1) - gdf2 = gd.from_pandas(pdf2) - gdf3 = gd.from_pandas(pdf3) - gdf_empty1 = gd.from_pandas(pdf_empty1) + gdf1 = cudf.from_pandas(pdf1) + gdf2 = cudf.from_pandas(pdf2) + gdf3 = cudf.from_pandas(pdf3) + gdf_empty1 = cudf.from_pandas(pdf_empty1) with _hide_concat_empty_dtype_warning(): assert_eq( @@ -864,7 +872,7 @@ def test_concat_join_many_df_and_empty_df(ignore_index, sort, join, axis): ignore_index=ignore_index, axis=axis, ), - gd.concat( + cudf.concat( [gdf1, gdf2, gdf3, gdf_empty1], sort=sort, join=join, @@ -888,11 +896,11 @@ def test_concat_join_one_df(ignore_index, sort, join, axis): } ) - gdf1 = gd.from_pandas(pdf1) + gdf1 = cudf.from_pandas(pdf1) expected = pd.concat( [pdf1], sort=sort, join=join, ignore_index=ignore_index, axis=axis ) - actual = gd.concat( + actual = cudf.concat( [gdf1], sort=sort, join=join, ignore_index=ignore_index, axis=axis ) @@ -923,8 +931,8 @@ def test_concat_join_one_df(ignore_index, sort, join, axis): def test_concat_join_no_overlapping_columns( pdf1, pdf2, ignore_index, sort, join, axis ): - gdf1 = gd.from_pandas(pdf1) - gdf2 = gd.from_pandas(pdf2) + gdf1 = cudf.from_pandas(pdf1) + gdf2 = cudf.from_pandas(pdf2) expected = pd.concat( [pdf1, pdf2], @@ -933,7 +941,7 @@ def test_concat_join_no_overlapping_columns( ignore_index=ignore_index, axis=axis, ) - actual = gd.concat( + actual = cudf.concat( [gdf1, gdf2], sort=sort, join=join, @@ -962,10 +970,10 @@ def test_concat_join_no_overlapping_columns_many_and_empty( ) pdf_empty = pd.DataFrame() - gdf4 = gd.from_pandas(pdf4) - gdf5 = gd.from_pandas(pdf5) - gdf6 = gd.from_pandas(pdf6) - gdf_empty = gd.from_pandas(pdf_empty) + gdf4 = cudf.from_pandas(pdf4) + gdf5 = cudf.from_pandas(pdf5) + gdf6 = cudf.from_pandas(pdf6) + gdf_empty = cudf.from_pandas(pdf_empty) with _hide_concat_empty_dtype_warning(): expected = pd.concat( @@ -975,7 +983,7 @@ def test_concat_join_no_overlapping_columns_many_and_empty( ignore_index=ignore_index, axis=axis, ) - actual = gd.concat( + actual = cudf.concat( [gdf4, gdf5, gdf6, gdf_empty], sort=sort, join=join, @@ -1038,7 +1046,7 @@ def test_concat_join_no_overlapping_columns_many_and_empty( def test_concat_join_no_overlapping_columns_many_and_empty2( objs, ignore_index, sort, join, axis ): - objs_gd = [gd.from_pandas(o) if o is not None else o for o in objs] + objs_gd = [cudf.from_pandas(o) if o is not None else o for o in objs] with _hide_concat_empty_dtype_warning(): expected = pd.concat( @@ -1048,7 +1056,7 @@ def test_concat_join_no_overlapping_columns_many_and_empty2( ignore_index=ignore_index, axis=axis, ) - actual = gd.concat( + actual = cudf.concat( objs_gd, sort=sort, join=join, @@ -1074,8 +1082,8 @@ def test_concat_join_no_overlapping_columns_empty_df_basic( ) pdf_empty = pd.DataFrame() - gdf6 = gd.from_pandas(pdf6) - gdf_empty = gd.from_pandas(pdf_empty) + gdf6 = cudf.from_pandas(pdf6) + gdf_empty = cudf.from_pandas(pdf_empty) with _hide_concat_empty_dtype_warning(): expected = pd.concat( @@ -1085,7 +1093,7 @@ def test_concat_join_no_overlapping_columns_empty_df_basic( ignore_index=ignore_index, axis=axis, ) - actual = gd.concat( + actual = cudf.concat( [gdf6, gdf_empty], sort=sort, join=join, @@ -1105,10 +1113,10 @@ def test_concat_join_no_overlapping_columns_empty_df_basic( @pytest.mark.parametrize("join", ["inner", "outer"]) @pytest.mark.parametrize("axis", [0, 1]) def test_concat_join_series(ignore_index, sort, join, axis): - s1 = gd.Series(["a", "b", "c"]) - s2 = gd.Series(["a", "b"]) - s3 = gd.Series(["a", "b", "c", "d"]) - s4 = gd.Series(dtype="str") + s1 = cudf.Series(["a", "b", "c"]) + s2 = cudf.Series(["a", "b"]) + s3 = cudf.Series(["a", "b", "c", "d"]) + s4 = cudf.Series(dtype="str") ps1 = s1.to_pandas() ps2 = s2.to_pandas() @@ -1123,7 +1131,7 @@ def test_concat_join_series(ignore_index, sort, join, axis): axis=axis, ) with expect_warning_if(axis == 1): - actual = gd.concat( + actual = cudf.concat( [s1, s2, s3, s4], sort=sort, join=join, @@ -1191,13 +1199,13 @@ def test_concat_join_empty_dataframes( df, other, ignore_index, axis, join, sort ): other_pd = [df] + other - gdf = gd.from_pandas(df) - other_gd = [gdf] + [gd.from_pandas(o) for o in other] + gdf = cudf.from_pandas(df) + other_gd = [gdf] + [cudf.from_pandas(o) for o in other] expected = pd.concat( other_pd, ignore_index=ignore_index, axis=axis, join=join, sort=sort ) - actual = gd.concat( + actual = cudf.concat( other_gd, ignore_index=ignore_index, axis=axis, join=join, sort=sort ) if expected.shape != df.shape: @@ -1302,8 +1310,8 @@ def test_concat_join_empty_dataframes_axis_1( ): # no duplicate columns other_pd = [df] + other - gdf = gd.from_pandas(df) - other_gd = [gdf] + [gd.from_pandas(o) for o in other] + gdf = cudf.from_pandas(df) + other_gd = [gdf] + [cudf.from_pandas(o) for o in other] with _hide_concat_empty_dtype_warning(): expected = pd.concat( @@ -1313,7 +1321,7 @@ def test_concat_join_empty_dataframes_axis_1( join=join, sort=sort, ) - actual = gd.concat( + actual = cudf.concat( other_gd, ignore_index=ignore_index, axis=axis, @@ -1356,18 +1364,18 @@ def test_concat_preserve_order(): assert_eq( pd.concat(dfs, join="inner"), - gd.concat([gd.DataFrame(df) for df in dfs], join="inner"), + cudf.concat([cudf.DataFrame(df) for df in dfs], join="inner"), check_index_type=True, ) @pytest.mark.parametrize("ignore_index", [True, False]) -@pytest.mark.parametrize("typ", [gd.DataFrame, gd.Series]) +@pytest.mark.parametrize("typ", [cudf.DataFrame, cudf.Series]) def test_concat_single_object(ignore_index, typ): """Ensure that concat on a single object does not change it.""" obj = typ([1, 2, 3]) assert_eq( - gd.concat([obj], ignore_index=ignore_index, axis=0), + cudf.concat([obj], ignore_index=ignore_index, axis=0), obj, check_index_type=True, ) @@ -1382,15 +1390,15 @@ def test_concat_single_object(ignore_index, typ): [ Decimal64Dtype(3, 2), Decimal64Dtype(8, 4), - gd.Decimal128Dtype(3, 2), - gd.Decimal32Dtype(8, 4), + cudf.Decimal128Dtype(3, 2), + cudf.Decimal32Dtype(8, 4), ], ) def test_concat_decimal_dataframe(ltype, rtype): - gdf1 = gd.DataFrame( + gdf1 = cudf.DataFrame( {"id": np.random.randint(0, 10, 3), "val": ["22.3", "59.5", "81.1"]} ) - gdf2 = gd.DataFrame( + gdf2 = cudf.DataFrame( {"id": np.random.randint(0, 10, 3), "val": ["2.35", "5.59", "8.14"]} ) @@ -1400,7 +1408,7 @@ def test_concat_decimal_dataframe(ltype, rtype): pdf1 = gdf1.to_pandas() pdf2 = gdf2.to_pandas() - got = gd.concat([gdf1, gdf2]) + got = cudf.concat([gdf1, gdf2]) expected = pd.concat([pdf1, pdf2]) assert_eq(expected, got, check_index_type=True) @@ -1417,13 +1425,13 @@ def test_concat_decimal_dataframe(ltype, rtype): ], ) def test_concat_decimal_series(ltype, rtype): - gs1 = gd.Series(["228.3", "559.5", "281.1"]).astype(ltype) - gs2 = gd.Series(["2.345", "5.259", "8.154"]).astype(rtype) + gs1 = cudf.Series(["228.3", "559.5", "281.1"]).astype(ltype) + gs2 = cudf.Series(["2.345", "5.259", "8.154"]).astype(rtype) ps1 = gs1.to_pandas() ps2 = gs2.to_pandas() - got = gd.concat([gs1, gs2]) + got = cudf.concat([gs1, gs2]) expected = pd.concat([ps1, ps2]) assert_eq(expected, got, check_index_type=True) @@ -1433,16 +1441,16 @@ def test_concat_decimal_series(ltype, rtype): "df1, df2, df3, expected", [ ( - gd.DataFrame( + cudf.DataFrame( {"val": [Decimal("42.5"), Decimal("8.7")]}, dtype=Decimal64Dtype(5, 2), ), - gd.DataFrame( + cudf.DataFrame( {"val": [Decimal("9.23"), Decimal("-67.49")]}, dtype=Decimal64Dtype(6, 4), ), - gd.DataFrame({"val": [8, -5]}, dtype="int32"), - gd.DataFrame( + cudf.DataFrame({"val": [8, -5]}, dtype="int32"), + cudf.DataFrame( { "val": [ Decimal("42.5"), @@ -1458,13 +1466,13 @@ def test_concat_decimal_series(ltype, rtype): ), ), ( - gd.DataFrame( + cudf.DataFrame( {"val": [Decimal("95.2"), Decimal("23.4")]}, dtype=Decimal64Dtype(5, 2), ), - gd.DataFrame({"val": [54, 509]}, dtype="uint16"), - gd.DataFrame({"val": [24, -48]}, dtype="int32"), - gd.DataFrame( + cudf.DataFrame({"val": [54, 509]}, dtype="uint16"), + cudf.DataFrame({"val": [24, -48]}, dtype="int32"), + cudf.DataFrame( { "val": [ Decimal("95.2"), @@ -1480,13 +1488,13 @@ def test_concat_decimal_series(ltype, rtype): ), ), ( - gd.DataFrame( + cudf.DataFrame( {"val": [Decimal("36.56"), Decimal("-59.24")]}, dtype=Decimal64Dtype(9, 4), ), - gd.DataFrame({"val": [403.21, 45.13]}, dtype="float32"), - gd.DataFrame({"val": [52.262, -49.25]}, dtype="float64"), - gd.DataFrame( + cudf.DataFrame({"val": [403.21, 45.13]}, dtype="float32"), + cudf.DataFrame({"val": [52.262, -49.25]}, dtype="float64"), + cudf.DataFrame( { "val": [ Decimal("36.56"), @@ -1502,13 +1510,13 @@ def test_concat_decimal_series(ltype, rtype): ), ), ( - gd.DataFrame( + cudf.DataFrame( {"val": [Decimal("9563.24"), Decimal("236.633")]}, dtype=Decimal64Dtype(9, 4), ), - gd.DataFrame({"val": [5393, -95832]}, dtype="int64"), - gd.DataFrame({"val": [-29.234, -31.945]}, dtype="float64"), - gd.DataFrame( + cudf.DataFrame({"val": [5393, -95832]}, dtype="int64"), + cudf.DataFrame({"val": [-29.234, -31.945]}, dtype="float64"), + cudf.DataFrame( { "val": [ Decimal("9563.24"), @@ -1524,13 +1532,13 @@ def test_concat_decimal_series(ltype, rtype): ), ), ( - gd.DataFrame( + cudf.DataFrame( {"val": [Decimal("95633.24"), Decimal("236.633")]}, dtype=Decimal128Dtype(19, 4), ), - gd.DataFrame({"val": [5393, -95832]}, dtype="int64"), - gd.DataFrame({"val": [-29.234, -31.945]}, dtype="float64"), - gd.DataFrame( + cudf.DataFrame({"val": [5393, -95832]}, dtype="int64"), + cudf.DataFrame({"val": [-29.234, -31.945]}, dtype="float64"), + cudf.DataFrame( { "val": [ Decimal("95633.24"), @@ -1548,7 +1556,7 @@ def test_concat_decimal_series(ltype, rtype): ], ) def test_concat_decimal_numeric_dataframe(df1, df2, df3, expected): - df = gd.concat([df1, df2, df3]) + df = cudf.concat([df1, df2, df3]) assert_eq(df, expected, check_index_type=True) assert_eq(df.val.dtype, expected.val.dtype) @@ -1557,15 +1565,15 @@ def test_concat_decimal_numeric_dataframe(df1, df2, df3, expected): "s1, s2, s3, expected", [ ( - gd.Series( + cudf.Series( [Decimal("32.8"), Decimal("-87.7")], dtype=Decimal64Dtype(6, 2) ), - gd.Series( + cudf.Series( [Decimal("101.243"), Decimal("-92.449")], dtype=Decimal64Dtype(9, 6), ), - gd.Series([94, -22], dtype="int32"), - gd.Series( + cudf.Series([94, -22], dtype="int32"), + cudf.Series( [ Decimal("32.8"), Decimal("-87.7"), @@ -1579,12 +1587,12 @@ def test_concat_decimal_numeric_dataframe(df1, df2, df3, expected): ), ), ( - gd.Series( + cudf.Series( [Decimal("7.2"), Decimal("122.1")], dtype=Decimal64Dtype(5, 2) ), - gd.Series([33, 984], dtype="uint32"), - gd.Series([593, -702], dtype="int32"), - gd.Series( + cudf.Series([33, 984], dtype="uint32"), + cudf.Series([593, -702], dtype="int32"), + cudf.Series( [ Decimal("7.2"), Decimal("122.1"), @@ -1598,13 +1606,13 @@ def test_concat_decimal_numeric_dataframe(df1, df2, df3, expected): ), ), ( - gd.Series( + cudf.Series( [Decimal("982.94"), Decimal("-493.626")], dtype=Decimal64Dtype(9, 4), ), - gd.Series([847.98, 254.442], dtype="float32"), - gd.Series([5299.262, -2049.25], dtype="float64"), - gd.Series( + cudf.Series([847.98, 254.442], dtype="float32"), + cudf.Series([5299.262, -2049.25], dtype="float64"), + cudf.Series( [ Decimal("982.94"), Decimal("-493.626"), @@ -1618,13 +1626,13 @@ def test_concat_decimal_numeric_dataframe(df1, df2, df3, expected): ), ), ( - gd.Series( + cudf.Series( [Decimal("492.204"), Decimal("-72824.455")], dtype=Decimal64Dtype(9, 4), ), - gd.Series([8438, -27462], dtype="int64"), - gd.Series([-40.292, 49202.953], dtype="float64"), - gd.Series( + cudf.Series([8438, -27462], dtype="int64"), + cudf.Series([-40.292, 49202.953], dtype="float64"), + cudf.Series( [ Decimal("492.204"), Decimal("-72824.455"), @@ -1638,19 +1646,19 @@ def test_concat_decimal_numeric_dataframe(df1, df2, df3, expected): ), ), ( - gd.Series( + cudf.Series( [Decimal("492.204"), Decimal("-72824.455")], dtype=Decimal64Dtype(10, 4), ), - gd.Series( + cudf.Series( [Decimal("8438"), Decimal("-27462")], dtype=Decimal32Dtype(9, 4), ), - gd.Series( + cudf.Series( [Decimal("-40.292"), Decimal("49202.953")], dtype=Decimal128Dtype(19, 4), ), - gd.Series( + cudf.Series( [ Decimal("492.204"), Decimal("-72824.455"), @@ -1666,7 +1674,7 @@ def test_concat_decimal_numeric_dataframe(df1, df2, df3, expected): ], ) def test_concat_decimal_numeric_series(s1, s2, s3, expected): - s = gd.concat([s1, s2, s3]) + s = cudf.concat([s1, s2, s3]) assert_eq(s, expected, check_index_type=True) @@ -1674,11 +1682,11 @@ def test_concat_decimal_numeric_series(s1, s2, s3, expected): "s1, s2, expected", [ ( - gd.Series( + cudf.Series( [Decimal("955.22"), Decimal("8.2")], dtype=Decimal64Dtype(5, 2) ), - gd.Series(["2007-06-12", "2006-03-14"], dtype="datetime64[s]"), - gd.Series( + cudf.Series(["2007-06-12", "2006-03-14"], dtype="datetime64[s]"), + cudf.Series( [ "955.22", "8.20", @@ -1689,17 +1697,17 @@ def test_concat_decimal_numeric_series(s1, s2, s3, expected): ), ), ( - gd.Series( + cudf.Series( [Decimal("-52.44"), Decimal("365.22")], dtype=Decimal64Dtype(5, 2), ), - gd.Series( + cudf.Series( np.arange( "2005-02-01T12", "2005-02-01T15", dtype="datetime64[h]" ), dtype="datetime64[s]", ), - gd.Series( + cudf.Series( [ "-52.44", "365.22", @@ -1711,25 +1719,25 @@ def test_concat_decimal_numeric_series(s1, s2, s3, expected): ), ), ( - gd.Series( + cudf.Series( [Decimal("753.0"), Decimal("94.22")], dtype=Decimal64Dtype(5, 2), ), - gd.Series([np.timedelta64(111, "s"), np.timedelta64(509, "s")]), - gd.Series( + cudf.Series([np.timedelta64(111, "s"), np.timedelta64(509, "s")]), + cudf.Series( ["753.00", "94.22", "0 days 00:01:51", "0 days 00:08:29"], index=[0, 1, 0, 1], ), ), ( - gd.Series( + cudf.Series( [Decimal("753.0"), Decimal("94.22")], dtype=Decimal64Dtype(5, 2), ), - gd.Series( + cudf.Series( [np.timedelta64(940252, "s"), np.timedelta64(758385, "s")] ), - gd.Series( + cudf.Series( ["753.00", "94.22", "10 days 21:10:52", "8 days 18:39:45"], index=[0, 1, 0, 1], ), @@ -1737,7 +1745,7 @@ def test_concat_decimal_numeric_series(s1, s2, s3, expected): ], ) def test_concat_decimal_non_numeric(s1, s2, expected): - s = gd.concat([s1, s2]) + s = cudf.concat([s1, s2]) assert_eq(s, expected, check_index_type=True) @@ -1745,9 +1753,9 @@ def test_concat_decimal_non_numeric(s1, s2, expected): "s1, s2, expected", [ ( - gd.Series([{"a": 5}, {"c": "hello"}, {"b": 7}]), - gd.Series([{"a": 5, "c": "hello", "b": 7}]), - gd.Series( + cudf.Series([{"a": 5}, {"c": "hello"}, {"b": 7}]), + cudf.Series([{"a": 5, "c": "hello", "b": 7}]), + cudf.Series( [ {"a": 5, "b": None, "c": None}, {"a": None, "b": None, "c": "hello"}, @@ -1760,7 +1768,7 @@ def test_concat_decimal_non_numeric(s1, s2, expected): ], ) def test_concat_struct_column(s1, s2, expected): - s = gd.concat([s1, s2]) + s = cudf.concat([s1, s2]) assert_eq(s, expected, check_index_type=True) @@ -1768,9 +1776,9 @@ def test_concat_struct_column(s1, s2, expected): "frame1, frame2, expected", [ ( - gd.Series([[{"b": 0}], [{"b": 1}], [{"b": 3}]]), - gd.Series([[{"b": 10}], [{"b": 12}], None]), - gd.Series( + cudf.Series([[{"b": 0}], [{"b": 1}], [{"b": 3}]]), + cudf.Series([[{"b": 10}], [{"b": 12}], None]), + cudf.Series( [ [{"b": 0}], [{"b": 1}], @@ -1783,9 +1791,9 @@ def test_concat_struct_column(s1, s2, expected): ), ), ( - gd.DataFrame({"a": [[{"b": 0}], [{"b": 1}], [{"b": 3}]]}), - gd.DataFrame({"a": [[{"b": 10}], [{"b": 12}], None]}), - gd.DataFrame( + cudf.DataFrame({"a": [[{"b": 0}], [{"b": 1}], [{"b": 3}]]}), + cudf.DataFrame({"a": [[{"b": 10}], [{"b": 12}], None]}), + cudf.DataFrame( { "a": [ [{"b": 0}], @@ -1802,7 +1810,7 @@ def test_concat_struct_column(s1, s2, expected): ], ) def test_concat_list_column(frame1, frame2, expected): - actual = gd.concat([frame1, frame2]) + actual = cudf.concat([frame1, frame2]) assert_eq(actual, expected, check_index_type=True) @@ -1814,10 +1822,10 @@ def test_concat_categorical_ordering(): sr = sr.cat.set_categories(["d", "a", "b", "c", "e"]) df = pd.DataFrame({"a": sr}) - gdf = gd.from_pandas(df) + gdf = cudf.from_pandas(df) expect = pd.concat([df, df, df]) - got = gd.concat([gdf, gdf, gdf]) + got = cudf.concat([gdf, gdf, gdf]) assert_eq(expect, got) @@ -1852,8 +1860,8 @@ def singleton_concat_obj(request, singleton_concat_index): def test_concat_singleton_sorting( axis, sort, ignore_index, singleton_concat_obj ): - gobj = gd.from_pandas(singleton_concat_obj) - gconcat = gd.concat( + gobj = cudf.from_pandas(singleton_concat_obj) + gconcat = cudf.concat( [gobj], axis=axis, sort=sort, ignore_index=ignore_index ) pconcat = pd.concat( @@ -1864,9 +1872,9 @@ def test_concat_singleton_sorting( @pytest.mark.parametrize("axis", [2, "invalid"]) def test_concat_invalid_axis(axis): - s = gd.Series([1, 2, 3]) + s = cudf.Series([1, 2, 3]) with pytest.raises(ValueError): - gd.concat([s], axis=axis) + cudf.concat([s], axis=axis) @pytest.mark.parametrize( @@ -1876,7 +1884,7 @@ def test_concat_invalid_axis(axis): ], ) def test_concat_mixed_list_types_error(s1, s2): - s1, s2 = gd.Series(s1), gd.Series(s2) + s1, s2 = cudf.Series(s1), cudf.Series(s2) with pytest.raises(NotImplementedError): - gd.concat([s1, s2], ignore_index=True) + cudf.concat([s1, s2], ignore_index=True) diff --git a/python/cudf/cudf/tests/test_custom_accessor.py b/python/cudf/cudf/tests/test_custom_accessor.py index 325be954fe4..5ffe255d0f8 100644 --- a/python/cudf/cudf/tests/test_custom_accessor.py +++ b/python/cudf/cudf/tests/test_custom_accessor.py @@ -1,13 +1,13 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. import pandas as pd import pytest -import cudf as gd +import cudf from cudf.testing._utils import assert_eq -@gd.api.extensions.register_dataframe_accessor("point") +@cudf.api.extensions.register_dataframe_accessor("point") @pd.api.extensions.register_dataframe_accessor("point") class PointsAccessor: def __init__(self, obj): @@ -29,7 +29,7 @@ def bounding_box(self): @pytest.mark.parametrize( - "gdf", [gd.datasets.randomdata(nrows=6, dtypes={"x": int, "y": int})] + "gdf", [cudf.datasets.randomdata(nrows=6, dtypes={"x": int, "y": int})] ) def test_dataframe_accessor(gdf): pdf = gdf.to_pandas() @@ -38,10 +38,10 @@ def test_dataframe_accessor(gdf): @pytest.mark.parametrize( - "gdf1", [gd.datasets.randomdata(nrows=1, dtypes={"x": int, "y": int})] + "gdf1", [cudf.datasets.randomdata(nrows=1, dtypes={"x": int, "y": int})] ) @pytest.mark.parametrize( - "gdf2", [gd.datasets.randomdata(nrows=1, dtypes={"x": int, "y": int})] + "gdf2", [cudf.datasets.randomdata(nrows=1, dtypes={"x": int, "y": int})] ) def test_dataframe_accessor_idendity(gdf1, gdf2): """Test for accessor identities @@ -55,8 +55,8 @@ def test_dataframe_accessor_idendity(gdf1, gdf2): @pd.api.extensions.register_index_accessor("odd") @pd.api.extensions.register_series_accessor("odd") -@gd.api.extensions.register_index_accessor("odd") -@gd.api.extensions.register_series_accessor("odd") +@cudf.api.extensions.register_index_accessor("odd") +@cudf.api.extensions.register_series_accessor("odd") class OddRowAccessor: def __init__(self, obj): self._obj = obj @@ -65,7 +65,7 @@ def __getitem__(self, i): return self._obj[2 * i - 1] -@pytest.mark.parametrize("gidx", [gd.Index(list(range(0, 50)))]) +@pytest.mark.parametrize("gidx", [cudf.Index(list(range(0, 50)))]) def test_index_accessor(gidx): pidx = gidx.to_pandas() @@ -73,7 +73,7 @@ def test_index_accessor(gidx): assert_eq(gidx.odd[i], pidx.odd[i]) -@pytest.mark.parametrize("gs", [gd.Series(list(range(1, 50)))]) +@pytest.mark.parametrize("gs", [cudf.Series(list(range(1, 50)))]) def test_series_accessor(gs): ps = gs.to_pandas() @@ -82,10 +82,10 @@ def test_series_accessor(gs): @pytest.mark.parametrize( - "gdf", [gd.datasets.randomdata(nrows=6, dtypes={"x": int, "y": int})] + "gdf", [cudf.datasets.randomdata(nrows=6, dtypes={"x": int, "y": int})] ) -@pytest.mark.parametrize("gidx", [gd.Index(list(range(1, 50)))]) -@pytest.mark.parametrize("gs", [gd.Series(list(range(1, 50)))]) +@pytest.mark.parametrize("gidx", [cudf.Index(list(range(1, 50)))]) +@pytest.mark.parametrize("gs", [cudf.Series(list(range(1, 50)))]) def test_accessor_space_separate(gdf, gidx, gs): assert not id(gdf._accessors) == id(gidx._accessors) assert not id(gidx._accessors) == id(gs._accessors) diff --git a/python/cudf/cudf/tests/test_datasets.py b/python/cudf/cudf/tests/test_datasets.py index 320c221fcb2..45629868ccc 100644 --- a/python/cudf/cudf/tests/test_datasets.py +++ b/python/cudf/cudf/tests/test_datasets.py @@ -2,15 +2,15 @@ import numpy as np -import cudf as gd +import cudf from cudf.testing._utils import assert_eq def test_dataset_timeseries(): - gdf1 = gd.datasets.timeseries( + gdf1 = cudf.datasets.timeseries( dtypes={"x": int, "y": float}, freq="120s", nulls_frequency=0.3, seed=1 ) - gdf2 = gd.datasets.timeseries( + gdf2 = cudf.datasets.timeseries( dtypes={"x": int, "y": float}, freq="120s", nulls_frequency=0.3, seed=1 ) @@ -20,7 +20,7 @@ def test_dataset_timeseries(): assert gdf1["y"].head().dtype == float assert gdf1.index.name == "timestamp" - gdf = gd.datasets.timeseries( + gdf = cudf.datasets.timeseries( "2000", "2010", freq="2H", @@ -33,13 +33,13 @@ def test_dataset_timeseries(): assert gdf["id"].head().dtype == int assert gdf["name"].head().dtype == "category" - gdf = gd.datasets.randomdata() + gdf = cudf.datasets.randomdata() assert gdf["id"].head().dtype == int assert gdf["x"].head().dtype == float assert gdf["y"].head().dtype == float assert len(gdf) == 10 - gdf = gd.datasets.randomdata( + gdf = cudf.datasets.randomdata( nrows=20, dtypes={"id": int, "a": int, "b": float} ) assert gdf["id"].head().dtype == int @@ -51,7 +51,7 @@ def test_dataset_timeseries(): def test_make_bool(): n = 10 state = np.random.RandomState(12) - arr = gd.datasets.make_bool(n, state) + arr = cudf.datasets.make_bool(n, state) assert np.all(np.isin(arr, [True, False])) assert arr.size == n assert arr.dtype == bool diff --git a/python/dask_cudf/dask_cudf/sorting.py b/python/dask_cudf/dask_cudf/sorting.py index d01ada92e33..f3774e20d32 100644 --- a/python/dask_cudf/dask_cudf/sorting.py +++ b/python/dask_cudf/dask_cudf/sorting.py @@ -16,7 +16,7 @@ from dask.highlevelgraph import HighLevelGraph from dask.utils import M -import cudf as gd +import cudf from cudf.api.types import _is_categorical_dtype from cudf.utils.nvtx_annotation import _dask_cudf_nvtx_annotate @@ -118,7 +118,7 @@ def _append_counts(val, count): return val # Sort by calculated quantile values, then number of observations. - combined_vals_counts = gd.core.reshape._merge_sorted( + combined_vals_counts = cudf.core.reshape._merge_sorted( [*map(_append_counts, vals, counts)] ) combined_counts = cupy.asnumpy(combined_vals_counts["_counts"].values) @@ -180,7 +180,7 @@ def finalize_tsk(tsk): if len(qs) == 0: name = "quantiles-" + token - empty_index = gd.Index([], dtype=float) + empty_index = cudf.Index([], dtype=float) return Series( { (name, 0): final_type( @@ -305,7 +305,7 @@ def sort_values( # Step 2 - Perform repartitioning shuffle meta = df._meta._constructor_sliced([0]) - if not isinstance(divisions, (gd.Series, gd.DataFrame)): + if not isinstance(divisions, (cudf.Series, cudf.DataFrame)): dtype = df[by[0]].dtype divisions = df._meta._constructor_sliced(divisions, dtype=dtype) @@ -330,7 +330,7 @@ def sort_values( # Step 3 - Return final sorted df df4 = df3.map_partitions(sort_function, **sort_kwargs) - if not isinstance(divisions, gd.DataFrame) and set_divisions: + if not isinstance(divisions, cudf.DataFrame) and set_divisions: # Can't have multi-column divisions elsewhere in dask (yet) df4.divisions = tuple(methods.tolist(divisions)) diff --git a/python/dask_cudf/dask_cudf/tests/test_accessor.py b/python/dask_cudf/dask_cudf/tests/test_accessor.py index a6a457d98a4..8c9ce45df59 100644 --- a/python/dask_cudf/dask_cudf/tests/test_accessor.py +++ b/python/dask_cudf/dask_cudf/tests/test_accessor.py @@ -11,7 +11,7 @@ from cudf import DataFrame, Series, date_range from cudf.testing._utils import assert_eq, does_not_raise -import dask_cudf as dgd +import dask_cudf ############################################################################# # Datetime Accessor # @@ -33,7 +33,7 @@ def data_dt_2(): def test_datetime_accessor_initialization(data): pdsr = pd.Series(data.copy()) sr = Series(pdsr) - dsr = dgd.from_cudf(sr, npartitions=5) + dsr = dask_cudf.from_cudf(sr, npartitions=5) with pytest.raises(AttributeError): dsr.dt @@ -42,7 +42,7 @@ def test_datetime_accessor_initialization(data): def test_series(data): pdsr = pd.Series(data.copy()) sr = Series(pdsr) - dsr = dgd.from_cudf(sr, npartitions=5) + dsr = dask_cudf.from_cudf(sr, npartitions=5) np.testing.assert_equal(np.array(pdsr), dsr.compute().values_host) @@ -52,7 +52,7 @@ def test_series(data): def test_dt_series(data, field): pdsr = pd.Series(data.copy()) sr = Series(pdsr) - dsr = dgd.from_cudf(sr, npartitions=5) + dsr = dask_cudf.from_cudf(sr, npartitions=5) base = getattr(pdsr.dt, field) test = getattr(dsr.dt, field).compute() assert_eq(base, test, check_dtype=False) @@ -61,7 +61,7 @@ def test_dt_series(data, field): @pytest.mark.parametrize("data", [data_dt_1()]) def test_dt_accessor(data): df = DataFrame({"dt_col": data.copy()}) - ddf = dgd.from_cudf(df, npartitions=5) + ddf = dask_cudf.from_cudf(df, npartitions=5) for i in ["year", "month", "day", "hour", "minute", "second", "weekday"]: assert i in dir(ddf.dt_col.dt) @@ -98,14 +98,14 @@ def data_cat_3(): @pytest.mark.parametrize("data", [data_cat_1()]) def test_categorical_accessor_initialization1(data): sr = Series(data.copy()) - dsr = dgd.from_cudf(sr, npartitions=5) + dsr = dask_cudf.from_cudf(sr, npartitions=5) dsr.cat @pytest.mark.parametrize("data", [data_cat_2()]) def test_categorical_accessor_initialization2(data): sr = Series(data.copy()) - dsr = dgd.from_cudf(sr, npartitions=5) + dsr = dask_cudf.from_cudf(sr, npartitions=5) with pytest.raises(AttributeError): dsr.cat @@ -115,7 +115,7 @@ def test_categorical_basic(data): cat = data.copy() pdsr = pd.Series(cat) sr = Series(cat) - dsr = dgd.from_cudf(sr, npartitions=2) + dsr = dask_cudf.from_cudf(sr, npartitions=2) result = dsr.compute() np.testing.assert_array_equal(cat.codes, result.cat.codes.values_host) @@ -143,7 +143,7 @@ def test_categorical_basic(data): df["a"] = ["xyz", "abc", "def"] * 10 pdf = df.to_pandas() - cddf = dgd.from_cudf(df, 1) + cddf = dask_cudf.from_cudf(df, 1) cddf["b"] = cddf["a"].astype("category") ddf = dd.from_pandas(pdf, 1) @@ -169,7 +169,7 @@ def test_categorical_compare_unordered(data): cat = data.copy() pdsr = pd.Series(cat) sr = Series(cat) - dsr = dgd.from_cudf(sr, npartitions=2) + dsr = dask_cudf.from_cudf(sr, npartitions=2) # Test equality out = dsr == dsr @@ -209,8 +209,8 @@ def test_categorical_compare_ordered(data): pdsr2 = pd.Series(cat2) sr1 = Series(cat1) sr2 = Series(cat2) - dsr1 = dgd.from_cudf(sr1, npartitions=2) - dsr2 = dgd.from_cudf(sr2, npartitions=2) + dsr1 = dask_cudf.from_cudf(sr1, npartitions=2) + dsr2 = dask_cudf.from_cudf(sr2, npartitions=2) # Test equality out = dsr1 == dsr1 @@ -248,7 +248,7 @@ def data_str_1(): def test_string_slicing(data): pdsr = pd.Series(data.copy()) sr = Series(pdsr) - dsr = dgd.from_cudf(sr, npartitions=2) + dsr = dask_cudf.from_cudf(sr, npartitions=2) base = pdsr.str.slice(0, 4) test = dsr.str.slice(0, 4).compute() assert_eq(base, test) @@ -261,7 +261,7 @@ def test_categorical_categories(): df["a"] = df["a"].astype("category") pdf = df.to_pandas(nullable=False) - ddf = dgd.from_cudf(df, 2) + ddf = dask_cudf.from_cudf(df, 2) dpdf = dd.from_pandas(pdf, 2) dd.assert_eq( @@ -272,7 +272,7 @@ def test_categorical_categories(): def test_categorical_as_known(): - df = dgd.from_cudf(DataFrame({"col_1": [0, 1, 2, 3]}), npartitions=2) + df = dask_cudf.from_cudf(DataFrame({"col_1": [0, 1, 2, 3]}), npartitions=2) df["col_1"] = df["col_1"].astype("category") actual = df["col_1"].cat.as_known() @@ -285,7 +285,7 @@ def test_categorical_as_known(): def test_str_slice(): df = DataFrame({"a": ["abc,def,123", "xyz,hi,bye"]}) - ddf = dgd.from_cudf(df, 1) + ddf = dask_cudf.from_cudf(df, 1) pdf = df.to_pandas() dd.assert_eq( @@ -345,7 +345,7 @@ def data_test_sort(): ) def test_create_list_series(data): expect = pd.Series(data) - ds_got = dgd.from_cudf(Series(data), 4) + ds_got = dask_cudf.from_cudf(Series(data), 4) assert_eq(expect, ds_got.compute()) @@ -355,7 +355,7 @@ def test_create_list_series(data): ) def test_unique(data): expect = Series(data).list.unique() - ds = dgd.from_cudf(Series(data), 5) + ds = dask_cudf.from_cudf(Series(data), 5) assert_eq(expect, ds.list.unique().compute()) @@ -365,7 +365,7 @@ def test_unique(data): ) def test_len(data): expect = Series(data).list.len() - ds = dgd.from_cudf(Series(data), 5) + ds = dask_cudf.from_cudf(Series(data), 5) assert_eq(expect, ds.list.len().compute()) @@ -375,7 +375,7 @@ def test_len(data): ) def test_contains(data, search_key): expect = Series(data).list.contains(search_key) - ds = dgd.from_cudf(Series(data), 5) + ds = dask_cudf.from_cudf(Series(data), 5) assert_eq(expect, ds.list.contains(search_key).compute()) @@ -388,7 +388,7 @@ def test_contains(data, search_key): ) def test_get(data, index): expect = Series(data).list.get(index) - ds = dgd.from_cudf(Series(data), 5) + ds = dask_cudf.from_cudf(Series(data), 5) assert_eq(expect, ds.list.get(index).compute()) @@ -398,7 +398,7 @@ def test_get(data, index): ) def test_leaves(data): expect = Series(data).list.leaves - ds = dgd.from_cudf(Series(data), 5) + ds = dask_cudf.from_cudf(Series(data), 5) got = ds.list.leaves.compute().reset_index(drop=True) assert_eq(expect, got) @@ -419,7 +419,7 @@ def test_take(data, list_indices, expectation): expect = Series(data).list.take(list_indices) if expectation == does_not_raise(): - ds = dgd.from_cudf(Series(data), 5) + ds = dask_cudf.from_cudf(Series(data), 5) assert_eq(expect, ds.list.take(list_indices).compute()) @@ -435,7 +435,7 @@ def test_sorting(data, ascending, na_position, ignore_index): ascending=ascending, na_position=na_position, ignore_index=ignore_index ) got = ( - dgd.from_cudf(Series(data), 5) + dask_cudf.from_cudf(Series(data), 5) .list.sort_values( ascending=ascending, na_position=na_position, @@ -464,7 +464,7 @@ def test_sorting(data, ascending, na_position, ignore_index): ) def test_create_struct_series(data): expect = pd.Series(data) - ds_got = dgd.from_cudf(Series(data), 2) + ds_got = dask_cudf.from_cudf(Series(data), 2) assert_eq(expect, ds_got.compute()) @@ -475,7 +475,7 @@ def test_create_struct_series(data): def test_struct_field_str(data): for test_key in ["a", "b"]: expect = Series(data).struct.field(test_key) - ds_got = dgd.from_cudf(Series(data), 2).struct.field(test_key) + ds_got = dask_cudf.from_cudf(Series(data), 2).struct.field(test_key) assert_eq(expect, ds_got.compute()) @@ -486,7 +486,7 @@ def test_struct_field_str(data): def test_struct_field_integer(data): for test_key in [0, 1]: expect = Series(data).struct.field(test_key) - ds_got = dgd.from_cudf(Series(data), 2).struct.field(test_key) + ds_got = dask_cudf.from_cudf(Series(data), 2).struct.field(test_key) assert_eq(expect, ds_got.compute()) @@ -495,7 +495,7 @@ def test_struct_field_integer(data): struct_accessor_data_params, ) def test_dask_struct_field_Key_Error(data): - got = dgd.from_cudf(Series(data), 2) + got = dask_cudf.from_cudf(Series(data), 2) with pytest.raises(KeyError): got.struct.field("notakey").compute() @@ -507,7 +507,7 @@ def test_dask_struct_field_Key_Error(data): ) def test_dask_struct_field_Int_Error(data): # breakpoint() - got = dgd.from_cudf(Series(data), 2) + got = dask_cudf.from_cudf(Series(data), 2) with pytest.raises(IndexError): got.struct.field(1000).compute() @@ -523,7 +523,7 @@ def test_dask_struct_field_Int_Error(data): ) def test_struct_explode(data): expect = Series(data).struct.explode() - got = dgd.from_cudf(Series(data), 2).struct.explode() + got = dask_cudf.from_cudf(Series(data), 2).struct.explode() # Output index will not agree for >1 partitions assert_eq(expect, got.compute().reset_index(drop=True)) @@ -533,7 +533,7 @@ def test_tz_localize(): expect = data.dt.tz_localize( "US/Eastern", ambiguous="NaT", nonexistent="NaT" ) - got = dgd.from_cudf(data, 2).dt.tz_localize( + got = dask_cudf.from_cudf(data, 2).dt.tz_localize( "US/Eastern", ambiguous="NaT", nonexistent="NaT" ) dd.assert_eq(expect, got) @@ -554,5 +554,5 @@ def test_tz_localize(): ) def test_tz_convert(data): expect = Series(data).dt.tz_convert("US/Pacific") - got = dgd.from_cudf(Series(data), 2).dt.tz_convert("US/Pacific") + got = dask_cudf.from_cudf(Series(data), 2).dt.tz_convert("US/Pacific") dd.assert_eq(expect, got) diff --git a/python/dask_cudf/dask_cudf/tests/test_core.py b/python/dask_cudf/dask_cudf/tests/test_core.py index afe2a050695..250256d3356 100644 --- a/python/dask_cudf/dask_cudf/tests/test_core.py +++ b/python/dask_cudf/dask_cudf/tests/test_core.py @@ -15,7 +15,7 @@ import cudf -import dask_cudf as dgd +import dask_cudf def test_from_dict_backend_dispatch(): @@ -28,7 +28,7 @@ def test_from_dict_backend_dispatch(): expect = cudf.DataFrame(data) with dask.config.set({"dataframe.backend": "cudf"}): ddf = dd.from_dict(data, npartitions=2) - assert isinstance(ddf, dgd.DataFrame) + assert isinstance(ddf, dask_cudf.DataFrame) dd.assert_eq(expect, ddf) @@ -43,7 +43,7 @@ def test_to_backend(): assert isinstance(ddf._meta, pd.DataFrame) gdf = ddf.to_backend("cudf") - assert isinstance(gdf, dgd.DataFrame) + assert isinstance(gdf, dask_cudf.DataFrame) dd.assert_eq(cudf.DataFrame(data), ddf) assert isinstance(gdf.to_backend()._meta, pd.DataFrame) @@ -58,13 +58,13 @@ def test_to_backend_kwargs(): # Using `nan_as_null=False` will result in a cudf-backed # Series with a NaN element (ranther than ) gser_nan = dser.to_backend("cudf", nan_as_null=False) - assert isinstance(gser_nan, dgd.Series) + assert isinstance(gser_nan, dask_cudf.Series) assert np.isnan(gser_nan.compute()).sum() == 1 # Using `nan_as_null=True` will result in a cudf-backed # Series with a element (ranther than NaN) gser_null = dser.to_backend("cudf", nan_as_null=True) - assert isinstance(gser_null, dgd.Series) + assert isinstance(gser_null, dask_cudf.Series) assert np.isnan(gser_null.compute()).sum() == 0 # Check `nullable` argument for `cudf.Series.to_pandas` @@ -110,7 +110,7 @@ def test_from_cudf_multiindex_raises(): with pytest.raises(NotImplementedError): # dask_cudf does not support MultiIndex yet - dgd.from_cudf(df.set_index(["x", "y"])) + dask_cudf.from_cudf(df.set_index(["x", "y"])) def test_from_cudf_with_generic_idx(): @@ -122,7 +122,7 @@ def test_from_cudf_with_generic_idx(): } ) - ddf = dgd.from_cudf(cdf, npartitions=2) + ddf = dask_cudf.from_cudf(cdf, npartitions=2) assert isinstance(ddf.index.compute(), cudf.RangeIndex) dd.assert_eq(ddf.loc[1:2, ["a"]], cdf.loc[1:2, ["a"]]) @@ -164,7 +164,7 @@ def test_query_local_dict(): {"x": np.random.randint(0, 5, size=10), "y": np.random.normal(size=10)} ) gdf = cudf.DataFrame.from_pandas(df) - ddf = dgd.from_cudf(gdf, npartitions=2) + ddf = dask_cudf.from_cudf(gdf, npartitions=2) val = 2 @@ -296,7 +296,7 @@ def test_set_index_sorted(): ddf1 = dd.from_pandas(df1, npartitions=2) gdf1 = cudf.from_pandas(df1) - gddf1 = dgd.from_cudf(gdf1, npartitions=2) + gddf1 = dask_cudf.from_cudf(gdf1, npartitions=2) expect = ddf1.set_index("id", sorted=True) got = gddf1.set_index("id", sorted=True) @@ -323,7 +323,9 @@ def test_rearrange_by_divisions(nelem, index): df["z"] = df["z"].astype("category") ddf1 = dd.from_pandas(df, npartitions=4) - gdf1 = dgd.from_cudf(cudf.DataFrame.from_pandas(df), npartitions=4) + gdf1 = dask_cudf.from_cudf( + cudf.DataFrame.from_pandas(df), npartitions=4 + ) ddf1.index.name = index gdf1.index.name = index divisions = (0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20) @@ -487,7 +489,7 @@ def test_repartition_hash_staged(npartitions): # WARNING: Specific npartitions-max_branch combination # was specifically chosen to cover changes in #4676 npartitions_initial = 17 - ddf = dgd.from_cudf(gdf, npartitions=npartitions_initial) + ddf = dask_cudf.from_cudf(gdf, npartitions=npartitions_initial) ddf_new = ddf.shuffle( on=by, ignore_index=True, npartitions=npartitions, max_branch=4 ) @@ -527,7 +529,7 @@ def test_repartition_hash(by, npartitions, max_branch): } ) gdf.d = gdf.d.astype("datetime64[ms]") - ddf = dgd.from_cudf(gdf, npartitions=npartitions_i) + ddf = dask_cudf.from_cudf(gdf, npartitions=npartitions_i) ddf_new = ddf.shuffle( on=by, ignore_index=True, @@ -554,7 +556,7 @@ def test_repartition_hash(by, npartitions, max_branch): def test_repartition_no_extra_row(): # see https://github.com/rapidsai/cudf/issues/11930 gdf = cudf.DataFrame({"a": [10, 20, 30], "b": [1, 2, 3]}).set_index("a") - ddf = dgd.from_cudf(gdf, npartitions=1) + ddf = dask_cudf.from_cudf(gdf, npartitions=1) ddf_new = ddf.repartition([0, 5, 10, 30], force=True) dd.assert_eq(ddf, ddf_new) dd.assert_eq(gdf, ddf_new) @@ -669,20 +671,20 @@ def test_hash_object_dispatch(index): # DataFrame result = dd.core.hash_object_dispatch(obj, index=index) - expected = dgd.backends.hash_object_cudf(obj, index=index) + expected = dask_cudf.backends.hash_object_cudf(obj, index=index) assert isinstance(result, cudf.Series) dd.assert_eq(result, expected) # Series result = dd.core.hash_object_dispatch(obj["x"], index=index) - expected = dgd.backends.hash_object_cudf(obj["x"], index=index) + expected = dask_cudf.backends.hash_object_cudf(obj["x"], index=index) assert isinstance(result, cudf.Series) dd.assert_eq(result, expected) # DataFrame with MultiIndex obj_multi = obj.set_index(["x", "z"], drop=True) result = dd.core.hash_object_dispatch(obj_multi, index=index) - expected = dgd.backends.hash_object_cudf(obj_multi, index=index) + expected = dask_cudf.backends.hash_object_cudf(obj_multi, index=index) assert isinstance(result, cudf.Series) dd.assert_eq(result, expected) @@ -729,7 +731,7 @@ def test_make_meta_backends(index): # Check dask code path if not MultiIndex if not isinstance(df.index, cudf.MultiIndex): - ddf = dgd.from_cudf(df, npartitions=1) + ddf = dask_cudf.from_cudf(df, npartitions=1) # Check "empty" metadata types dd.assert_eq(ddf._meta.dtypes, df.dtypes) @@ -751,7 +753,7 @@ def test_dataframe_series_replace(data): pdf = data.copy() gdf = cudf.from_pandas(pdf) - ddf = dgd.from_cudf(gdf, npartitions=5) + ddf = dask_cudf.from_cudf(gdf, npartitions=5) dd.assert_eq(ddf.replace(1, 2), pdf.replace(1, 2)) @@ -760,7 +762,7 @@ def test_dataframe_assign_col(): df = cudf.DataFrame(list(range(100))) pdf = pd.DataFrame(list(range(100))) - ddf = dgd.from_cudf(df, npartitions=4) + ddf = dask_cudf.from_cudf(df, npartitions=4) ddf["fold"] = 0 ddf["fold"] = ddf["fold"].map_partitions( lambda cudf_df: cp.random.randint(0, 4, len(cudf_df)) @@ -783,7 +785,7 @@ def test_dataframe_set_index(): pdf = df.to_pandas() with dask.config.set({"dataframe.convert-string": False}): - ddf = dgd.from_cudf(df, npartitions=4) + ddf = dask_cudf.from_cudf(df, npartitions=4) ddf = ddf.set_index("str") pddf = dd.from_pandas(pdf, npartitions=4) @@ -799,7 +801,7 @@ def test_series_describe(): sr = cudf.datasets.randomdata(20)["x"] psr = sr.to_pandas() - dsr = dgd.from_cudf(sr, npartitions=4) + dsr = dask_cudf.from_cudf(sr, npartitions=4) pdsr = dd.from_pandas(psr, npartitions=4) dd.assert_eq( @@ -814,7 +816,7 @@ def test_dataframe_describe(): df = cudf.datasets.randomdata(20) pdf = df.to_pandas() - ddf = dgd.from_cudf(df, npartitions=4) + ddf = dask_cudf.from_cudf(df, npartitions=4) pddf = dd.from_pandas(pdf, npartitions=4) dd.assert_eq( @@ -831,7 +833,7 @@ def test_zero_std_describe(): } ) pdf = df.to_pandas() - ddf = dgd.from_cudf(df, npartitions=4) + ddf = dask_cudf.from_cudf(df, npartitions=4) pddf = dd.from_pandas(pdf, npartitions=4) dd.assert_eq(ddf.describe(), pddf.describe(), rtol=1e-3) @@ -846,7 +848,7 @@ def test_large_numbers_var(): } ) pdf = df.to_pandas() - ddf = dgd.from_cudf(df, npartitions=4) + ddf = dask_cudf.from_cudf(df, npartitions=4) pddf = dd.from_pandas(pdf, npartitions=4) dd.assert_eq(ddf.var(), pddf.var(), rtol=1e-3) @@ -858,7 +860,7 @@ def test_index_map_partitions(): ddf = dd.from_pandas(pd.DataFrame({"a": range(10)}), npartitions=2) mins_pd = ddf.index.map_partitions(M.min, meta=ddf.index).compute() - gddf = dgd.from_cudf(cudf.DataFrame({"a": range(10)}), npartitions=2) + gddf = dask_cudf.from_cudf(cudf.DataFrame({"a": range(10)}), npartitions=2) mins_gd = gddf.index.map_partitions(M.min, meta=gddf.index).compute() dd.assert_eq(mins_pd, mins_gd) @@ -878,7 +880,7 @@ def test_merging_categorical_columns(): {"id_1": [0, 1, 2, 3], "cat_col": ["a", "b", "f", "f"]} ) - ddf_1 = dgd.from_cudf(df_1, npartitions=2) + ddf_1 = dask_cudf.from_cudf(df_1, npartitions=2) ddf_1 = dd.categorical.categorize(ddf_1, columns=["cat_col"]) @@ -886,7 +888,7 @@ def test_merging_categorical_columns(): {"id_2": [111, 112, 113], "cat_col": ["g", "h", "f"]} ) - ddf_2 = dgd.from_cudf(df_2, npartitions=2) + ddf_2 = dask_cudf.from_cudf(df_2, npartitions=2) ddf_2 = dd.categorical.categorize(ddf_2, columns=["cat_col"]) expected = cudf.DataFrame( @@ -930,7 +932,7 @@ def test_categorical_dtype_round_trip(): s = cudf.Series(4 * ["foo"], dtype="category") assert s.dtype.ordered is False - ds = dgd.from_cudf(s, npartitions=2) + ds = dask_cudf.from_cudf(s, npartitions=2) pds = dd.from_pandas(s.to_pandas(), npartitions=2) dd.assert_eq(ds, pds) assert ds.dtype.ordered is False diff --git a/python/dask_cudf/dask_cudf/tests/test_delayed_io.py b/python/dask_cudf/dask_cudf/tests/test_delayed_io.py index 6c68d92a8df..e6fb58ad6df 100644 --- a/python/dask_cudf/dask_cudf/tests/test_delayed_io.py +++ b/python/dask_cudf/dask_cudf/tests/test_delayed_io.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2022, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. """ Test IO with dask.delayed API @@ -10,14 +10,14 @@ from dask.delayed import delayed -import cudf as gd +import cudf -import dask_cudf as dgd +import dask_cudf @delayed def load_data(nelem, ident): - df = gd.DataFrame() + df = cudf.DataFrame() df["x"] = np.arange(nelem) df["ident"] = np.asarray([ident] * nelem) return df @@ -30,39 +30,39 @@ def get_combined_column(df): def test_dataframe_from_delayed(): delays = [load_data(10 * i, i) for i in range(1, 3)] - out = dgd.from_delayed(delays) + out = dask_cudf.from_delayed(delays) res = out.compute() - assert isinstance(res, gd.DataFrame) + assert isinstance(res, cudf.DataFrame) - expected = gd.concat([d.compute() for d in delays]) + expected = cudf.concat([d.compute() for d in delays]) assert_frame_equal(res.to_pandas(), expected.to_pandas()) def test_series_from_delayed(): delays = [get_combined_column(load_data(10 * i, i)) for i in range(1, 3)] - out = dgd.from_delayed(delays) + out = dask_cudf.from_delayed(delays) res = out.compute() - assert isinstance(res, gd.Series) + assert isinstance(res, cudf.Series) - expected = gd.concat([d.compute() for d in delays]) + expected = cudf.concat([d.compute() for d in delays]) np.testing.assert_array_equal(res.to_pandas(), expected.to_pandas()) def test_dataframe_to_delayed(): nelem = 100 - df = gd.DataFrame() + df = cudf.DataFrame() df["x"] = np.arange(nelem) df["y"] = np.random.randint(nelem, size=nelem) - ddf = dgd.from_cudf(df, npartitions=5) + ddf = dask_cudf.from_cudf(df, npartitions=5) delays = ddf.to_delayed() assert len(delays) == 5 # Concat the delayed partitions - got = gd.concat([d.compute() for d in delays]) + got = cudf.concat([d.compute() for d in delays]) assert_frame_equal(got.to_pandas(), df.to_pandas()) # Check individual partitions @@ -81,17 +81,17 @@ def test_dataframe_to_delayed(): def test_series_to_delayed(): nelem = 100 - sr = gd.Series(np.random.randint(nelem, size=nelem)) + sr = cudf.Series(np.random.randint(nelem, size=nelem)) - dsr = dgd.from_cudf(sr, npartitions=5) + dsr = dask_cudf.from_cudf(sr, npartitions=5) delays = dsr.to_delayed() assert len(delays) == 5 # Concat the delayed partitions - got = gd.concat([d.compute() for d in delays]) - assert isinstance(got, gd.Series) + got = cudf.concat([d.compute() for d in delays]) + assert isinstance(got, cudf.Series) np.testing.assert_array_equal(got.to_pandas(), sr.to_pandas()) # Check individual partitions @@ -110,15 +110,15 @@ def test_series_to_delayed(): def test_mixing_series_frame_error(): nelem = 20 - df = gd.DataFrame() + df = cudf.DataFrame() df["x"] = np.arange(nelem) df["y"] = np.random.randint(nelem, size=nelem) - ddf = dgd.from_cudf(df, npartitions=5) + ddf = dask_cudf.from_cudf(df, npartitions=5) delay_frame = ddf.to_delayed() delay_series = ddf.x.to_delayed() - combined = dgd.from_delayed(delay_frame + delay_series) + combined = dask_cudf.from_delayed(delay_frame + delay_series) with pytest.raises(ValueError) as raises: combined.compute() @@ -129,15 +129,15 @@ def test_mixing_series_frame_error(): def test_frame_extra_columns_error(): nelem = 20 - df = gd.DataFrame() + df = cudf.DataFrame() df["x"] = np.arange(nelem) df["y"] = np.random.randint(nelem, size=nelem) - ddf1 = dgd.from_cudf(df, npartitions=5) + ddf1 = dask_cudf.from_cudf(df, npartitions=5) df["z"] = np.arange(nelem) - ddf2 = dgd.from_cudf(df, npartitions=5) + ddf2 = dask_cudf.from_cudf(df, npartitions=5) - combined = dgd.from_delayed(ddf1.to_delayed() + ddf2.to_delayed()) + combined = dask_cudf.from_delayed(ddf1.to_delayed() + ddf2.to_delayed()) with pytest.raises(ValueError) as raises: combined.compute() @@ -150,18 +150,18 @@ def test_frame_extra_columns_error(): def test_frame_dtype_error(): nelem = 20 - df1 = gd.DataFrame() + df1 = cudf.DataFrame() df1["bad"] = np.arange(nelem) df1["bad"] = np.arange(nelem, dtype=np.float64) - df2 = gd.DataFrame() + df2 = cudf.DataFrame() df2["bad"] = np.arange(nelem) df2["bad"] = np.arange(nelem, dtype=np.float32) - ddf1 = dgd.from_cudf(df1, npartitions=5) - ddf2 = dgd.from_cudf(df2, npartitions=5) + ddf1 = dask_cudf.from_cudf(df1, npartitions=5) + ddf2 = dask_cudf.from_cudf(df2, npartitions=5) - combined = dgd.from_delayed(ddf1.to_delayed() + ddf2.to_delayed()) + combined = dask_cudf.from_delayed(ddf1.to_delayed() + ddf2.to_delayed()) with pytest.raises(ValueError) as raises: combined.compute() diff --git a/python/dask_cudf/dask_cudf/tests/test_join.py b/python/dask_cudf/dask_cudf/tests/test_join.py index 73fd37df6fa..eb500ad2462 100644 --- a/python/dask_cudf/dask_cudf/tests/test_join.py +++ b/python/dask_cudf/dask_cudf/tests/test_join.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2023, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. from functools import partial @@ -10,7 +10,7 @@ import cudf -import dask_cudf as dgd +import dask_cudf param_nrows = [5, 10, 50, 100] @@ -44,8 +44,8 @@ def test_join_inner(left_nrows, right_nrows, left_nkeys, right_nkeys): expect = expect.to_pandas() # dask_cudf - left = dgd.from_cudf(left, chunksize=chunksize) - right = dgd.from_cudf(right, chunksize=chunksize) + left = dask_cudf.from_cudf(left, chunksize=chunksize) + right = dask_cudf.from_cudf(right, chunksize=chunksize) joined = left.set_index("x").join( right.set_index("x"), how="inner", lsuffix="l", rsuffix="r" @@ -102,8 +102,8 @@ def test_join_left(left_nrows, right_nrows, left_nkeys, right_nkeys, how): expect = expect.to_pandas() # dask_cudf - left = dgd.from_cudf(left, chunksize=chunksize) - right = dgd.from_cudf(right, chunksize=chunksize) + left = dask_cudf.from_cudf(left, chunksize=chunksize) + right = dask_cudf.from_cudf(right, chunksize=chunksize) joined = left.set_index("x").join( right.set_index("x"), how=how, lsuffix="l", rsuffix="r" @@ -173,8 +173,8 @@ def normalize(df): ) # dask_cudf - left = dgd.from_cudf(left, chunksize=chunksize) - right = dgd.from_cudf(right, chunksize=chunksize) + left = dask_cudf.from_cudf(left, chunksize=chunksize) + right = dask_cudf.from_cudf(right, chunksize=chunksize) result = left.merge(right, on=("x", "y"), how=how).compute( scheduler="single-threaded" @@ -216,8 +216,8 @@ def test_merge_1col_left( ) # dask_cudf - left = dgd.from_cudf(left, chunksize=chunksize) - right = dgd.from_cudf(right, chunksize=chunksize) + left = dask_cudf.from_cudf(left, chunksize=chunksize) + right = dask_cudf.from_cudf(right, chunksize=chunksize) joined = left.merge(right, on=["x"], how=how) @@ -238,8 +238,8 @@ def test_merge_should_fail(): df2["a"] = [7, 2, 3, 8, 5, 9] * 2 df2["c"] = np.random.randint(0, 12, 12) - left = dgd.from_cudf(df1, 1).groupby("a").b.min().to_frame() - right = dgd.from_cudf(df2, 1).groupby("a").c.min().to_frame() + left = dask_cudf.from_cudf(df1, 1).groupby("a").b.min().to_frame() + right = dask_cudf.from_cudf(df2, 1).groupby("a").c.min().to_frame() with pytest.raises(KeyError): left.merge(right, how="left", on=["nonCol"]) @@ -250,7 +250,7 @@ def test_merge_should_fail(): # Same column names df2["b"] = np.random.randint(0, 12, 12) - right = dgd.from_cudf(df2, 1).groupby("a").b.min().to_frame() + right = dask_cudf.from_cudf(df2, 1).groupby("a").b.min().to_frame() with pytest.raises(KeyError): left.merge(right, how="left", on="NonCol") diff --git a/python/dask_cudf/dask_cudf/tests/test_reductions.py b/python/dask_cudf/dask_cudf/tests/test_reductions.py index e347e8be9e4..8688f830dcb 100644 --- a/python/dask_cudf/dask_cudf/tests/test_reductions.py +++ b/python/dask_cudf/dask_cudf/tests/test_reductions.py @@ -9,7 +9,7 @@ import cudf -import dask_cudf as dgd +import dask_cudf def _make_random_frame(nelem, npartitions=2): @@ -20,7 +20,7 @@ def _make_random_frame(nelem, npartitions=2): } ) gdf = cudf.DataFrame.from_pandas(df) - dgf = dgd.from_cudf(gdf, npartitions=npartitions) + dgf = dask_cudf.from_cudf(gdf, npartitions=npartitions) return df, dgf @@ -67,7 +67,7 @@ def test_series_reduce(reducer): "op", ["max", "min", "sum", "prod", "mean", "var", "std"] ) def test_rowwise_reductions(data, op): - gddf = dgd.from_cudf(data, npartitions=10) + gddf = dask_cudf.from_cudf(data, npartitions=10) pddf = gddf.to_dask_dataframe() with dask.config.set({"dataframe.convert-string": False}): From bbfe1c376ae5958771648ff51be8a64971d2a1e9 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 1 Feb 2024 13:47:56 -0600 Subject: [PATCH 190/384] Update cudf.pandas FAQ. (#14940) This PR updates the cudf.pandas docs to reflect cudf using pandas 2. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Ashwin Srinath (https://github.com/shwina) URL: https://github.com/rapidsai/cudf/pull/14940 --- docs/cudf/source/cudf_pandas/benchmarks.md | 2 +- docs/cudf/source/cudf_pandas/faq.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/cudf/source/cudf_pandas/benchmarks.md b/docs/cudf/source/cudf_pandas/benchmarks.md index 1a916dbee6a..9c747ed9c8f 100644 --- a/docs/cudf/source/cudf_pandas/benchmarks.md +++ b/docs/cudf/source/cudf_pandas/benchmarks.md @@ -46,7 +46,7 @@ source pandas/py-pandas/bin/activate pip install --extra-index-url=https://pypi.nvidia.com cudf-cu12 # or cudf-cu11 ``` -5. Modify pandas join/group code to use `cudf.pandas` and be compatible with pandas 1.5 APIs: +5. Modify pandas join/group code to use `cudf.pandas` and remove the `dtype_backend` keyword argument (not supported): ```bash diff --git a/pandas/groupby-pandas.py b/pandas/groupby-pandas.py diff --git a/docs/cudf/source/cudf_pandas/faq.md b/docs/cudf/source/cudf_pandas/faq.md index bbeaf0a5f00..dde7afb1360 100644 --- a/docs/cudf/source/cudf_pandas/faq.md +++ b/docs/cudf/source/cudf_pandas/faq.md @@ -136,7 +136,8 @@ There are a few known limitations that you should be aware of: print(lst) # lst is unchanged, as this specific UDF could not run on the GPU [10] ``` -- `cudf.pandas` (and cuDF in general) is currently only compatible with pandas 1.5.x. +- `cudf.pandas` (and cuDF in general) is only compatible with pandas 2. Version + 24.02 of cudf was the last to support pandas 1.5.x. ## Can I force running on the CPU? From e4cd20ebfc38124e6ee0cf217335c2479c41aa57 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 1 Feb 2024 14:00:15 -0600 Subject: [PATCH 191/384] Make codecov only informational (always pass). (#14952) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR makes the codecov status always pass ✔️ so that it doesn't distract from actual CI failures in the commit CI summary. https://docs.codecov.com/docs/commit-status#informational cc: @davidwendt @mroeschke Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Matthew Roeschke (https://github.com/mroeschke) - Vyas Ramasubramani (https://github.com/vyasr) - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/14952 --- codecov.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/codecov.yml b/codecov.yml index 344d4f3f04e..bb4115d7383 100644 --- a/codecov.yml +++ b/codecov.yml @@ -6,6 +6,7 @@ coverage: default: target: auto threshold: 5% + informational: true # The coverage will always pass github_checks: annotations: true From 2b0d9876ed02c1e4ea50907ac0d917669d51348c Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 1 Feb 2024 14:12:54 -0600 Subject: [PATCH 192/384] Update tests for pandas 2. (#14941) A few cleanups in test files following #14916. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) - Charles Blackmon-Luca (https://github.com/charlesbluca) URL: https://github.com/rapidsai/cudf/pull/14941 --- python/cudf/cudf/tests/test_duplicates.py | 231 ++++++++---------- python/cudf/cudf/tests/test_string.py | 3 +- python/dask_cudf/dask_cudf/tests/test_core.py | 9 - 3 files changed, 106 insertions(+), 137 deletions(-) diff --git a/python/cudf/cudf/tests/test_duplicates.py b/python/cudf/cudf/tests/test_duplicates.py index 447b2b3c4f5..161b245953b 100644 --- a/python/cudf/cudf/tests/test_duplicates.py +++ b/python/cudf/cudf/tests/test_duplicates.py @@ -11,26 +11,6 @@ from cudf import concat from cudf.testing._utils import assert_eq, assert_exceptions_equal -# TODO: PANDAS 1.0 support -# Revisit drop_duplicates() tests to update parameters like ignore_index. - - -def assert_df(g, p): - # assert_eq() with sorted index of dataframes - g = g.sort_index() - p = p.sort_index() - return assert_eq(g, p) - - -def assert_df2(g, p): - assert g.index.dtype == p.index.dtype - np.testing.assert_equal(g.index.to_numpy(), p.index) - assert tuple(g.columns) == tuple(p.columns) - for k in g.columns: - assert g[k].dtype == p[k].dtype - np.testing.assert_equal(g[k].to_numpy(), p[k]) - - # most tests are similar to pandas drop_duplicates @@ -48,6 +28,7 @@ def test_duplicated_with_misspelled_column_name(subset): @pytest.mark.parametrize("keep", ["first", "last", False]) +@pytest.mark.parametrize("ignore_index", [True, False]) @pytest.mark.parametrize( "data", [ @@ -57,14 +38,18 @@ def test_duplicated_with_misspelled_column_name(subset): pd.Series(["aaa"] * 10, dtype="object"), ], ) -def test_drop_duplicates_series(data, keep): +def test_drop_duplicates_series(data, keep, ignore_index): pds = pd.Series(data) gds = cudf.from_pandas(pds) - assert_df(pds.drop_duplicates(keep=keep), gds.drop_duplicates(keep=keep)) - pds.drop_duplicates(keep=keep, inplace=True) - gds.drop_duplicates(keep=keep, inplace=True) - assert_df(pds, gds) + assert_eq( + pds.drop_duplicates(keep=keep, ignore_index=ignore_index), + gds.drop_duplicates(keep=keep, ignore_index=ignore_index), + ) + + pds.drop_duplicates(keep=keep, inplace=True, ignore_index=ignore_index) + gds.drop_duplicates(keep=keep, inplace=True, ignore_index=ignore_index) + assert_eq(pds, gds) def test_drop_duplicates(): @@ -82,31 +67,31 @@ def test_drop_duplicates(): result.drop_duplicates("AAA", inplace=True) expected = pdf.copy() expected.drop_duplicates("AAA", inplace=True) - assert_df(result, expected) + assert_eq(result, expected) result = gdf.drop_duplicates("AAA", keep="last") expected = pdf.drop_duplicates("AAA", keep="last") - assert_df(result, expected) + assert_eq(result, expected) result = gdf.drop_duplicates("AAA", keep=False) expected = pdf.drop_duplicates("AAA", keep=False) - assert_df(result, expected) + assert_eq(result, expected) assert len(result) == 0 # multi column expected = pdf.loc[[0, 1, 2, 3]] result = gdf.drop_duplicates(np.array(["AAA", "B"])) - assert_df(result, expected) + assert_eq(result, expected) result = pdf.drop_duplicates(np.array(["AAA", "B"])) - assert_df(result, expected) + assert_eq(result, expected) result = gdf.drop_duplicates(("AAA", "B"), keep="last") expected = pdf.drop_duplicates(("AAA", "B"), keep="last") - assert_df(result, expected) + assert_eq(result, expected) result = gdf.drop_duplicates(("AAA", "B"), keep=False) expected = pdf.drop_duplicates(("AAA", "B"), keep=False) - assert_df(result, expected) + assert_eq(result, expected) # consider everything df2 = gdf.loc[:, ["AAA", "B", "C"]] @@ -114,60 +99,60 @@ def test_drop_duplicates(): result = df2.drop_duplicates() # in this case only expected = df2.drop_duplicates(["AAA", "B"]) - assert_df(result, expected) + assert_eq(result, expected) result = df2.drop_duplicates(keep="last") expected = df2.drop_duplicates(["AAA", "B"], keep="last") - assert_df(result, expected) + assert_eq(result, expected) result = df2.drop_duplicates(keep=False) expected = df2.drop_duplicates(["AAA", "B"], keep=False) - assert_df(result, expected) + assert_eq(result, expected) # integers result = gdf.drop_duplicates("C") expected = pdf.drop_duplicates("C") - assert_df(result, expected) + assert_eq(result, expected) result = gdf.drop_duplicates("C", keep="last") expected = pdf.drop_duplicates("C", keep="last") - assert_df(result, expected) + assert_eq(result, expected) gdf["E"] = gdf["C"].astype("int8") result = gdf.drop_duplicates("E") pdf["E"] = pdf["C"].astype("int8") expected = pdf.drop_duplicates("E") - assert_df(result, expected) + assert_eq(result, expected) result = gdf.drop_duplicates("E", keep="last") expected = pdf.drop_duplicates("E", keep="last") - assert_df(result, expected) + assert_eq(result, expected) pdf = pd.DataFrame( {"x": [7, 6, 3, 3, 4, 8, 0], "y": [0, 6, 5, 5, 9, 1, 2]} ) gdf = cudf.DataFrame.from_pandas(pdf) - assert_df(gdf.drop_duplicates(), pdf.drop_duplicates()) + assert_eq(gdf.drop_duplicates(), pdf.drop_duplicates()) pdf = pd.DataFrame([[1, 0], [0, 2]]) gdf = cudf.DataFrame.from_pandas(pdf) - assert_df(gdf.drop_duplicates(), pdf.drop_duplicates()) + assert_eq(gdf.drop_duplicates(), pdf.drop_duplicates()) pdf = pd.DataFrame([[-2, 0], [0, -4]]) gdf = cudf.DataFrame.from_pandas(pdf) - assert_df(gdf.drop_duplicates(), pdf.drop_duplicates()) + assert_eq(gdf.drop_duplicates(), pdf.drop_duplicates()) x = np.iinfo(np.int64).max / 3 * 2 pdf = pd.DataFrame([[-x, x], [0, x + 4]]) gdf = cudf.DataFrame.from_pandas(pdf) - assert_df(gdf.drop_duplicates(), pdf.drop_duplicates()) + assert_eq(gdf.drop_duplicates(), pdf.drop_duplicates()) pdf = pd.DataFrame([[-x, x], [x, x + 4]]) gdf = cudf.DataFrame.from_pandas(pdf) - assert_df(gdf.drop_duplicates(), pdf.drop_duplicates()) + assert_eq(gdf.drop_duplicates(), pdf.drop_duplicates()) pdf = pd.DataFrame([i] * 9 for i in range(16)) pdf = pd.concat([pdf, pd.DataFrame([[1] + [0] * 8])], ignore_index=True) gdf = cudf.DataFrame.from_pandas(pdf) - assert_df(gdf.drop_duplicates(), pdf.drop_duplicates()) + assert_eq(gdf.drop_duplicates(), pdf.drop_duplicates()) @pytest.mark.skip(reason="cudf does not support duplicate column names yet") @@ -178,11 +163,11 @@ def test_drop_duplicates_with_duplicate_column_names(): df = cudf.DataFrame.from_pandas(df) result0 = df.drop_duplicates() - assert_df(result0, df) + assert_eq(result0, df) result1 = df.drop_duplicates("a") expected1 = df[:2] - assert_df(result1, expected1) + assert_eq(result1, expected1) def test_drop_duplicates_for_take_all(): @@ -198,28 +183,28 @@ def test_drop_duplicates_for_take_all(): # single column result = gdf.drop_duplicates("AAA") expected = pdf.drop_duplicates("AAA") - assert_df(result, expected) + assert_eq(result, expected) result = gdf.drop_duplicates("AAA", keep="last") expected = pdf.drop_duplicates("AAA", keep="last") - assert_df(result, expected) + assert_eq(result, expected) result = gdf.drop_duplicates("AAA", keep=False) expected = pdf.drop_duplicates("AAA", keep=False) - assert_df(result, expected) + assert_eq(result, expected) # multiple columns result = gdf.drop_duplicates(["AAA", "B"]) expected = pdf.drop_duplicates(["AAA", "B"]) - assert_df(result, expected) + assert_eq(result, expected) result = gdf.drop_duplicates(["AAA", "B"], keep="last") expected = pdf.drop_duplicates(["AAA", "B"], keep="last") - assert_df(result, expected) + assert_eq(result, expected) result = gdf.drop_duplicates(["AAA", "B"], keep=False) expected = pdf.drop_duplicates(["AAA", "B"], keep=False) - assert_df(result, expected) + assert_eq(result, expected) def test_drop_duplicates_tuple(): @@ -244,21 +229,21 @@ def test_drop_duplicates_tuple(): # single column result = gdf.drop_duplicates(("AA", "AB")) expected = pdf.drop_duplicates(("AA", "AB")) - assert_df(result, expected) + assert_eq(result, expected) result = gdf.drop_duplicates(("AA", "AB"), keep="last") expected = pdf.drop_duplicates(("AA", "AB"), keep="last") - assert_df(result, expected) + assert_eq(result, expected) result = gdf.drop_duplicates(("AA", "AB"), keep=False) expected = pdf.drop_duplicates(("AA", "AB"), keep=False) # empty df assert len(result) == 0 - assert_df(result, expected) + assert_eq(result, expected) # multi column expected = pdf.drop_duplicates((("AA", "AB"), "B")) result = gdf.drop_duplicates((("AA", "AB"), "B")) - assert_df(result, expected) + assert_eq(result, expected) @pytest.mark.parametrize( @@ -274,11 +259,11 @@ def test_drop_duplicates_tuple(): def test_drop_duplicates_empty(df): df = cudf.DataFrame.from_pandas(df) result = df.drop_duplicates() - assert_df(result, df) + assert_eq(result, df) result = df.copy() result.drop_duplicates(inplace=True) - assert_df(result, df) + assert_eq(result, df) @pytest.mark.parametrize("num_columns", [3, 4, 5]) @@ -296,18 +281,18 @@ def get_pdf(n_dup): for i in range(5): pdf = get_pdf(i) gdf = cudf.DataFrame.from_pandas(pdf) - assert_df(gdf.drop_duplicates(), pdf.drop_duplicates()) + assert_eq(gdf.drop_duplicates(), pdf.drop_duplicates()) # subset columns, single columns - assert_df( + assert_eq( gdf.drop_duplicates(pdf.columns[:-1]), pdf.drop_duplicates(pdf.columns[:-1]), ) - assert_df( + assert_eq( gdf.drop_duplicates(pdf.columns[-1]), pdf.drop_duplicates(pdf.columns[-1]), ) - assert_df( + assert_eq( gdf.drop_duplicates(pdf.columns[0]), pdf.drop_duplicates(pdf.columns[0]), ) @@ -315,12 +300,12 @@ def get_pdf(n_dup): # subset columns shuffled cols = list(pdf.columns) random.Random(3).shuffle(cols) - assert_df(gdf.drop_duplicates(cols), pdf.drop_duplicates(cols)) + assert_eq(gdf.drop_duplicates(cols), pdf.drop_duplicates(cols)) random.Random(3).shuffle(cols) - assert_df(gdf.drop_duplicates(cols[:-1]), pdf.drop_duplicates(cols[:-1])) + assert_eq(gdf.drop_duplicates(cols[:-1]), pdf.drop_duplicates(cols[:-1])) random.Random(3).shuffle(cols) - assert_df(gdf.drop_duplicates(cols[-1]), pdf.drop_duplicates(cols[-1])) - assert_df( + assert_eq(gdf.drop_duplicates(cols[-1]), pdf.drop_duplicates(cols[-1])) + assert_eq( gdf.drop_duplicates(cols, keep="last"), pdf.drop_duplicates(cols, keep="last"), ) @@ -332,7 +317,7 @@ def test_dataframe_drop_duplicates_method(): columns=["n1", "n2", "s1"], ) gdf = cudf.DataFrame.from_pandas(pdf) - assert_df(gdf.drop_duplicates(), pdf.drop_duplicates()) + assert_eq(gdf.drop_duplicates(), pdf.drop_duplicates()) assert_eq( gdf.drop_duplicates("n1")["n1"].reset_index(drop=True), @@ -355,13 +340,13 @@ def test_dataframe_drop_duplicates_method(): assert gdf.drop_duplicates("s1", inplace=True) is None gdf = cudf.DataFrame.from_pandas(pdf) - assert_df(gdf.drop_duplicates("n1"), pdf.drop_duplicates("n1")) - assert_df(gdf.drop_duplicates("n2"), pdf.drop_duplicates("n2")) - assert_df(gdf.drop_duplicates("s1"), pdf.drop_duplicates("s1")) - assert_df( + assert_eq(gdf.drop_duplicates("n1"), pdf.drop_duplicates("n1")) + assert_eq(gdf.drop_duplicates("n2"), pdf.drop_duplicates("n2")) + assert_eq(gdf.drop_duplicates("s1"), pdf.drop_duplicates("s1")) + assert_eq( gdf.drop_duplicates(["n1", "n2"]), pdf.drop_duplicates(["n1", "n2"]) ) - assert_df( + assert_eq( gdf.drop_duplicates(["n1", "s1"]), pdf.drop_duplicates(["n1", "s1"]) ) @@ -387,13 +372,13 @@ def test_datetime_drop_duplicates(): date_df["value"] = np.random.sample(len(date_df)) df = concat([date_df, date_df[:4]]) - assert_df(df[:-4], df.drop_duplicates()) + assert_eq(df[:-4], df.drop_duplicates()) df2 = df.reset_index() - assert_df(df2[:-4], df2.drop_duplicates()) + assert_eq(df2[:-4], df2.drop_duplicates()) df3 = df.set_index("date") - assert_df(df3[:-4], df3.drop_duplicates()) + assert_eq(df3[:-4], df3.drop_duplicates()) def test_drop_duplicates_NA(): @@ -410,29 +395,29 @@ def test_drop_duplicates_NA(): # single column result = df.drop_duplicates("A") expected = df.to_pandas().loc[[0, 2, 3]] - assert_df(result, expected) + assert_eq(result, expected) result = df.drop_duplicates("A", keep="last") expected = df.to_pandas().loc[[1, 6, 7]] - assert_df(result, expected) + assert_eq(result, expected) result = df.drop_duplicates("A", keep=False) expected = df.to_pandas().loc[[]] # empty df - assert_df(result, expected) + assert_eq(result, expected) assert len(result) == 0 # multi column result = df.drop_duplicates(["A", "B"]) expected = df.to_pandas().loc[[0, 2, 3, 6]] - assert_df(result, expected) + assert_eq(result, expected) result = df.drop_duplicates(["A", "B"], keep="last") expected = df.to_pandas().loc[[1, 5, 6, 7]] - assert_df(result, expected) + assert_eq(result, expected) result = df.drop_duplicates(["A", "B"], keep=False) expected = df.to_pandas().loc[[6]] - assert_df(result, expected) + assert_eq(result, expected) # nan df = pd.DataFrame( @@ -447,83 +432,77 @@ def test_drop_duplicates_NA(): # single column result = df.drop_duplicates("C") expected = df[:2] - assert_df(result, expected) + assert_eq(result, expected) result = df.drop_duplicates("C", keep="last") expected = df.to_pandas().loc[[3, 7]] - assert_df(result, expected) + assert_eq(result, expected) result = df.drop_duplicates("C", keep=False) expected = df.to_pandas().loc[[]] # empty df - assert_df(result, expected) + assert_eq(result, expected) assert len(result) == 0 # multi column result = df.drop_duplicates(["C", "B"]) expected = df.to_pandas().loc[[0, 1, 2, 4]] - assert_df(result, expected) + assert_eq(result, expected) result = df.drop_duplicates(["C", "B"], keep="last") expected = df.to_pandas().loc[[1, 3, 6, 7]] - assert_df(result, expected) + assert_eq(result, expected) result = df.drop_duplicates(["C", "B"], keep=False) expected = df.to_pandas().loc[[1]] - assert_df(result, expected) + assert_eq(result, expected) def test_drop_duplicates_NA_for_take_all(): - # TODO: PANDAS 1.0 support - add ignore_index for - # pandas drop_duplicates calls in this function. - - # none pdf = pd.DataFrame( { "A": [None, None, "foo", "bar", "foo", "baz", "bar", "qux"], "C": [1.0, np.nan, np.nan, np.nan, 1.0, 2.0, 3, 1.0], } ) - df = cudf.DataFrame.from_pandas(pdf) - # single column + + # single column with None result = df.drop_duplicates("A") - expected = pdf.iloc[[0, 2, 3, 5, 7]] - assert_df(result, expected) - assert_df( + expected = pdf.drop_duplicates("A") + assert_eq(result, expected) + assert_eq( df.drop_duplicates("A", ignore_index=True), - result.reset_index(drop=True), + pdf.drop_duplicates("A", ignore_index=True), ) result = df.drop_duplicates("A", keep="last") - expected = pdf.iloc[[1, 4, 5, 6, 7]] - assert_df(result, expected) - assert_df( + expected = pdf.drop_duplicates("A", keep="last") + assert_eq(result, expected) + assert_eq( df.drop_duplicates("A", ignore_index=True, keep="last"), - result.reset_index(drop=True), + pdf.drop_duplicates("A", ignore_index=True, keep="last"), ) result = df.drop_duplicates("A", keep=False) - expected = pdf.iloc[[5, 7]] - assert_df(result, expected) - assert_df( + expected = pdf.drop_duplicates("A", keep=False) + assert_eq(result, expected) + assert_eq( df.drop_duplicates("A", ignore_index=True, keep=False), - result.reset_index(drop=True), + pdf.drop_duplicates("A", ignore_index=True, keep=False), ) - # nan - - # single column + # single column with nan result = df.drop_duplicates("C") - expected = pdf.iloc[[0, 1, 5, 6]] - assert_df(result, expected) + expected = pdf.drop_duplicates("C") + assert_eq(result, expected) result = df.drop_duplicates("C", keep="last") - expected = pdf.iloc[[3, 5, 6, 7]] - assert_df(result, expected) + expected = pdf.drop_duplicates("C", keep="last") + assert_eq(result, expected) result = df.drop_duplicates("C", keep=False) - expected = pdf.iloc[[5, 6]] - assert_df(result, expected) + expected = pdf.drop_duplicates("C", keep=False) + assert_eq(result, expected) def test_drop_duplicates_inplace(): @@ -541,19 +520,19 @@ def test_drop_duplicates_inplace(): df.drop_duplicates("A", inplace=True) expected = orig[:2] result = df - assert_df(result, expected) + assert_eq(result, expected) df = orig.copy() df.drop_duplicates("A", keep="last", inplace=True) expected = orig.loc[[6, 7]] result = df - assert_df(result, expected) + assert_eq(result, expected) df = orig.copy() df.drop_duplicates("A", keep=False, inplace=True) expected = orig.loc[[]] result = df - assert_df(result, expected) + assert_eq(result, expected) assert len(df) == 0 # multi column @@ -561,19 +540,19 @@ def test_drop_duplicates_inplace(): df.drop_duplicates(["A", "B"], inplace=True) expected = orig.loc[[0, 1, 2, 3]] result = df - assert_df(result, expected) + assert_eq(result, expected) df = orig.copy() df.drop_duplicates(["A", "B"], keep="last", inplace=True) expected = orig.loc[[0, 5, 6, 7]] result = df - assert_df(result, expected) + assert_eq(result, expected) df = orig.copy() df.drop_duplicates(["A", "B"], keep=False, inplace=True) expected = orig.loc[[0]] result = df - assert_df(result, expected) + assert_eq(result, expected) # consider everything orig2 = orig.loc[:, ["A", "B", "C"]].copy() @@ -583,19 +562,19 @@ def test_drop_duplicates_inplace(): # in this case only expected = orig2.drop_duplicates(["A", "B"]) result = df2 - assert_df(result, expected) + assert_eq(result, expected) df2 = orig2.copy() df2.drop_duplicates(keep="last", inplace=True) expected = orig2.drop_duplicates(["A", "B"], keep="last") result = df2 - assert_df(result, expected) + assert_eq(result, expected) df2 = orig2.copy() df2.drop_duplicates(keep=False, inplace=True) expected = orig2.drop_duplicates(["A", "B"], keep=False) result = df2 - assert_df(result, expected) + assert_eq(result, expected) def test_drop_duplicates_multi_index(): @@ -610,11 +589,11 @@ def test_drop_duplicates_multi_index(): expected = pdf.drop_duplicates() result = gdf.drop_duplicates() - assert_df(result.to_pandas(), expected) + assert_eq(result.to_pandas(), expected) # FIXME: to_pandas needed until sort_index support for MultiIndex for col in gdf.columns: - assert_df( + assert_eq( gdf[col].drop_duplicates().to_pandas(), pdf[col].drop_duplicates(), ) diff --git a/python/cudf/cudf/tests/test_string.py b/python/cudf/cudf/tests/test_string.py index b2bf687ba06..a9ba80a395d 100644 --- a/python/cudf/cudf/tests/test_string.py +++ b/python/cudf/cudf/tests/test_string.py @@ -972,8 +972,7 @@ def test_string_split_re(data, pat, n, expand): ps = pd.Series(data, dtype="str") gs = cudf.Series(data, dtype="str") - # Pandas does not support the regex parameter until 1.4.0 - expect = ps.str.split(pat=pat, n=n, expand=expand) + expect = ps.str.split(pat=pat, n=n, expand=expand, regex=True) got = gs.str.split(pat=pat, n=n, expand=expand, regex=True) assert_eq(expect, got) diff --git a/python/dask_cudf/dask_cudf/tests/test_core.py b/python/dask_cudf/dask_cudf/tests/test_core.py index 250256d3356..ecad2220ba5 100644 --- a/python/dask_cudf/dask_cudf/tests/test_core.py +++ b/python/dask_cudf/dask_cudf/tests/test_core.py @@ -6,7 +6,6 @@ import numpy as np import pandas as pd import pytest -from packaging import version import dask from dask import dataframe as dd @@ -598,14 +597,6 @@ def test_unary_ops(func, gdf, gddf): p = func(gdf) g = func(gddf) - # Fixed in https://github.com/dask/dask/pull/4657 - if isinstance(p, cudf.Index): - if version.parse(dask.__version__) < version.parse("1.1.6"): - pytest.skip( - "dask.dataframe assert_eq index check hardcoded to " - "pandas prior to 1.1.6 release" - ) - dd.assert_eq(p, g, check_names=False) From 9916395201c8544692d1aeb191fa370e0dc8e630 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Thu, 1 Feb 2024 18:44:47 -0500 Subject: [PATCH 193/384] Use offsetalator in cudf::strings::findall (#14745) Use `make_offsets_child_column` and `offsetalator_iterator` to build/access offsets instead of hardcoded types. This cleans up the code nicely by automatically handling offset overflow and computing the total number of matches. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Mike Wilson (https://github.com/hyperbolic2346) URL: https://github.com/rapidsai/cudf/pull/14745 --- .../cudf/strings/detail/strings_children.cuh | 2 +- cpp/src/strings/search/findall.cu | 27 +++++++------------ 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/cpp/include/cudf/strings/detail/strings_children.cuh b/cpp/include/cudf/strings/detail/strings_children.cuh index 42a180c27c1..8e2b6055a5c 100644 --- a/cpp/include/cudf/strings/detail/strings_children.cuh +++ b/cpp/include/cudf/strings/detail/strings_children.cuh @@ -165,7 +165,7 @@ std::pair, int64_t> make_offsets_child_column( auto input_itr = cudf::detail::make_counting_transform_iterator(0, map_fn); // Use the sizes-to-offsets iterator to compute the total number of elements auto const total_elements = - sizes_to_offsets(input_itr, input_itr + strings_count + 1, d_offsets, stream); + cudf::detail::sizes_to_offsets(input_itr, input_itr + strings_count + 1, d_offsets, stream); // TODO: replace exception with if-statement when enabling creating INT64 offsets CUDF_EXPECTS(total_elements <= size_type_max, diff --git a/cpp/src/strings/search/findall.cu b/cpp/src/strings/search/findall.cu index acea4ff1c51..8df1a67d56d 100644 --- a/cpp/src/strings/search/findall.cu +++ b/cpp/src/strings/search/findall.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,11 +20,10 @@ #include #include -#include #include #include +#include #include -#include #include #include #include @@ -34,7 +33,6 @@ #include #include -#include namespace cudf { namespace strings { @@ -50,7 +48,7 @@ namespace { */ struct findall_fn { column_device_view const d_strings; - size_type const* d_offsets; + cudf::detail::input_offsetalator const d_offsets; string_index_pair* d_indices; __device__ void operator()(size_type const idx, reprog_device const prog, int32_t const prog_idx) @@ -77,8 +75,8 @@ struct findall_fn { std::unique_ptr findall_util(column_device_view const& d_strings, reprog_device& d_prog, - size_type total_matches, - size_type const* d_offsets, + int64_t total_matches, + cudf::detail::input_offsetalator const d_offsets, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { @@ -105,17 +103,12 @@ std::unique_ptr findall(strings_column_view const& input, auto d_prog = regex_device_builder::create_prog_device(prog, stream); // Create lists offsets column - auto offsets = count_matches(*d_strings, *d_prog, strings_count + 1, stream, mr); - auto d_offsets = offsets->mutable_view().data(); - - // Convert counts into offsets - thrust::exclusive_scan( - rmm::exec_policy(stream), d_offsets, d_offsets + strings_count + 1, d_offsets); - - // Create indices vector with the total number of groups that will be extracted - auto const total_matches = - cudf::detail::get_value(offsets->view(), strings_count, stream); + auto const sizes = count_matches(*d_strings, *d_prog, strings_count, stream, mr); + auto [offsets, total_matches] = cudf::strings::detail::make_offsets_child_column( + sizes->view().begin(), sizes->view().end(), stream, mr); + auto const d_offsets = cudf::detail::offsetalator_factory::make_input_iterator(offsets->view()); + // Build strings column of the matches auto strings_output = findall_util(*d_strings, *d_prog, total_matches, d_offsets, stream, mr); // Build the lists column from the offsets and the strings From b80f4e2e8ef15bbaad7dfde1ac6c8adc93cfd46d Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 1 Feb 2024 17:53:01 -0800 Subject: [PATCH 194/384] Implement groupby in pylibcudf (#14945) This PR implements groupby in pylibcudf along with the minimal set of aggregation logic to support groupby. To limit its scope, this PR does not include other aggregation logic for e.g. non-groupby reductions and scans. Due to the large scale of what's already in this PR, I have also omitted the changes required to leverage pylibcudf in the current cudf Cython code from this PR. That will be done in a follow-up. This PR's diff is misleadingly large, a large chunk of it is adding documentation and function declarations that shouldn't impose too heavy a cognitive load in review. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Ashwin Srinath (https://github.com/shwina) - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/14945 --- .../api_docs/pylibcudf/aggregation.rst | 6 + .../user_guide/api_docs/pylibcudf/groupby.rst | 6 + .../user_guide/api_docs/pylibcudf/index.rst | 2 + python/cudf/cudf/_lib/aggregation.pyx | 24 +- python/cudf/cudf/_lib/cpp/CMakeLists.txt | 2 +- python/cudf/cudf/_lib/cpp/aggregation.pxd | 131 ++--- python/cudf/cudf/_lib/cpp/aggregation.pyx | 0 python/cudf/cudf/_lib/cpp/groupby.pxd | 18 +- python/cudf/cudf/_lib/cpp/replace.pxd | 9 +- .../cudf/cudf/_lib/pylibcudf/CMakeLists.txt | 4 +- python/cudf/cudf/_lib/pylibcudf/__init__.pxd | 7 +- python/cudf/cudf/_lib/pylibcudf/__init__.py | 5 +- .../cudf/cudf/_lib/pylibcudf/aggregation.pxd | 89 +++ .../cudf/cudf/_lib/pylibcudf/aggregation.pyx | 513 ++++++++++++++++++ python/cudf/cudf/_lib/pylibcudf/copying.pyx | 18 +- python/cudf/cudf/_lib/pylibcudf/groupby.pxd | 47 ++ python/cudf/cudf/_lib/pylibcudf/groupby.pyx | 251 +++++++++ python/cudf/cudf/_lib/pylibcudf/types.pxd | 14 +- python/cudf/cudf/_lib/pylibcudf/types.pyx | 9 +- python/cudf/cudf/_lib/pylibcudf/utils.pxd | 7 +- python/cudf/cudf/_lib/pylibcudf/utils.pyx | 25 +- 21 files changed, 1079 insertions(+), 108 deletions(-) create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/aggregation.rst create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/groupby.rst create mode 100644 python/cudf/cudf/_lib/cpp/aggregation.pyx create mode 100644 python/cudf/cudf/_lib/pylibcudf/aggregation.pxd create mode 100644 python/cudf/cudf/_lib/pylibcudf/aggregation.pyx create mode 100644 python/cudf/cudf/_lib/pylibcudf/groupby.pxd create mode 100644 python/cudf/cudf/_lib/pylibcudf/groupby.pyx diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/aggregation.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/aggregation.rst new file mode 100644 index 00000000000..739305af5d4 --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/aggregation.rst @@ -0,0 +1,6 @@ +=========== +aggregation +=========== + +.. automodule:: cudf._lib.pylibcudf.aggregation + :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/groupby.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/groupby.rst new file mode 100644 index 00000000000..d6e994f7dbc --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/groupby.rst @@ -0,0 +1,6 @@ +======= +groupby +======= + +.. automodule:: cudf._lib.pylibcudf.groupby + :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst index 7504295de92..4735b0d9414 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst @@ -8,10 +8,12 @@ This page provides API documentation for pylibcudf. :maxdepth: 1 :caption: API Documentation + aggregation binaryop column copying gpumemoryview + groupby scalar table types diff --git a/python/cudf/cudf/_lib/aggregation.pyx b/python/cudf/cudf/_lib/aggregation.pyx index 72c5e288f0b..b202d08ac2e 100644 --- a/python/cudf/cudf/_lib/aggregation.pyx +++ b/python/cudf/cudf/_lib/aggregation.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from enum import Enum, IntEnum @@ -51,7 +51,7 @@ class AggregationKind(Enum): NUNIQUE = libcudf_aggregation.aggregation.Kind.NUNIQUE NTH = libcudf_aggregation.aggregation.Kind.NTH_ELEMENT RANK = libcudf_aggregation.aggregation.Kind.RANK - COLLECT = libcudf_aggregation.aggregation.Kind.COLLECT + COLLECT = libcudf_aggregation.aggregation.Kind.COLLECT_LIST UNIQUE = libcudf_aggregation.aggregation.Kind.COLLECT_SET PTX = libcudf_aggregation.aggregation.Kind.PTX CUDA = libcudf_aggregation.aggregation.Kind.CUDA @@ -191,7 +191,7 @@ cdef class RollingAggregation: cdef RollingAggregation agg = cls() agg.c_obj = move( libcudf_aggregation.make_collect_list_aggregation[ - rolling_aggregation]()) + rolling_aggregation](libcudf_types.null_policy.INCLUDE)) return agg @classmethod @@ -335,7 +335,9 @@ cdef class GroupbyAggregation: cdef GroupbyAggregation agg = cls() agg.c_obj = move( libcudf_aggregation. - make_collect_list_aggregation[groupby_aggregation]()) + make_collect_list_aggregation[groupby_aggregation]( + libcudf_types.null_policy.INCLUDE + )) return agg @classmethod @@ -343,7 +345,9 @@ cdef class GroupbyAggregation: cdef GroupbyAggregation agg = cls() agg.c_obj = move( libcudf_aggregation. - make_nunique_aggregation[groupby_aggregation]()) + make_nunique_aggregation[groupby_aggregation]( + libcudf_types.null_policy.EXCLUDE + )) return agg @classmethod @@ -422,7 +426,11 @@ cdef class GroupbyAggregation: cdef GroupbyAggregation agg = cls() agg.c_obj = move( libcudf_aggregation. - make_collect_set_aggregation[groupby_aggregation]()) + make_collect_set_aggregation[groupby_aggregation]( + libcudf_types.null_policy.INCLUDE, + libcudf_types.null_equality.EQUAL, + libcudf_types.nan_equality.ALL_EQUAL, + )) return agg @classmethod @@ -724,7 +732,9 @@ cdef class ReduceAggregation: def nunique(cls): cdef ReduceAggregation agg = cls() agg.c_obj = move( - libcudf_aggregation.make_nunique_aggregation[reduce_aggregation]()) + libcudf_aggregation.make_nunique_aggregation[reduce_aggregation]( + libcudf_types.null_policy.EXCLUDE + )) return agg @classmethod diff --git a/python/cudf/cudf/_lib/cpp/CMakeLists.txt b/python/cudf/cudf/_lib/cpp/CMakeLists.txt index 764f28add0e..316541c9bc5 100644 --- a/python/cudf/cudf/_lib/cpp/CMakeLists.txt +++ b/python/cudf/cudf/_lib/cpp/CMakeLists.txt @@ -12,7 +12,7 @@ # the License. # ============================================================================= -set(cython_sources binaryop.pyx copying.pyx types.pyx) +set(cython_sources aggregation.pyx binaryop.pyx copying.pyx types.pyx) set(linked_libraries cudf::cudf) diff --git a/python/cudf/cudf/_lib/cpp/aggregation.pxd b/python/cudf/cudf/_lib/cpp/aggregation.pxd index a1d1485e1e8..16f48b30a50 100644 --- a/python/cudf/cudf/_lib/cpp/aggregation.pxd +++ b/python/cudf/cudf/_lib/cpp/aggregation.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from libc.stdint cimport int32_t from libcpp cimport bool from libcpp.memory cimport unique_ptr @@ -8,6 +8,8 @@ from libcpp.vector cimport vector from cudf._lib.cpp.types cimport ( data_type, interpolation, + nan_equality, + null_equality, null_order, null_policy, order, @@ -19,71 +21,74 @@ ctypedef int32_t underlying_type_t_rank_method cdef extern from "cudf/aggregation.hpp" namespace "cudf" nogil: - cdef cppclass aggregation: - ctypedef enum Kind: - SUM 'cudf::aggregation::SUM' - PRODUCT 'cudf::aggregation::PRODUCT' - MIN 'cudf::aggregation::MIN' - MAX 'cudf::aggregation::MAX' - COUNT_VALID 'cudf::aggregation::COUNT_VALID' - COUNT_ALL 'cudf::aggregation::COUNT_ALL' - ANY 'cudf::aggregation::ANY' - ALL 'cudf::aggregation::ALL' - SUM_OF_SQUARES 'cudf::aggregation::SUM_OF_SQUARES' - MEAN 'cudf::aggregation::MEAN' - VARIANCE 'cudf::aggregation::VARIANCE' - STD 'cudf::aggregation::STD' - MEDIAN 'cudf::aggregation::MEDIAN' - QUANTILE 'cudf::aggregation::QUANTILE' - ARGMAX 'cudf::aggregation::ARGMAX' - ARGMIN 'cudf::aggregation::ARGMIN' - NUNIQUE 'cudf::aggregation::NUNIQUE' - NTH_ELEMENT 'cudf::aggregation::NTH_ELEMENT' - RANK 'cudf::aggregation::RANK' - COLLECT 'cudf::aggregation::COLLECT_LIST' - COLLECT_SET 'cudf::aggregation::COLLECT_SET' - PTX 'cudf::aggregation::PTX' - CUDA 'cudf::aggregation::CUDA' - CORRELATION 'cudf::aggregation::CORRELATION' - COVARIANCE 'cudf::aggregation::COVARIANCE' + # Cython doesn't appear to support enum class nested inside a class, so + # have to namespace it manually + cpdef enum class Kind "cudf::aggregation::Kind": + SUM + PRODUCT + MIN + MAX + COUNT_VALID + COUNT_ALL + ANY + ALL + SUM_OF_SQUARES + MEAN + VARIANCE + STD + MEDIAN + QUANTILE + ARGMAX + ARGMIN + NUNIQUE + NTH_ELEMENT + RANK + COLLECT_LIST + COLLECT_SET + PTX + CUDA + CORRELATION + COVARIANCE + cdef cppclass aggregation: Kind kind + unique_ptr[aggregation] clone() - cdef cppclass rolling_aggregation: - aggregation.Kind kind + cdef cppclass rolling_aggregation(aggregation): + pass - cdef cppclass groupby_aggregation: - aggregation.Kind kind + cdef cppclass groupby_aggregation(aggregation): + pass - cdef cppclass groupby_scan_aggregation: - aggregation.Kind kind + cdef cppclass groupby_scan_aggregation(aggregation): + pass - cdef cppclass reduce_aggregation: - aggregation.Kind kind + cdef cppclass reduce_aggregation(aggregation): + pass - cdef cppclass scan_aggregation: - aggregation.Kind kind + cdef cppclass scan_aggregation(aggregation): + pass - ctypedef enum udf_type: - CUDA 'cudf::udf_type::CUDA' - PTX 'cudf::udf_type::PTX' + cpdef enum class udf_type(bool): + CUDA + PTX - ctypedef enum correlation_type: - PEARSON 'cudf::correlation_type::PEARSON' - KENDALL 'cudf::correlation_type::KENDALL' - SPEARMAN 'cudf::correlation_type::SPEARMAN' + cpdef enum class correlation_type(int32_t): + PEARSON + KENDALL + SPEARMAN - ctypedef enum rank_method: - FIRST "cudf::rank_method::FIRST" - AVERAGE "cudf::rank_method::AVERAGE" - MIN "cudf::rank_method::MIN" - MAX "cudf::rank_method::MAX" - DENSE "cudf::rank_method::DENSE" + cpdef enum class rank_method(int32_t): + FIRST + AVERAGE + MIN + MAX + DENSE - ctypedef enum rank_percentage: - NONE "cudf::rank_percentage::NONE" - ZERO_NORMALIZED "cudf::rank_percentage::ZERO_NORMALIZED" - ONE_NORMALIZED "cudf::rank_percentage::ONE_NORMALIZED" + cpdef enum class rank_percentage(int32_t): + NONE + ZERO_NORMALIZED + ONE_NORMALIZED cdef unique_ptr[T] make_sum_aggregation[T]() except + @@ -93,8 +98,6 @@ cdef extern from "cudf/aggregation.hpp" namespace "cudf" nogil: cdef unique_ptr[T] make_max_aggregation[T]() except + - cdef unique_ptr[T] make_count_aggregation[T]() except + - cdef unique_ptr[T] make_count_aggregation[T](null_policy) except + cdef unique_ptr[T] make_any_aggregation[T]() except + @@ -119,20 +122,20 @@ cdef extern from "cudf/aggregation.hpp" namespace "cudf" nogil: cdef unique_ptr[T] make_argmin_aggregation[T]() except + - cdef unique_ptr[T] make_nunique_aggregation[T]() except + - - cdef unique_ptr[T] make_nth_element_aggregation[T]( - size_type n - ) except + + cdef unique_ptr[T] make_nunique_aggregation[T](null_policy null_handling) except + cdef unique_ptr[T] make_nth_element_aggregation[T]( size_type n, null_policy null_handling ) except + - cdef unique_ptr[T] make_collect_list_aggregation[T]() except + + cdef unique_ptr[T] make_collect_list_aggregation[T]( + null_policy null_handling + ) except + - cdef unique_ptr[T] make_collect_set_aggregation[T]() except + + cdef unique_ptr[T] make_collect_set_aggregation[T]( + null_policy null_handling, null_equality nulls_equal, nan_equality nans_equal + ) except + cdef unique_ptr[T] make_udf_aggregation[T]( udf_type type, diff --git a/python/cudf/cudf/_lib/cpp/aggregation.pyx b/python/cudf/cudf/_lib/cpp/aggregation.pyx new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/cudf/cudf/_lib/cpp/groupby.pxd b/python/cudf/cudf/_lib/cpp/groupby.pxd index 0266404fc50..8bbefcde0dd 100644 --- a/python/cudf/cudf/_lib/cpp/groupby.pxd +++ b/python/cudf/cudf/_lib/cpp/groupby.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp cimport bool from libcpp.functional cimport reference_wrapper @@ -16,7 +16,13 @@ from cudf._lib.cpp.replace cimport replace_policy from cudf._lib.cpp.scalar.scalar cimport scalar from cudf._lib.cpp.table.table cimport table from cudf._lib.cpp.table.table_view cimport table_view -from cudf._lib.cpp.types cimport null_order, null_policy, order, size_type +from cudf._lib.cpp.types cimport ( + null_order, + null_policy, + order, + size_type, + sorted, +) from cudf._lib.cpp.utilities.host_span cimport host_span # workaround for https://github.com/cython/cython/issues/3885 @@ -55,20 +61,20 @@ cdef extern from "cudf/groupby.hpp" \ groupby( const table_view& keys, null_policy include_null_keys, - bool keys_are_sorted, + sorted keys_are_sorted, ) except + groupby( const table_view& keys, null_policy include_null_keys, - bool keys_are_sorted, + sorted keys_are_sorted, const vector[order]& column_order, ) except + groupby( const table_view& keys, null_policy include_null_keys, - bool keys_are_sorted, + sorted keys_are_sorted, const vector[order]& column_order, const vector[null_order]& null_precedence ) except + @@ -100,6 +106,6 @@ cdef extern from "cudf/groupby.hpp" \ groups get_groups(table_view values) except + pair[unique_ptr[table], unique_ptr[table]] replace_nulls( - const table_view& value, + const table_view& values, const vector[replace_policy] replace_policy ) except + diff --git a/python/cudf/cudf/_lib/cpp/replace.pxd b/python/cudf/cudf/_lib/cpp/replace.pxd index c1ec89a6233..74bc9c2bb4c 100644 --- a/python/cudf/cudf/_lib/cpp/replace.pxd +++ b/python/cudf/cudf/_lib/cpp/replace.pxd @@ -1,5 +1,6 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. +from libcpp cimport bool from libcpp.memory cimport unique_ptr from cudf._lib.types import cudf_to_np_types, np_to_cudf_types @@ -11,9 +12,9 @@ from cudf._lib.cpp.scalar.scalar cimport scalar cdef extern from "cudf/replace.hpp" namespace "cudf" nogil: - ctypedef enum replace_policy: - PRECEDING 'cudf::replace_policy::PRECEDING', - FOLLOWING 'cudf::replace_policy::FOLLOWING' + cdef enum class replace_policy(bool): + PRECEDING + FOLLOWING cdef unique_ptr[column] replace_nulls( column_view source_column, diff --git a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt index acb013c8b8c..0ca0c122c38 100644 --- a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt +++ b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt @@ -12,8 +12,8 @@ # the License. # ============================================================================= -set(cython_sources binaryop.pyx column.pyx copying.pyx gpumemoryview.pyx interop.pyx scalar.pyx - table.pyx types.pyx utils.pyx +set(cython_sources aggregation.pyx binaryop.pyx column.pyx copying.pyx gpumemoryview.pyx + groupby.pyx interop.pyx scalar.pyx table.pyx types.pyx utils.pyx ) set(linked_libraries cudf::cudf) rapids_cython_create_modules( diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd index f4b8c50eecc..14c98af3fff 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd @@ -1,22 +1,25 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. # TODO: Verify consistent usage of relative/absolute imports in pylibcudf. -from . cimport binaryop, copying, interop +from . cimport aggregation, binaryop, copying, groupby, interop from .column cimport Column from .gpumemoryview cimport gpumemoryview from .scalar cimport Scalar from .table cimport Table # TODO: cimport type_id once # https://github.com/cython/cython/issues/5609 is resolved -from .types cimport DataType +from .types cimport DataType, type_id __all__ = [ "Column", "DataType", "Scalar", "Table", + "aggregation", "binaryop", "copying", "gpumemoryview", + "groupby", "interop", + "types", ] diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.py b/python/cudf/cudf/_lib/pylibcudf/__init__.py index a27d80fc5a2..07612d76540 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.py +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.py @@ -1,6 +1,6 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. -from . import binaryop, copying, interop +from . import aggregation, binaryop, copying, groupby, interop from .column import Column from .gpumemoryview import gpumemoryview from .scalar import Scalar @@ -13,8 +13,11 @@ "Scalar", "Table", "TypeId", + "aggregation", "binaryop", "copying", "gpumemoryview", + "groupby", "interop", + "types", ] diff --git a/python/cudf/cudf/_lib/pylibcudf/aggregation.pxd b/python/cudf/cudf/_lib/pylibcudf/aggregation.pxd new file mode 100644 index 00000000000..8eda16c4165 --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/aggregation.pxd @@ -0,0 +1,89 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from libcpp.memory cimport unique_ptr + +from cudf._lib.cpp.aggregation cimport ( + Kind as kind_t, + aggregation, + correlation_type, + groupby_aggregation, + groupby_scan_aggregation, + rank_method, + rank_percentage, +) +from cudf._lib.cpp.types cimport ( + interpolation, + nan_equality, + null_equality, + null_order, + null_policy, + order, + size_type, +) + +from .types cimport DataType + + +cdef class Aggregation: + cdef unique_ptr[aggregation] c_obj + cpdef kind(self) + cdef unique_ptr[groupby_aggregation] clone_underlying_as_groupby(self) except * + cdef unique_ptr[groupby_scan_aggregation] clone_underlying_as_groupby_scan( + self + ) except * + + @staticmethod + cdef Aggregation from_libcudf(unique_ptr[aggregation] agg) + + +cpdef Aggregation sum() + +cpdef Aggregation product() + +cpdef Aggregation min() + +cpdef Aggregation max() + +cpdef Aggregation count(null_policy null_handling = *) + +cpdef Aggregation any() + +cpdef Aggregation all() + +cpdef Aggregation sum_of_squares() + +cpdef Aggregation mean() + +cpdef Aggregation variance(size_type ddof = *) + +cpdef Aggregation std(size_type ddof = *) + +cpdef Aggregation median() + +cpdef Aggregation quantile(list quantiles, interpolation interp = *) + +cpdef Aggregation argmax() + +cpdef Aggregation argmin() + +cpdef Aggregation nunique(null_policy null_handling = *) + +cpdef Aggregation nth_element(size_type n, null_policy null_handling = *) + +cpdef Aggregation collect_list(null_policy null_handling = *) + +cpdef Aggregation collect_set(null_handling = *, nulls_equal = *, nans_equal = *) + +cpdef Aggregation udf(str operation, DataType output_type) + +cpdef Aggregation correlation(correlation_type type, size_type min_periods) + +cpdef Aggregation covariance(size_type min_periods, size_type ddof) + +cpdef Aggregation rank( + rank_method method, + order column_order = *, + null_policy null_handling = *, + null_order null_precedence = *, + rank_percentage percentage = *, +) diff --git a/python/cudf/cudf/_lib/pylibcudf/aggregation.pyx b/python/cudf/cudf/_lib/pylibcudf/aggregation.pyx new file mode 100644 index 00000000000..0b91263d720 --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/aggregation.pyx @@ -0,0 +1,513 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from cython.operator cimport dereference +from libcpp.cast cimport dynamic_cast +from libcpp.memory cimport unique_ptr +from libcpp.utility cimport move + +from cudf._lib.cpp.aggregation cimport ( + aggregation, + correlation_type, + groupby_aggregation, + groupby_scan_aggregation, + make_all_aggregation, + make_any_aggregation, + make_argmax_aggregation, + make_argmin_aggregation, + make_collect_list_aggregation, + make_collect_set_aggregation, + make_correlation_aggregation, + make_count_aggregation, + make_covariance_aggregation, + make_max_aggregation, + make_mean_aggregation, + make_median_aggregation, + make_min_aggregation, + make_nth_element_aggregation, + make_nunique_aggregation, + make_product_aggregation, + make_quantile_aggregation, + make_rank_aggregation, + make_std_aggregation, + make_sum_aggregation, + make_sum_of_squares_aggregation, + make_udf_aggregation, + make_variance_aggregation, + rank_method, + rank_percentage, +) +from cudf._lib.cpp.types cimport ( + interpolation, + nan_equality, + null_equality, + null_order, + null_policy, + order, + size_type, +) + +from cudf._lib.cpp.aggregation import Kind # no-cython-lint +from cudf._lib.cpp.aggregation import \ + correlation_type as CorrelationType # no-cython-lint +from cudf._lib.cpp.aggregation import \ + rank_method as RankMethod # no-cython-lint +from cudf._lib.cpp.aggregation import \ + rank_percentage as RankPercentage # no-cython-lint +from cudf._lib.cpp.aggregation import udf_type as UdfType # no-cython-lint + +from .types cimport DataType + +# workaround for https://github.com/cython/cython/issues/3885 +ctypedef groupby_aggregation * gba_ptr +ctypedef groupby_scan_aggregation * gbsa_ptr + + +cdef class Aggregation: + """A type of aggregation to perform. + + Aggregations are passed to APIs like + :py:func:`~cudf._lib.pylibcudf.groupby.GroupBy.aggregate` to indicate what + operations to perform. Using a class for aggregations provides a unified + API for handling parametrizable aggregations. This class should never be + instantiated directly, only via one of the factory functions. + """ + def __init__(self): + raise ValueError( + "Aggregations should not be constructed directly. Use one of the factories." + ) + + # TODO: Ideally we would include the return type here, but we need to do so + # in a way that Sphinx understands (currently have issues due to + # https://github.com/cython/cython/issues/5609). + cpdef kind(self): + """Get the kind of the aggregation.""" + return dereference(self.c_obj).kind + + cdef unique_ptr[groupby_aggregation] clone_underlying_as_groupby(self) except *: + """Make a copy of the underlying aggregation that can be used in a groupby. + + This function will raise an exception if the aggregation is not supported as a + groupby aggregation. This failure to cast translates the per-algorithm + aggregation logic encoded in libcudf's type hierarchy into Python. + """ + cdef unique_ptr[aggregation] agg = dereference(self.c_obj).clone() + cdef groupby_aggregation *agg_cast = dynamic_cast[gba_ptr](agg.get()) + if agg_cast is NULL: + agg_repr = str(self.kind()).split(".")[1].title() + raise TypeError(f"{agg_repr} aggregations are not supported by groupby") + agg.release() + return unique_ptr[groupby_aggregation](agg_cast) + + # Ideally this function could reuse the code above, but Cython lacks the + # first-class support for type-aliasing and templates that would make it possible. + cdef unique_ptr[groupby_scan_aggregation] clone_underlying_as_groupby_scan( + self + ) except *: + """Make a copy of the underlying aggregation that can be used in a groupby scan. + + This function will raise an exception if the aggregation is not supported as a + groupby scan aggregation. This failure to cast translates the per-algorithm + aggregation logic encoded in libcudf's type hierarchy into Python. + """ + cdef unique_ptr[aggregation] agg = dereference(self.c_obj).clone() + cdef groupby_scan_aggregation *agg_cast = dynamic_cast[gbsa_ptr](agg.get()) + if agg_cast is NULL: + agg_repr = str(self.kind()).split(".")[1].title() + raise TypeError(f"{agg_repr} scans are not supported by groupby") + agg.release() + return unique_ptr[groupby_scan_aggregation](agg_cast) + + @staticmethod + cdef Aggregation from_libcudf(unique_ptr[aggregation] agg): + """Create a Python Aggregation from a libcudf aggregation.""" + cdef Aggregation out = Aggregation.__new__(Aggregation) + out.c_obj = move(agg) + return out + + +cpdef Aggregation sum(): + """Create a sum aggregation. + + Returns + ------- + Aggregation + The sum aggregation. + """ + return Aggregation.from_libcudf(move(make_sum_aggregation[aggregation]())) + + +cpdef Aggregation product(): + """Create a product aggregation. + + Returns + ------- + Aggregation + The product aggregation. + """ + return Aggregation.from_libcudf(move(make_product_aggregation[aggregation]())) + + +cpdef Aggregation min(): + """Create a min aggregation. + + Returns + ------- + Aggregation + The min aggregation. + """ + return Aggregation.from_libcudf(move(make_min_aggregation[aggregation]())) + + +cpdef Aggregation max(): + """Create a max aggregation. + + Returns + ------- + Aggregation + The max aggregation. + """ + return Aggregation.from_libcudf(move(make_max_aggregation[aggregation]())) + + +cpdef Aggregation count(null_policy null_handling = null_policy.EXCLUDE): + """Create a count aggregation. + + Parameters + ---------- + null_handling : null_policy, default EXCLUDE + Whether or not nulls should be included. + + Returns + ------- + Aggregation + The count aggregation. + """ + return Aggregation.from_libcudf( + move(make_count_aggregation[aggregation](null_handling)) + ) + + +cpdef Aggregation any(): + """Create an any aggregation. + + Returns + ------- + Aggregation + The any aggregation. + """ + return Aggregation.from_libcudf(move(make_any_aggregation[aggregation]())) + + +cpdef Aggregation all(): + """Create an all aggregation. + + Returns + ------- + Aggregation + The all aggregation. + """ + return Aggregation.from_libcudf(move(make_all_aggregation[aggregation]())) + + +cpdef Aggregation sum_of_squares(): + """Create a sum_of_squares aggregation. + + Returns + ------- + Aggregation + The sum_of_squares aggregation. + """ + return Aggregation.from_libcudf( + move(make_sum_of_squares_aggregation[aggregation]()) + ) + + +cpdef Aggregation mean(): + """Create a mean aggregation. + + Returns + ------- + Aggregation + The mean aggregation. + """ + return Aggregation.from_libcudf(move(make_mean_aggregation[aggregation]())) + + +cpdef Aggregation variance(size_type ddof=1): + """Create a variance aggregation. + + Parameters + ---------- + ddof : int, default 1 + Delta degrees of freedom. + + Returns + ------- + Aggregation + The variance aggregation. + """ + return Aggregation.from_libcudf(move(make_variance_aggregation[aggregation](ddof))) + + +cpdef Aggregation std(size_type ddof=1): + """Create a std aggregation. + + Parameters + ---------- + ddof : int, default 1 + Delta degrees of freedom. The default value is 1. + + Returns + ------- + Aggregation + The std aggregation. + """ + return Aggregation.from_libcudf(move(make_std_aggregation[aggregation](ddof))) + + +cpdef Aggregation median(): + """Create a median aggregation. + + Returns + ------- + Aggregation + The median aggregation. + """ + return Aggregation.from_libcudf(move(make_median_aggregation[aggregation]())) + + +cpdef Aggregation quantile(list quantiles, interpolation interp = interpolation.LINEAR): + """Create a quantile aggregation. + + Parameters + ---------- + quantiles : list + List of quantiles to compute, should be between 0 and 1. + interp : interpolation, default LINEAR + Interpolation technique to use when the desired quantile lies between + two data points. + + Returns + ------- + Aggregation + The quantile aggregation. + """ + return Aggregation.from_libcudf( + move(make_quantile_aggregation[aggregation](quantiles, interp)) + ) + + +cpdef Aggregation argmax(): + """Create an argmax aggregation. + + Returns + ------- + Aggregation + The argmax aggregation. + """ + return Aggregation.from_libcudf(move(make_argmax_aggregation[aggregation]())) + + +cpdef Aggregation argmin(): + """Create an argmin aggregation. + + Returns + ------- + Aggregation + The argmin aggregation. + """ + return Aggregation.from_libcudf(move(make_argmin_aggregation[aggregation]())) + + +cpdef Aggregation nunique(null_policy null_handling = null_policy.EXCLUDE): + """Create a nunique aggregation. + + Parameters + ---------- + null_handling : null_policy, default EXCLUDE + Whether or not nulls should be included. + + Returns + ------- + Aggregation + The nunique aggregation. + """ + return Aggregation.from_libcudf( + move(make_nunique_aggregation[aggregation](null_handling)) + ) + + +cpdef Aggregation nth_element( + size_type n, null_policy null_handling = null_policy.INCLUDE +): + """Create a nth_element aggregation. + + Parameters + ---------- + null_handling : null_policy, default INCLUDE + Whether or not nulls should be included. + + Returns + ------- + Aggregation + The nth_element aggregation. + """ + return Aggregation.from_libcudf( + move(make_nth_element_aggregation[aggregation](n, null_handling)) + ) + + +cpdef Aggregation collect_list(null_policy null_handling = null_policy.INCLUDE): + """Create a collect_list aggregation. + + Parameters + ---------- + null_handling : null_policy, default INCLUDE + Whether or not nulls should be included. + + Returns + ------- + Aggregation + The collect_list aggregation. + """ + return Aggregation.from_libcudf( + move(make_collect_list_aggregation[aggregation](null_handling)) + ) + + +cpdef Aggregation collect_set( + null_handling = null_policy.INCLUDE, + nulls_equal = null_equality.EQUAL, + nans_equal = nan_equality.ALL_EQUAL, +): + """Create a collect_set aggregation. + + Parameters + ---------- + null_handling : null_policy, default INCLUDE + Whether or not nulls should be included. + nulls_equal : null_equality, default EQUAL + Whether or not nulls should be considered equal. + nans_equal : nan_equality, default ALL_EQUAL + Whether or not NaNs should be considered equal. + + Returns + ------- + Aggregation + The collect_set aggregation. + """ + return Aggregation.from_libcudf( + move( + make_collect_set_aggregation[aggregation]( + null_handling, nulls_equal, nans_equal + ) + ) + ) + +cpdef Aggregation udf(str operation, DataType output_type): + """Create a udf aggregation. + + Parameters + ---------- + operation : str + The operation to perform as a string of PTX code. + output_type : DataType + The output type of the aggregation. + + Returns + ------- + Aggregation + The udf aggregation. + """ + return Aggregation.from_libcudf( + move( + make_udf_aggregation[aggregation]( + UdfType.PTX, + operation.encode("utf-8"), + output_type.c_obj, + ) + ) + ) + + +cpdef Aggregation correlation(correlation_type type, size_type min_periods): + """Create a correlation aggregation. + + Parameters + ---------- + type : correlation_type + The type of correlation to compute. + min_periods : int + The minimum number of observations to consider for computing the + correlation. + + Returns + ------- + Aggregation + The correlation aggregation. + """ + return Aggregation.from_libcudf( + move(make_correlation_aggregation[aggregation](type, min_periods)) + ) + + +cpdef Aggregation covariance(size_type min_periods, size_type ddof): + """Create a covariance aggregation. + + Parameters + ---------- + min_periods : int + The minimum number of observations to consider for computing the + covariance. + ddof : int + Delta degrees of freedom. + + Returns + ------- + Aggregation + The covariance aggregation. + """ + return Aggregation.from_libcudf( + move(make_covariance_aggregation[aggregation](min_periods, ddof)) + ) + + +cpdef Aggregation rank( + rank_method method, + order column_order = order.ASCENDING, + null_policy null_handling = null_policy.EXCLUDE, + null_order null_precedence = null_order.AFTER, + rank_percentage percentage = rank_percentage.NONE, +): + """Create a rank aggregation. + + Parameters + ---------- + method : rank_method + The method to use for ranking. + column_order : order, default ASCENDING + The order in which to sort the column. + null_handling : null_policy, default EXCLUDE + Whether or not nulls should be included. + null_precedence : null_order, default AFTER + Whether nulls should come before or after non-nulls. + percentage : rank_percentage, default NONE + Whether or not ranks should be converted to percentages, and if so, + the type of normalization to use. + + Returns + ------- + Aggregation + The rank aggregation. + """ + return Aggregation.from_libcudf( + move( + make_rank_aggregation[aggregation]( + method, + column_order, + null_handling, + null_precedence, + percentage, + ) + ) + ) diff --git a/python/cudf/cudf/_lib/pylibcudf/copying.pyx b/python/cudf/cudf/_lib/pylibcudf/copying.pyx index 65f8c7a1854..12e592f3a92 100644 --- a/python/cudf/cudf/_lib/pylibcudf/copying.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/copying.pyx @@ -26,23 +26,9 @@ from cudf._lib.cpp.copying import \ out_of_bounds_policy as OutOfBoundsPolicy # no-cython-lint from .column cimport Column +from .scalar cimport Scalar from .table cimport Table - -# This is a workaround for -# https://github.com/cython/cython/issues/4180 -# when creating reference_wrapper[constscalar] in the constructor -ctypedef const scalar constscalar - - -cdef vector[reference_wrapper[const scalar]] _as_vector(list source): - """Make a vector of reference_wrapper[const scalar] from a list of scalars.""" - cdef vector[reference_wrapper[const scalar]] c_scalars - c_scalars.reserve(len(source)) - cdef Scalar slr - for slr in source: - c_scalars.push_back( - reference_wrapper[constscalar](dereference((slr).c_obj))) - return c_scalars +from .utils cimport _as_vector cpdef Table gather( diff --git a/python/cudf/cudf/_lib/pylibcudf/groupby.pxd b/python/cudf/cudf/_lib/pylibcudf/groupby.pxd new file mode 100644 index 00000000000..ce472e3c990 --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/groupby.pxd @@ -0,0 +1,47 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from libcpp.memory cimport unique_ptr +from libcpp.pair cimport pair +from libcpp.vector cimport vector + +from cudf._lib.cpp.aggregation cimport ( + aggregation, + groupby_aggregation, + groupby_scan_aggregation, +) +from cudf._lib.cpp.groupby cimport ( + aggregation_request, + aggregation_result, + groupby, + scan_request, +) +from cudf._lib.cpp.table.table cimport table + +from .column cimport Column +from .table cimport Table + + +cdef class GroupByRequest: + # The groupby APIs accept vectors of unique_ptrs to aggregation requests. + # This ownership model means that if GroupByRequest owned the + # corresponding C++ object, that object would have to be copied by e.g. + # each groupby.aggregate call to avoid invalidating this object. Therefore, + # this class instead stores only Python/Cython objects and constructs the + # C++ object on the fly as requested. + cdef Column _values + cdef list _aggregations + + cdef aggregation_request _to_libcudf_agg_request(self) except * + cdef scan_request _to_libcudf_scan_request(self) except * + + +cdef class GroupBy: + cdef unique_ptr[groupby] c_obj + cpdef tuple aggregate(self, list requests) + cpdef tuple scan(self, list requests) + cpdef tuple shift(self, Table values, list offset, list fill_values) + cpdef tuple replace_nulls(self, Table values, list replace_policy) + cpdef tuple get_groups(self, Table values=*) + + @staticmethod + cdef tuple _parse_outputs(pair[unique_ptr[table], vector[aggregation_result]] c_res) diff --git a/python/cudf/cudf/_lib/pylibcudf/groupby.pyx b/python/cudf/cudf/_lib/pylibcudf/groupby.pyx new file mode 100644 index 00000000000..f442aafa4bd --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/groupby.pyx @@ -0,0 +1,251 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from cython.operator cimport dereference +from libcpp.functional cimport reference_wrapper +from libcpp.memory cimport unique_ptr +from libcpp.pair cimport pair +from libcpp.utility cimport move +from libcpp.vector cimport vector + +from cudf._lib.cpp.groupby cimport ( + aggregation_request, + aggregation_result, + groupby, + groups, + scan_request, +) +from cudf._lib.cpp.scalar.scalar cimport scalar +from cudf._lib.cpp.table.table cimport table +from cudf._lib.cpp.types cimport size_type + +from .aggregation cimport Aggregation +from .column cimport Column +from .table cimport Table +from .types cimport null_policy, sorted +from .utils cimport _as_vector + + +cdef class GroupByRequest: + """A request for a groupby aggregation or scan. + + Parameters + ---------- + values : Column + The column to aggregate. + aggregations : List[Aggregation] + The list of aggregations to perform. + """ + def __init__(self, Column values, list aggregations): + self._values = values + self._aggregations = aggregations + + cdef aggregation_request _to_libcudf_agg_request(self) except *: + """Convert to a libcudf aggregation_request object. + + This method is for internal use only. It creates a new libcudf + :cpp:class:`cudf::groupby::aggregation_request` object each time it is + called. + """ + cdef aggregation_request c_obj + c_obj.values = self._values.view() + + cdef Aggregation agg + for agg in self._aggregations: + c_obj.aggregations.push_back(move(agg.clone_underlying_as_groupby())) + return move(c_obj) + + cdef scan_request _to_libcudf_scan_request(self) except *: + """Convert to a libcudf scan_request object. + + This method is for internal use only. It creates a new libcudf + :cpp:class:`cudf::groupby::scan_request` object each time it is + called. + """ + cdef scan_request c_obj + c_obj.values = self._values.view() + + cdef Aggregation agg + for agg in self._aggregations: + c_obj.aggregations.push_back(move(agg.clone_underlying_as_groupby_scan())) + return move(c_obj) + + +cdef class GroupBy: + """Group values by keys and compute various aggregate quantities. + + Parameters + ---------- + keys : Table + The columns to group by. + null_handling : null_policy, optional + Whether or not to include null rows in ``keys``. Default is null_policy.EXCLUDE. + keys_are_sorted : sorted, optional + Whether the keys are already sorted. Default is sorted.NO. + """ + def __init__( + self, + Table keys, + null_policy null_handling=null_policy.EXCLUDE, + sorted keys_are_sorted=sorted.NO + ): + self.c_obj.reset(new groupby(keys.view(), null_handling, keys_are_sorted)) + + @staticmethod + cdef tuple _parse_outputs( + pair[unique_ptr[table], vector[aggregation_result]] c_res + ): + # Convert libcudf aggregation/scan outputs into pylibcudf objects. + # This function is for internal use only. + cdef Table group_keys = Table.from_libcudf(move(c_res.first)) + + cdef int i, j + cdef list results = [] + cdef list inner_results + for i in range(c_res.second.size()): + inner_results = [] + for j in range(c_res.second[i].results.size()): + inner_results.append( + Column.from_libcudf(move(c_res.second[i].results[j])) + ) + results.append(Table(inner_results)) + return group_keys, results + + cpdef tuple aggregate(self, list requests): + """Compute aggregations on columns. + + Parameters + ---------- + requests : List[GroupByRequest] + The list of `~.cudf._lib.pylibcudf.groupby.GroupByRequest` , each + representing a set of aggregations to perform on a given column of values. + + Returns + ------- + Tuple[Table, List[Table, ...]] + A tuple whose first element is the unique keys and whose second + element is a table of aggregation results. One table is returned + for each aggregation request, with the columns corresponding to the + sequence of aggregations in the request. + """ + cdef GroupByRequest request + cdef vector[aggregation_request] c_requests + for request in requests: + c_requests.push_back(move(request._to_libcudf_agg_request())) + + cdef pair[unique_ptr[table], vector[aggregation_result]] c_res = move( + dereference(self.c_obj).aggregate(c_requests) + ) + return GroupBy._parse_outputs(move(c_res)) + + cpdef tuple scan(self, list requests): + """Compute scans on columns. + + Parameters + ---------- + requests : List[GroupByRequest] + The list of `~.cudf._lib.pylibcudf.groupby.GroupByRequest` , each + representing a set of aggregations to perform on a given column of values. + + Returns + ------- + Tuple[Table, List[Table, ...]] + A tuple whose first element is the unique keys and whose second + element is a table of aggregation results. One table is returned + for each aggregation request, with the columns corresponding to the + sequence of aggregations in the request. + """ + cdef GroupByRequest request + cdef vector[scan_request] c_requests + for request in requests: + c_requests.push_back(move(request._to_libcudf_scan_request())) + + cdef pair[unique_ptr[table], vector[aggregation_result]] c_res = move( + dereference(self.c_obj).scan(c_requests) + ) + return GroupBy._parse_outputs(move(c_res)) + + cpdef tuple shift(self, Table values, list offset, list fill_values): + """Compute shifts on columns. + + Parameters + ---------- + values : Table + The columns to shift. + offset : List[int] + The offsets to shift by. + fill_values : List[Scalar] + The values to use to fill in missing values. + + Returns + ------- + Tuple[Table, Table] + A tuple whose first element is the group's keys and whose second + element is a table of shifted values. + """ + cdef vector[reference_wrapper[const scalar]] c_fill_values = \ + _as_vector(fill_values) + + cdef vector[size_type] c_offset = offset + cdef pair[unique_ptr[table], unique_ptr[table]] c_res = move( + dereference(self.c_obj).shift(values.view(), c_offset, c_fill_values) + ) + + return ( + Table.from_libcudf(move(c_res.first)), + Table.from_libcudf(move(c_res.second)), + ) + + cpdef tuple replace_nulls(self, Table value, list replace_policies): + """Replace nulls in columns. + + Parameters + ---------- + values : Table + The columns to replace nulls in. + replace_policies : List[replace_policy] + The policies to use to replace nulls. + + Returns + ------- + Tuple[Table, Table] + A tuple whose first element is the group's keys and whose second + element is a table of values with nulls replaced. + """ + cdef pair[unique_ptr[table], unique_ptr[table]] c_res = move( + dereference(self.c_obj).replace_nulls(value.view(), replace_policies) + ) + + return ( + Table.from_libcudf(move(c_res.first)), + Table.from_libcudf(move(c_res.second)), + ) + + cpdef tuple get_groups(self, Table values=None): + """Get the grouped keys and values labels for each row. + + Parameters + ---------- + values : Table, optional + The columns to get group labels for. If not specified, the group + labels for the group keys are returned. + + Returns + ------- + Tuple[Table, Table, List[int]] + A tuple of tables containing three items: + - A table of group keys + - A table of group values + - A list of integer offsets into the tables + """ + + cdef groups c_groups + if values: + c_groups = dereference(self.c_obj).get_groups(values.view()) + else: + c_groups = dereference(self.c_obj).get_groups() + + return ( + Table.from_libcudf(move(c_groups.keys)), + Table.from_libcudf(move(c_groups.values)), + c_groups.offsets, + ) diff --git a/python/cudf/cudf/_lib/pylibcudf/types.pxd b/python/cudf/cudf/_lib/pylibcudf/types.pxd index 80baa484be7..1ad3d19f15c 100644 --- a/python/cudf/cudf/_lib/pylibcudf/types.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/types.pxd @@ -1,9 +1,19 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. from libc.stdint cimport int32_t from libcpp cimport bool as cbool -from cudf._lib.cpp.types cimport data_type, type_id +from cudf._lib.cpp.types cimport ( + data_type, + interpolation, + nan_equality, + null_equality, + null_order, + null_policy, + order, + sorted, + type_id, +) cdef class DataType: diff --git a/python/cudf/cudf/_lib/pylibcudf/types.pyx b/python/cudf/cudf/_lib/pylibcudf/types.pyx index 931ab9fde39..5b25e7674e2 100644 --- a/python/cudf/cudf/_lib/pylibcudf/types.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/types.pyx @@ -4,7 +4,14 @@ from libc.stdint cimport int32_t from cudf._lib.cpp.types cimport data_type, type_id -from cudf._lib.cpp.types import type_id as TypeId # no-cython-lint +from cudf._lib.cpp.types import type_id as TypeId # no-cython-lint, isort:skip +from cudf._lib.cpp.types import null_policy as NullPolicy # no-cython-lint, isort:skip +from cudf._lib.cpp.types import interpolation as Interpolation # no-cython-lint, isort:skip +from cudf._lib.cpp.types import nan_equality as NanEquality # no-cython-lint, isort:skip +from cudf._lib.cpp.types import null_equality as NullEquality # no-cython-lint, isort:skip +from cudf._lib.cpp.types import null_order as NullOrder # no-cython-lint, isort:skip +from cudf._lib.cpp.types import order as Order # no-cython-lint, isort:skip +from cudf._lib.cpp.types import sorted as Sorted # no-cython-lint, isort:skip cdef class DataType: diff --git a/python/cudf/cudf/_lib/pylibcudf/utils.pxd b/python/cudf/cudf/_lib/pylibcudf/utils.pxd index 18bcd9cc91a..7efeaaf7e24 100644 --- a/python/cudf/cudf/_lib/pylibcudf/utils.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/utils.pxd @@ -1,7 +1,12 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +from libcpp.functional cimport reference_wrapper +from libcpp.vector cimport vector + +from cudf._lib.cpp.scalar.scalar cimport scalar from cudf._lib.cpp.types cimport bitmask_type cdef void * int_to_void_ptr(Py_ssize_t ptr) nogil cdef bitmask_type * int_to_bitmask_ptr(Py_ssize_t ptr) nogil +cdef vector[reference_wrapper[const scalar]] _as_vector(list source) diff --git a/python/cudf/cudf/_lib/pylibcudf/utils.pyx b/python/cudf/cudf/_lib/pylibcudf/utils.pyx index ccf9ea2bd70..ea34a87a72a 100644 --- a/python/cudf/cudf/_lib/pylibcudf/utils.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/utils.pyx @@ -1,9 +1,21 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +from cython.operator import dereference from libc.stdint cimport uintptr_t +from libcpp.functional cimport reference_wrapper +from libcpp.vector cimport vector +from cudf._lib.cpp.scalar.scalar cimport scalar from cudf._lib.cpp.types cimport bitmask_type +from .scalar cimport Scalar + +# This is a workaround for +# https://github.com/cython/cython/issues/4180 +# when creating reference_wrapper[constscalar] in the constructor +ctypedef const scalar constscalar + cdef void * int_to_void_ptr(Py_ssize_t ptr) nogil: return (ptr) @@ -11,3 +23,14 @@ cdef void * int_to_void_ptr(Py_ssize_t ptr) nogil: cdef bitmask_type * int_to_bitmask_ptr(Py_ssize_t ptr) nogil: return (ptr) + + +cdef vector[reference_wrapper[const scalar]] _as_vector(list source): + """Make a vector of reference_wrapper[const scalar] from a list of scalars.""" + cdef vector[reference_wrapper[const scalar]] c_scalars + c_scalars.reserve(len(source)) + cdef Scalar slr + for slr in source: + c_scalars.push_back( + reference_wrapper[constscalar](dereference((slr).c_obj))) + return c_scalars From 6cbe7a0fa5518d63347cf9556fd00442544b60e7 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Fri, 2 Feb 2024 20:13:41 -0500 Subject: [PATCH 195/384] Remove deprecated strings functions (#14848) Removes the functions deprecated in 24.02 in #14202. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Yunsong Wang (https://github.com/PointKernel) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/14848 --- cpp/include/cudf/column/column_factories.hpp | 80 ------------------- .../cudf/strings/strings_column_view.hpp | 10 --- cpp/src/strings/strings_column_factories.cu | 70 ---------------- cpp/src/strings/strings_column_view.cpp | 7 -- 4 files changed, 167 deletions(-) diff --git a/cpp/include/cudf/column/column_factories.hpp b/cpp/include/cudf/column/column_factories.hpp index a6167d983c5..96322159f0f 100644 --- a/cpp/include/cudf/column/column_factories.hpp +++ b/cpp/include/cudf/column/column_factories.hpp @@ -410,63 +410,6 @@ std::unique_ptr make_strings_column( rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()); -/** - * @brief Construct a STRING type column given a device span of chars encoded as UTF-8, a device - * span of byte offsets identifying individual strings within the char vector, and an optional - * null bitmask. - * - * @deprecated Since 24.02 - * - * `offsets.front()` must always be zero. - * - * The total number of char bytes must not exceed the maximum size of size_type. Use the - * strings_column_view class to perform strings operations on this type of column. - * - * This function makes a deep copy of the strings, offsets, null_mask to create a new column. - * - * @param strings The device span of chars in device memory. This char vector is expected to be - * UTF-8 encoded characters. - * @param offsets The device span of byte offsets in device memory. The number of elements is - * one more than the total number of strings so the `offsets.back()` is the total number of bytes - * in the strings array. `offsets.front()` must always be 0 to point to the beginning of `strings`. - * @param null_mask Device span containing the null element indicator bitmask. Arrow format for - * nulls is used for interpreting this bitmask. - * @param null_count The number of null string entries - * @param stream CUDA stream used for device memory operations and kernel launches - * @param mr Device memory resource used for allocation of the column's `null_mask` and children - * columns' device memory - * @return Constructed strings column - */ -[[deprecated]] std::unique_ptr make_strings_column( - cudf::device_span strings, - cudf::device_span offsets, - cudf::device_span null_mask, - size_type null_count, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()); - -/** - * @brief Construct a STRING type column given offsets column, chars columns, and null mask and null - * count. - * - * The columns and mask are moved into the resulting strings column. - * - * @param num_strings The number of strings the column represents. - * @param offsets_column The column of offset values for this column. The number of elements is - * one more than the total number of strings so the `offset[last] - offset[0]` is the total number - * of bytes in the strings vector. - * @param chars_column The column of char bytes for all the strings for this column. Individual - * strings are identified by the offsets and the nullmask. - * @param null_count The number of null string entries. - * @param null_mask The bits specifying the null strings in device memory. Arrow format for - * nulls is used for interpreting this bitmask. - * @return Constructed strings column - */ -[[deprecated]] std::unique_ptr make_strings_column(size_type num_strings, - std::unique_ptr offsets_column, - std::unique_ptr chars_column, - size_type null_count, - rmm::device_buffer&& null_mask); /** * @brief Construct a STRING type column given offsets column, chars columns, and null mask and null * count. @@ -490,29 +433,6 @@ std::unique_ptr make_strings_column(size_type num_strings, size_type null_count, rmm::device_buffer&& null_mask); -/** - * @brief Construct a STRING type column given offsets, columns, and optional null count and null - * mask. - * - * @deprecated Since 24.02 - * - * @param[in] num_strings The number of strings the column represents. - * @param[in] offsets The offset values for this column. The number of elements is one more than the - * total number of strings so the `offset[last] - offset[0]` is the total number of bytes in the - * strings vector. - * @param[in] chars The char bytes for all the strings for this column. Individual strings are - * identified by the offsets and the nullmask. - * @param[in] null_mask The bits specifying the null strings in device memory. Arrow format for - * nulls is used for interpreting this bitmask. - * @param[in] null_count The number of null string entries. - * @return Constructed strings column - */ -[[deprecated]] std::unique_ptr make_strings_column(size_type num_strings, - rmm::device_uvector&& offsets, - rmm::device_uvector&& chars, - rmm::device_buffer&& null_mask, - size_type null_count); - /** * @brief Construct a LIST type column given offsets column, child column, null mask and null * count. diff --git a/cpp/include/cudf/strings/strings_column_view.hpp b/cpp/include/cudf/strings/strings_column_view.hpp index e6546777f3f..840a2dd1165 100644 --- a/cpp/include/cudf/strings/strings_column_view.hpp +++ b/cpp/include/cudf/strings/strings_column_view.hpp @@ -103,16 +103,6 @@ class strings_column_view : private column_view { */ [[nodiscard]] offset_iterator offsets_end() const; - /** - * @brief Returns the internal column of chars - * - * @throw cudf::logic_error if this is an empty column - * @param stream CUDA stream used for device memory operations and kernel launches - * @return The chars column - */ - [[deprecated]] [[nodiscard]] column_view chars( - rmm::cuda_stream_view stream = cudf::get_default_stream()) const; - /** * @brief Returns the number of bytes in the chars child column. * diff --git a/cpp/src/strings/strings_column_factories.cu b/cpp/src/strings/strings_column_factories.cu index 5ba4d8d3132..0f1b9e3baae 100644 --- a/cpp/src/strings/strings_column_factories.cu +++ b/cpp/src/strings/strings_column_factories.cu @@ -56,25 +56,6 @@ std::unique_ptr make_strings_column( return cudf::strings::detail::make_strings_column(strings.begin(), strings.end(), stream, mr); } -std::unique_ptr make_strings_column(device_span chars, - device_span offsets, - size_type null_count, - rmm::device_buffer&& null_mask, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) -{ - CUDF_FUNC_RANGE(); - - return cudf::strings::detail::make_strings_column(chars.begin(), - chars.end(), - offsets.begin(), - offsets.end(), - null_count, - std::move(null_mask), - stream, - mr); -} - std::unique_ptr make_strings_column(device_span string_views, string_view null_placeholder, rmm::cuda_stream_view stream, @@ -88,57 +69,6 @@ std::unique_ptr make_strings_column(device_span strin it_pair, it_pair + string_views.size(), stream, mr); } -// Create a strings-type column from device vector of chars and vector of offsets. -std::unique_ptr make_strings_column(cudf::device_span strings, - cudf::device_span offsets, - cudf::device_span valid_mask, - size_type null_count, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) -{ - CUDF_FUNC_RANGE(); - - // build null bitmask - rmm::device_buffer null_mask{ - valid_mask.data(), valid_mask.size() * sizeof(bitmask_type), stream, mr}; - - return cudf::strings::detail::make_strings_column(strings.begin(), - strings.end(), - offsets.begin(), - offsets.end(), - null_count, - std::move(null_mask), - stream, - mr); -} - -// -std::unique_ptr make_strings_column(size_type num_strings, - std::unique_ptr offsets_column, - std::unique_ptr chars_column, - size_type null_count, - rmm::device_buffer&& null_mask) -{ - CUDF_FUNC_RANGE(); - - if (num_strings == 0) { return make_empty_column(type_id::STRING); } - - if (null_count > 0) CUDF_EXPECTS(null_mask.size() > 0, "Column with nulls must be nullable."); - CUDF_EXPECTS(num_strings == offsets_column->size() - 1, - "Invalid offsets column size for strings column."); - CUDF_EXPECTS(offsets_column->null_count() == 0, "Offsets column should not contain nulls"); - CUDF_EXPECTS(chars_column->null_count() == 0, "Chars column should not contain nulls"); - - std::vector> children; - children.emplace_back(std::move(offsets_column)); - return std::make_unique(data_type{type_id::STRING}, - num_strings, - std::move(*(chars_column->release().data.release())), - std::move(null_mask), - null_count, - std::move(children)); -} - std::unique_ptr make_strings_column(size_type num_strings, std::unique_ptr offsets_column, rmm::device_buffer&& chars_buffer, diff --git a/cpp/src/strings/strings_column_view.cpp b/cpp/src/strings/strings_column_view.cpp index 27a8c6fb17f..6be22d8e729 100644 --- a/cpp/src/strings/strings_column_view.cpp +++ b/cpp/src/strings/strings_column_view.cpp @@ -45,13 +45,6 @@ strings_column_view::offset_iterator strings_column_view::offsets_end() const return offsets_begin() + size() + 1; } -column_view strings_column_view::chars(rmm::cuda_stream_view stream) const -{ - CUDF_EXPECTS(num_children() > 0, "strings column has no children"); - return column_view( - data_type{type_id::INT8}, chars_size(stream), chars_begin(stream), nullptr, 0, 0); -} - size_type strings_column_view::chars_size(rmm::cuda_stream_view stream) const noexcept { if (size() == 0) return 0; From 6b911db70a661440c843985b50464653bef6109c Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Fri, 2 Feb 2024 20:14:20 -0500 Subject: [PATCH 196/384] Use offsetalator in cudf::strings::url_decode (#14744) Removes hardcoded size-type for offset variables and replaces them with offsetalator iterator. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Vukasin Milovanovic (https://github.com/vuule) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/14744 --- cpp/src/strings/convert/convert_urls.cu | 39 +++++++------------------ 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/cpp/src/strings/convert/convert_urls.cu b/cpp/src/strings/convert/convert_urls.cu index a9ddcfa12a2..b96c799cf4d 100644 --- a/cpp/src/strings/convert/convert_urls.cu +++ b/cpp/src/strings/convert/convert_urls.cu @@ -16,9 +16,9 @@ #include #include -#include #include #include +#include #include #include #include @@ -34,10 +34,6 @@ #include -#include - -#include - namespace cudf { namespace strings { namespace detail { @@ -282,7 +278,7 @@ CUDF_KERNEL void url_decode_char_counter(column_device_view const in_strings, template CUDF_KERNEL void url_decode_char_replacer(column_device_view const in_strings, char* const out_chars, - size_type const* const out_offsets) + cudf::detail::input_offsetalator const out_offsets) { constexpr int halo_size = 2; __shared__ char temporary_buffer[num_warps_per_threadblock][char_block_size + halo_size * 2]; @@ -384,38 +380,25 @@ std::unique_ptr url_decode(strings_column_view const& strings, auto const num_threadblocks = std::min(65536, cudf::util::div_rounding_up_unsafe(strings_count, num_warps_per_threadblock)); - auto offset_count = strings_count + 1; auto const d_strings = column_device_view::create(strings.parent(), stream); - // build offsets column - auto offsets_column = make_numeric_column( - data_type{type_to_id()}, offset_count, mask_state::UNALLOCATED, stream, mr); - - // count number of bytes in each string after decoding and store it in offsets_column - auto offsets_view = offsets_column->view(); - auto offsets_mutable_view = offsets_column->mutable_view(); + // build offsets column by computing the output row sizes and scanning the results + auto row_sizes = rmm::device_uvector(strings_count, stream); url_decode_char_counter - <<>>( - *d_strings, offsets_mutable_view.begin()); - - // use scan to transform number of bytes into offsets - thrust::exclusive_scan(rmm::exec_policy(stream), - offsets_view.begin(), - offsets_view.end(), - offsets_mutable_view.begin()); - - // copy the total number of characters of all strings combined (last element of the offset column) - // to the host memory - auto out_chars_bytes = cudf::detail::get_value(offsets_view, offset_count - 1, stream); + <<>>(*d_strings, row_sizes.data()); + // performs scan on the sizes and builds the appropriate offsets column + auto [offsets_column, out_chars_bytes] = cudf::strings::detail::make_offsets_child_column( + row_sizes.begin(), row_sizes.end(), stream, mr); // create the chars column rmm::device_uvector chars(out_chars_bytes, stream, mr); auto d_out_chars = chars.data(); + auto const offsets = + cudf::detail::offsetalator_factory::make_input_iterator(offsets_column->view()); // decode and copy the characters from the input column to the output column url_decode_char_replacer - <<>>( - *d_strings, d_out_chars, offsets_column->view().begin()); + <<>>(*d_strings, d_out_chars, offsets); // copy null mask rmm::device_buffer null_mask = cudf::detail::copy_bitmask(strings.parent(), stream, mr); From 6cebf2294ff3cb5ed4d4712ccbb26f6a27687ad5 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 2 Feb 2024 19:46:11 -0600 Subject: [PATCH 197/384] unset `CUDF_SPILL` after a pytest (#14958) This PR updates spilling tests to limit the scope of all environment and cudf option modifications to the scope of the test to avoid interfering with other tests. This PR also temporary skips a test that is not currently safe to run in an environment where other tests may already have modified related global state (the rmm default memory resource). Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/14958 --- python/cudf/cudf/tests/test_spilling.py | 104 +++++++++++++++--------- 1 file changed, 65 insertions(+), 39 deletions(-) diff --git a/python/cudf/cudf/tests/test_spilling.py b/python/cudf/cudf/tests/test_spilling.py index 7e66a7ab4ba..f18cb32a091 100644 --- a/python/cudf/cudf/tests/test_spilling.py +++ b/python/cudf/cudf/tests/test_spilling.py @@ -1,5 +1,6 @@ # Copyright (c) 2022-2024, NVIDIA CORPORATION. +import contextlib import importlib import random import time @@ -215,45 +216,66 @@ def test_spilling_buffer(manager: SpillManager): buf.spill(target="cpu") -def test_environment_variables(monkeypatch): - def reload_options(): - # In order to enabling monkey patching of the environment variables - # mark the global manager as uninitialized. - set_global_manager(None) - cudf.core.buffer.spill_manager._global_manager_uninitialized = True - importlib.reload(cudf.options) - - monkeypatch.setenv("CUDF_SPILL_ON_DEMAND", "off") - monkeypatch.setenv("CUDF_SPILL", "off") - reload_options() - assert get_global_manager() is None - - monkeypatch.setenv("CUDF_SPILL", "on") - reload_options() - manager = get_global_manager() - assert isinstance(manager, SpillManager) - assert manager._spill_on_demand is False - assert manager._device_memory_limit is None - assert manager.statistics.level == 0 - - monkeypatch.setenv("CUDF_SPILL_DEVICE_LIMIT", "1000") - reload_options() - manager = get_global_manager() - assert isinstance(manager, SpillManager) - assert manager._device_memory_limit == 1000 - assert manager.statistics.level == 0 - - monkeypatch.setenv("CUDF_SPILL_STATS", "1") - reload_options() - manager = get_global_manager() - assert isinstance(manager, SpillManager) - assert manager.statistics.level == 1 - - monkeypatch.setenv("CUDF_SPILL_STATS", "2") - reload_options() - manager = get_global_manager() - assert isinstance(manager, SpillManager) - assert manager.statistics.level == 2 +def _reload_options(): + # In order to enabling monkey patching of the environment variables + # mark the global manager as uninitialized. + set_global_manager(None) + cudf.core.buffer.spill_manager._global_manager_uninitialized = True + importlib.reload(cudf.options) + + +@contextlib.contextmanager +def _get_manager_in_env(monkeypatch, var_vals): + with monkeypatch.context() as m: + for var, val in var_vals: + m.setenv(var, val) + _reload_options() + yield get_global_manager() + _reload_options() + + +def test_environment_variables_spill_off(monkeypatch): + with _get_manager_in_env( + monkeypatch, + [("CUDF_SPILL", "off"), ("CUDF_SPILL_ON_DEMAND", "off")], + ) as manager: + assert manager is None + + +def test_environment_variables_spill_on(monkeypatch): + with _get_manager_in_env( + monkeypatch, + [("CUDF_SPILL", "on")], + ) as manager: + assert isinstance(manager, SpillManager) + assert manager._spill_on_demand is True + assert manager._device_memory_limit is None + assert manager.statistics.level == 0 + + +def test_environment_variables_device_limit(monkeypatch): + with _get_manager_in_env( + monkeypatch, + [("CUDF_SPILL", "on"), ("CUDF_SPILL_DEVICE_LIMIT", "1000")], + ) as manager: + assert isinstance(manager, SpillManager) + assert manager._device_memory_limit == 1000 + assert manager.statistics.level == 0 + + +@pytest.mark.parametrize("level", (1, 2)) +def test_environment_variables_spill_stats(monkeypatch, level): + with _get_manager_in_env( + monkeypatch, + [ + ("CUDF_SPILL", "on"), + ("CUDF_SPILL_DEVICE_LIMIT", "1000"), + ("CUDF_SPILL_STATS", f"{level}"), + ], + ) as manager: + assert isinstance(manager, SpillManager) + assert manager._device_memory_limit == 1000 + assert manager.statistics.level == level def test_spill_device_memory(manager: SpillManager): @@ -507,6 +529,10 @@ def test_serialize_cuda_dataframe(manager: SpillManager): assert_eq(df1, df2) +@pytest.mark.skip( + reason="This test is not safe because other tests may have enabled" + "spilling and already modified rmm's global state" +) def test_get_rmm_memory_resource_stack(): mr1 = rmm.mr.get_current_device_resource() assert all( From 690f7df991e2b04096404f0ef0747af97f55284c Mon Sep 17 00:00:00 2001 From: Karthikeyan <6488848+karthikeyann@users.noreply.github.com> Date: Mon, 5 Feb 2024 17:16:06 +0530 Subject: [PATCH 198/384] [DOC] Update typo in docs example of structs_column_wrapper (#14949) Replace struct_column_wrapper with structs_column_wrapper in example given in documentation of `structs_column_wrapper` Authors: - Karthikeyan (https://github.com/karthikeyann) Approvers: - David Wendt (https://github.com/davidwendt) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/14949 --- cpp/include/cudf_test/column_wrapper.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/include/cudf_test/column_wrapper.hpp b/cpp/include/cudf_test/column_wrapper.hpp index c4fa4be0f89..e7ca8400246 100644 --- a/cpp/include/cudf_test/column_wrapper.hpp +++ b/cpp/include/cudf_test/column_wrapper.hpp @@ -1820,12 +1820,12 @@ class structs_column_wrapper : public detail::column_wrapper { * child_columns.push_back(std::move(child_int_col)); * child_columns.push_back(std::move(child_string_col)); * - * struct_column_wrapper struct_column_wrapper{ + * structs_column_wrapper structs_col{ * child_cols, * {1,0,1,0,1} // Validity. * }; * - * auto struct_col {struct_column_wrapper.release()}; + * auto struct_col {structs_col.release()}; * @endcode * * @param child_columns The vector of pre-constructed child columns @@ -1846,12 +1846,12 @@ class structs_column_wrapper : public detail::column_wrapper { * fixed_width_column_wrapper child_int_col_wrapper{ 1, 2, 3, 4, 5 }; * string_column_wrapper child_string_col_wrapper {"All", "the", "leaves", "are", "brown"}; * - * struct_column_wrapper struct_column_wrapper{ + * structs_column_wrapper structs_col{ * {child_int_col_wrapper, child_string_col_wrapper} * {1,0,1,0,1} // Validity. * }; * - * auto struct_col {struct_column_wrapper.release()}; + * auto struct_col {structs_col.release()}; * @endcode * * @param child_column_wrappers The list of child column wrappers @@ -1882,12 +1882,12 @@ class structs_column_wrapper : public detail::column_wrapper { * fixed_width_column_wrapper child_int_col_wrapper{ 1, 2, 3, 4, 5 }; * string_column_wrapper child_string_col_wrapper {"All", "the", "leaves", "are", "brown"}; * - * struct_column_wrapper struct_column_wrapper{ + * structs_column_wrapper structs_col{ * {child_int_col_wrapper, child_string_col_wrapper} * cudf::detail::make_counting_transform_iterator(0, [](auto i){ return i%2; }) // Validity. * }; * - * auto struct_col {struct_column_wrapper.release()}; + * auto struct_col {structs_col.release()}; * @endcode * * @param child_column_wrappers The list of child column wrappers From 9e9f2b957bfa7c171b0996b98e21629cc4fb5c7a Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 5 Feb 2024 06:23:36 -1000 Subject: [PATCH 199/384] Replace _is_datetime64tz/interval_dtype with isinstance (#14943) This is more explicit than the methods which may allow array objects where we don't want to Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Bradley Dice (https://github.com/bdice) - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/14943 --- python/cudf/cudf/core/column/column.py | 18 +++++++++++++----- python/cudf/cudf/core/column/datetime.py | 9 ++------- python/cudf/cudf/core/column/interval.py | 10 +--------- python/cudf/cudf/core/dtypes.py | 5 ++++- 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 9143c7f5e9e..2bb0ac7bf12 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -53,8 +53,6 @@ from cudf._typing import ColumnLike, Dtype, ScalarLike from cudf.api.types import ( _is_categorical_dtype, - _is_datetime64tz_dtype, - _is_interval_dtype, _is_non_decimal_numeric_dtype, _is_pandas_nullable_extension_dtype, infer_dtype, @@ -2263,9 +2261,17 @@ def as_column( np_type = None try: if dtype is not None: - if _is_categorical_dtype(dtype) or _is_interval_dtype(dtype): + if dtype in {"category", "interval"} or isinstance( + dtype, + ( + cudf.CategoricalDtype, + cudf.IntervalDtype, + pd.IntervalDtype, + pd.CategoricalDtype, + ), + ): raise TypeError - if _is_datetime64tz_dtype(dtype): + if isinstance(dtype, pd.DatetimeTZDtype): raise NotImplementedError( "Use `tz_localize()` to construct " "timezone aware data." @@ -2413,7 +2419,9 @@ def as_column( elif np_type == np.str_: sr = pd.Series(arbitrary, dtype="str") data = as_column(sr, nan_as_null=nan_as_null) - elif _is_interval_dtype(dtype): + elif dtype == "interval" or isinstance( + dtype, (pd.IntervalDtype, cudf.IntervalDtype) + ): sr = pd.Series(arbitrary, dtype="interval") data = as_column(sr, nan_as_null=nan_as_null, dtype=dtype) elif ( diff --git a/python/cudf/cudf/core/column/datetime.py b/python/cudf/cudf/core/column/datetime.py index 6682bbb333b..7df22c7d8ea 100644 --- a/python/cudf/cudf/core/column/datetime.py +++ b/python/cudf/cudf/core/column/datetime.py @@ -22,12 +22,7 @@ DtypeObj, ScalarLike, ) -from cudf.api.types import ( - _is_datetime64tz_dtype, - is_datetime64_dtype, - is_scalar, - is_timedelta64_dtype, -) +from cudf.api.types import is_datetime64_dtype, is_scalar, is_timedelta64_dtype from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_220 from cudf.core.buffer import Buffer, cuda_array_interface_wrapper from cudf.core.column import ColumnBase, as_column, column, string @@ -702,7 +697,7 @@ def can_cast_safely(self, to_dtype: Dtype) -> bool: return False def _with_type_metadata(self, dtype): - if _is_datetime64tz_dtype(dtype): + if isinstance(dtype, pd.DatetimeTZDtype): return DatetimeTZColumn( data=self.base_data, dtype=dtype, diff --git a/python/cudf/cudf/core/column/interval.py b/python/cudf/cudf/core/column/interval.py index f5d527ad201..5d93fa26298 100644 --- a/python/cudf/cudf/core/column/interval.py +++ b/python/cudf/cudf/core/column/interval.py @@ -5,7 +5,6 @@ import pyarrow as pa import cudf -from cudf.api.types import _is_interval_dtype from cudf.core.column import StructColumn from cudf.core.dtypes import CategoricalDtype, IntervalDtype @@ -94,20 +93,13 @@ def as_interval_column(self, dtype): new_struct = self._get_decategorized_column() return IntervalColumn.from_struct_column(new_struct) else: - # a user can directly input the string `interval` as the dtype - # when creating an interval series or interval dataframe - if _is_interval_dtype(dtype): - dtype = IntervalDtype( - self.dtype.subtype, self.dtype.closed - ) - children = self.children return IntervalColumn( size=self.size, dtype=dtype, mask=self.mask, offset=self.offset, null_count=self.null_count, - children=children, + children=self.children, ) else: raise ValueError("dtype must be IntervalDtype") diff --git a/python/cudf/cudf/core/dtypes.py b/python/cudf/cudf/core/dtypes.py index 7892f8065d0..26d2ea3e992 100644 --- a/python/cudf/cudf/core/dtypes.py +++ b/python/cudf/cudf/core/dtypes.py @@ -257,7 +257,10 @@ def to_pandas(self) -> pd.CategoricalDtype: def _init_categories(self, categories: Any): if categories is None: return categories - if len(categories) == 0 and not _is_interval_dtype(categories): + if len(categories) == 0 and not isinstance( + getattr(categories, "dtype", None), + (cudf.IntervalDtype, pd.IntervalDtype), + ): dtype = "object" # type: Any else: dtype = None From fc83eff33b0c67aab19445a42f352a38bbb41ca5 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 5 Feb 2024 10:38:47 -0800 Subject: [PATCH 200/384] Migrate unary operations to pylibcudf (#14850) This PR migrates the unary operations in cuDF Python to pylibcudf. Authors: - Vyas Ramasubramani (https://github.com/vyasr) - Bradley Dice (https://github.com/bdice) Approvers: - Ashwin Srinath (https://github.com/shwina) URL: https://github.com/rapidsai/cudf/pull/14850 --- .../user_guide/api_docs/pylibcudf/index.rst | 1 + .../user_guide/api_docs/pylibcudf/unary.rst | 6 + python/cudf/cudf/_lib/cpp/CMakeLists.txt | 2 +- python/cudf/cudf/_lib/cpp/unary.pxd | 50 +++--- python/cudf/cudf/_lib/cpp/unary.pyx | 0 .../cudf/cudf/_lib/pylibcudf/CMakeLists.txt | 2 +- python/cudf/cudf/_lib/pylibcudf/__init__.pxd | 3 +- python/cudf/cudf/_lib/pylibcudf/__init__.py | 3 +- python/cudf/cudf/_lib/pylibcudf/binaryop.pxd | 2 +- python/cudf/cudf/_lib/pylibcudf/binaryop.pyx | 12 +- python/cudf/cudf/_lib/pylibcudf/unary.pxd | 19 +++ python/cudf/cudf/_lib/pylibcudf/unary.pyx | 156 ++++++++++++++++++ python/cudf/cudf/_lib/unary.pyx | 117 +++---------- python/cudf/cudf/core/column/numerical.py | 12 +- 14 files changed, 257 insertions(+), 128 deletions(-) create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/unary.rst create mode 100644 python/cudf/cudf/_lib/cpp/unary.pyx create mode 100644 python/cudf/cudf/_lib/pylibcudf/unary.pxd create mode 100644 python/cudf/cudf/_lib/pylibcudf/unary.pyx diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst index 4735b0d9414..3bc56ddffc3 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst @@ -17,3 +17,4 @@ This page provides API documentation for pylibcudf. scalar table types + unary diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/unary.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/unary.rst new file mode 100644 index 00000000000..add4baa0a54 --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/unary.rst @@ -0,0 +1,6 @@ +===== +unary +===== + +.. automodule:: cudf._lib.pylibcudf.unary + :members: diff --git a/python/cudf/cudf/_lib/cpp/CMakeLists.txt b/python/cudf/cudf/_lib/cpp/CMakeLists.txt index 316541c9bc5..e79fef98448 100644 --- a/python/cudf/cudf/_lib/cpp/CMakeLists.txt +++ b/python/cudf/cudf/_lib/cpp/CMakeLists.txt @@ -12,7 +12,7 @@ # the License. # ============================================================================= -set(cython_sources aggregation.pyx binaryop.pyx copying.pyx types.pyx) +set(cython_sources aggregation.pyx binaryop.pyx copying.pyx types.pyx unary.pyx) set(linked_libraries cudf::cudf) diff --git a/python/cudf/cudf/_lib/cpp/unary.pxd b/python/cudf/cudf/_lib/cpp/unary.pxd index 83a5701eaf0..cc07290b6c4 100644 --- a/python/cudf/cudf/_lib/cpp/unary.pxd +++ b/python/cudf/cudf/_lib/cpp/unary.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from libc.stdint cimport int32_t from libcpp.memory cimport unique_ptr @@ -7,34 +7,32 @@ from cudf._lib.cpp.column.column cimport column from cudf._lib.cpp.column.column_view cimport column_view from cudf._lib.cpp.types cimport data_type -ctypedef int32_t underlying_type_t_unary_op - cdef extern from "cudf/unary.hpp" namespace "cudf" nogil: - ctypedef enum unary_operator: - SIN "cudf::unary_operator::SIN" - COS "cudf::unary_operator::COS" - TAN "cudf::unary_operator::TAN" - ARCSIN "cudf::unary_operator::ARCSIN" - ARCCOS "cudf::unary_operator::ARCCOS" - ARCTAN "cudf::unary_operator::ARCTAN" - SINH "cudf::unary_operator::SINH" - COSH "cudf::unary_operator::COSH" - TANH "cudf::unary_operator::TANH" - ARCSINH "cudf::unary_operator::ARCSINH" - ARCCOSH "cudf::unary_operator::ARCCOSH" - ARCTANH "cudf::unary_operator::ARCTANH" - EXP "cudf::unary_operator::EXP" - LOG "cudf::unary_operator::LOG" - SQRT "cudf::unary_operator::SQRT" - CBRT "cudf::unary_operator::CBRT" - CEIL "cudf::unary_operator::CEIL" - FLOOR "cudf::unary_operator::FLOOR" - ABS "cudf::unary_operator::ABS" - RINT "cudf::unary_operator::RINT" - BIT_INVERT "cudf::unary_operator::BIT_INVERT" - NOT "cudf::unary_operator::NOT" + cpdef enum class unary_operator(int32_t): + SIN + COS + TAN + ARCSIN + ARCCOS + ARCTAN + SINH + COSH + TANH + ARCSINH + ARCCOSH + ARCTANH + EXP + LOG + SQRT + CBRT + CEIL + FLOOR + ABS + RINT + BIT_INVERT + NOT cdef extern unique_ptr[column] unary_operation( column_view input, diff --git a/python/cudf/cudf/_lib/cpp/unary.pyx b/python/cudf/cudf/_lib/cpp/unary.pyx new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt index 0ca0c122c38..432617681db 100644 --- a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt +++ b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt @@ -13,7 +13,7 @@ # ============================================================================= set(cython_sources aggregation.pyx binaryop.pyx column.pyx copying.pyx gpumemoryview.pyx - groupby.pyx interop.pyx scalar.pyx table.pyx types.pyx utils.pyx + groupby.pyx interop.pyx scalar.pyx table.pyx types.pyx unary.pyx utils.pyx ) set(linked_libraries cudf::cudf) rapids_cython_create_modules( diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd index 14c98af3fff..5cd8f017372 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd @@ -1,7 +1,7 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. # TODO: Verify consistent usage of relative/absolute imports in pylibcudf. -from . cimport aggregation, binaryop, copying, groupby, interop +from . cimport aggregation, binaryop, copying, groupby, interop, unary from .column cimport Column from .gpumemoryview cimport gpumemoryview from .scalar cimport Scalar @@ -21,5 +21,6 @@ __all__ = [ "gpumemoryview", "groupby", "interop", + "unary", "types", ] diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.py b/python/cudf/cudf/_lib/pylibcudf/__init__.py index 07612d76540..6f1eb0b6b67 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.py +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.py @@ -1,6 +1,6 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. -from . import aggregation, binaryop, copying, groupby, interop +from . import aggregation, binaryop, copying, groupby, interop, unary from .column import Column from .gpumemoryview import gpumemoryview from .scalar import Scalar @@ -19,5 +19,6 @@ "gpumemoryview", "groupby", "interop", + "unary", "types", ] diff --git a/python/cudf/cudf/_lib/pylibcudf/binaryop.pxd b/python/cudf/cudf/_lib/pylibcudf/binaryop.pxd index 56b98333757..87f0cf0f91e 100644 --- a/python/cudf/cudf/_lib/pylibcudf/binaryop.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/binaryop.pxd @@ -10,5 +10,5 @@ cpdef Column binary_operation( object lhs, object rhs, binary_operator op, - DataType data_type + DataType output_type ) diff --git a/python/cudf/cudf/_lib/pylibcudf/binaryop.pyx b/python/cudf/cudf/_lib/pylibcudf/binaryop.pyx index af248ba2071..05671bc310e 100644 --- a/python/cudf/cudf/_lib/pylibcudf/binaryop.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/binaryop.pyx @@ -21,7 +21,7 @@ cpdef Column binary_operation( object lhs, object rhs, binary_operator op, - DataType data_type + DataType output_type ): """Perform a binary operation between a column and another column or scalar. @@ -40,8 +40,8 @@ cpdef Column binary_operation( The right hand side argument. op : BinaryOperator The operation to perform. - data_type : DataType - The output to use for the output. + output_type : DataType + The data type to use for the output. Returns ------- @@ -57,7 +57,7 @@ cpdef Column binary_operation( ( lhs).view(), ( rhs).view(), op, - data_type.c_obj + output_type.c_obj ) ) elif isinstance(lhs, Column) and isinstance(rhs, Scalar): @@ -67,7 +67,7 @@ cpdef Column binary_operation( ( lhs).view(), dereference(( rhs).c_obj), op, - data_type.c_obj + output_type.c_obj ) ) elif isinstance(lhs, Scalar) and isinstance(rhs, Column): @@ -77,7 +77,7 @@ cpdef Column binary_operation( dereference(( lhs).c_obj), ( rhs).view(), op, - data_type.c_obj + output_type.c_obj ) ) else: diff --git a/python/cudf/cudf/_lib/pylibcudf/unary.pxd b/python/cudf/cudf/_lib/pylibcudf/unary.pxd new file mode 100644 index 00000000000..b4372db4ae2 --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/unary.pxd @@ -0,0 +1,19 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from cudf._lib.cpp.unary cimport unary_operator + +from .column cimport Column +from .types cimport DataType + + +cpdef Column unary_operation(Column input, unary_operator op) + +cpdef Column is_null(Column input) + +cpdef Column is_valid(Column input) + +cpdef Column cast(Column input, DataType data_type) + +cpdef Column is_nan(Column input) + +cpdef Column is_not_nan(Column input) diff --git a/python/cudf/cudf/_lib/pylibcudf/unary.pyx b/python/cudf/cudf/_lib/pylibcudf/unary.pyx new file mode 100644 index 00000000000..437dd313e85 --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/unary.pyx @@ -0,0 +1,156 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from libcpp.memory cimport unique_ptr +from libcpp.utility cimport move + +from cudf._lib.cpp cimport unary as cpp_unary +from cudf._lib.cpp.column.column cimport column +from cudf._lib.cpp.unary cimport unary_operator + +from cudf._lib.cpp.unary import \ + unary_operator as UnaryOperator # no-cython-lint + +from .column cimport Column +from .types cimport DataType + + +cpdef Column unary_operation(Column input, unary_operator op): + """Perform a unary operation on a column. + + For details, see :cpp:func:`unary_operation`. + + Parameters + ---------- + input : Column + The column to operate on. + op : UnaryOperator + The operation to perform. + + Returns + ------- + pylibcudf.Column + The result of the unary operation + """ + cdef unique_ptr[column] result + + with nogil: + result = move(cpp_unary.unary_operation(input.view(), op)) + + return Column.from_libcudf(move(result)) + + +cpdef Column is_null(Column input): + """Check whether elements of a column are null. + + For details, see :cpp:func:`is_null`. + + Parameters + ---------- + input : Column + The column to check. + + Returns + ------- + pylibcudf.Column + A boolean column with ``True`` representing null values. + """ + cdef unique_ptr[column] result + + with nogil: + result = move(cpp_unary.is_null(input.view())) + + return Column.from_libcudf(move(result)) + + +cpdef Column is_valid(Column input): + """Check whether elements of a column are valid. + + For details, see :cpp:func:`is_valid`. + + Parameters + ---------- + input : Column + The column to check. + + Returns + ------- + pylibcudf.Column + A boolean column with ``True`` representing valid values. + """ + cdef unique_ptr[column] result + + with nogil: + result = move(cpp_unary.is_valid(input.view())) + + return Column.from_libcudf(move(result)) + + +cpdef Column cast(Column input, DataType data_type): + """Cast a column to a different data type. + + For details, see :cpp:func:`cast`. + + Parameters + ---------- + input : Column + The column to check. + data_type : DataType + The data type to cast to. + + Returns + ------- + pylibcudf.Column + A boolean column with ``True`` representing null values. + """ + cdef unique_ptr[column] result + + with nogil: + result = move(cpp_unary.cast(input.view(), data_type.c_obj)) + + return Column.from_libcudf(move(result)) + + +cpdef Column is_nan(Column input): + """Check whether elements of a column are nan. + + For details, see :cpp:func:`is_nan`. + + Parameters + ---------- + input : Column + The column to check. + + Returns + ------- + pylibcudf.Column + A boolean column with ``True`` representing nan values. + """ + cdef unique_ptr[column] result + + with nogil: + result = move(cpp_unary.is_nan(input.view())) + + return Column.from_libcudf(move(result)) + + +cpdef Column is_not_nan(Column input): + """Check whether elements of a column are not nan. + + For details, see :cpp:func:`is_not_nan`. + + Parameters + ---------- + input : Column + The column to check. + + Returns + ------- + pylibcudf.Column + A boolean column with ``True`` representing non-nan values. + """ + cdef unique_ptr[column] result + + with nogil: + result = move(cpp_unary.is_not_nan(input.view())) + + return Column.from_libcudf(move(result)) diff --git a/python/cudf/cudf/_lib/unary.pyx b/python/cudf/cudf/_lib/unary.pyx index 7ef4d00b9ff..2f58c4512d6 100644 --- a/python/cudf/cudf/_lib/unary.pyx +++ b/python/cudf/cudf/_lib/unary.pyx @@ -1,100 +1,45 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. -from enum import IntEnum - -from cudf.api.types import is_decimal_dtype -from cudf.core.buffer import acquire_spill_lock - -from libcpp.memory cimport unique_ptr -from libcpp.utility cimport move +from cudf._lib.column cimport Column +from cudf._lib.types cimport dtype_to_pylibcudf_type import numpy as np -cimport cudf._lib.cpp.unary as libcudf_unary -from cudf._lib.column cimport Column -from cudf._lib.cpp.column.column cimport column -from cudf._lib.cpp.column.column_view cimport column_view -from cudf._lib.cpp.types cimport data_type -from cudf._lib.cpp.unary cimport unary_operator, underlying_type_t_unary_op -from cudf._lib.types cimport dtype_to_data_type - - -class UnaryOp(IntEnum): - SIN = unary_operator.SIN - COS = unary_operator.COS - TAN = unary_operator.TAN - ASIN = unary_operator.ARCSIN - ACOS = unary_operator.ARCCOS - ATAN = unary_operator.ARCTAN - SINH = unary_operator.SINH - COSH = unary_operator.COSH - TANH = unary_operator.TANH - ARCSINH = unary_operator.ARCSINH - ARCCOSH = unary_operator.ARCCOSH - ARCTANH = unary_operator.ARCTANH - EXP = unary_operator.EXP - LOG = unary_operator.LOG - SQRT = unary_operator.SQRT - CBRT = unary_operator.CBRT - CEIL = unary_operator.CEIL - FLOOR = unary_operator.FLOOR - ABS = unary_operator.ABS - RINT = unary_operator.RINT - INVERT = unary_operator.BIT_INVERT - NOT = unary_operator.NOT +from cudf._lib import pylibcudf +from cudf.api.types import is_decimal_dtype +from cudf.core.buffer import acquire_spill_lock @acquire_spill_lock() def unary_operation(Column input, object op): - cdef column_view c_input = input.view() - cdef unary_operator c_op = ( - op) - cdef unique_ptr[column] c_result - - with nogil: - c_result = move( - libcudf_unary.unary_operation( - c_input, - c_op - ) - ) - - return Column.from_unique_ptr(move(c_result)) + return Column.from_pylibcudf( + pylibcudf.unary.unary_operation(input.to_pylibcudf(mode="read"), op) + ) @acquire_spill_lock() def is_null(Column input): - cdef column_view c_input = input.view() - cdef unique_ptr[column] c_result - - with nogil: - c_result = move(libcudf_unary.is_null(c_input)) - - return Column.from_unique_ptr(move(c_result)) + return Column.from_pylibcudf( + pylibcudf.unary.is_null(input.to_pylibcudf(mode="read")) + ) @acquire_spill_lock() def is_valid(Column input): - cdef column_view c_input = input.view() - cdef unique_ptr[column] c_result - - with nogil: - c_result = move(libcudf_unary.is_valid(c_input)) - - return Column.from_unique_ptr(move(c_result)) + return Column.from_pylibcudf( + pylibcudf.unary.is_valid(input.to_pylibcudf(mode="read")) + ) @acquire_spill_lock() def cast(Column input, object dtype=np.float64): - cdef column_view c_input = input.view() - cdef data_type c_dtype = dtype_to_data_type(dtype) - - cdef unique_ptr[column] c_result - - with nogil: - c_result = move(libcudf_unary.cast(c_input, c_dtype)) + result = Column.from_pylibcudf( + pylibcudf.unary.cast( + input.to_pylibcudf(mode="read"), + dtype_to_pylibcudf_type(dtype) + ) + ) - result = Column.from_unique_ptr(move(c_result)) if is_decimal_dtype(result.dtype): result.dtype.precision = dtype.precision return result @@ -102,21 +47,13 @@ def cast(Column input, object dtype=np.float64): @acquire_spill_lock() def is_nan(Column input): - cdef column_view c_input = input.view() - cdef unique_ptr[column] c_result - - with nogil: - c_result = move(libcudf_unary.is_nan(c_input)) - - return Column.from_unique_ptr(move(c_result)) + return Column.from_pylibcudf( + pylibcudf.unary.is_nan(input.to_pylibcudf(mode="read")) + ) @acquire_spill_lock() def is_non_nan(Column input): - cdef column_view c_input = input.view() - cdef unique_ptr[column] c_result - - with nogil: - c_result = move(libcudf_unary.is_not_nan(c_input)) - - return Column.from_unique_ptr(move(c_result)) + return Column.from_pylibcudf( + pylibcudf.unary.is_not_nan(input.to_pylibcudf(mode="read")) + ) diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index ae4ad9c5136..b80dd626066 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -20,6 +20,7 @@ import cudf from cudf import _lib as libcudf +from cudf._lib import pylibcudf from cudf._lib.types import size_type_dtype from cudf._typing import ( ColumnBinaryOperand, @@ -56,6 +57,13 @@ from .numerical_base import NumericalBaseColumn +_unaryop_map = { + "ASIN": "ARCSIN", + "ACOS": "ARCCOS", + "ATAN": "ARCTAN", + "INVERT": "BIT_INVERT", +} + class NumericalColumn(NumericalBaseColumn): """ @@ -214,7 +222,9 @@ def unary_operator(self, unaryop: Union[str, Callable]) -> ColumnBase: if callable(unaryop): return libcudf.transform.transform(self, unaryop) - unaryop = libcudf.unary.UnaryOp[unaryop.upper()] + unaryop = unaryop.upper() + unaryop = _unaryop_map.get(unaryop, unaryop) + unaryop = pylibcudf.unary.UnaryOperator[unaryop] return libcudf.unary.unary_operation(self, unaryop) def _binaryop(self, other: ColumnBinaryOperand, op: str) -> ColumnBase: From de1da2b25f26dc44f23d34b40099b503600cf85c Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 5 Feb 2024 10:48:21 -0800 Subject: [PATCH 201/384] Rewrite cudf internals using pylibcudf groupby (#14946) This PR builds on #14945 to use pylibcudf's groupby in cudf's internals. It should not be merged until after that PR. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Ashwin Srinath (https://github.com/shwina) URL: https://github.com/rapidsai/cudf/pull/14946 --- python/cudf/cudf/_lib/aggregation.pxd | 9 +- python/cudf/cudf/_lib/aggregation.pyx | 368 ++++-------------- python/cudf/cudf/_lib/groupby.pyx | 313 ++++----------- .../cudf/cudf/_lib/pylibcudf/aggregation.pyx | 48 +++ python/cudf/cudf/_lib/pylibcudf/groupby.pxd | 2 +- python/cudf/cudf/_lib/pylibcudf/groupby.pyx | 48 ++- 6 files changed, 221 insertions(+), 567 deletions(-) diff --git a/python/cudf/cudf/_lib/aggregation.pxd b/python/cudf/cudf/_lib/aggregation.pxd index ad2c978801f..f83f170c7c2 100644 --- a/python/cudf/cudf/_lib/aggregation.pxd +++ b/python/cudf/cudf/_lib/aggregation.pxd @@ -1,7 +1,8 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr +from cudf._lib cimport pylibcudf from cudf._lib.cpp.aggregation cimport ( groupby_aggregation, groupby_scan_aggregation, @@ -15,10 +16,7 @@ cdef class RollingAggregation: cdef unique_ptr[rolling_aggregation] c_obj cdef class GroupbyAggregation: - cdef unique_ptr[groupby_aggregation] c_obj - -cdef class GroupbyScanAggregation: - cdef unique_ptr[groupby_scan_aggregation] c_obj + cdef pylibcudf.aggregation.Aggregation c_obj cdef class ReduceAggregation: cdef unique_ptr[reduce_aggregation] c_obj @@ -28,6 +26,5 @@ cdef class ScanAggregation: cdef RollingAggregation make_rolling_aggregation(op, kwargs=*) cdef GroupbyAggregation make_groupby_aggregation(op, kwargs=*) -cdef GroupbyScanAggregation make_groupby_scan_aggregation(op, kwargs=*) cdef ReduceAggregation make_reduce_aggregation(op, kwargs=*) cdef ScanAggregation make_scan_aggregation(op, kwargs=*) diff --git a/python/cudf/cudf/_lib/aggregation.pyx b/python/cudf/cudf/_lib/aggregation.pyx index b202d08ac2e..127580a6ec6 100644 --- a/python/cudf/cudf/_lib/aggregation.pyx +++ b/python/cudf/cudf/_lib/aggregation.pyx @@ -23,13 +23,14 @@ from cudf._lib.types import Interpolation cimport cudf._lib.cpp.aggregation as libcudf_aggregation cimport cudf._lib.cpp.types as libcudf_types -from cudf._lib.cpp.aggregation cimport ( - underlying_type_t_correlation_type, - underlying_type_t_rank_method, -) +from cudf._lib.cpp.aggregation cimport underlying_type_t_correlation_type import cudf +from cudf._lib cimport pylibcudf + +from cudf._lib import pylibcudf + class AggregationKind(Enum): SUM = libcudf_aggregation.aggregation.Kind.SUM @@ -257,226 +258,120 @@ cdef class GroupbyAggregation: like `df.agg(lambda x: x.sum())`; such functions are called with this class as an argument to generation the desired aggregation. """ + def __init__(self, pylibcudf.aggregation.Aggregation agg): + self.c_obj = agg + @property def kind(self): - return AggregationKind(self.c_obj.get()[0].kind).name + return AggregationKind(int(self.c_obj.kind())).name @classmethod def sum(cls): - cdef GroupbyAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_sum_aggregation[groupby_aggregation]()) - return agg + return cls(pylibcudf.aggregation.sum()) @classmethod def min(cls): - cdef GroupbyAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_min_aggregation[groupby_aggregation]()) - return agg + return cls(pylibcudf.aggregation.min()) @classmethod def max(cls): - cdef GroupbyAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_max_aggregation[groupby_aggregation]()) - return agg + return cls(pylibcudf.aggregation.max()) @classmethod def idxmin(cls): - cdef GroupbyAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_argmin_aggregation[ - groupby_aggregation]()) - return agg + return cls(pylibcudf.aggregation.argmin()) @classmethod def idxmax(cls): - cdef GroupbyAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_argmax_aggregation[ - groupby_aggregation]()) - return agg + return cls(pylibcudf.aggregation.argmax()) @classmethod def mean(cls): - cdef GroupbyAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_mean_aggregation[groupby_aggregation]()) - return agg + return cls(pylibcudf.aggregation.mean()) @classmethod def count(cls, dropna=True): - cdef libcudf_types.null_policy c_null_handling - if dropna: - c_null_handling = libcudf_types.null_policy.EXCLUDE - else: - c_null_handling = libcudf_types.null_policy.INCLUDE - - cdef GroupbyAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_count_aggregation[groupby_aggregation]( - c_null_handling - )) - return agg + return cls(pylibcudf.aggregation.count( + pylibcudf.types.NullPolicy.EXCLUDE + if dropna else pylibcudf.types.NullPolicy.INCLUDE + )) @classmethod def size(cls): - cdef GroupbyAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_count_aggregation[groupby_aggregation]( - ( - NullHandling.INCLUDE) - )) - return agg + return cls(pylibcudf.aggregation.count(pylibcudf.types.NullPolicy.INCLUDE)) @classmethod def collect(cls): - cdef GroupbyAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_collect_list_aggregation[groupby_aggregation]( - libcudf_types.null_policy.INCLUDE - )) - return agg + return cls( + pylibcudf.aggregation.collect_list(pylibcudf.types.NullPolicy.INCLUDE) + ) @classmethod def nunique(cls): - cdef GroupbyAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_nunique_aggregation[groupby_aggregation]( - libcudf_types.null_policy.EXCLUDE - )) - return agg + return cls(pylibcudf.aggregation.nunique(pylibcudf.types.NullPolicy.EXCLUDE)) @classmethod def nth(cls, libcudf_types.size_type size): - cdef GroupbyAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_nth_element_aggregation[groupby_aggregation](size)) - return agg + return cls(pylibcudf.aggregation.nth_element(size)) @classmethod def product(cls): - cdef GroupbyAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_product_aggregation[groupby_aggregation]()) - return agg + return cls(pylibcudf.aggregation.product()) prod = product @classmethod def sum_of_squares(cls): - cdef GroupbyAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_sum_of_squares_aggregation[groupby_aggregation]() - ) - return agg + return cls(pylibcudf.aggregation.sum_of_squares()) @classmethod def var(cls, ddof=1): - cdef GroupbyAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_variance_aggregation[groupby_aggregation](ddof)) - return agg + return cls(pylibcudf.aggregation.variance(ddof)) @classmethod def std(cls, ddof=1): - cdef GroupbyAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_std_aggregation[groupby_aggregation](ddof)) - return agg + return cls(pylibcudf.aggregation.std(ddof)) @classmethod def median(cls): - cdef GroupbyAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_median_aggregation[groupby_aggregation]()) - return agg + return cls(pylibcudf.aggregation.median()) @classmethod def quantile(cls, q=0.5, interpolation="linear"): - cdef GroupbyAggregation agg = cls() - if not pd.api.types.is_list_like(q): q = [q] - cdef vector[double] c_q = q - cdef libcudf_types.interpolation c_interp = ( - ( - ( - Interpolation[interpolation.upper()] - ) - ) - ) - agg.c_obj = move( - libcudf_aggregation.make_quantile_aggregation[groupby_aggregation]( - c_q, c_interp) - ) - return agg + return cls(pylibcudf.aggregation.quantile( + q, pylibcudf.types.Interpolation[interpolation.upper()] + )) @classmethod def unique(cls): - cdef GroupbyAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_collect_set_aggregation[groupby_aggregation]( - libcudf_types.null_policy.INCLUDE, - libcudf_types.null_equality.EQUAL, - libcudf_types.nan_equality.ALL_EQUAL, - )) - return agg + return cls(pylibcudf.aggregation.collect_set( + pylibcudf.types.NullPolicy.INCLUDE, + pylibcudf.types.NullEquality.EQUAL, + pylibcudf.types.NanEquality.ALL_EQUAL, + + )) @classmethod def first(cls): - cdef GroupbyAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_nth_element_aggregation[groupby_aggregation]( - 0, - ( - NullHandling.EXCLUDE - ) - ) + return cls( + pylibcudf.aggregation.nth_element(0, pylibcudf.types.NullPolicy.EXCLUDE) ) - return agg @classmethod def last(cls): - cdef GroupbyAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_nth_element_aggregation[groupby_aggregation]( - -1, - ( - NullHandling.EXCLUDE - ) - ) + return cls( + pylibcudf.aggregation.nth_element(-1, pylibcudf.types.NullPolicy.EXCLUDE) ) - return agg @classmethod def corr(cls, method, libcudf_types.size_type min_periods): - cdef GroupbyAggregation agg = cls() - cdef libcudf_aggregation.correlation_type c_method = ( - ( - ( - CorrelationType[method.upper()] - ) - ) - ) - agg.c_obj = move( - libcudf_aggregation. - make_correlation_aggregation[groupby_aggregation]( - c_method, min_periods - )) - return agg + return cls(pylibcudf.aggregation.correlation( + pylibcudf.aggregation.CorrelationType[method.upper()], + min_periods + + )) @classmethod def cov( @@ -484,125 +379,36 @@ cdef class GroupbyAggregation: libcudf_types.size_type min_periods, libcudf_types.size_type ddof=1 ): - cdef GroupbyAggregation agg = cls() - - agg.c_obj = move( - libcudf_aggregation. - make_covariance_aggregation[groupby_aggregation]( - min_periods, ddof - )) - return agg - - -cdef class GroupbyScanAggregation: - """A Cython wrapper for groupby scan aggregations. - - **This class should never be instantiated using a standard constructor, - only using one of its many factories.** These factories handle mapping - different cudf operations to their libcudf analogs, e.g. - `cudf.DataFrame.idxmin` -> `libcudf.argmin`. Additionally, they perform - any additional configuration needed to translate Python arguments into - their corresponding C++ types (for instance, C++ enumerations used for - flag arguments). The factory approach is necessary to support operations - like `df.agg(lambda x: x.sum())`; such functions are called with this - class as an argument to generation the desired aggregation. - """ - @property - def kind(self): - return AggregationKind(self.c_obj.get()[0].kind).name - - @classmethod - def sum(cls): - cdef GroupbyScanAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_sum_aggregation[groupby_scan_aggregation]()) - return agg - - @classmethod - def min(cls): - cdef GroupbyScanAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_min_aggregation[groupby_scan_aggregation]()) - return agg - - @classmethod - def max(cls): - cdef GroupbyScanAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_max_aggregation[groupby_scan_aggregation]()) - return agg - - @classmethod - def count(cls, dropna=True): - cdef libcudf_types.null_policy c_null_handling - if dropna: - c_null_handling = libcudf_types.null_policy.EXCLUDE - else: - c_null_handling = libcudf_types.null_policy.INCLUDE - - cdef GroupbyScanAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_count_aggregation[groupby_scan_aggregation](c_null_handling)) - return agg - - @classmethod - def size(cls): - cdef GroupbyScanAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_count_aggregation[groupby_scan_aggregation]( - ( - NullHandling.INCLUDE) - )) - return agg + return cls(pylibcudf.aggregation.covariance( + min_periods, + ddof + )) + # scan aggregations @classmethod def cumcount(cls): - cdef GroupbyScanAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_count_aggregation[groupby_scan_aggregation]( - libcudf_types.null_policy.INCLUDE - )) - return agg + return cls.count(False) - # scan aggregations - # TODO: update this after adding per algorithm aggregation derived types - # https://github.com/rapidsai/cudf/issues/7106 cumsum = sum cummin = min cummax = max @classmethod def rank(cls, method, ascending, na_option, pct): - cdef GroupbyScanAggregation agg = cls() - cdef libcudf_aggregation.rank_method c_method = ( - ( - ( - RankMethod[method.upper()] - ) - ) - ) - agg.c_obj = move( - libcudf_aggregation. - make_rank_aggregation[groupby_scan_aggregation]( - c_method, - (libcudf_types.order.ASCENDING if ascending else - libcudf_types.order.DESCENDING), - (libcudf_types.null_policy.EXCLUDE if na_option == "keep" else - libcudf_types.null_policy.INCLUDE), - (libcudf_types.null_order.BEFORE - if (na_option == "top") == ascending else - libcudf_types.null_order.AFTER), - (libcudf_aggregation.rank_percentage.ZERO_NORMALIZED - if pct else - libcudf_aggregation.rank_percentage.NONE) - )) - return agg + return cls(pylibcudf.aggregation.rank( + pylibcudf.aggregation.RankMethod[method.upper()], + (pylibcudf.types.Order.ASCENDING if ascending else + pylibcudf.types.Order.DESCENDING), + (pylibcudf.types.NullPolicy.EXCLUDE if na_option == "keep" else + pylibcudf.types.NullPolicy.INCLUDE), + (pylibcudf.types.NullOrder.BEFORE + if (na_option == "top") == ascending else + pylibcudf.types.NullOrder.AFTER), + (pylibcudf.aggregation.RankPercentage.ZERO_NORMALIZED + if pct else + pylibcudf.aggregation.RankPercentage.NONE) + + )) cdef class ReduceAggregation: @@ -878,44 +684,6 @@ cdef GroupbyAggregation make_groupby_aggregation(op, kwargs=None): raise TypeError(f"Unknown aggregation {op}") return agg -cdef GroupbyScanAggregation make_groupby_scan_aggregation(op, kwargs=None): - r""" - Parameters - ---------- - op : str or callable - If callable, must meet one of the following requirements: - - * Is of the form lambda x: x.agg(*args, **kwargs), where - `agg` is the name of a supported aggregation. Used to - to specify aggregations that take arguments, e.g., - `lambda x: x.quantile(0.5)`. - * Is a user defined aggregation function that operates on - grouped, scannable values. In this case, the output dtype must be - specified in the `kwargs` dictionary. - \*\*kwargs : dict, optional - Any keyword arguments to be passed to the op. - - Returns - ------- - GroupbyScanAggregation - """ - if kwargs is None: - kwargs = {} - - cdef GroupbyScanAggregation agg - if isinstance(op, str): - agg = getattr(GroupbyScanAggregation, op)(**kwargs) - elif callable(op): - if op is list: - agg = GroupbyScanAggregation.collect() - elif "dtype" in kwargs: - agg = GroupbyScanAggregation.from_udf(op, **kwargs) - else: - agg = op(GroupbyScanAggregation) - else: - raise TypeError(f"Unknown aggregation {op}") - return agg - cdef ReduceAggregation make_reduce_aggregation(op, kwargs=None): r""" Parameters diff --git a/python/cudf/cudf/_lib/groupby.pyx b/python/cudf/cudf/_lib/groupby.pyx index db4c5e6173a..3493d1c4f33 100644 --- a/python/cudf/cudf/_lib/groupby.pyx +++ b/python/cudf/cudf/_lib/groupby.pyx @@ -13,33 +13,16 @@ from cudf.core.dtypes import ( StructDtype, ) -from libcpp cimport bool -from libcpp.memory cimport unique_ptr -from libcpp.pair cimport pair -from libcpp.utility cimport move -from libcpp.vector cimport vector - -from cudf._lib.column cimport Column from cudf._lib.scalar cimport DeviceScalar -from cudf._lib.utils cimport columns_from_unique_ptr, table_view_from_columns +from cudf._lib.utils cimport columns_from_pylibcudf_table from cudf._lib.scalar import as_device_scalar -from libcpp.functional cimport reference_wrapper - -cimport cudf._lib.cpp.groupby as libcudf_groupby -cimport cudf._lib.cpp.types as libcudf_types -from cudf._lib.aggregation cimport ( - GroupbyAggregation, - GroupbyScanAggregation, - make_groupby_aggregation, - make_groupby_scan_aggregation, -) -from cudf._lib.cpp.column.column cimport column +from cudf._lib.aggregation cimport make_groupby_aggregation from cudf._lib.cpp.replace cimport replace_policy from cudf._lib.cpp.scalar.scalar cimport scalar -from cudf._lib.cpp.table.table cimport table, table_view -from cudf._lib.cpp.types cimport size_type + +from cudf._lib import pylibcudf # The sets below define the possible aggregations that can be performed on # different dtypes. These strings must be elements of the AggregationKind enum. @@ -111,67 +94,24 @@ def _(dtype: DecimalDtype): return _DECIMAL_AGGS -cdef _agg_result_from_columns( - vector[libcudf_groupby.aggregation_result]& c_result_columns, - set column_included, - int n_input_columns -): - """Construct the list of result columns from libcudf result. The result - contains the same number of lists as the number of input columns. Result - for an input column that has no applicable aggregations is an empty list. - """ - cdef: - int i - int j - int result_index = 0 - vector[unique_ptr[column]]* c_result - result_columns = [] - for i in range(n_input_columns): - if i in column_included: - c_result = &c_result_columns[result_index].results - result_columns.append([ - Column.from_unique_ptr(move(c_result[0][j])) - for j in range(c_result[0].size()) - ]) - result_index += 1 - else: - result_columns.append([]) - return result_columns - cdef class GroupBy: - cdef unique_ptr[libcudf_groupby.groupby] c_obj cdef dict __dict__ - def __cinit__(self, list keys, bool dropna=True): - cdef libcudf_types.null_policy c_null_handling - cdef table_view keys_view - - if dropna: - c_null_handling = libcudf_types.null_policy.EXCLUDE - else: - c_null_handling = libcudf_types.null_policy.INCLUDE - + def __init__(self, keys, dropna=True): with acquire_spill_lock() as spill_lock: - keys_view = table_view_from_columns(keys) - # We spill lock the columns while this GroupBy instance is alive. - self._spill_lock = spill_lock - - with nogil: - self.c_obj.reset( - new libcudf_groupby.groupby( - keys_view, - c_null_handling, - ) + self._groupby = pylibcudf.groupby.GroupBy( + pylibcudf.table.Table([c.to_pylibcudf(mode="read") for c in keys]), + pylibcudf.types.NullPolicy.EXCLUDE if dropna + else pylibcudf.types.NullPolicy.INCLUDE ) - def __init__(self, list keys, bool dropna=True): - self.keys = keys - self.dropna = dropna + # We spill lock the columns while this GroupBy instance is alive. + self._spill_lock = spill_lock def groups(self, list values): """ - Perform a sort groupby, using ``self.keys`` as the key columns - and ``values`` as the value columns. + Perform a sort groupby, using the keys used to construct the Groupby as the key + columns and ``values`` as the value columns. Parameters ---------- @@ -188,145 +128,17 @@ cdef class GroupBy: Integer offsets such that offsets[i+1] - offsets[i] represents the size of group `i`. """ - cdef table_view values_view = table_view_from_columns(values) - - with nogil: - c_groups = move(self.c_obj.get()[0].get_groups(values_view)) - - grouped_key_cols = columns_from_unique_ptr(move(c_groups.keys)) - - if values: - grouped_value_cols = columns_from_unique_ptr(move(c_groups.values)) - else: - grouped_value_cols = [] - return grouped_key_cols, grouped_value_cols, c_groups.offsets - - def aggregate_internal(self, values, aggregations): - """`values` is a list of columns and `aggregations` is a list of list - of aggregations. `aggregations[i]` is a list of aggregations for - `values[i]`. Returns a tuple containing 1) list of list of aggregation - results, 2) a list of grouped keys, and 3) a list of list of - aggregations performed. - """ - cdef vector[libcudf_groupby.aggregation_request] c_agg_requests - cdef libcudf_groupby.aggregation_request c_agg_request - cdef Column col - cdef GroupbyAggregation agg_obj - - cdef pair[ - unique_ptr[table], - vector[libcudf_groupby.aggregation_result] - ] c_result - - allow_empty = all(len(v) == 0 for v in aggregations) - - included_aggregations = [] - column_included = set() - for i, (col, aggs) in enumerate(zip(values, aggregations)): - dtype = col.dtype - - valid_aggregations = get_valid_aggregation(dtype) - included_aggregations_i = [] - - c_agg_request = move(libcudf_groupby.aggregation_request()) - for agg in aggs: - agg_obj = make_groupby_aggregation(agg) - if (valid_aggregations == "ALL" - or agg_obj.kind in valid_aggregations): - included_aggregations_i.append((agg, agg_obj.kind)) - c_agg_request.aggregations.push_back( - move(agg_obj.c_obj) - ) - included_aggregations.append(included_aggregations_i) - if not c_agg_request.aggregations.empty(): - c_agg_request.values = col.view() - c_agg_requests.push_back( - move(c_agg_request) - ) - column_included.add(i) - if c_agg_requests.empty() and not allow_empty: - raise DataError("All requested aggregations are unsupported.") - - with nogil: - c_result = move( - self.c_obj.get()[0].aggregate( - c_agg_requests - ) - ) - - grouped_keys = columns_from_unique_ptr( - move(c_result.first) - ) - - result_columns = _agg_result_from_columns( - c_result.second, column_included, len(values) - ) - - return result_columns, grouped_keys, included_aggregations - - def scan_internal(self, values, aggregations): - """`values` is a list of columns and `aggregations` is a list of list - of aggregations. `aggregations[i]` is a list of aggregations for - `values[i]`. Returns a tuple containing 1) list of list of aggregation - results, 2) a list of grouped keys, and 3) a list of list of - aggregations performed. - """ - cdef vector[libcudf_groupby.scan_request] c_agg_requests - cdef libcudf_groupby.scan_request c_agg_request - cdef Column col - cdef GroupbyScanAggregation agg_obj - - cdef pair[ - unique_ptr[table], - vector[libcudf_groupby.aggregation_result] - ] c_result - - allow_empty = all(len(v) == 0 for v in aggregations) - - included_aggregations = [] - column_included = set() - for i, (col, aggs) in enumerate(zip(values, aggregations)): - dtype = col.dtype - - valid_aggregations = get_valid_aggregation(dtype) - included_aggregations_i = [] - - c_agg_request = move(libcudf_groupby.scan_request()) - for agg in aggs: - agg_obj = make_groupby_scan_aggregation(agg) - if (valid_aggregations == "ALL" - or agg_obj.kind in valid_aggregations): - included_aggregations_i.append((agg, agg_obj.kind)) - c_agg_request.aggregations.push_back( - move(agg_obj.c_obj) - ) - included_aggregations.append(included_aggregations_i) - if not c_agg_request.aggregations.empty(): - c_agg_request.values = col.view() - c_agg_requests.push_back( - move(c_agg_request) - ) - column_included.add(i) - if c_agg_requests.empty() and not allow_empty: - raise DataError("All requested aggregations are unsupported.") - - with nogil: - c_result = move( - self.c_obj.get()[0].scan( - c_agg_requests - ) - ) - - grouped_keys = columns_from_unique_ptr( - move(c_result.first) + grouped_keys, grouped_values, offsets = self._groupby.get_groups( + pylibcudf.table.Table([c.to_pylibcudf(mode="read") for c in values]) + if values else None ) - result_columns = _agg_result_from_columns( - c_result.second, column_included, len(values) + return ( + columns_from_pylibcudf_table(grouped_keys), + columns_from_pylibcudf_table(grouped_values), + offsets, ) - return result_columns, grouped_keys, included_aggregations - def aggregate(self, values, aggregations): """ Parameters @@ -344,56 +156,61 @@ cdef class GroupBy: ------- Frame of aggregated values """ - if _is_all_scan_aggregate(aggregations): - return self.scan_internal(values, aggregations) + included_aggregations = [] + column_included = [] + requests = [] + for i, (col, aggs) in enumerate(zip(values, aggregations)): + valid_aggregations = get_valid_aggregation(col.dtype) + included_aggregations_i = [] + col_aggregations = [] + for agg in aggs: + agg_obj = make_groupby_aggregation(agg) + if valid_aggregations == "ALL" or agg_obj.kind in valid_aggregations: + included_aggregations_i.append((agg, agg_obj.kind)) + col_aggregations.append(agg_obj.c_obj) + included_aggregations.append(included_aggregations_i) + if col_aggregations: + requests.append(pylibcudf.groupby.GroupByRequest( + col.to_pylibcudf(mode="read"), col_aggregations + )) + column_included.append(i) - return self.aggregate_internal(values, aggregations) + if not requests and any(len(v) > 0 for v in aggregations): + raise DataError("All requested aggregations are unsupported.") - def shift(self, list values, int periods, list fill_values): - cdef table_view view = table_view_from_columns(values) - cdef size_type num_col = view.num_columns() - cdef vector[size_type] offsets = vector[size_type](num_col, periods) - - cdef vector[reference_wrapper[constscalar]] c_fill_values - cdef DeviceScalar d_slr - d_slrs = [] - c_fill_values.reserve(num_col) - for val, col in zip(fill_values, values): - d_slr = as_device_scalar(val, dtype=col.dtype) - d_slrs.append(d_slr) - c_fill_values.push_back( - reference_wrapper[constscalar](d_slr.get_raw_ptr()[0]) - ) + keys, results = self._groupby.scan(requests) if \ + _is_all_scan_aggregate(aggregations) else self._groupby.aggregate(requests) - cdef pair[unique_ptr[table], unique_ptr[table]] c_result + result_columns = [[] for _ in range(len(values))] + for i, result in zip(column_included, results): + result_columns[i] = columns_from_pylibcudf_table(result) - with nogil: - c_result = move( - self.c_obj.get()[0].shift(view, offsets, c_fill_values) - ) + return result_columns, columns_from_pylibcudf_table(keys), included_aggregations - grouped_keys = columns_from_unique_ptr(move(c_result.first)) - shifted = columns_from_unique_ptr(move(c_result.second)) + def shift(self, list values, int periods, list fill_values): + keys, shifts = self._groupby.shift( + pylibcudf.table.Table([c.to_pylibcudf(mode="read") for c in values]), + [periods] * len(values), + [ + ( as_device_scalar(val, dtype=col.dtype)).c_value + for val, col in zip(fill_values, values) + ], + ) - return shifted, grouped_keys + return columns_from_pylibcudf_table(shifts), columns_from_pylibcudf_table(keys) def replace_nulls(self, list values, object method): - cdef table_view val_view = table_view_from_columns(values) - cdef pair[unique_ptr[table], unique_ptr[table]] c_result - cdef replace_policy policy = ( - replace_policy.PRECEDING - if method == 'ffill' else replace_policy.FOLLOWING - ) - cdef vector[replace_policy] policies = vector[replace_policy]( - val_view.num_columns(), policy + # TODO: This is using an enum (replace_policy) that has not been exposed in + # pylibcudf yet. We'll want to fix that import once it is in pylibcudf. + _, replaced = self._groupby.replace_nulls( + pylibcudf.table.Table([c.to_pylibcudf(mode="read") for c in values]), + [ + replace_policy.PRECEDING + if method == 'ffill' else replace_policy.FOLLOWING + ] * len(values), ) - with nogil: - c_result = move( - self.c_obj.get()[0].replace_nulls(val_view, policies) - ) - - return columns_from_unique_ptr(move(c_result.second)) + return columns_from_pylibcudf_table(replaced) _GROUPBY_SCANS = {"cumcount", "cumsum", "cummin", "cummax", "rank"} diff --git a/python/cudf/cudf/_lib/pylibcudf/aggregation.pyx b/python/cudf/cudf/_lib/pylibcudf/aggregation.pyx index 0b91263d720..bde2643d5b1 100644 --- a/python/cudf/cudf/_lib/pylibcudf/aggregation.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/aggregation.pyx @@ -70,6 +70,8 @@ cdef class Aggregation: operations to perform. Using a class for aggregations provides a unified API for handling parametrizable aggregations. This class should never be instantiated directly, only via one of the factory functions. + + For details, see :cpp:class:`cudf::aggregation`. """ def __init__(self): raise ValueError( @@ -128,6 +130,8 @@ cdef class Aggregation: cpdef Aggregation sum(): """Create a sum aggregation. + For details, see :cpp:func:`make_sum_aggregation`. + Returns ------- Aggregation @@ -139,6 +143,8 @@ cpdef Aggregation sum(): cpdef Aggregation product(): """Create a product aggregation. + For details, see :cpp:func:`make_product_aggregation`. + Returns ------- Aggregation @@ -150,6 +156,8 @@ cpdef Aggregation product(): cpdef Aggregation min(): """Create a min aggregation. + For details, see :cpp:func:`make_min_aggregation`. + Returns ------- Aggregation @@ -161,6 +169,8 @@ cpdef Aggregation min(): cpdef Aggregation max(): """Create a max aggregation. + For details, see :cpp:func:`make_max_aggregation`. + Returns ------- Aggregation @@ -172,6 +182,8 @@ cpdef Aggregation max(): cpdef Aggregation count(null_policy null_handling = null_policy.EXCLUDE): """Create a count aggregation. + For details, see :cpp:func:`make_count_aggregation`. + Parameters ---------- null_handling : null_policy, default EXCLUDE @@ -190,6 +202,8 @@ cpdef Aggregation count(null_policy null_handling = null_policy.EXCLUDE): cpdef Aggregation any(): """Create an any aggregation. + For details, see :cpp:func:`make_any_aggregation`. + Returns ------- Aggregation @@ -201,6 +215,8 @@ cpdef Aggregation any(): cpdef Aggregation all(): """Create an all aggregation. + For details, see :cpp:func:`make_all_aggregation`. + Returns ------- Aggregation @@ -212,6 +228,8 @@ cpdef Aggregation all(): cpdef Aggregation sum_of_squares(): """Create a sum_of_squares aggregation. + For details, see :cpp:func:`make_sum_of_squares_aggregation`. + Returns ------- Aggregation @@ -225,6 +243,8 @@ cpdef Aggregation sum_of_squares(): cpdef Aggregation mean(): """Create a mean aggregation. + For details, see :cpp:func:`make_mean_aggregation`. + Returns ------- Aggregation @@ -236,6 +256,8 @@ cpdef Aggregation mean(): cpdef Aggregation variance(size_type ddof=1): """Create a variance aggregation. + For details, see :cpp:func:`make_variance_aggregation`. + Parameters ---------- ddof : int, default 1 @@ -252,6 +274,8 @@ cpdef Aggregation variance(size_type ddof=1): cpdef Aggregation std(size_type ddof=1): """Create a std aggregation. + For details, see :cpp:func:`make_std_aggregation`. + Parameters ---------- ddof : int, default 1 @@ -268,6 +292,8 @@ cpdef Aggregation std(size_type ddof=1): cpdef Aggregation median(): """Create a median aggregation. + For details, see :cpp:func:`make_median_aggregation`. + Returns ------- Aggregation @@ -279,6 +305,8 @@ cpdef Aggregation median(): cpdef Aggregation quantile(list quantiles, interpolation interp = interpolation.LINEAR): """Create a quantile aggregation. + For details, see :cpp:func:`make_quantile_aggregation`. + Parameters ---------- quantiles : list @@ -300,6 +328,8 @@ cpdef Aggregation quantile(list quantiles, interpolation interp = interpolation. cpdef Aggregation argmax(): """Create an argmax aggregation. + For details, see :cpp:func:`make_argmax_aggregation`. + Returns ------- Aggregation @@ -311,6 +341,8 @@ cpdef Aggregation argmax(): cpdef Aggregation argmin(): """Create an argmin aggregation. + For details, see :cpp:func:`make_argmin_aggregation`. + Returns ------- Aggregation @@ -322,6 +354,8 @@ cpdef Aggregation argmin(): cpdef Aggregation nunique(null_policy null_handling = null_policy.EXCLUDE): """Create a nunique aggregation. + For details, see :cpp:func:`make_nunique_aggregation`. + Parameters ---------- null_handling : null_policy, default EXCLUDE @@ -342,6 +376,8 @@ cpdef Aggregation nth_element( ): """Create a nth_element aggregation. + For details, see :cpp:func:`make_nth_element_aggregation`. + Parameters ---------- null_handling : null_policy, default INCLUDE @@ -360,6 +396,8 @@ cpdef Aggregation nth_element( cpdef Aggregation collect_list(null_policy null_handling = null_policy.INCLUDE): """Create a collect_list aggregation. + For details, see :cpp:func:`make_collect_list_aggregation`. + Parameters ---------- null_handling : null_policy, default INCLUDE @@ -382,6 +420,8 @@ cpdef Aggregation collect_set( ): """Create a collect_set aggregation. + For details, see :cpp:func:`make_collect_set_aggregation`. + Parameters ---------- null_handling : null_policy, default INCLUDE @@ -407,6 +447,8 @@ cpdef Aggregation collect_set( cpdef Aggregation udf(str operation, DataType output_type): """Create a udf aggregation. + For details, see :cpp:func:`make_udf_aggregation`. + Parameters ---------- operation : str @@ -433,6 +475,8 @@ cpdef Aggregation udf(str operation, DataType output_type): cpdef Aggregation correlation(correlation_type type, size_type min_periods): """Create a correlation aggregation. + For details, see :cpp:func:`make_correlation_aggregation`. + Parameters ---------- type : correlation_type @@ -454,6 +498,8 @@ cpdef Aggregation correlation(correlation_type type, size_type min_periods): cpdef Aggregation covariance(size_type min_periods, size_type ddof): """Create a covariance aggregation. + For details, see :cpp:func:`make_covariance_aggregation`. + Parameters ---------- min_periods : int @@ -481,6 +527,8 @@ cpdef Aggregation rank( ): """Create a rank aggregation. + For details, see :cpp:func:`make_rank_aggregation`. + Parameters ---------- method : rank_method diff --git a/python/cudf/cudf/_lib/pylibcudf/groupby.pxd b/python/cudf/cudf/_lib/pylibcudf/groupby.pxd index ce472e3c990..d06959b3c31 100644 --- a/python/cudf/cudf/_lib/pylibcudf/groupby.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/groupby.pxd @@ -40,7 +40,7 @@ cdef class GroupBy: cpdef tuple aggregate(self, list requests) cpdef tuple scan(self, list requests) cpdef tuple shift(self, Table values, list offset, list fill_values) - cpdef tuple replace_nulls(self, Table values, list replace_policy) + cpdef tuple replace_nulls(self, Table values, list replace_policies) cpdef tuple get_groups(self, Table values=*) @staticmethod diff --git a/python/cudf/cudf/_lib/pylibcudf/groupby.pyx b/python/cudf/cudf/_lib/pylibcudf/groupby.pyx index f442aafa4bd..d6ce9825ed3 100644 --- a/python/cudf/cudf/_lib/pylibcudf/groupby.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/groupby.pyx @@ -14,6 +14,7 @@ from cudf._lib.cpp.groupby cimport ( groups, scan_request, ) +from cudf._lib.cpp.replace cimport replace_policy from cudf._lib.cpp.scalar.scalar cimport scalar from cudf._lib.cpp.table.table cimport table from cudf._lib.cpp.types cimport size_type @@ -28,6 +29,12 @@ from .utils cimport _as_vector cdef class GroupByRequest: """A request for a groupby aggregation or scan. + This class is functionally polymorphic and can represent either an + aggregation or a scan depending on the algorithm it is used with. For + details on the libcudf types it converts to, see + :cpp:class:`cudf::groupby::aggregation_request` and + :cpp:class:`cudf::groupby::scan_request`. + Parameters ---------- values : Column @@ -73,6 +80,8 @@ cdef class GroupByRequest: cdef class GroupBy: """Group values by keys and compute various aggregate quantities. + For details, see :cpp:class:`cudf::groupby::groupby`. + Parameters ---------- keys : Table @@ -113,6 +122,8 @@ cdef class GroupBy: cpdef tuple aggregate(self, list requests): """Compute aggregations on columns. + For details, see :cpp:func:`cudf::groupby::groupby::aggregate`. + Parameters ---------- requests : List[GroupByRequest] @@ -132,14 +143,16 @@ cdef class GroupBy: for request in requests: c_requests.push_back(move(request._to_libcudf_agg_request())) - cdef pair[unique_ptr[table], vector[aggregation_result]] c_res = move( - dereference(self.c_obj).aggregate(c_requests) - ) + cdef pair[unique_ptr[table], vector[aggregation_result]] c_res + with nogil: + c_res = move(dereference(self.c_obj).aggregate(c_requests)) return GroupBy._parse_outputs(move(c_res)) cpdef tuple scan(self, list requests): """Compute scans on columns. + For details, see :cpp:func:`cudf::groupby::groupby::scan`. + Parameters ---------- requests : List[GroupByRequest] @@ -159,14 +172,16 @@ cdef class GroupBy: for request in requests: c_requests.push_back(move(request._to_libcudf_scan_request())) - cdef pair[unique_ptr[table], vector[aggregation_result]] c_res = move( - dereference(self.c_obj).scan(c_requests) - ) + cdef pair[unique_ptr[table], vector[aggregation_result]] c_res + with nogil: + c_res = move(dereference(self.c_obj).scan(c_requests)) return GroupBy._parse_outputs(move(c_res)) cpdef tuple shift(self, Table values, list offset, list fill_values): """Compute shifts on columns. + For details, see :cpp:func:`cudf::groupby::groupby::shift`. + Parameters ---------- values : Table @@ -186,9 +201,11 @@ cdef class GroupBy: _as_vector(fill_values) cdef vector[size_type] c_offset = offset - cdef pair[unique_ptr[table], unique_ptr[table]] c_res = move( - dereference(self.c_obj).shift(values.view(), c_offset, c_fill_values) - ) + cdef pair[unique_ptr[table], unique_ptr[table]] c_res + with nogil: + c_res = move( + dereference(self.c_obj).shift(values.view(), c_offset, c_fill_values) + ) return ( Table.from_libcudf(move(c_res.first)), @@ -198,6 +215,8 @@ cdef class GroupBy: cpdef tuple replace_nulls(self, Table value, list replace_policies): """Replace nulls in columns. + For details, see :cpp:func:`cudf::groupby::groupby::replace_nulls`. + Parameters ---------- values : Table @@ -211,9 +230,12 @@ cdef class GroupBy: A tuple whose first element is the group's keys and whose second element is a table of values with nulls replaced. """ - cdef pair[unique_ptr[table], unique_ptr[table]] c_res = move( - dereference(self.c_obj).replace_nulls(value.view(), replace_policies) - ) + cdef pair[unique_ptr[table], unique_ptr[table]] c_res + cdef vector[replace_policy] c_replace_policies = replace_policies + with nogil: + c_res = move( + dereference(self.c_obj).replace_nulls(value.view(), c_replace_policies) + ) return ( Table.from_libcudf(move(c_res.first)), @@ -223,6 +245,8 @@ cdef class GroupBy: cpdef tuple get_groups(self, Table values=None): """Get the grouped keys and values labels for each row. + For details, see :cpp:func:`cudf::groupby::groupby::get_groups`. + Parameters ---------- values : Table, optional From dfc7f257cce9a7d094cec12b829992e52e8c40d0 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 5 Feb 2024 11:46:49 -0800 Subject: [PATCH 202/384] Use fused types for overloaded function signatures (#14969) This change makes the pylibcudf API more convenient and a more faithful reproduction of the underlying libcudf APIs that offer overloaded signatures. In cases like binary ops where we were previously using runtime instance checks, this change also removes unnecessary runtime overhead if the calling code is Cython since in those cases the types at the call site are known at compile time. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Ashwin Srinath (https://github.com/shwina) URL: https://github.com/rapidsai/cudf/pull/14969 --- docs/cudf/source/developer_guide/pylibcudf.md | 23 + python/cudf/cudf/_lib/copying.pyx | 52 +-- python/cudf/cudf/_lib/pylibcudf/binaryop.pxd | 14 +- python/cudf/cudf/_lib/pylibcudf/binaryop.pyx | 32 +- python/cudf/cudf/_lib/pylibcudf/copying.pxd | 52 ++- python/cudf/cudf/_lib/pylibcudf/copying.pyx | 395 ++++++------------ 6 files changed, 241 insertions(+), 327 deletions(-) diff --git a/docs/cudf/source/developer_guide/pylibcudf.md b/docs/cudf/source/developer_guide/pylibcudf.md index 1b321dbb1fe..0120cbb286e 100644 --- a/docs/cudf/source/developer_guide/pylibcudf.md +++ b/docs/cudf/source/developer_guide/pylibcudf.md @@ -153,3 +153,26 @@ from cudf._lib.cpp.copying cimport out_of_bounds_policy from cudf._lib.cpp.copying import \ out_of_bounds_policy as OutOfBoundsPolicy # no-cython-lint ``` + +### Handling overloaded functions in libcudf +As a C++ library, libcudf makes extensive use of function overloading. +For example, both of the following functions exist in libcudf: +```cpp +std::unique_ptr empty_like(table_view const& input_table); +std::unique_ptr empty_like(column_view const& input); +``` + +However, Cython does not directly support overloading in this way, instead following Pythonic semantics where every function name must uniquely identify the function. +Therefore, Cython's [fused types](https://cython.readthedocs.io/en/latest/src/userguide/fusedtypes.html) should be used when implementing pylibcudf wrappers of overloaded functions like the above. +Fused types are Cython's version of generic programming and in this case amount to writing templated functions that compile into separate copies corresponding to the different C++ overloads. +For the above functions, the equivalent Cython function is +```cython +ctypedef fused ColumnOrTable: + Table + Column + +cpdef ColumnOrTable empty_like(ColumnOrTable input) +``` + +[Cython supports specializing the contents of fused-type functions based on the argument types](https://cython.readthedocs.io/en/latest/src/userguide/fusedtypes.html#type-checking-specializations), so any type-specific logic may be encoded using the appropriate conditionals. +See the pylibcudf source for examples of how to implement such functions. diff --git a/python/cudf/cudf/_lib/copying.pyx b/python/cudf/cudf/_lib/copying.pyx index 8eb0500617f..6a52af520f0 100644 --- a/python/cudf/cudf/_lib/copying.pyx +++ b/python/cudf/cudf/_lib/copying.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. import pickle @@ -184,18 +184,13 @@ def scatter(list sources, Column scatter_map, list target_columns, f"index out of bounds for column of size {n_rows}" ) - if isinstance(sources[0], Column): - tbl = pylibcudf.copying.scatter_table( - pylibcudf.Table([col.to_pylibcudf(mode="read") for col in sources]), - scatter_map.to_pylibcudf(mode="read"), - pylibcudf.Table([col.to_pylibcudf(mode="read") for col in target_columns]), - ) - else: - tbl = pylibcudf.copying.scatter_scalars( - [( as_device_scalar(slr)).c_value for slr in sources], - scatter_map.to_pylibcudf(mode="read"), - pylibcudf.Table([col.to_pylibcudf(mode="read") for col in target_columns]), - ) + tbl = pylibcudf.copying.scatter( + pylibcudf.Table([col.to_pylibcudf(mode="read") for col in sources]) + if isinstance(sources[0], Column) + else [( as_device_scalar(slr)).c_value for slr in sources], + scatter_map.to_pylibcudf(mode="read"), + pylibcudf.Table([col.to_pylibcudf(mode="read") for col in target_columns]), + ) return columns_from_pylibcudf_table(tbl) @@ -203,7 +198,7 @@ def scatter(list sources, Column scatter_map, list target_columns, @acquire_spill_lock() def column_empty_like(Column input_column): return Column.from_pylibcudf( - pylibcudf.copying.empty_column_like( + pylibcudf.copying.empty_like( input_column.to_pylibcudf(mode="read") ) ) @@ -222,7 +217,7 @@ def column_allocate_like(Column input_column, size=None): @acquire_spill_lock() def columns_empty_like(list input_columns): return columns_from_pylibcudf_table( - pylibcudf.copying.empty_table_like( + pylibcudf.copying.empty_like( pylibcudf.Table([col.to_pylibcudf(mode="read") for col in input_columns]) ) ) @@ -232,7 +227,7 @@ def columns_empty_like(list input_columns): def column_slice(Column input_column, object indices): return [ Column.from_pylibcudf(c) - for c in pylibcudf.copying.column_slice( + for c in pylibcudf.copying.slice( input_column.to_pylibcudf(mode="read"), list(indices), ) @@ -243,7 +238,7 @@ def column_slice(Column input_column, object indices): def columns_slice(list input_columns, object indices): return [ columns_from_pylibcudf_table(tbl) - for tbl in pylibcudf.copying.table_slice( + for tbl in pylibcudf.copying.slice( pylibcudf.Table([col.to_pylibcudf(mode="read") for col in input_columns]), list(indices), ) @@ -254,7 +249,7 @@ def columns_slice(list input_columns, object indices): def column_split(Column input_column, object splits): return [ Column.from_pylibcudf(c) - for c in pylibcudf.copying.column_split( + for c in pylibcudf.copying.split( input_column.to_pylibcudf(mode="read"), list(splits), ) @@ -265,7 +260,7 @@ def column_split(Column input_column, object splits): def columns_split(list input_columns, object splits): return [ columns_from_pylibcudf_table(tbl) - for tbl in pylibcudf.copying.table_split( + for tbl in pylibcudf.copying.split( pylibcudf.Table([col.to_pylibcudf(mode="read") for col in input_columns]), list(splits), ) @@ -303,18 +298,13 @@ def boolean_mask_scatter(list input_, list target_columns, if len(input_) == 0: return [] - if isinstance(input_[0], Column): - tbl = pylibcudf.copying.boolean_mask_table_scatter( - pylibcudf.Table([col.to_pylibcudf(mode="read") for col in input_]), - pylibcudf.Table([col.to_pylibcudf(mode="read") for col in target_columns]), - boolean_mask.to_pylibcudf(mode="read"), - ) - else: - tbl = pylibcudf.copying.boolean_mask_scalars_scatter( - [( as_device_scalar(i)).c_value for i in input_], - pylibcudf.Table([col.to_pylibcudf(mode="read") for col in target_columns]), - boolean_mask.to_pylibcudf(mode="read"), - ) + tbl = pylibcudf.copying.boolean_mask_scatter( + pylibcudf.Table([col.to_pylibcudf(mode="read") for col in input_]) + if isinstance(input_[0], Column) + else [( as_device_scalar(i)).c_value for i in input_], + pylibcudf.Table([col.to_pylibcudf(mode="read") for col in target_columns]), + boolean_mask.to_pylibcudf(mode="read"), + ) return columns_from_pylibcudf_table(tbl) diff --git a/python/cudf/cudf/_lib/pylibcudf/binaryop.pxd b/python/cudf/cudf/_lib/pylibcudf/binaryop.pxd index 87f0cf0f91e..0aa6aac7b39 100644 --- a/python/cudf/cudf/_lib/pylibcudf/binaryop.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/binaryop.pxd @@ -3,12 +3,22 @@ from cudf._lib.cpp.binaryop cimport binary_operator from .column cimport Column +from .scalar cimport Scalar from .types cimport DataType +# Need two separate fused types to generate the cartesian product of signatures. +ctypedef fused LeftBinaryOperand: + Column + Scalar + +ctypedef fused RightBinaryOperand: + Column + Scalar + cpdef Column binary_operation( - object lhs, - object rhs, + LeftBinaryOperand lhs, + RightBinaryOperand rhs, binary_operator op, DataType output_type ) diff --git a/python/cudf/cudf/_lib/pylibcudf/binaryop.pyx b/python/cudf/cudf/_lib/pylibcudf/binaryop.pyx index 05671bc310e..16de7757469 100644 --- a/python/cudf/cudf/_lib/pylibcudf/binaryop.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/binaryop.pyx @@ -18,25 +18,25 @@ from .types cimport DataType cpdef Column binary_operation( - object lhs, - object rhs, + LeftBinaryOperand lhs, + RightBinaryOperand rhs, binary_operator op, DataType output_type ): """Perform a binary operation between a column and another column or scalar. - Either ``lhs`` or ``rhs`` must be a - :py:class:`~cudf._lib.pylibcudf.column.Column`. The other may be a + ``lhs`` and ``rhs`` may be a :py:class:`~cudf._lib.pylibcudf.column.Column` or a - :py:class:`~cudf._lib.pylibcudf.scalar.Scalar`. + :py:class:`~cudf._lib.pylibcudf.scalar.Scalar`, but at least one must be a + :py:class:`~cudf._lib.pylibcudf.column.Column`. For details, see :cpp:func:`binary_operation`. Parameters ---------- - lhs : Column or Scalar + lhs : Union[Column, Scalar] The left hand side argument. - rhs : Column or Scalar + rhs : Union[Column, Scalar] The right hand side argument. op : BinaryOperator The operation to perform. @@ -50,32 +50,32 @@ cpdef Column binary_operation( """ cdef unique_ptr[column] result - if isinstance(lhs, Column) and isinstance(rhs, Column): + if LeftBinaryOperand is Column and RightBinaryOperand is Column: with nogil: result = move( cpp_binaryop.binary_operation( - ( lhs).view(), - ( rhs).view(), + lhs.view(), + rhs.view(), op, output_type.c_obj ) ) - elif isinstance(lhs, Column) and isinstance(rhs, Scalar): + elif LeftBinaryOperand is Column and RightBinaryOperand is Scalar: with nogil: result = move( cpp_binaryop.binary_operation( - ( lhs).view(), - dereference(( rhs).c_obj), + lhs.view(), + dereference(rhs.c_obj), op, output_type.c_obj ) ) - elif isinstance(lhs, Scalar) and isinstance(rhs, Column): + elif LeftBinaryOperand is Scalar and RightBinaryOperand is Column: with nogil: result = move( cpp_binaryop.binary_operation( - dereference(( lhs).c_obj), - ( rhs).view(), + dereference(lhs.c_obj), + rhs.view(), op, output_type.c_obj ) diff --git a/python/cudf/cudf/_lib/pylibcudf/copying.pxd b/python/cudf/cudf/_lib/pylibcudf/copying.pxd index 3567df9ac9c..7b5f1e70ea3 100644 --- a/python/cudf/cudf/_lib/pylibcudf/copying.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/copying.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. from libcpp cimport bool as cbool @@ -9,6 +9,26 @@ from .column cimport Column from .scalar cimport Scalar from .table cimport Table +ctypedef fused ColumnOrTable: + Table + Column + + +ctypedef fused TableOrListOfScalars: + Table + # The contents of the list must be validated as Scalars at runtime. + list + + +# Need two separate fused types to generate the cartesian product of signatures. +ctypedef fused LeftCopyIfElseOperand: + Column + Scalar + +ctypedef fused RightCopyIfElseOperand: + Column + Scalar + cpdef Table gather( Table source_table, @@ -16,13 +36,9 @@ cpdef Table gather( out_of_bounds_policy bounds_policy ) -cpdef Table scatter_table(Table source, Column scatter_map, Table target_table) - -cpdef Table scatter_scalars(list source, Column scatter_map, Table target_table) +cpdef Table scatter(TableOrListOfScalars source, Column scatter_map, Table target_table) -cpdef object empty_column_like(Column input) - -cpdef object empty_table_like(Table input) +cpdef ColumnOrTable empty_like(ColumnOrTable input) cpdef Column allocate_like(Column input_column, mask_allocation_policy policy, size=*) @@ -44,18 +60,20 @@ cpdef Column copy_range( cpdef Column shift(Column input, size_type offset, Scalar fill_values) -cpdef list column_split(Column input_column, list splits) - -cpdef list table_split(Table input_table, list splits) - -cpdef list column_slice(Column input_column, list indices) +cpdef list split(ColumnOrTable input, list splits) -cpdef list table_slice(Table input_table, list indices) +cpdef list slice(ColumnOrTable input, list indices) -cpdef Column copy_if_else(object lhs, object rhs, Column boolean_mask) - -cpdef Table boolean_mask_table_scatter(Table input, Table target, Column boolean_mask) +cpdef Column copy_if_else( + LeftCopyIfElseOperand lhs, + RightCopyIfElseOperand rhs, + Column boolean_mask +) -cpdef Table boolean_mask_scalars_scatter(list input, Table target, Column boolean_mask) +cpdef Table boolean_mask_scatter( + TableOrListOfScalars input, + Table target, + Column boolean_mask +) cpdef Scalar get_element(Column input_column, size_type index) diff --git a/python/cudf/cudf/_lib/pylibcudf/copying.pyx b/python/cudf/cudf/_lib/pylibcudf/copying.pyx index 12e592f3a92..d78955dc325 100644 --- a/python/cudf/cudf/_lib/pylibcudf/copying.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/copying.pyx @@ -67,49 +67,22 @@ cpdef Table gather( return Table.from_libcudf(move(c_result)) -cpdef Table scatter_table(Table source, Column scatter_map, Table target_table): - """Scatter rows from source into target_table according to scatter_map. - - For details, see :cpp:func:`scatter`. - - Parameters - ---------- - source : Table - The table object from which to pull data. - scatter_map : Column - A mapping from rows in source to rows in target_table. - target_table : Table - The table object into which to scatter data. - - Returns - ------- - pylibcudf.Table - The result of the scatter - """ - cdef unique_ptr[table] c_result - - with nogil: - c_result = move( - cpp_copying.scatter( - source.view(), - scatter_map.view(), - target_table.view(), - ) - ) - - return Table.from_libcudf(move(c_result)) - +cpdef Table scatter( + TableOrListOfScalars source, + Column scatter_map, + Table target_table +): + """Scatter from source into target_table according to scatter_map. -# TODO: Could generalize list to sequence -cpdef Table scatter_scalars(list source, Column scatter_map, Table target_table): - """Scatter scalars from source into target_table according to scatter_map. + If source is a table, it specifies rows to scatter. If source is a list, + each scalar is scattered into the corresponding column in the ``target_table``. For details, see :cpp:func:`scatter`. Parameters ---------- - source : List[Scalar] - A list of scalars to scatter into target_table. + source : Union[Table, List[Scalar]] + The table object or list of scalars from which to pull data. scatter_map : Column A mapping from rows in source to rows in target_table. target_table : Table @@ -117,73 +90,58 @@ cpdef Table scatter_scalars(list source, Column scatter_map, Table target_table) Returns ------- - pylibcudf.Table + Table The result of the scatter """ - cdef vector[reference_wrapper[const scalar]] source_scalars = \ - _as_vector(source) - cdef unique_ptr[table] c_result - with nogil: - c_result = move( - cpp_copying.scatter( - source_scalars, - scatter_map.view(), - target_table.view(), + cdef vector[reference_wrapper[const scalar]] source_scalars + if TableOrListOfScalars is Table: + with nogil: + c_result = move( + cpp_copying.scatter( + source.view(), + scatter_map.view(), + target_table.view(), + ) ) - ) - - return Table.from_libcudf(move(c_result)) - - -cpdef object empty_column_like(Column input): - """Create an empty column with the same type as input. - - For details, see :cpp:func:`empty_like`. - - Parameters - ---------- - input : Column - The column to use as a template for the output. - - Returns - ------- - pylibcudf.Column - An empty column with the same type as input. - """ - cdef unique_ptr[column] c_column_result - with nogil: - c_column_result = move( - cpp_copying.empty_like( - ( input).view(), + else: + source_scalars = _as_vector(source) + with nogil: + c_result = move( + cpp_copying.scatter( + source_scalars, + scatter_map.view(), + target_table.view(), + ) ) - ) - return Column.from_libcudf(move(c_column_result)) + return Table.from_libcudf(move(c_result)) -cpdef object empty_table_like(Table input): - """Create an empty table with the same type as input. +cpdef ColumnOrTable empty_like(ColumnOrTable input): + """Create an empty column or table with the same type as ``input``. For details, see :cpp:func:`empty_like`. Parameters ---------- - input : Table - The table to use as a template for the output. + input : Union[Column, Table] + The column or table to use as a template for the output. Returns ------- - pylibcudf.Table - An empty table with the same type as input. + Union[Column, Table] + An empty column or table with the same type(s) as ``input``. """ - cdef unique_ptr[table] c_table_result - with nogil: - c_table_result = move( - cpp_copying.empty_like( - (
input).view(), - ) - ) - return Table.from_libcudf(move(c_table_result)) + cdef unique_ptr[table] c_tbl_result + cdef unique_ptr[column] c_col_result + if ColumnOrTable is Column: + with nogil: + c_col_result = move(cpp_copying.empty_like(input.view())) + return Column.from_libcudf(move(c_col_result)) + else: + with nogil: + c_tbl_result = move(cpp_copying.empty_like(input.view())) + return Table.from_libcudf(move(c_tbl_result)) cpdef Column allocate_like( @@ -340,157 +298,100 @@ cpdef Column shift(Column input, size_type offset, Scalar fill_values): return Column.from_libcudf(move(c_result)) -cpdef list column_split(Column input_column, list splits): - """Split input_column into multiple columns. +cpdef list split(ColumnOrTable input, list splits): + """Split input into multiple. For details on the implementation, see :cpp:func:`split`. Parameters ---------- - input_column : Column + input : Union[Column, Table] The column to split. splits : List[int] The indices at which to split the column. Returns ------- - List[pylibcudf.Column] - The result of splitting input_column. + List[Union[Column, Table]] + The result of splitting input. """ - cdef vector[size_type] c_splits - cdef int split - for split in splits: - c_splits.push_back(split) - - cdef vector[column_view] c_result - with nogil: - c_result = move( - cpp_copying.split( - input_column.view(), - c_splits - ) - ) - + cdef vector[size_type] c_splits = splits + cdef vector[column_view] c_col_result + cdef vector[table_view] c_tbl_result cdef int i - return [ - Column.from_column_view(c_result[i], input_column) - for i in range(c_result.size()) - ] + if ColumnOrTable is Column: + with nogil: + c_col_result = move(cpp_copying.split(input.view(), c_splits)) -cpdef list table_split(Table input_table, list splits): - """Split input_table into multiple tables. - - For details on the implementation, see :cpp:func:`split`. - - Parameters - ---------- - input_table : Table - The table to split. - splits : List[int] - The indices at which to split the table. - - Returns - ------- - List[pylibcudf.Table] - The result of splitting input_table. - """ - cdef vector[size_type] c_splits = splits - cdef vector[table_view] c_result - with nogil: - c_result = move( - cpp_copying.split( - input_table.view(), - c_splits - ) - ) + return [ + Column.from_column_view(c_col_result[i], input) + for i in range(c_col_result.size()) + ] + else: + with nogil: + c_tbl_result = move(cpp_copying.split(input.view(), c_splits)) - cdef int i - return [ - Table.from_table_view(c_result[i], input_table) - for i in range(c_result.size()) - ] + return [ + Table.from_table_view(c_tbl_result[i], input) + for i in range(c_tbl_result.size()) + ] -cpdef list column_slice(Column input_column, list indices): - """Slice input_column according to indices. +cpdef list slice(ColumnOrTable input, list indices): + """Slice input according to indices. For details on the implementation, see :cpp:func:`slice`. Parameters ---------- - input_column : Column - The column to slice. + input_column : Union[Column, Table] + The column or table to slice. indices : List[int] - The indices to select from input_column. + The indices to select from input. Returns ------- - List[pylibcudf.Column] - The result of slicing input_column. + List[Union[Column, Table]] + The result of slicing ``input``. """ cdef vector[size_type] c_indices = indices - cdef vector[column_view] c_result - with nogil: - c_result = move( - cpp_copying.slice( - input_column.view(), - c_indices - ) - ) - + cdef vector[column_view] c_col_result + cdef vector[table_view] c_tbl_result cdef int i - return [ - Column.from_column_view(c_result[i], input_column) - for i in range(c_result.size()) - ] - - -cpdef list table_slice(Table input_table, list indices): - """Slice input_table according to indices. - - For details on the implementation, see :cpp:func:`slice`. - - Parameters - ---------- - input_table : Table - The table to slice. - indices : List[int] - The indices to select from input_table. + if ColumnOrTable is Column: + with nogil: + c_col_result = move(cpp_copying.slice(input.view(), c_indices)) - Returns - ------- - List[pylibcudf.Table] - The result of slicing input_table. - """ - cdef vector[size_type] c_indices = indices - cdef vector[table_view] c_result - with nogil: - c_result = move( - cpp_copying.slice( - input_table.view(), - c_indices - ) - ) + return [ + Column.from_column_view(c_col_result[i], input) + for i in range(c_col_result.size()) + ] + else: + with nogil: + c_tbl_result = move(cpp_copying.slice(input.view(), c_indices)) - cdef int i - return [ - Table.from_table_view(c_result[i], input_table) - for i in range(c_result.size()) - ] + return [ + Table.from_table_view(c_tbl_result[i], input) + for i in range(c_tbl_result.size()) + ] -cpdef Column copy_if_else(object lhs, object rhs, Column boolean_mask): +cpdef Column copy_if_else( + LeftCopyIfElseOperand lhs, + RightCopyIfElseOperand rhs, + Column boolean_mask +): """Copy elements from lhs or rhs into a new column according to boolean_mask. For details on the implementation, see :cpp:func:`copy_if_else`. Parameters ---------- - lhs : Column or Scalar + lhs : Union[Column, Scalar] The column or scalar to copy from if the corresponding element in boolean_mask is True. - rhs : Column or Scalar + rhs : Union[Column, Scalar] The column or scalar to copy from if the corresponding element in boolean_mask is False. boolean_mask : Column @@ -503,56 +404,51 @@ cpdef Column copy_if_else(object lhs, object rhs, Column boolean_mask): """ cdef unique_ptr[column] result - if isinstance(lhs, Column) and isinstance(rhs, Column): + if LeftCopyIfElseOperand is Column and RightCopyIfElseOperand is Column: with nogil: result = move( - cpp_copying.copy_if_else( - ( lhs).view(), - ( rhs).view(), - boolean_mask.view() - ) + cpp_copying.copy_if_else(lhs.view(), rhs.view(), boolean_mask.view()) ) - elif isinstance(lhs, Column) and isinstance(rhs, Scalar): + elif LeftCopyIfElseOperand is Column and RightCopyIfElseOperand is Scalar: with nogil: result = move( cpp_copying.copy_if_else( - ( lhs).view(), - dereference(( rhs).c_obj), - boolean_mask.view() + lhs.view(), dereference(rhs.c_obj), boolean_mask.view() ) ) - elif isinstance(lhs, Scalar) and isinstance(rhs, Column): + elif LeftCopyIfElseOperand is Scalar and RightCopyIfElseOperand is Column: with nogil: result = move( cpp_copying.copy_if_else( - dereference(( lhs).c_obj), - ( rhs).view(), - boolean_mask.view() + dereference(lhs.c_obj), rhs.view(), boolean_mask.view() ) ) - elif isinstance(lhs, Scalar) and isinstance(rhs, Scalar): + else: with nogil: result = move( cpp_copying.copy_if_else( - dereference(( lhs).c_obj), - dereference(( rhs).c_obj), - boolean_mask.view() + dereference(lhs.c_obj), dereference(rhs.c_obj), boolean_mask.view() ) ) - else: - raise ValueError(f"Invalid arguments {lhs} and {rhs}") return Column.from_libcudf(move(result)) -cpdef Table boolean_mask_table_scatter(Table input, Table target, Column boolean_mask): +cpdef Table boolean_mask_scatter( + TableOrListOfScalars input, + Table target, + Column boolean_mask +): """Scatter rows from input into target according to boolean_mask. + If source is a table, it specifies rows to scatter. If source is a list, + each scalar is scattered into the corresponding column in the ``target_table``. + For details on the implementation, see :cpp:func:`boolean_mask_scatter`. Parameters ---------- - input : Table + input : Union[Table, List[Scalar]] The table object from which to pull data. target : Table The table object into which to scatter data. @@ -561,54 +457,31 @@ cpdef Table boolean_mask_table_scatter(Table input, Table target, Column boolean Returns ------- - pylibcudf.Table + Table The result of the scatter """ cdef unique_ptr[table] result + cdef vector[reference_wrapper[const scalar]] source_scalars - with nogil: - result = move( - cpp_copying.boolean_mask_scatter( - (
input).view(), - target.view(), - boolean_mask.view() + if TableOrListOfScalars is Table: + with nogil: + result = move( + cpp_copying.boolean_mask_scatter( + input.view(), + target.view(), + boolean_mask.view() + ) ) - ) - - return Table.from_libcudf(move(result)) - - -# TODO: Could generalize list to sequence -cpdef Table boolean_mask_scalars_scatter(list input, Table target, Column boolean_mask): - """Scatter scalars from input into target according to boolean_mask. - - For details on the implementation, see :cpp:func:`boolean_mask_scatter`. - - Parameters - ---------- - input : List[Scalar] - A list of scalars to scatter into target. - target : Table - The table object into which to scatter data. - boolean_mask : Column - A mapping from rows in input to rows in target. - - Returns - ------- - pylibcudf.Table - The result of the scatter - """ - cdef vector[reference_wrapper[const scalar]] source_scalars = _as_vector(input) - - cdef unique_ptr[table] result - with nogil: - result = move( - cpp_copying.boolean_mask_scatter( - source_scalars, - target.view(), - boolean_mask.view(), + else: + source_scalars = _as_vector(input) + with nogil: + result = move( + cpp_copying.boolean_mask_scatter( + source_scalars, + target.view(), + boolean_mask.view(), + ) ) - ) return Table.from_libcudf(move(result)) From 6b989f4ade91b218b460429fc4ce7ed66c39ad13 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 5 Feb 2024 15:52:20 -0600 Subject: [PATCH 203/384] Update copyrights for 24.04. (#14964) `branch-24.04` was opened without updating the copyrights on a few files. This fixes those missing copyright updates, which keep getting updated by our pre-commit hooks for me locally. Authors: - Bradley Dice (https://github.com/bdice) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) - Ray Douglass (https://github.com/raydouglass) --- ci/check_style.sh | 2 +- cpp/examples/fetch_dependencies.cmake | 2 +- cpp/libcudf_kafka/CMakeLists.txt | 2 +- fetch_rapids.cmake | 2 +- java/src/main/native/CMakeLists.txt | 2 +- python/cudf/CMakeLists.txt | 2 +- python/cudf_kafka/CMakeLists.txt | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ci/check_style.sh b/ci/check_style.sh index da598a58880..8d882743fcc 100755 --- a/ci/check_style.sh +++ b/ci/check_style.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. set -euo pipefail diff --git a/cpp/examples/fetch_dependencies.cmake b/cpp/examples/fetch_dependencies.cmake index 34db0bcdb8c..a03f84ae142 100644 --- a/cpp/examples/fetch_dependencies.cmake +++ b/cpp/examples/fetch_dependencies.cmake @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at diff --git a/cpp/libcudf_kafka/CMakeLists.txt b/cpp/libcudf_kafka/CMakeLists.txt index 5080091664e..be2c85d6bd3 100644 --- a/cpp/libcudf_kafka/CMakeLists.txt +++ b/cpp/libcudf_kafka/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2018-2023, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at diff --git a/fetch_rapids.cmake b/fetch_rapids.cmake index 463caa5088b..6942b257c3f 100644 --- a/fetch_rapids.cmake +++ b/fetch_rapids.cmake @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2018-2023, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at diff --git a/java/src/main/native/CMakeLists.txt b/java/src/main/native/CMakeLists.txt index e42eff19895..1406cc3c3a7 100644 --- a/java/src/main/native/CMakeLists.txt +++ b/java/src/main/native/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2019-2023, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at diff --git a/python/cudf/CMakeLists.txt b/python/cudf/CMakeLists.txt index 77771afe0e6..481d6194a03 100644 --- a/python/cudf/CMakeLists.txt +++ b/python/cudf/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at diff --git a/python/cudf_kafka/CMakeLists.txt b/python/cudf_kafka/CMakeLists.txt index db18d901ba6..81be80121dd 100644 --- a/python/cudf_kafka/CMakeLists.txt +++ b/python/cudf_kafka/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at From 20ed009003944be776e28c26301354be287726f9 Mon Sep 17 00:00:00 2001 From: "Richard (Rick) Zamora" Date: Tue, 6 Feb 2024 08:15:52 -0600 Subject: [PATCH 204/384] Direct ``SeriesGroupBy.aggregate`` to ``SeriesGroupBy.agg`` (#14971) Calling `SeriesGroupBy.aggregate` is currently directed to `GroupBy.agg` instead of `SeriesGroupBy.agg`. This means that `SeriesGroupBy.aggregate` currently produces a `DataFrame` in many cases that it *should* produce a `Series`. This PR corrects the underlying problem. Authors: - Richard (Rick) Zamora (https://github.com/rjzamora) Approvers: - Bradley Dice (https://github.com/bdice) - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/14971 --- python/cudf/cudf/core/groupby/groupby.py | 2 ++ python/cudf/cudf/tests/groupby/test_agg.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 1f08abdc7fc..78593f20421 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -2640,6 +2640,8 @@ def agg(self, func): return result + aggregate = agg + def apply(self, func, *args): result = super().apply(func, *args) diff --git a/python/cudf/cudf/tests/groupby/test_agg.py b/python/cudf/cudf/tests/groupby/test_agg.py index 7919ee4a9f1..f8847f02d5a 100644 --- a/python/cudf/cudf/tests/groupby/test_agg.py +++ b/python/cudf/cudf/tests/groupby/test_agg.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. import numpy as np import pytest @@ -16,3 +16,13 @@ def test_agg_count_dtype(empty): df = df.iloc[:0] result = df.groupby("a").agg({"c": "count"}) assert result["c"].dtype == np.dtype("int64") + + +@pytest.mark.parametrize("attr", ["agg", "aggregate"]) +def test_series_agg(attr): + df = cudf.DataFrame({"a": [1, 2, 1, 2], "b": [0, 0, 0, 0]}) + pdf = df.to_pandas() + agg = getattr(df.groupby("a")["a"], attr)("count") + pd_agg = getattr(pdf.groupby(["a"])["a"], attr)("count") + + assert agg.ndim == pd_agg.ndim From 06655753dc8d6007bd8286454c9742f68a4b1a61 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 6 Feb 2024 11:31:00 -0500 Subject: [PATCH 205/384] Use offsetalator in nvtext::byte_pair_encoding (#14888) Replaces hardcoded offset types as size-type with the offsetalator or int64 (for temporary vectors). Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Karthikeyan (https://github.com/karthikeyann) URL: https://github.com/rapidsai/cudf/pull/14888 --- cpp/src/text/bpe/byte_pair_encoding.cu | 77 ++++++++++++-------------- 1 file changed, 36 insertions(+), 41 deletions(-) diff --git a/cpp/src/text/bpe/byte_pair_encoding.cu b/cpp/src/text/bpe/byte_pair_encoding.cu index c6d299424d2..62d91054c14 100644 --- a/cpp/src/text/bpe/byte_pair_encoding.cu +++ b/cpp/src/text/bpe/byte_pair_encoding.cu @@ -23,9 +23,12 @@ #include #include #include +#include #include +#include #include #include +#include #include #include @@ -76,9 +79,9 @@ constexpr int block_size = 512; template struct bpe_unpairable_offsets_fn { cudf::device_span d_chars; - cudf::size_type offset; + int64_t offset; MapRefType const d_map; - __device__ cudf::size_type operator()(cudf::size_type idx) + __device__ int64_t operator()(int64_t idx) { if (!cudf::strings::detail::is_begin_utf8_char(d_chars[idx])) { return 0; } @@ -86,7 +89,7 @@ struct bpe_unpairable_offsets_fn { auto const end = d_chars.end(); auto const lhs = cudf::string_view(itr, cudf::strings::detail::bytes_in_utf8_byte(*itr)); auto const next = itr + lhs.size_bytes(); - auto output = 0; + auto output = 0L; if (next < end) { auto const rhs = cudf::string_view(next, cudf::strings::detail::bytes_in_utf8_byte(*next)); // see if both halves exist anywhere in the table, if not these are unpairable @@ -123,6 +126,7 @@ struct bpe_unpairable_offsets_fn { */ template CUDF_KERNEL void bpe_parallel_fn(cudf::column_device_view const d_strings, + char const* d_input_chars, MapRefType const d_map, int8_t* d_spaces_data, // working memory cudf::size_type* d_ranks_data, // more working memory @@ -134,10 +138,8 @@ CUDF_KERNEL void bpe_parallel_fn(cudf::column_device_view const d_strings, static_cast(cudf::detail::grid_1d::global_thread_id() / block_size); auto const lane_idx = static_cast(threadIdx.x); - auto const d_str = d_strings.element(str_idx); - auto const offsets = - d_strings.child(cudf::strings_column_view::offsets_column_index).data(); - auto const offset = offsets[str_idx + d_strings.offset()] - offsets[d_strings.offset()]; + auto const d_str = d_strings.element(str_idx); + auto const offset = thrust::distance(d_input_chars, d_str.data()); auto const d_spaces = d_spaces_data + offset; auto const end_spaces = d_spaces + d_str.size_bytes(); @@ -292,6 +294,7 @@ CUDF_KERNEL void bpe_parallel_fn(cudf::column_device_view const d_strings, * @param d_sizes Output sizes of each row */ CUDF_KERNEL void bpe_finalize(cudf::column_device_view const d_strings, + char const* d_input_chars, int8_t* d_spaces_data, // where separators are inserted cudf::size_type* d_sizes // output sizes of encoded strings ) @@ -311,9 +314,7 @@ CUDF_KERNEL void bpe_finalize(cudf::column_device_view const d_strings, return; } - auto const offsets = - d_strings.child(cudf::strings_column_view::offsets_column_index).data(); - auto const offset = offsets[str_idx + d_strings.offset()] - offsets[d_strings.offset()]; + auto const offset = thrust::distance(d_input_chars, d_str.data()); auto const d_spaces = d_spaces_data + offset; auto const end_spaces = d_spaces + d_str.size_bytes(); @@ -352,27 +353,22 @@ std::unique_ptr byte_pair_encoding(cudf::strings_column_view const auto const d_strings = cudf::column_device_view::create(input.parent(), stream); - auto const first_offset = (input.offset() == 0) ? 0 - : cudf::detail::get_value( + auto const first_offset = (input.offset() == 0) ? 0L + : cudf::strings::detail::get_offset_value( input.offsets(), input.offset(), stream); auto const last_offset = (input.offset() == 0 && input.size() == input.offsets().size() - 1) - ? input.chars_size(stream) - : cudf::detail::get_value( + ? static_cast(input.chars_size(stream)) + : cudf::strings::detail::get_offset_value( input.offsets(), input.size() + input.offset(), stream); auto const chars_size = last_offset - first_offset; auto const d_input_chars = input.chars_begin(stream) + first_offset; - auto const offset_data_type = cudf::data_type{cudf::type_to_id()}; - auto offsets = cudf::make_numeric_column( - offset_data_type, input.size() + 1, cudf::mask_state::UNALLOCATED, stream, mr); - auto d_offsets = offsets->mutable_view().data(); - rmm::device_uvector d_spaces(chars_size, stream); // identifies non-merged pairs // used for various purposes below: unpairable-offsets, pair ranks, separator insert positions - rmm::device_uvector d_working(chars_size, stream); + rmm::device_uvector d_working(chars_size, stream); - auto const chars_begin = thrust::counting_iterator(0); - auto const chars_end = thrust::counting_iterator(chars_size); + auto const chars_begin = thrust::counting_iterator(0); + auto const chars_end = thrust::counting_iterator(chars_size); { // this kernel locates unpairable sections of strings to create artificial string row @@ -383,14 +379,16 @@ std::unique_ptr byte_pair_encoding(cudf::strings_column_view const auto up_fn = bpe_unpairable_offsets_fn{d_chars_span, first_offset, mp_map}; thrust::transform(rmm::exec_policy_nosync(stream), chars_begin, chars_end, d_up_offsets, up_fn); auto const up_end = // remove all but the unpairable offsets - thrust::remove(rmm::exec_policy_nosync(stream), d_up_offsets, d_up_offsets + chars_size, 0); + thrust::remove(rmm::exec_policy_nosync(stream), d_up_offsets, d_up_offsets + chars_size, 0L); auto const unpairables = thrust::distance(d_up_offsets, up_end); // number of unpairables // new string boundaries created by combining unpairable offsets with the existing offsets - auto tmp_offsets = rmm::device_uvector(unpairables + input.size() + 1, stream); + auto tmp_offsets = rmm::device_uvector(unpairables + input.size() + 1, stream); + auto input_offsets = + cudf::detail::offsetalator_factory::make_input_iterator(input.offsets(), input.offset()); thrust::merge(rmm::exec_policy_nosync(stream), - input.offsets_begin(), - input.offsets_end(), + input_offsets, + input_offsets + input.size() + 1, d_up_offsets, up_end, tmp_offsets.begin()); @@ -402,31 +400,28 @@ std::unique_ptr byte_pair_encoding(cudf::strings_column_view const tmp_offsets.resize(offsets_total, stream); // temp column created with the merged offsets and the original chars data - auto const col_offsets = - cudf::column_view(cudf::device_span(tmp_offsets)); - auto const tmp_size = offsets_total - 1; - auto const tmp_input = cudf::column_view( + auto const col_offsets = cudf::column_view(cudf::device_span(tmp_offsets)); + auto const tmp_size = offsets_total - 1; + auto const tmp_input = cudf::column_view( input.parent().type(), tmp_size, input.chars_begin(stream), nullptr, 0, 0, {col_offsets}); auto const d_tmp_strings = cudf::column_device_view::create(tmp_input, stream); // launch the byte-pair-encoding kernel on the temp column rmm::device_uvector d_rerank(chars_size, stream); // more working memory; - auto const d_ranks = d_working.data(); // store pair ranks here + rmm::device_uvector d_ranks(chars_size, stream); auto const pair_map = get_bpe_merge_pairs_impl(merge_pairs)->get_merge_pairs_ref(); bpe_parallel_fn<<>>( - *d_tmp_strings, pair_map, d_spaces.data(), d_ranks, d_rerank.data()); + *d_tmp_strings, d_input_chars, pair_map, d_spaces.data(), d_ranks.data(), d_rerank.data()); } - // compute the output sizes and store them in the d_offsets vector + // compute the output sizes + auto output_sizes = rmm::device_uvector(input.size(), stream); bpe_finalize<<>>( - *d_strings, d_spaces.data(), d_offsets); + *d_strings, d_input_chars, d_spaces.data(), output_sizes.data()); // convert sizes to offsets in-place - auto const bytes = - cudf::detail::sizes_to_offsets(d_offsets, d_offsets + input.size() + 1, d_offsets, stream); - CUDF_EXPECTS(bytes <= static_cast(std::numeric_limits::max()), - "Size of output exceeds the column size limit", - std::overflow_error); + auto [offsets, bytes] = cudf::strings::detail::make_offsets_child_column( + output_sizes.begin(), output_sizes.end(), stream, mr); // build the output: inserting separators to the input character data rmm::device_uvector chars(bytes, stream, mr); @@ -436,8 +431,8 @@ std::unique_ptr byte_pair_encoding(cudf::strings_column_view const auto offsets_at_non_zero = [d_spaces = d_spaces.data()] __device__(auto idx) { return d_spaces[idx] > 0; // separator to be inserted here }; - auto const copy_end = thrust::copy_if( - rmm::exec_policy_nosync(stream), chars_begin + 1, chars_end, d_inserts, offsets_at_non_zero); + auto const copy_end = + cudf::detail::copy_if_safe(chars_begin + 1, chars_end, d_inserts, offsets_at_non_zero, stream); // this will insert the single-byte separator into positions specified in d_inserts auto const sep_char = thrust::constant_iterator(separator.to_string(stream)[0]); From cf32049b36578570419f458e436aec1a51dcc640 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 6 Feb 2024 11:31:53 -0600 Subject: [PATCH 206/384] Deprecate certain frequency strings (#14967) This PR deprecates "H", "N", "T", "L", "U" and "S" as frequencies in all datetime APIs. This PR prepares `branch-24.04` for `pandas-2.2` support. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/14967 --- python/cudf/cudf/_lib/datetime.pyx | 32 ++++- python/cudf/cudf/core/_compat.py | 1 + .../indexes/datetime/test_time_specific.py | 6 +- .../cudf/tests/series/test_datetimelike.py | 6 +- python/cudf/cudf/tests/test_datasets.py | 2 +- python/cudf/cudf/tests/test_datetime.py | 120 ++++++++++-------- python/cudf/cudf/tests/test_index.py | 8 +- python/cudf/cudf/tests/test_interval.py | 7 +- python/cudf/cudf/tests/test_replace.py | 18 ++- python/cudf/cudf/tests/test_resampling.py | 49 +++---- python/cudf/cudf/tests/test_serialize.py | 4 +- python/cudf/cudf/tests/test_sorting.py | 6 + 12 files changed, 158 insertions(+), 101 deletions(-) diff --git a/python/cudf/cudf/_lib/datetime.pyx b/python/cudf/cudf/_lib/datetime.pyx index 3d96f59c4d6..c777a3ff766 100644 --- a/python/cudf/cudf/_lib/datetime.pyx +++ b/python/cudf/cudf/_lib/datetime.pyx @@ -1,4 +1,6 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. + +import warnings from cudf.core.buffer import acquire_spill_lock @@ -85,19 +87,35 @@ cdef libcudf_datetime.rounding_frequency _get_rounding_frequency(object freq): cdef libcudf_datetime.rounding_frequency freq_val # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Timedelta.resolution_string.html + old_to_new_freq_map = { + "H": "h", + "N": "ns", + "T": "min", + "L": "ms", + "U": "us", + "S": "s", + } + if freq in old_to_new_freq_map: + warnings.warn( + f"FutureWarning: {freq} is deprecated and will be " + "removed in a future version, please use " + f"{old_to_new_freq_map[freq]} instead.", + FutureWarning + ) + freq = old_to_new_freq_map.get(freq) if freq == "D": freq_val = libcudf_datetime.rounding_frequency.DAY - elif freq == "H": + elif freq == "h": freq_val = libcudf_datetime.rounding_frequency.HOUR - elif freq in ("T", "min"): + elif freq == "min": freq_val = libcudf_datetime.rounding_frequency.MINUTE - elif freq == "S": + elif freq == "s": freq_val = libcudf_datetime.rounding_frequency.SECOND - elif freq in ("L", "ms"): + elif freq == "ms": freq_val = libcudf_datetime.rounding_frequency.MILLISECOND - elif freq in ("U", "us"): + elif freq == "us": freq_val = libcudf_datetime.rounding_frequency.MICROSECOND - elif freq == "N": + elif freq == "ns": freq_val = libcudf_datetime.rounding_frequency.NANOSECOND else: raise ValueError(f"Invalid resolution: '{freq}'") diff --git a/python/cudf/cudf/core/_compat.py b/python/cudf/cudf/core/_compat.py index 5aa685560c8..3e2890e2ac4 100644 --- a/python/cudf/cudf/core/_compat.py +++ b/python/cudf/cudf/core/_compat.py @@ -11,4 +11,5 @@ PANDAS_GE_214 = PANDAS_VERSION >= version.parse("2.1.4") PANDAS_GE_220 = PANDAS_VERSION >= version.parse("2.2.0") PANDAS_LT_203 = PANDAS_VERSION < version.parse("2.0.3") +PANDAS_GE_220 = PANDAS_VERSION >= version.parse("2.2.0") PANDAS_LT_300 = PANDAS_VERSION < version.parse("3.0.0") diff --git a/python/cudf/cudf/tests/indexes/datetime/test_time_specific.py b/python/cudf/cudf/tests/indexes/datetime/test_time_specific.py index 1ed1e23f1ab..b28ef131025 100644 --- a/python/cudf/cudf/tests/indexes/datetime/test_time_specific.py +++ b/python/cudf/cudf/tests/indexes/datetime/test_time_specific.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. import pandas as pd import cudf @@ -17,7 +17,7 @@ def test_tz_localize(): def test_tz_convert(): - pidx = pd.date_range("2023-01-01", periods=3, freq="H") + pidx = pd.date_range("2023-01-01", periods=3, freq="h") idx = cudf.from_pandas(pidx) pidx = pidx.tz_localize("UTC") idx = idx.tz_localize("UTC") @@ -27,6 +27,6 @@ def test_tz_convert(): def test_delocalize_naive(): - pidx = pd.date_range("2023-01-01", periods=3, freq="H") + pidx = pd.date_range("2023-01-01", periods=3, freq="h") idx = cudf.from_pandas(pidx) assert_eq(pidx.tz_localize(None), idx.tz_localize(None)) diff --git a/python/cudf/cudf/tests/series/test_datetimelike.py b/python/cudf/cudf/tests/series/test_datetimelike.py index df68eaca399..98be7045923 100644 --- a/python/cudf/cudf/tests/series/test_datetimelike.py +++ b/python/cudf/cudf/tests/series/test_datetimelike.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. import os @@ -130,7 +130,7 @@ def test_delocalize_naive(): "to_tz", ["Europe/London", "America/Chicago", "UTC", None] ) def test_convert(from_tz, to_tz): - ps = pd.Series(pd.date_range("2023-01-01", periods=3, freq="H")) + ps = pd.Series(pd.date_range("2023-01-01", periods=3, freq="h")) gs = cudf.from_pandas(ps) ps = ps.dt.tz_localize(from_tz) gs = gs.dt.tz_localize(from_tz) @@ -140,7 +140,7 @@ def test_convert(from_tz, to_tz): def test_convert_from_naive(): - gs = cudf.Series(cudf.date_range("2023-01-01", periods=3, freq="H")) + gs = cudf.Series(cudf.date_range("2023-01-01", periods=3, freq="h")) with pytest.raises(TypeError): gs.dt.tz_convert("America/New_York") diff --git a/python/cudf/cudf/tests/test_datasets.py b/python/cudf/cudf/tests/test_datasets.py index 45629868ccc..8e5e5ab66c4 100644 --- a/python/cudf/cudf/tests/test_datasets.py +++ b/python/cudf/cudf/tests/test_datasets.py @@ -23,7 +23,7 @@ def test_dataset_timeseries(): gdf = cudf.datasets.timeseries( "2000", "2010", - freq="2H", + freq="2h", dtypes={"value": float, "name": "category", "id": int}, nulls_frequency=0.7, seed=1, diff --git a/python/cudf/cudf/tests/test_datetime.py b/python/cudf/cudf/tests/test_datetime.py index 24d8aa052e8..1f24337d28b 100644 --- a/python/cudf/cudf/tests/test_datetime.py +++ b/python/cudf/cudf/tests/test_datetime.py @@ -13,7 +13,12 @@ import cudf import cudf.testing.dataset_generator as dataset_generator from cudf import DataFrame, Series -from cudf.core._compat import PANDAS_EQ_200, PANDAS_GE_200, PANDAS_GE_210 +from cudf.core._compat import ( + PANDAS_EQ_200, + PANDAS_GE_200, + PANDAS_GE_210, + PANDAS_GE_220, +) from cudf.core.index import DatetimeIndex from cudf.testing._utils import ( DATETIME_TYPES, @@ -39,7 +44,7 @@ def data1(): def data2(): return pd.date_range( - "20010101", freq="243434324423423234N", name="times", periods=10 + "20010101", freq="243434324423423234ns", name="times", periods=10 ) @@ -1497,10 +1502,10 @@ def test_is_month_start(data, dtype): {"hours": 10, "days": 57, "nanoseconds": 3}, "83D", "17h", - "-680T", + "-680min", "110546s", - "110546789L", - "110546789248U", + "110546789ms", + "110546789248us", ] @@ -1540,7 +1545,7 @@ def test_date_range_start_end_freq(request, start, end, freq): condition=( start == "1831-05-08 15:23:21" and end == "1996-11-21 04:05:30" - and freq == "110546789L" + and freq == "110546789ms" ), reason="https://github.com/rapidsai/cudf/issues/12133", ) @@ -1653,7 +1658,8 @@ def test_date_range_end_freq_periods(request, end, freq, periods): request.applymarker( pytest.mark.xfail( condition=( - isinstance(freq, dict) + not PANDAS_GE_220 + and isinstance(freq, dict) and freq.get("hours", None) == 10 and freq.get("days", None) == 57 and freq.get("nanoseconds", None) == 3 @@ -1723,30 +1729,34 @@ def test_date_range_raise_overflow(): @pytest.mark.parametrize( "freqstr_unsupported", [ - "1M", - "2SM", + "1ME", + "2SME", "3MS", - "4BM", - "5CBM", + "4BME", + "5CBME", "6SMS", "7BMS", "8CBMS", - "Q", - "2BQ", + "QE", + "2BQE", "3BQS", - "10A", - "10Y", - "9BA", - "9BY", - "8AS", + "10YE", + "9BYE", "8YS", - "7BAS", "7BYS", - "BH", + "bh", "B", ], ) -def test_date_range_raise_unsupported(freqstr_unsupported): +def test_date_range_raise_unsupported(request, freqstr_unsupported): + request.applymarker( + pytest.mark.xfail( + condition=( + not PANDAS_GE_220 and freqstr_unsupported.endswith("E") + ), + reason="TODO: Remove this once pandas-2.2 support is added", + ) + ) s, e = "2001-01-01", "2008-01-31" pd.date_range(start=s, end=e, freq=freqstr_unsupported) with pytest.raises(ValueError, match="does not yet support"): @@ -1757,9 +1767,9 @@ def test_date_range_raise_unsupported(freqstr_unsupported): # is a valid frequency for every 3 milliseconds. if freqstr_unsupported != "3MS": freqstr_unsupported = freqstr_unsupported.lower() - pd.date_range(start=s, end=e, freq=freqstr_unsupported) with pytest.raises(ValueError, match="does not yet support"): - cudf.date_range(start=s, end=e, freq=freqstr_unsupported) + with expect_warning_if(PANDAS_GE_220): + cudf.date_range(start=s, end=e, freq=freqstr_unsupported) ################################################################## @@ -1957,7 +1967,7 @@ def test_error_values(): ) @pytest.mark.parametrize("time_type", DATETIME_TYPES) @pytest.mark.parametrize( - "resolution", ["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"] + "resolution", ["D", "h", "min", "min", "s", "ms", "us", "ns"] ) def test_ceil(request, data, time_type, resolution): alias_map = {"L": "ms", "U": "us", "N": "ns"} @@ -2002,7 +2012,7 @@ def test_ceil(request, data, time_type, resolution): ) @pytest.mark.parametrize("time_type", DATETIME_TYPES) @pytest.mark.parametrize( - "resolution", ["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"] + "resolution", ["D", "h", "min", "min", "s", "ms", "us", "ns"] ) def test_floor(request, data, time_type, resolution): alias_map = {"L": "ms", "U": "us", "N": "ns"} @@ -2048,25 +2058,9 @@ def test_floor(request, data, time_type, resolution): ) @pytest.mark.parametrize("time_type", DATETIME_TYPES) @pytest.mark.parametrize( - "resolution", ["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"] + "resolution", ["D", "h", "min", "min", "s", "ms", "us", "ns"] ) -def test_round(request, data, time_type, resolution): - alias_map = {"L": "ms", "U": "us", "N": "ns"} - request.applymarker( - pytest.mark.xfail( - condition=( - PANDAS_EQ_200 - and resolution in {"L", "ms", "U", "us", "N"} - and np.dtype( - f"datetime64[{alias_map.get(resolution, resolution)}]" - ) - > np.dtype(time_type) - ), - reason="https://github.com/pandas-dev/pandas/issues/52761", - strict=True, - ) - ) - +def test_round(data, time_type, resolution): gs = cudf.Series(data, dtype=time_type) ps = gs.to_pandas() @@ -2284,20 +2278,20 @@ def test_daterange_pandas_compatibility(): @pytest.mark.parametrize( "data,dtype,freq", [ - ([10], "datetime64[ns]", "2N"), - ([10, 12, 14, 16], "datetime64[ns]", "2N"), - ([10, 11, 12, 13], "datetime64[ns]", "1N"), + ([10], "datetime64[ns]", "2ns"), + ([10, 12, 14, 16], "datetime64[ns]", "2ns"), + ([10, 11, 12, 13], "datetime64[ns]", "1ns"), ([100, 200, 300, 400], "datetime64[s]", "100s"), ([101, 201, 301, 401], "datetime64[ms]", "100ms"), ], ) def test_datetime_index_with_freq(request, data, dtype, freq): - request.applymarker( - pytest.mark.xfail( - condition=(not PANDAS_GE_200 and dtype != "datetime64[ns]"), - reason="Pandas < 2.0 lacks non-nano-second dtype support.", - ) - ) + # request.applymarker( + # pytest.mark.xfail( + # condition=(not PANDAS_GE_200 and dtype != "datetime64[ns]"), + # reason="Pandas < 2.0 lacks non-nano-second dtype support.", + # ) + # ) actual = cudf.DatetimeIndex(data, dtype=dtype, freq=freq) expected = pd.DatetimeIndex(data, dtype=dtype, freq=freq) assert_eq(actual, expected) @@ -2306,7 +2300,7 @@ def test_datetime_index_with_freq(request, data, dtype, freq): @pytest.mark.parametrize( "data,dtype,freq", [ - ([10, 1232, 13244, 13426], "datetime64[ns]", "2N"), + ([10, 1232, 13244, 13426], "datetime64[ns]", "2ns"), ([10, 11, 12, 13], "datetime64[ns]", "1s"), ([10000, 200, 300, 400], "datetime64[s]", "100s"), ([107871, 201, 301, 401], "datetime64[ms]", "100ns"), @@ -2454,3 +2448,23 @@ def test_dateimeindex_from_noniso_string(): def test_to_datetime_errors_non_scalar_not_implemented(errors): with pytest.raises(NotImplementedError): cudf.to_datetime([1, ""], unit="s", errors=errors) + + +@pytest.mark.parametrize( + "freqstr", + [ + "H", + "N", + "T", + "L", + "U", + "S", + ], +) +def test_datetime_raise_warning(freqstr): + t = cudf.Series( + ["2001-01-01 00:04:45", "2001-01-01 00:04:58", "2001-01-01 00:05:04"], + dtype="datetime64[ns]", + ) + with pytest.warns(FutureWarning): + t.dt.ceil(freqstr) diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index 996b651b9fe..7a190fb428a 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -2422,7 +2422,7 @@ def test_index_type_methods(data, func): @pytest.mark.parametrize( - "resolution", ["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"] + "resolution", ["D", "h", "min", "s", "ms", "us", "ns"] ) def test_index_datetime_ceil(resolution): cuidx = cudf.DatetimeIndex([1000000, 2000000, 3000000, 4000000, 5000000]) @@ -2435,7 +2435,7 @@ def test_index_datetime_ceil(resolution): @pytest.mark.parametrize( - "resolution", ["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"] + "resolution", ["D", "h", "min", "s", "ms", "us", "ns"] ) def test_index_datetime_floor(resolution): cuidx = cudf.DatetimeIndex([1000000, 2000000, 3000000, 4000000, 5000000]) @@ -2448,7 +2448,7 @@ def test_index_datetime_floor(resolution): @pytest.mark.parametrize( - "resolution", ["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"] + "resolution", ["D", "h", "min", "s", "ms", "us", "ns"] ) def test_index_datetime_round(resolution): cuidx = cudf.DatetimeIndex([1000000, 2000000, 3000000, 4000000, 5000000]) @@ -2490,7 +2490,7 @@ def test_index_nan_as_null(data, nan_idx, NA_idx, nan_as_null): pd.Series( range(25), index=pd.date_range( - start="2019-01-01", end="2019-01-02", freq="H" + start="2019-01-01", end="2019-01-02", freq="h" ), ), ], diff --git a/python/cudf/cudf/tests/test_interval.py b/python/cudf/cudf/tests/test_interval.py index ef853a23004..1c61b378d68 100644 --- a/python/cudf/cudf/tests/test_interval.py +++ b/python/cudf/cudf/tests/test_interval.py @@ -6,6 +6,7 @@ import pytest import cudf +from cudf.core._compat import PANDAS_GE_220 from cudf.testing._utils import assert_eq @@ -166,13 +167,17 @@ def test_interval_index_unique(): assert_eq(expected, actual) +@pytest.mark.xfail( + condition=not PANDAS_GE_220, + reason="TODO: Remove this once pandas-2.2 support is added", +) @pytest.mark.parametrize("box", [pd.Series, pd.IntervalIndex]) @pytest.mark.parametrize("tz", ["US/Eastern", None]) def test_interval_with_datetime(tz, box): dti = pd.date_range( start=pd.Timestamp("20180101", tz=tz), end=pd.Timestamp("20181231", tz=tz), - freq="M", + freq="ME", ) pobj = box(pd.IntervalIndex.from_breaks(dti)) if tz is None: diff --git a/python/cudf/cudf/tests/test_replace.py b/python/cudf/cudf/tests/test_replace.py index 6db1c97b9fd..0f8f8de36a1 100644 --- a/python/cudf/cudf/tests/test_replace.py +++ b/python/cudf/cudf/tests/test_replace.py @@ -8,7 +8,7 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210 +from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210, PANDAS_GE_220 from cudf.core.dtypes import Decimal32Dtype, Decimal64Dtype, Decimal128Dtype from cudf.testing._utils import ( INTEGER_TYPES, @@ -484,7 +484,13 @@ def test_fillna_categorical(psr_data, fill_value, inplace): @pytest.mark.parametrize( "psr_data", [ - pd.Series(pd.date_range("2010-01-01", "2020-01-10", freq="1y")), + pd.Series( + pd.date_range( + "2010-01-01", + "2020-01-10", + freq="1YE" if PANDAS_GE_220 else "1y", + ) + ), pd.Series(["2010-01-01", None, "2011-10-10"], dtype="datetime64[ns]"), pd.Series( [ @@ -525,7 +531,13 @@ def test_fillna_categorical(psr_data, fill_value, inplace): "fill_value", [ pd.Timestamp("2010-01-02"), - pd.Series(pd.date_range("2010-01-01", "2020-01-10", freq="1y")) + pd.Series( + pd.date_range( + "2010-01-01", + "2020-01-10", + freq="1YE" if PANDAS_GE_220 else "1y", + ) + ) + pd.Timedelta("1d"), pd.Series(["2010-01-01", None, "2011-10-10"], dtype="datetime64[ns]"), pd.Series( diff --git a/python/cudf/cudf/tests/test_resampling.py b/python/cudf/cudf/tests/test_resampling.py index 6281d54aa60..ce0fbbfada8 100644 --- a/python/cudf/cudf/tests/test_resampling.py +++ b/python/cudf/cudf/tests/test_resampling.py @@ -24,31 +24,31 @@ def assert_resample_results_equal(lhs, rhs, **kwargs): def test_series_downsample_simple(ts_resolution): # Series with and index of 5min intervals: - index = pd.date_range(start="2001-01-01", periods=10, freq="1T") + index = pd.date_range(start="2001-01-01", periods=10, freq="1min") psr = pd.Series(range(10), index=index) gsr = cudf.from_pandas(psr) gsr.index = gsr.index.astype(f"datetime64[{ts_resolution}]") assert_resample_results_equal( - psr.resample("3T").sum(), - gsr.resample("3T").sum(), + psr.resample("3min").sum(), + gsr.resample("3min").sum(), ) def test_series_upsample_simple(): # Series with and index of 5min intervals: - index = pd.date_range(start="2001-01-01", periods=10, freq="1T") + index = pd.date_range(start="2001-01-01", periods=10, freq="1min") psr = pd.Series(range(10), index=index) gsr = cudf.from_pandas(psr) assert_resample_results_equal( - psr.resample("3T").sum(), - gsr.resample("3T").sum(), + psr.resample("3min").sum(), + gsr.resample("3min").sum(), ) -@pytest.mark.parametrize("rule", ["2S", "10S"]) +@pytest.mark.parametrize("rule", ["2s", "10s"]) def test_series_resample_ffill(rule): - rng = pd.date_range("1/1/2012", periods=10, freq="5S") + rng = pd.date_range("1/1/2012", periods=10, freq="5s") ts = pd.Series(np.random.randint(0, 500, len(rng)), index=rng) gts = cudf.from_pandas(ts) assert_resample_results_equal( @@ -56,9 +56,9 @@ def test_series_resample_ffill(rule): ) -@pytest.mark.parametrize("rule", ["2S", "10S"]) +@pytest.mark.parametrize("rule", ["2s", "10s"]) def test_series_resample_bfill(rule): - rng = pd.date_range("1/1/2012", periods=10, freq="5S") + rng = pd.date_range("1/1/2012", periods=10, freq="5s") ts = pd.Series(np.random.randint(0, 500, len(rng)), index=rng) gts = cudf.from_pandas(ts) assert_resample_results_equal( @@ -66,9 +66,9 @@ def test_series_resample_bfill(rule): ) -@pytest.mark.parametrize("rule", ["2S", "10S"]) +@pytest.mark.parametrize("rule", ["2s", "10s"]) def test_series_resample_asfreq(rule): - rng = pd.date_range("1/1/2012", periods=100, freq="5S") + rng = pd.date_range("1/1/2012", periods=100, freq="5s") ts = pd.Series(np.random.randint(0, 500, len(rng)), index=rng) gts = cudf.from_pandas(ts) assert_resample_results_equal( @@ -79,25 +79,25 @@ def test_series_resample_asfreq(rule): def test_dataframe_resample_aggregation_simple(): pdf = pd.DataFrame( np.random.randn(1000, 3), - index=pd.date_range("1/1/2012", freq="S", periods=1000), + index=pd.date_range("1/1/2012", freq="s", periods=1000), columns=["A", "B", "C"], ) gdf = cudf.from_pandas(pdf) assert_resample_results_equal( - pdf.resample("3T").mean(), gdf.resample("3T").mean() + pdf.resample("3min").mean(), gdf.resample("3min").mean() ) def test_dataframe_resample_multiagg(): pdf = pd.DataFrame( np.random.randn(1000, 3), - index=pd.date_range("1/1/2012", freq="S", periods=1000), + index=pd.date_range("1/1/2012", freq="s", periods=1000), columns=["A", "B", "C"], ) gdf = cudf.from_pandas(pdf) assert_resample_results_equal( - pdf.resample("3T").agg(["sum", "mean", "std"]), - gdf.resample("3T").agg(["sum", "mean", "std"]), + pdf.resample("3min").agg(["sum", "mean", "std"]), + gdf.resample("3min").agg(["sum", "mean", "std"]), ) @@ -106,12 +106,13 @@ def test_dataframe_resample_on(): pdf = pd.DataFrame( { "x": np.random.randn(1000), - "y": pd.date_range("1/1/2012", freq="S", periods=1000), + "y": pd.date_range("1/1/2012", freq="s", periods=1000), } ) gdf = cudf.from_pandas(pdf) assert_resample_results_equal( - pdf.resample("3T", on="y").mean(), gdf.resample("3T", on="y").mean() + pdf.resample("3min", on="y").mean(), + gdf.resample("3min", on="y").mean(), ) @@ -120,15 +121,15 @@ def test_dataframe_resample_level(): pdf = pd.DataFrame( { "x": np.random.randn(1000), - "y": pd.date_range("1/1/2012", freq="S", periods=1000), + "y": pd.date_range("1/1/2012", freq="s", periods=1000), } ) pdi = pd.MultiIndex.from_frame(pdf) pdf = pd.DataFrame({"a": np.random.randn(1000)}, index=pdi) gdf = cudf.from_pandas(pdf) assert_resample_results_equal( - pdf.resample("3T", level="y").mean(), - gdf.resample("3T", level="y").mean(), + pdf.resample("3min", level="y").mean(), + gdf.resample("3min", level="y").mean(), ) @@ -139,8 +140,8 @@ def test_dataframe_resample_level(): ("1us", "10us", "us"), ("ms", "100us", "us"), ("ms", "1s", "s"), - ("s", "1T", "s"), - ("1T", "30s", "s"), + ("s", "1min", "s"), + ("1min", "30s", "s"), ("1D", "10D", "s"), ("10D", "1D", "s"), ], diff --git a/python/cudf/cudf/tests/test_serialize.py b/python/cudf/cudf/tests/test_serialize.py index 4e2a9f581c3..f26d78e7783 100644 --- a/python/cudf/cudf/tests/test_serialize.py +++ b/python/cudf/cudf/tests/test_serialize.py @@ -352,9 +352,9 @@ def test_serialize_seriesgroupby(): def test_serialize_seriesresampler(): - index = cudf.date_range(start="2001-01-01", periods=10, freq="1T") + index = cudf.date_range(start="2001-01-01", periods=10, freq="1min") sr = cudf.Series(range(10), index=index) - re_sampler = sr.resample("3T") + re_sampler = sr.resample("3min") actual = re_sampler.sum() recreated = re_sampler.__class__.deserialize(*re_sampler.serialize()) expected = recreated.sum() diff --git a/python/cudf/cudf/tests/test_sorting.py b/python/cudf/cudf/tests/test_sorting.py index f30c14373bf..dd545da4243 100644 --- a/python/cudf/cudf/tests/test_sorting.py +++ b/python/cudf/cudf/tests/test_sorting.py @@ -8,6 +8,7 @@ import pytest from cudf import DataFrame, Series +from cudf.core._compat import PANDAS_GE_220 from cudf.core.column import NumericalColumn from cudf.testing._utils import ( DATETIME_TYPES, @@ -48,6 +49,11 @@ def test_dataframe_sort_values(nelem, dtype): @pytest.mark.parametrize("ignore_index", [True, False]) @pytest.mark.parametrize("index", ["a", "b", ["a", "b"]]) def test_dataframe_sort_values_ignore_index(index, ignore_index): + if not PANDAS_GE_220 and isinstance(index, list) and not ignore_index: + pytest.skip( + reason="TODO: Remove this once pandas-2.2 support is added", + ) + gdf = DataFrame( {"a": [1, 3, 5, 2, 4], "b": [1, 1, 2, 2, 3], "c": [9, 7, 7, 7, 1]} ) From 72ecbe96788f6dc76d6f9f45dd7607dd38c90978 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 6 Feb 2024 14:37:47 -0500 Subject: [PATCH 207/384] Use offsetalator in nvtext tokenize functions (#14783) Adds offsetalator in place of hardcoded offset type arrays to the strings split functions: - `nvtext::tokenize()` - `nvtext::count_tokens()` - `nvtext::character_tokenize()` - `nvtext::ngrams_tokenize()` - `nvtext::tokenize_with_vocabulary()` Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Karthikeyan (https://github.com/karthikeyann) URL: https://github.com/rapidsai/cudf/pull/14783 --- cpp/src/text/ngrams_tokenize.cu | 86 ++++++++++--------------- cpp/src/text/tokenize.cu | 47 ++++++-------- cpp/src/text/utilities/tokenize_ops.cuh | 20 +++--- cpp/src/text/vocabulary_tokenize.cu | 11 ++-- 4 files changed, 71 insertions(+), 93 deletions(-) diff --git a/cpp/src/text/ngrams_tokenize.cu b/cpp/src/text/ngrams_tokenize.cu index 642dca5fc47..3444786ff80 100644 --- a/cpp/src/text/ngrams_tokenize.cu +++ b/cpp/src/text/ngrams_tokenize.cu @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -37,12 +38,9 @@ #include #include #include -#include #include -#include - namespace nvtext { namespace detail { namespace { @@ -60,10 +58,10 @@ namespace { * member. */ struct string_tokens_positions_fn { - cudf::column_device_view const d_strings; // strings to tokenize - cudf::string_view const d_delimiter; // delimiter to tokenize around - cudf::size_type const* d_token_offsets; // offsets into the d_token_positions for each string - position_pair* d_token_positions; // token positions in each string + cudf::column_device_view const d_strings; // strings to tokenize + cudf::string_view const d_delimiter; // delimiter to tokenize around + cudf::detail::input_offsetalator d_token_offsets; // offsets of d_token_positions for each string + position_pair* d_token_positions; // token positions in each string __device__ void operator()(cudf::size_type idx) { @@ -95,12 +93,12 @@ struct ngram_builder_fn { cudf::column_device_view const d_strings; // strings to generate ngrams from cudf::string_view const d_separator; // separator to place between them 'grams cudf::size_type const ngrams; // ngram number to generate (2=bi-gram, 3=tri-gram) - cudf::size_type const* d_token_offsets; // offsets for token position for each string - position_pair const* d_token_positions; // token positions for each string - cudf::size_type const* d_chars_offsets{}; // offsets for each string's ngrams - char* d_chars{}; // write ngram strings to here - cudf::size_type const* d_ngram_offsets{}; // offsets for sizes of each string's ngrams - cudf::size_type* d_ngram_sizes{}; // write ngram sizes to here + cudf::detail::input_offsetalator d_token_offsets; // offsets for token position for each string + position_pair const* d_token_positions; // token positions for each string + cudf::detail::input_offsetalator d_chars_offsets{}; // offsets for each string's ngrams + char* d_chars{}; // write ngram strings to here + cudf::size_type const* d_ngram_offsets{}; // offsets for sizes of each string's ngrams + cudf::size_type* d_ngram_sizes{}; // write ngram sizes to here __device__ cudf::size_type operator()(cudf::size_type idx) { @@ -165,16 +163,12 @@ std::unique_ptr ngrams_tokenize(cudf::strings_column_view const& s // first, get the number of tokens per string to get the token-offsets // Ex. token-counts = [3,2]; token-offsets = [0,3,5] - rmm::device_uvector token_offsets(strings_count + 1, stream); - auto d_token_offsets = token_offsets.data(); - thrust::transform_inclusive_scan(rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(strings_count), - d_token_offsets + 1, - strings_tokenizer{d_strings, d_delimiter}, - thrust::plus()); - token_offsets.set_element_to_zero_async(0, stream); - auto const total_tokens = token_offsets.back_element(stream); // Ex. 5 tokens + auto const count_itr = + cudf::detail::make_counting_transform_iterator(0, strings_tokenizer{d_strings, d_delimiter}); + auto [token_offsets, total_tokens] = cudf::strings::detail::make_offsets_child_column( + count_itr, count_itr + strings_count, stream, rmm::mr::get_current_device_resource()); + auto d_token_offsets = + cudf::detail::offsetalator_factory::make_input_iterator(token_offsets->view()); // get the token positions (in bytes) per string // Ex. start/end pairs: [(0,1),(2,4),(5,8), (0,2),(3,4)] @@ -188,21 +182,17 @@ std::unique_ptr ngrams_tokenize(cudf::strings_column_view const& s // compute the number of ngrams per string to get the total number of ngrams to generate // Ex. ngram-counts = [2,1]; ngram-offsets = [0,2,3]; total = 3 bigrams - rmm::device_uvector ngram_offsets(strings_count + 1, stream); - auto d_ngram_offsets = ngram_offsets.data(); - thrust::transform_inclusive_scan( - rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(strings_count), - d_ngram_offsets + 1, + auto const ngram_counts = cudf::detail::make_counting_transform_iterator( + 0, cuda::proclaim_return_type( [d_token_offsets, ngrams] __device__(cudf::size_type idx) { - auto token_count = d_token_offsets[idx + 1] - d_token_offsets[idx]; + auto token_count = + static_cast(d_token_offsets[idx + 1] - d_token_offsets[idx]); return (token_count >= ngrams) ? token_count - ngrams + 1 : 0; - }), - thrust::plus{}); - ngram_offsets.set_element_to_zero_async(0, stream); - auto const total_ngrams = ngram_offsets.back_element(stream); + })); + auto [ngram_offsets, total_ngrams] = cudf::detail::make_offsets_child_column( + ngram_counts, ngram_counts + strings_count, stream, rmm::mr::get_current_device_resource()); + auto d_ngram_offsets = ngram_offsets->view().begin(); // Compute the total size of the ngrams for each string (not for each ngram) // Ex. 2 bigrams in 1st string total to 10 bytes; 1 bigram in 2nd string is 4 bytes @@ -212,21 +202,14 @@ std::unique_ptr ngrams_tokenize(cudf::strings_column_view const& s // ngrams for each string. // Ex. bigram for first string produces 2 bigrams ("a_bb","bb_ccc") which // is built in memory like this: "a_bbbb_ccc" - rmm::device_uvector chars_offsets(strings_count + 1, stream); - // First compute the output sizes for each string (this not the final output result) - thrust::transform( - rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(strings_count), - chars_offsets.begin(), - ngram_builder_fn{d_strings, d_separator, ngrams, d_token_offsets, d_token_positions}); - // Convert the sizes to offsets - auto const output_chars_size = cudf::detail::sizes_to_offsets( - chars_offsets.begin(), chars_offsets.end(), chars_offsets.begin(), stream); - CUDF_EXPECTS( - output_chars_size <= static_cast(std::numeric_limits::max()), - "Size of output exceeds the column size limit", - std::overflow_error); + + // First compute the output sizes for each string (this not the final output result) + auto const sizes_itr = cudf::detail::make_counting_transform_iterator( + 0, ngram_builder_fn{d_strings, d_separator, ngrams, d_token_offsets, d_token_positions}); + auto [chars_offsets, output_chars_size] = cudf::strings::detail::make_offsets_child_column( + sizes_itr, sizes_itr + strings_count, stream, rmm::mr::get_current_device_resource()); + auto d_chars_offsets = + cudf::detail::offsetalator_factory::make_input_iterator(chars_offsets->view()); // This will contain the size in bytes of each ngram to generate rmm::device_uvector ngram_sizes(total_ngrams, stream); @@ -245,14 +228,13 @@ std::unique_ptr ngrams_tokenize(cudf::strings_column_view const& s ngrams, d_token_offsets, d_token_positions, - chars_offsets.data(), + d_chars_offsets, d_chars, d_ngram_offsets, ngram_sizes.data()}); // build the offsets column -- converting the ngram sizes into offsets auto offsets_column = std::get<0>( cudf::detail::make_offsets_child_column(ngram_sizes.begin(), ngram_sizes.end(), stream, mr)); - offsets_column->set_null_count(0); // create the output strings column return make_strings_column( total_ngrams, std::move(offsets_column), chars.release(), 0, rmm::device_buffer{}); diff --git a/cpp/src/text/tokenize.cu b/cpp/src/text/tokenize.cu index c256607fb23..97896f20f4f 100644 --- a/cpp/src/text/tokenize.cu +++ b/cpp/src/text/tokenize.cu @@ -22,8 +22,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -38,7 +38,6 @@ #include #include #include -#include #include namespace nvtext { @@ -80,18 +79,17 @@ std::unique_ptr tokenize_fn(cudf::size_type strings_count, token_count_fn(strings_count, tokenizer, stream, rmm::mr::get_current_device_resource()); auto d_token_counts = token_counts->view(); // create token-index offsets from the counts - rmm::device_uvector token_offsets(strings_count + 1, stream); - thrust::inclusive_scan(rmm::exec_policy(stream), - d_token_counts.template begin(), - d_token_counts.template end(), - token_offsets.begin() + 1); - token_offsets.set_element_to_zero_async(0, stream); - auto const total_tokens = token_offsets.back_element(stream); - // build a list of pointers to each token + auto [token_offsets, total_tokens] = + cudf::detail::make_offsets_child_column(d_token_counts.template begin(), + d_token_counts.template end(), + stream, + rmm::mr::get_current_device_resource()); + // build a list of pointers to each token rmm::device_uvector tokens(total_tokens, stream); // now go get the tokens - tokenizer.d_offsets = token_offsets.data(); - tokenizer.d_tokens = tokens.data(); + tokenizer.d_offsets = + cudf::detail::offsetalator_factory::make_input_iterator(token_offsets->view()); + tokenizer.d_tokens = tokens.data(); thrust::for_each_n(rmm::exec_policy(stream), thrust::make_counting_iterator(0), strings_count, @@ -178,8 +176,8 @@ std::unique_ptr character_tokenize(cudf::strings_column_view const } auto offsets = strings_column.offsets(); - auto offset = cudf::detail::get_value(offsets, strings_column.offset(), stream); - auto chars_bytes = cudf::detail::get_value( + auto offset = cudf::strings::detail::get_offset_value(offsets, strings_column.offset(), stream); + auto chars_bytes = cudf::strings::detail::get_offset_value( offsets, strings_column.offset() + strings_count, stream) - offset; auto d_chars = @@ -202,22 +200,19 @@ std::unique_ptr character_tokenize(cudf::strings_column_view const // create output offsets column // -- conditionally copy a counting iterator where // the first byte of each character is located - auto offsets_column = - cudf::make_numeric_column(cudf::data_type{cudf::type_to_id()}, - num_characters + 1, - cudf::mask_state::UNALLOCATED, - stream, - mr); - auto d_new_offsets = offsets_column->mutable_view().begin(); - thrust::copy_if( - rmm::exec_policy(stream), - thrust::counting_iterator(0), - thrust::counting_iterator(chars_bytes + 1), + auto offsets_column = cudf::make_numeric_column( + offsets.type(), num_characters + 1, cudf::mask_state::UNALLOCATED, stream, mr); + auto d_new_offsets = + cudf::detail::offsetalator_factory::make_output_iterator(offsets_column->mutable_view()); + cudf::detail::copy_if_safe( + thrust::counting_iterator(0), + thrust::counting_iterator(chars_bytes + 1), d_new_offsets, [d_chars, chars_bytes] __device__(auto idx) { // this will also set the final value to the size chars_bytes return idx < chars_bytes ? cudf::strings::detail::is_begin_utf8_char(d_chars[idx]) : true; - }); + }, + stream); // create the output chars buffer -- just a copy of the input's chars rmm::device_uvector output_chars(chars_bytes, stream, mr); diff --git a/cpp/src/text/utilities/tokenize_ops.cuh b/cpp/src/text/utilities/tokenize_ops.cuh index a84e94a6924..0901dc37e56 100644 --- a/cpp/src/text/utilities/tokenize_ops.cuh +++ b/cpp/src/text/utilities/tokenize_ops.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -147,10 +147,10 @@ struct characters_tokenizer { * positions into the d_tokens vector. */ struct strings_tokenizer { - cudf::column_device_view const d_strings; ///< strings to tokenize - cudf::string_view const d_delimiter; ///< delimiter characters to tokenize around - cudf::size_type* d_offsets{}; ///< offsets into the d_tokens vector for each string - string_index_pair* d_tokens{}; ///< token positions in device memory + cudf::column_device_view const d_strings; ///< strings to tokenize + cudf::string_view const d_delimiter; ///< delimiter characters to tokenize around + cudf::detail::input_offsetalator d_offsets; ///< offsets into the d_tokens vector for each string + string_index_pair* d_tokens{}; ///< token positions in device memory /** * @brief Identifies the token positions within each string. @@ -191,11 +191,11 @@ using delimiterator = cudf::column_device_view::const_iterator #include #include -#include #include #include #include +#include #include #include #include +#include #include #include #include @@ -297,7 +298,7 @@ struct vocabulary_tokenizer_fn { cudf::string_view const d_delimiter; MapRefType d_map; cudf::size_type const default_id; - cudf::size_type const* d_offsets; + cudf::detail::input_offsetalator d_offsets; cudf::size_type* d_results; __device__ void operator()(cudf::size_type idx) const @@ -378,7 +379,7 @@ std::unique_ptr tokenize_with_vocabulary(cudf::strings_column_view auto tokens = cudf::make_numeric_column( output_type, total_count, cudf::mask_state::UNALLOCATED, stream, mr); auto d_tokens = tokens->mutable_view().data(); - auto d_offsets = token_offsets->view().data(); + auto d_offsets = cudf::detail::offsetalator_factory::make_input_iterator(token_offsets->view()); vocabulary_tokenizer_fn tokenizer{ *d_strings, d_delimiter, map_ref, default_id, d_offsets, d_tokens}; thrust::for_each_n(rmm::exec_policy(stream), zero_itr, input.size(), tokenizer); @@ -394,11 +395,11 @@ std::unique_ptr tokenize_with_vocabulary(cudf::strings_column_view // longer strings perform better with warp-parallel approach auto const first_offset = (input.offset() == 0) ? 0 - : cudf::detail::get_value( + : cudf::strings::detail::get_offset_value( input.offsets(), input.offset(), stream); auto const last_offset = (input.offset() == 0 && input.size() == input.offsets().size() - 1) ? input.chars_size(stream) - : cudf::detail::get_value( + : cudf::strings::detail::get_offset_value( input.offsets(), input.size() + input.offset(), stream); auto const chars_size = last_offset - first_offset; auto const d_input_chars = input.chars_begin(stream) + first_offset; From 506d575917f92f461c53b1caf4f43b0959736b6c Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 6 Feb 2024 15:12:16 -0600 Subject: [PATCH 208/384] Deprecate non-integer `periods` in `date_range` and `interval_range` (#14976) This PR deprecates non-integer `periods` in `date_range` and `interval_range` to match pandas-2.2 deprecations. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/14976 --- python/cudf/cudf/core/index.py | 6 +++ python/cudf/cudf/core/tools/datetimes.py | 7 ++++ .../cudf/cudf/tests/indexes/test_interval.py | 38 +++++++++++++++---- 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index c8eedae200b..c05d89e7279 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -2698,6 +2698,12 @@ def interval_range( start = cudf.Scalar(start) if start is not None else start end = cudf.Scalar(end) if end is not None else end + if periods is not None and not cudf.api.types.is_integer(periods): + warnings.warn( + "Non-integer 'periods' in cudf.date_range, and cudf.interval_range" + " are deprecated and will raise in a future version.", + FutureWarning, + ) periods = cudf.Scalar(int(periods)) if periods is not None else periods freq = cudf.Scalar(freq) if freq is not None else freq diff --git a/python/cudf/cudf/core/tools/datetimes.py b/python/cudf/cudf/core/tools/datetimes.py index faa7407daaf..928154e10fd 100644 --- a/python/cudf/cudf/core/tools/datetimes.py +++ b/python/cudf/cudf/core/tools/datetimes.py @@ -869,6 +869,13 @@ def date_range( "three must be specified" ) + if periods is not None and not cudf.api.types.is_integer(periods): + warnings.warn( + "Non-integer 'periods' in cudf.date_range, and cudf.interval_range" + " are deprecated and will raise in a future version.", + FutureWarning, + ) + dtype = np.dtype(" Date: Tue, 6 Feb 2024 13:23:18 -0800 Subject: [PATCH 209/384] Implement joins in pylibcudf (#14972) Contributes to #13921 Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Ashwin Srinath (https://github.com/shwina) URL: https://github.com/rapidsai/cudf/pull/14972 --- .../user_guide/api_docs/pylibcudf/index.rst | 1 + .../user_guide/api_docs/pylibcudf/join.rst | 6 + python/cudf/cudf/_lib/cpp/join.pxd | 9 +- python/cudf/cudf/_lib/join.pyx | 76 +++------ .../cudf/cudf/_lib/pylibcudf/CMakeLists.txt | 5 +- python/cudf/cudf/_lib/pylibcudf/__init__.pxd | 3 +- python/cudf/cudf/_lib/pylibcudf/__init__.py | 3 +- python/cudf/cudf/_lib/pylibcudf/groupby.pyx | 3 + python/cudf/cudf/_lib/pylibcudf/join.pxd | 15 ++ python/cudf/cudf/_lib/pylibcudf/join.pyx | 159 ++++++++++++++++++ 10 files changed, 218 insertions(+), 62 deletions(-) create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/join.rst create mode 100644 python/cudf/cudf/_lib/pylibcudf/join.pxd create mode 100644 python/cudf/cudf/_lib/pylibcudf/join.pyx diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst index 3bc56ddffc3..6e596151871 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst @@ -14,6 +14,7 @@ This page provides API documentation for pylibcudf. copying gpumemoryview groupby + join scalar table types diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/join.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/join.rst new file mode 100644 index 00000000000..05b9709d116 --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/join.rst @@ -0,0 +1,6 @@ +==== +join +==== + +.. automodule:: cudf._lib.pylibcudf.join + :members: diff --git a/python/cudf/cudf/_lib/cpp/join.pxd b/python/cudf/cudf/_lib/cpp/join.pxd index 171658c78ee..ea05256430a 100644 --- a/python/cudf/cudf/_lib/cpp/join.pxd +++ b/python/cudf/cudf/_lib/cpp/join.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp cimport bool from libcpp.memory cimport unique_ptr @@ -13,19 +13,20 @@ from cudf._lib.cpp.table.table_view cimport table_view from cudf._lib.cpp.types cimport size_type ctypedef unique_ptr[device_uvector[size_type]] gather_map_type +ctypedef pair[gather_map_type, gather_map_type] gather_map_pair_type cdef extern from "cudf/join.hpp" namespace "cudf" nogil: - cdef pair[gather_map_type, gather_map_type] inner_join( + cdef gather_map_pair_type inner_join( const table_view left_keys, const table_view right_keys, ) except + - cdef pair[gather_map_type, gather_map_type] left_join( + cdef gather_map_pair_type left_join( const table_view left_keys, const table_view right_keys, ) except + - cdef pair[gather_map_type, gather_map_type] full_join( + cdef gather_map_pair_type full_join( const table_view left_keys, const table_view right_keys, ) except + diff --git a/python/cudf/cudf/_lib/join.pyx b/python/cudf/cudf/_lib/join.pyx index 378be978cc0..65f2f8cdcc8 100644 --- a/python/cudf/cudf/_lib/join.pyx +++ b/python/cudf/cudf/_lib/join.pyx @@ -1,19 +1,10 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from cudf.core.buffer import acquire_spill_lock -from libcpp.memory cimport make_unique, unique_ptr -from libcpp.pair cimport pair -from libcpp.utility cimport move - -from rmm._lib.device_buffer cimport device_buffer - -cimport cudf._lib.cpp.join as cpp_join from cudf._lib.column cimport Column -from cudf._lib.cpp.column.column cimport column -from cudf._lib.cpp.table.table_view cimport table_view -from cudf._lib.cpp.types cimport data_type, size_type, type_id -from cudf._lib.utils cimport table_view_from_columns + +from cudf._lib import pylibcudf # The functions below return the *gathermaps* that represent # the join result when joining on the keys `lhs` and `rhs`. @@ -21,53 +12,30 @@ from cudf._lib.utils cimport table_view_from_columns @acquire_spill_lock() def join(list lhs, list rhs, how=None): - cdef pair[cpp_join.gather_map_type, cpp_join.gather_map_type] c_result - cdef table_view c_lhs = table_view_from_columns(lhs) - cdef table_view c_rhs = table_view_from_columns(rhs) - - if how == "inner": - with nogil: - c_result = move(cpp_join.inner_join(c_lhs, c_rhs)) - elif how == "left": - with nogil: - c_result = move(cpp_join.left_join(c_lhs, c_rhs)) - elif how == "outer": - with nogil: - c_result = move(cpp_join.full_join(c_lhs, c_rhs)) - else: + if how == "outer": + how = "full" + if (join_func := getattr(pylibcudf.join, f"{how}_join", None)) is None: raise ValueError(f"Invalid join type {how}") - cdef Column left_rows = _gather_map_as_column(move(c_result.first)) - cdef Column right_rows = _gather_map_as_column(move(c_result.second)) - return left_rows, right_rows + left_rows, right_rows = join_func( + pylibcudf.Table([c.to_pylibcudf(mode="read") for c in lhs]), + pylibcudf.Table([c.to_pylibcudf(mode="read") for c in rhs]), + ) + return Column.from_pylibcudf(left_rows), Column.from_pylibcudf(right_rows) @acquire_spill_lock() def semi_join(list lhs, list rhs, how=None): - # left-semi and left-anti joins - cdef cpp_join.gather_map_type c_result - cdef table_view c_lhs = table_view_from_columns(lhs) - cdef table_view c_rhs = table_view_from_columns(rhs) - - if how == "leftsemi": - with nogil: - c_result = move(cpp_join.left_semi_join(c_lhs, c_rhs)) - elif how == "leftanti": - with nogil: - c_result = move(cpp_join.left_anti_join(c_lhs, c_rhs)) - else: + if ( + join_func := getattr( + pylibcudf.join, f"{how.replace('left', 'left_')}_join", None + ) + ) is None: raise ValueError(f"Invalid join type {how}") - cdef Column left_rows = _gather_map_as_column(move(c_result)) - return left_rows, None - - -cdef Column _gather_map_as_column(cpp_join.gather_map_type gather_map): - # help to convert a gather map to a Column - cdef device_buffer c_empty - cdef size_type size = gather_map.get()[0].size() - cdef unique_ptr[column] c_col = move(make_unique[column]( - data_type(type_id.INT32), - size, - gather_map.get()[0].release(), move(c_empty), 0)) - return Column.from_unique_ptr(move(c_col)) + return Column.from_pylibcudf( + join_func( + pylibcudf.Table([c.to_pylibcudf(mode="read") for c in lhs]), + pylibcudf.Table([c.to_pylibcudf(mode="read") for c in rhs]), + ) + ), None diff --git a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt index 432617681db..da5645b5947 100644 --- a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt +++ b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt @@ -12,8 +12,9 @@ # the License. # ============================================================================= -set(cython_sources aggregation.pyx binaryop.pyx column.pyx copying.pyx gpumemoryview.pyx - groupby.pyx interop.pyx scalar.pyx table.pyx types.pyx unary.pyx utils.pyx +set(cython_sources + aggregation.pyx binaryop.pyx column.pyx copying.pyx gpumemoryview.pyx groupby.pyx interop.pyx + join.pyx scalar.pyx table.pyx types.pyx unary.pyx utils.pyx ) set(linked_libraries cudf::cudf) rapids_cython_create_modules( diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd index 5cd8f017372..bbe491f43e3 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd @@ -1,7 +1,7 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. # TODO: Verify consistent usage of relative/absolute imports in pylibcudf. -from . cimport aggregation, binaryop, copying, groupby, interop, unary +from . cimport aggregation, binaryop, copying, groupby, interop, join, unary from .column cimport Column from .gpumemoryview cimport gpumemoryview from .scalar cimport Scalar @@ -21,6 +21,7 @@ __all__ = [ "gpumemoryview", "groupby", "interop", + "join", "unary", "types", ] diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.py b/python/cudf/cudf/_lib/pylibcudf/__init__.py index 6f1eb0b6b67..35812b65046 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.py +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.py @@ -1,6 +1,6 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. -from . import aggregation, binaryop, copying, groupby, interop, unary +from . import aggregation, binaryop, copying, groupby, interop, join, unary from .column import Column from .gpumemoryview import gpumemoryview from .scalar import Scalar @@ -19,6 +19,7 @@ "gpumemoryview", "groupby", "interop", + "join", "unary", "types", ] diff --git a/python/cudf/cudf/_lib/pylibcudf/groupby.pyx b/python/cudf/cudf/_lib/pylibcudf/groupby.pyx index d6ce9825ed3..b8cc59eed09 100644 --- a/python/cudf/cudf/_lib/pylibcudf/groupby.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/groupby.pyx @@ -144,6 +144,9 @@ cdef class GroupBy: c_requests.push_back(move(request._to_libcudf_agg_request())) cdef pair[unique_ptr[table], vector[aggregation_result]] c_res + # TODO: Need to capture C++ exceptions indicating that an invalid type was used. + # We rely on libcudf to tell us this rather than checking the types beforehand + # ourselves. with nogil: c_res = move(dereference(self.c_obj).aggregate(c_requests)) return GroupBy._parse_outputs(move(c_res)) diff --git a/python/cudf/cudf/_lib/pylibcudf/join.pxd b/python/cudf/cudf/_lib/pylibcudf/join.pxd new file mode 100644 index 00000000000..4014dd4a399 --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/join.pxd @@ -0,0 +1,15 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from .column cimport Column +from .table cimport Table + + +cpdef tuple inner_join(Table left_keys, Table right_keys) + +cpdef tuple left_join(Table left_keys, Table right_keys) + +cpdef tuple full_join(Table left_keys, Table right_keys) + +cpdef Column left_semi_join(Table left_keys, Table right_keys) + +cpdef Column left_anti_join(Table left_keys, Table right_keys) diff --git a/python/cudf/cudf/_lib/pylibcudf/join.pyx b/python/cudf/cudf/_lib/pylibcudf/join.pyx new file mode 100644 index 00000000000..e1b61dabe22 --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/join.pyx @@ -0,0 +1,159 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from cython.operator import dereference + +from libcpp.memory cimport make_unique +from libcpp.utility cimport move + +from rmm._lib.device_buffer cimport device_buffer + +from cudf._lib.cpp cimport join as cpp_join +from cudf._lib.cpp.column.column cimport column +from cudf._lib.cpp.types cimport data_type, size_type, type_id + +from .column cimport Column +from .table cimport Table + + +cdef Column _column_from_gather_map(cpp_join.gather_map_type gather_map): + # helper to convert a gather map to a Column + cdef device_buffer c_empty + cdef size_type size = dereference(gather_map.get()).size() + return Column.from_libcudf( + move( + make_unique[column]( + data_type(type_id.INT32), + size, + dereference(gather_map.get()).release(), + move(c_empty), + 0 + ) + ) + ) + + +cpdef tuple inner_join(Table left_keys, Table right_keys): + """Perform an inner join between two tables. + + For details, see :cpp:func:`inner_join`. + + Parameters + ---------- + left_keys : Table + The left table to join. + right_keys : Table + The right table to join. + + Returns + ------- + Tuple[Column, Column] + A tuple containing the row indices from the left and right tables after the + join. + """ + cdef cpp_join.gather_map_pair_type c_result + with nogil: + c_result = cpp_join.inner_join(left_keys.view(), right_keys.view()) + return ( + _column_from_gather_map(move(c_result.first)), + _column_from_gather_map(move(c_result.second)), + ) + + +cpdef tuple left_join(Table left_keys, Table right_keys): + """Perform a left join between two tables. + + For details, see :cpp:func:`left_join`. + + Parameters + ---------- + left_keys : Table + The left table to join. + right_keys : Table + The right table to join. + + Returns + ------- + Tuple[Column, Column] + A tuple containing the row indices from the left and right tables after the + join. + """ + cdef cpp_join.gather_map_pair_type c_result + with nogil: + c_result = cpp_join.left_join(left_keys.view(), right_keys.view()) + return ( + _column_from_gather_map(move(c_result.first)), + _column_from_gather_map(move(c_result.second)), + ) + + +cpdef tuple full_join(Table left_keys, Table right_keys): + """Perform a full join between two tables. + + For details, see :cpp:func:`full_join`. + + Parameters + ---------- + left_keys : Table + The left table to join. + right_keys : Table + The right table to join. + + Returns + ------- + Tuple[Column, Column] + A tuple containing the row indices from the left and right tables after the + join. + """ + cdef cpp_join.gather_map_pair_type c_result + with nogil: + c_result = cpp_join.full_join(left_keys.view(), right_keys.view()) + return ( + _column_from_gather_map(move(c_result.first)), + _column_from_gather_map(move(c_result.second)), + ) + + +cpdef Column left_semi_join(Table left_keys, Table right_keys): + """Perform a left semi join between two tables. + + For details, see :cpp:func:`left_semi_join`. + + Parameters + ---------- + left_keys : Table + The left table to join. + right_keys : Table + The right table to join. + + Returns + ------- + Column + A column containing the row indices from the left table after the join. + """ + cdef cpp_join.gather_map_type c_result + with nogil: + c_result = cpp_join.left_semi_join(left_keys.view(), right_keys.view()) + return _column_from_gather_map(move(c_result)) + + +cpdef Column left_anti_join(Table left_keys, Table right_keys): + """Perform a left anti join between two tables. + + For details, see :cpp:func:`left_anti_join`. + + Parameters + ---------- + left_keys : Table + The left table to join. + right_keys : Table + The right table to join. + + Returns + ------- + Column + A column containing the row indices from the left table after the join. + """ + cdef cpp_join.gather_map_type c_result + with nogil: + c_result = cpp_join.left_anti_join(left_keys.view(), right_keys.view()) + return _column_from_gather_map(move(c_result)) From d29b8a8edff5317f43023cb59c905d9bb21a21d3 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 6 Feb 2024 15:35:01 -0800 Subject: [PATCH 210/384] Implement scans and reductions in pylibcudf (#14970) Contributes to #13921 Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Ashwin Srinath (https://github.com/shwina) URL: https://github.com/rapidsai/cudf/pull/14970 --- .../user_guide/api_docs/pylibcudf/index.rst | 1 + .../user_guide/api_docs/pylibcudf/reduce.rst | 6 + python/cudf/cudf/_lib/aggregation.pxd | 20 +- python/cudf/cudf/_lib/aggregation.pyx | 302 +----------------- python/cudf/cudf/_lib/cpp/CMakeLists.txt | 2 +- python/cudf/cudf/_lib/cpp/reduce.pxd | 5 +- python/cudf/cudf/_lib/cpp/reduce.pyx | 0 python/cudf/cudf/_lib/groupby.pyx | 4 +- .../cudf/cudf/_lib/pylibcudf/CMakeLists.txt | 2 +- python/cudf/cudf/_lib/pylibcudf/__init__.pxd | 12 +- python/cudf/cudf/_lib/pylibcudf/__init__.py | 12 +- .../cudf/cudf/_lib/pylibcudf/aggregation.pxd | 11 + .../cudf/cudf/_lib/pylibcudf/aggregation.pyx | 49 +-- python/cudf/cudf/_lib/pylibcudf/reduce.pxd | 15 + python/cudf/cudf/_lib/pylibcudf/reduce.pyx | 108 +++++++ python/cudf/cudf/_lib/reduce.pyx | 94 ++---- 16 files changed, 239 insertions(+), 404 deletions(-) create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/reduce.rst create mode 100644 python/cudf/cudf/_lib/cpp/reduce.pyx create mode 100644 python/cudf/cudf/_lib/pylibcudf/reduce.pxd create mode 100644 python/cudf/cudf/_lib/pylibcudf/reduce.pyx diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst index 6e596151871..4772d654a3c 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst @@ -15,6 +15,7 @@ This page provides API documentation for pylibcudf. gpumemoryview groupby join + reduce scalar table types diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/reduce.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/reduce.rst new file mode 100644 index 00000000000..e6f1b02331d --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/reduce.rst @@ -0,0 +1,6 @@ +====== +reduce +====== + +.. automodule:: cudf._lib.pylibcudf.reduce + :members: diff --git a/python/cudf/cudf/_lib/aggregation.pxd b/python/cudf/cudf/_lib/aggregation.pxd index f83f170c7c2..7a2a2b022fb 100644 --- a/python/cudf/cudf/_lib/aggregation.pxd +++ b/python/cudf/cudf/_lib/aggregation.pxd @@ -3,28 +3,14 @@ from libcpp.memory cimport unique_ptr from cudf._lib cimport pylibcudf -from cudf._lib.cpp.aggregation cimport ( - groupby_aggregation, - groupby_scan_aggregation, - reduce_aggregation, - rolling_aggregation, - scan_aggregation, -) +from cudf._lib.cpp.aggregation cimport rolling_aggregation cdef class RollingAggregation: cdef unique_ptr[rolling_aggregation] c_obj -cdef class GroupbyAggregation: +cdef class Aggregation: cdef pylibcudf.aggregation.Aggregation c_obj -cdef class ReduceAggregation: - cdef unique_ptr[reduce_aggregation] c_obj - -cdef class ScanAggregation: - cdef unique_ptr[scan_aggregation] c_obj - cdef RollingAggregation make_rolling_aggregation(op, kwargs=*) -cdef GroupbyAggregation make_groupby_aggregation(op, kwargs=*) -cdef ReduceAggregation make_reduce_aggregation(op, kwargs=*) -cdef ScanAggregation make_scan_aggregation(op, kwargs=*) +cdef Aggregation make_aggregation(op, kwargs=*) diff --git a/python/cudf/cudf/_lib/aggregation.pyx b/python/cudf/cudf/_lib/aggregation.pyx index 127580a6ec6..036c922e128 100644 --- a/python/cudf/cudf/_lib/aggregation.pyx +++ b/python/cudf/cudf/_lib/aggregation.pyx @@ -6,21 +6,17 @@ import pandas as pd from libcpp.string cimport string from libcpp.utility cimport move -from libcpp.vector cimport vector from cudf._lib.types import SUPPORTED_NUMPY_TO_LIBCUDF_TYPES, NullHandling from cudf.utils import cudautils from cudf._lib.types cimport ( - underlying_type_t_interpolation, underlying_type_t_null_policy, underlying_type_t_type_id, ) from numba.np import numpy_support -from cudf._lib.types import Interpolation - cimport cudf._lib.cpp.aggregation as libcudf_aggregation cimport cudf._lib.cpp.types as libcudf_types from cudf._lib.cpp.aggregation cimport underlying_type_t_correlation_type @@ -245,19 +241,7 @@ cdef class RollingAggregation: )) return agg -cdef class GroupbyAggregation: - """A Cython wrapper for groupby aggregations. - - **This class should never be instantiated using a standard constructor, - only using one of its many factories.** These factories handle mapping - different cudf operations to their libcudf analogs, e.g. - `cudf.DataFrame.idxmin` -> `libcudf.argmin`. Additionally, they perform - any additional configuration needed to translate Python arguments into - their corresponding C++ types (for instance, C++ enumerations used for - flag arguments). The factory approach is necessary to support operations - like `df.agg(lambda x: x.sum())`; such functions are called with this - class as an argument to generation the desired aggregation. - """ +cdef class Aggregation: def __init__(self, pylibcudf.aggregation.Aggregation agg): self.c_obj = agg @@ -410,202 +394,14 @@ cdef class GroupbyAggregation: )) - -cdef class ReduceAggregation: - """A Cython wrapper for reduce aggregations. - - **This class should never be instantiated using a standard constructor, - only using one of its many factories.** These factories handle mapping - different cudf operations to their libcudf analogs, e.g. - `cudf.DataFrame.idxmin` -> `libcudf.argmin`. Additionally, they perform - any additional configuration needed to translate Python arguments into - their corresponding C++ types (for instance, C++ enumerations used for - flag arguments). The factory approach is necessary to support operations - like `df.agg(lambda x: x.sum())`; such functions are called with this - class as an argument to generation the desired aggregation. - """ - @property - def kind(self): - return AggregationKind(self.c_obj.get()[0].kind).name - - @classmethod - def sum(cls): - cdef ReduceAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_sum_aggregation[reduce_aggregation]()) - return agg - - @classmethod - def product(cls): - cdef ReduceAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_product_aggregation[ - reduce_aggregation]()) - return agg - prod = product - - @classmethod - def min(cls): - cdef ReduceAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_min_aggregation[reduce_aggregation]()) - return agg - - @classmethod - def max(cls): - cdef ReduceAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_max_aggregation[reduce_aggregation]()) - return agg - + # Reduce aggregations @classmethod def any(cls): - cdef ReduceAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_any_aggregation[reduce_aggregation]()) - return agg + return cls(pylibcudf.aggregation.any()) @classmethod def all(cls): - cdef ReduceAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_all_aggregation[reduce_aggregation]()) - return agg - - @classmethod - def sum_of_squares(cls): - cdef ReduceAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_sum_of_squares_aggregation[ - reduce_aggregation]() - ) - return agg - - @classmethod - def mean(cls): - cdef ReduceAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_mean_aggregation[reduce_aggregation]()) - return agg - - @classmethod - def var(cls, ddof=1): - cdef ReduceAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_variance_aggregation[ - reduce_aggregation](ddof)) - return agg - - @classmethod - def std(cls, ddof=1): - cdef ReduceAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_std_aggregation[reduce_aggregation](ddof)) - return agg - - @classmethod - def median(cls): - cdef ReduceAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_median_aggregation[reduce_aggregation]()) - return agg - - @classmethod - def quantile(cls, q=0.5, interpolation="linear"): - cdef ReduceAggregation agg = cls() - - if not pd.api.types.is_list_like(q): - q = [q] - - cdef vector[double] c_q = q - cdef libcudf_types.interpolation c_interp = ( - ( - ( - Interpolation[interpolation.upper()] - ) - ) - ) - agg.c_obj = move( - libcudf_aggregation.make_quantile_aggregation[reduce_aggregation]( - c_q, c_interp) - ) - return agg - - @classmethod - def nunique(cls): - cdef ReduceAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_nunique_aggregation[reduce_aggregation]( - libcudf_types.null_policy.EXCLUDE - )) - return agg - - @classmethod - def nth(cls, libcudf_types.size_type size): - cdef ReduceAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_nth_element_aggregation[ - reduce_aggregation](size)) - return agg - -cdef class ScanAggregation: - """A Cython wrapper for scan aggregations. - - **This class should never be instantiated using a standard constructor, - only using one of its many factories.** These factories handle mapping - different cudf operations to their libcudf analogs, e.g. - `cudf.DataFrame.idxmin` -> `libcudf.argmin`. Additionally, they perform - any additional configuration needed to translate Python arguments into - their corresponding C++ types (for instance, C++ enumerations used for - flag arguments). The factory approach is necessary to support operations - like `df.agg(lambda x: x.sum())`; such functions are called with this - class as an argument to generation the desired aggregation. - """ - @property - def kind(self): - return AggregationKind(self.c_obj.get()[0].kind).name - - @classmethod - def sum(cls): - cdef ScanAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_sum_aggregation[scan_aggregation]()) - return agg - - @classmethod - def product(cls): - cdef ScanAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_product_aggregation[scan_aggregation]()) - return agg - prod = product - - @classmethod - def min(cls): - cdef ScanAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_min_aggregation[scan_aggregation]()) - return agg - - @classmethod - def max(cls): - cdef ScanAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation. - make_max_aggregation[scan_aggregation]()) - return agg - - # scan aggregations - # TODO: update this after adding per algorithm aggregation derived types - # https://github.com/rapidsai/cudf/issues/7106 - cumsum = sum - cummin = min - cummax = max + return cls(pylibcudf.aggregation.all()) cdef RollingAggregation make_rolling_aggregation(op, kwargs=None): @@ -646,7 +442,7 @@ cdef RollingAggregation make_rolling_aggregation(op, kwargs=None): raise TypeError(f"Unknown aggregation {op}") return agg -cdef GroupbyAggregation make_groupby_aggregation(op, kwargs=None): +cdef Aggregation make_aggregation(op, kwargs=None): r""" Parameters ---------- @@ -665,97 +461,21 @@ cdef GroupbyAggregation make_groupby_aggregation(op, kwargs=None): Returns ------- - GroupbyAggregation - """ - if kwargs is None: - kwargs = {} - - cdef GroupbyAggregation agg - if isinstance(op, str): - agg = getattr(GroupbyAggregation, op)(**kwargs) - elif callable(op): - if op is list: - agg = GroupbyAggregation.collect() - elif "dtype" in kwargs: - agg = GroupbyAggregation.from_udf(op, **kwargs) - else: - agg = op(GroupbyAggregation) - else: - raise TypeError(f"Unknown aggregation {op}") - return agg - -cdef ReduceAggregation make_reduce_aggregation(op, kwargs=None): - r""" - Parameters - ---------- - op : str or callable - If callable, must meet one of the following requirements: - - * Is of the form lambda x: x.agg(*args, **kwargs), where - `agg` is the name of a supported aggregation. Used to - to specify aggregations that take arguments, e.g., - `lambda x: x.quantile(0.5)`. - * Is a user defined aggregation function that operates on - reducible values. In this case, the output dtype must be - specified in the `kwargs` dictionary. - \*\*kwargs : dict, optional - Any keyword arguments to be passed to the op. - - Returns - ------- - ReduceAggregation - """ - if kwargs is None: - kwargs = {} - - cdef ReduceAggregation agg - if isinstance(op, str): - agg = getattr(ReduceAggregation, op)(**kwargs) - elif callable(op): - if op is list: - agg = ReduceAggregation.collect() - elif "dtype" in kwargs: - agg = ReduceAggregation.from_udf(op, **kwargs) - else: - agg = op(ReduceAggregation) - else: - raise TypeError(f"Unknown aggregation {op}") - return agg - -cdef ScanAggregation make_scan_aggregation(op, kwargs=None): - r""" - Parameters - ---------- - op : str or callable - If callable, must meet one of the following requirements: - - * Is of the form lambda x: x.agg(*args, **kwargs), where - `agg` is the name of a supported aggregation. Used to - to specify aggregations that take arguments, e.g., - `lambda x: x.quantile(0.5)`. - * Is a user defined aggregation function that operates on - scannable values. In this case, the output dtype must be - specified in the `kwargs` dictionary. - \*\*kwargs : dict, optional - Any keyword arguments to be passed to the op. - - Returns - ------- - ScanAggregation + Aggregation """ if kwargs is None: kwargs = {} - cdef ScanAggregation agg + cdef Aggregation agg if isinstance(op, str): - agg = getattr(ScanAggregation, op)(**kwargs) + agg = getattr(Aggregation, op)(**kwargs) elif callable(op): if op is list: - agg = ScanAggregation.collect() + agg = Aggregation.collect() elif "dtype" in kwargs: - agg = ScanAggregation.from_udf(op, **kwargs) + agg = Aggregation.from_udf(op, **kwargs) else: - agg = op(ScanAggregation) + agg = op(Aggregation) else: raise TypeError(f"Unknown aggregation {op}") return agg diff --git a/python/cudf/cudf/_lib/cpp/CMakeLists.txt b/python/cudf/cudf/_lib/cpp/CMakeLists.txt index e79fef98448..da06cf225e9 100644 --- a/python/cudf/cudf/_lib/cpp/CMakeLists.txt +++ b/python/cudf/cudf/_lib/cpp/CMakeLists.txt @@ -12,7 +12,7 @@ # the License. # ============================================================================= -set(cython_sources aggregation.pyx binaryop.pyx copying.pyx types.pyx unary.pyx) +set(cython_sources aggregation.pyx binaryop.pyx copying.pyx reduce.pxd types.pyx unary.pyx) set(linked_libraries cudf::cudf) diff --git a/python/cudf/cudf/_lib/cpp/reduce.pxd b/python/cudf/cudf/_lib/cpp/reduce.pxd index 997782dec6c..9c893fe9bcb 100644 --- a/python/cudf/cudf/_lib/cpp/reduce.pxd +++ b/python/cudf/cudf/_lib/cpp/reduce.pxd @@ -1,5 +1,6 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. +from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.utility cimport pair @@ -17,7 +18,7 @@ cdef extern from "cudf/reduction.hpp" namespace "cudf" nogil: data_type type ) except + - ctypedef enum scan_type: + cpdef enum class scan_type(bool): INCLUSIVE "cudf::scan_type::INCLUSIVE", EXCLUSIVE "cudf::scan_type::EXCLUSIVE", diff --git a/python/cudf/cudf/_lib/cpp/reduce.pyx b/python/cudf/cudf/_lib/cpp/reduce.pyx new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/cudf/cudf/_lib/groupby.pyx b/python/cudf/cudf/_lib/groupby.pyx index 3493d1c4f33..eb0f784de17 100644 --- a/python/cudf/cudf/_lib/groupby.pyx +++ b/python/cudf/cudf/_lib/groupby.pyx @@ -18,7 +18,7 @@ from cudf._lib.utils cimport columns_from_pylibcudf_table from cudf._lib.scalar import as_device_scalar -from cudf._lib.aggregation cimport make_groupby_aggregation +from cudf._lib.aggregation cimport make_aggregation from cudf._lib.cpp.replace cimport replace_policy from cudf._lib.cpp.scalar.scalar cimport scalar @@ -164,7 +164,7 @@ cdef class GroupBy: included_aggregations_i = [] col_aggregations = [] for agg in aggs: - agg_obj = make_groupby_aggregation(agg) + agg_obj = make_aggregation(agg) if valid_aggregations == "ALL" or agg_obj.kind in valid_aggregations: included_aggregations_i.append((agg, agg_obj.kind)) col_aggregations.append(agg_obj.c_obj) diff --git a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt index da5645b5947..6144fd07ac0 100644 --- a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt +++ b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt @@ -14,7 +14,7 @@ set(cython_sources aggregation.pyx binaryop.pyx column.pyx copying.pyx gpumemoryview.pyx groupby.pyx interop.pyx - join.pyx scalar.pyx table.pyx types.pyx unary.pyx utils.pyx + join.pyx reduce.pyx scalar.pyx table.pyx types.pyx unary.pyx utils.pyx ) set(linked_libraries cudf::cudf) rapids_cython_create_modules( diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd index bbe491f43e3..74afa2dbacd 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd @@ -1,7 +1,16 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. # TODO: Verify consistent usage of relative/absolute imports in pylibcudf. -from . cimport aggregation, binaryop, copying, groupby, interop, join, unary +from . cimport ( + aggregation, + binaryop, + copying, + groupby, + interop, + join, + reduce, + unary, +) from .column cimport Column from .gpumemoryview cimport gpumemoryview from .scalar cimport Scalar @@ -23,5 +32,6 @@ __all__ = [ "interop", "join", "unary", + "reduce", "types", ] diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.py b/python/cudf/cudf/_lib/pylibcudf/__init__.py index 35812b65046..96663d365a8 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.py +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.py @@ -1,6 +1,15 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. -from . import aggregation, binaryop, copying, groupby, interop, join, unary +from . import ( + aggregation, + binaryop, + copying, + groupby, + interop, + join, + reduce, + unary, +) from .column import Column from .gpumemoryview import gpumemoryview from .scalar import Scalar @@ -21,5 +30,6 @@ "interop", "join", "unary", + "reduce", "types", ] diff --git a/python/cudf/cudf/_lib/pylibcudf/aggregation.pxd b/python/cudf/cudf/_lib/pylibcudf/aggregation.pxd index 8eda16c4165..1b7da5a5532 100644 --- a/python/cudf/cudf/_lib/pylibcudf/aggregation.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/aggregation.pxd @@ -10,6 +10,8 @@ from cudf._lib.cpp.aggregation cimport ( groupby_scan_aggregation, rank_method, rank_percentage, + reduce_aggregation, + scan_aggregation, ) from cudf._lib.cpp.types cimport ( interpolation, @@ -23,14 +25,23 @@ from cudf._lib.cpp.types cimport ( from .types cimport DataType +# workaround for https://github.com/cython/cython/issues/3885 +ctypedef groupby_aggregation * gba_ptr +ctypedef groupby_scan_aggregation * gbsa_ptr +ctypedef reduce_aggregation * ra_ptr +ctypedef scan_aggregation * sa_ptr + cdef class Aggregation: cdef unique_ptr[aggregation] c_obj cpdef kind(self) + cdef void _unsupported_agg_error(self, str alg) cdef unique_ptr[groupby_aggregation] clone_underlying_as_groupby(self) except * cdef unique_ptr[groupby_scan_aggregation] clone_underlying_as_groupby_scan( self ) except * + cdef const reduce_aggregation* view_underlying_as_reduce(self) except * + cdef const scan_aggregation* view_underlying_as_scan(self) except * @staticmethod cdef Aggregation from_libcudf(unique_ptr[aggregation] agg) diff --git a/python/cudf/cudf/_lib/pylibcudf/aggregation.pyx b/python/cudf/cudf/_lib/pylibcudf/aggregation.pyx index bde2643d5b1..0020a0c681d 100644 --- a/python/cudf/cudf/_lib/pylibcudf/aggregation.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/aggregation.pyx @@ -35,6 +35,8 @@ from cudf._lib.cpp.aggregation cimport ( make_variance_aggregation, rank_method, rank_percentage, + reduce_aggregation, + scan_aggregation, ) from cudf._lib.cpp.types cimport ( interpolation, @@ -57,10 +59,6 @@ from cudf._lib.cpp.aggregation import udf_type as UdfType # no-cython-lint from .types cimport DataType -# workaround for https://github.com/cython/cython/issues/3885 -ctypedef groupby_aggregation * gba_ptr -ctypedef groupby_scan_aggregation * gbsa_ptr - cdef class Aggregation: """A type of aggregation to perform. @@ -85,40 +83,47 @@ cdef class Aggregation: """Get the kind of the aggregation.""" return dereference(self.c_obj).kind - cdef unique_ptr[groupby_aggregation] clone_underlying_as_groupby(self) except *: - """Make a copy of the underlying aggregation that can be used in a groupby. + cdef void _unsupported_agg_error(self, str alg): + # Te functions calling this all use a dynamic cast between aggregation types, + # and the cast returning a null pointer is how we capture whether or not + # libcudf supports a given aggregation for a particular algorithm. + agg_repr = str(self.kind()).split(".")[1].title() + raise TypeError(f"{agg_repr} aggregations are not supported by {alg}") - This function will raise an exception if the aggregation is not supported as a - groupby aggregation. This failure to cast translates the per-algorithm - aggregation logic encoded in libcudf's type hierarchy into Python. - """ + cdef unique_ptr[groupby_aggregation] clone_underlying_as_groupby(self) except *: + """Make a copy of the aggregation that can be used in a groupby.""" cdef unique_ptr[aggregation] agg = dereference(self.c_obj).clone() cdef groupby_aggregation *agg_cast = dynamic_cast[gba_ptr](agg.get()) if agg_cast is NULL: - agg_repr = str(self.kind()).split(".")[1].title() - raise TypeError(f"{agg_repr} aggregations are not supported by groupby") + self._unsupported_agg_error("groupby") agg.release() return unique_ptr[groupby_aggregation](agg_cast) - # Ideally this function could reuse the code above, but Cython lacks the - # first-class support for type-aliasing and templates that would make it possible. cdef unique_ptr[groupby_scan_aggregation] clone_underlying_as_groupby_scan( self ) except *: - """Make a copy of the underlying aggregation that can be used in a groupby scan. - - This function will raise an exception if the aggregation is not supported as a - groupby scan aggregation. This failure to cast translates the per-algorithm - aggregation logic encoded in libcudf's type hierarchy into Python. - """ + """Make a copy of the aggregation that can be used in a groupby scan.""" cdef unique_ptr[aggregation] agg = dereference(self.c_obj).clone() cdef groupby_scan_aggregation *agg_cast = dynamic_cast[gbsa_ptr](agg.get()) if agg_cast is NULL: - agg_repr = str(self.kind()).split(".")[1].title() - raise TypeError(f"{agg_repr} scans are not supported by groupby") + self._unsupported_agg_error("groupby_scan") agg.release() return unique_ptr[groupby_scan_aggregation](agg_cast) + cdef const reduce_aggregation* view_underlying_as_reduce(self) except *: + """View the underlying aggregation as a reduce_aggregation.""" + cdef reduce_aggregation *agg_cast = dynamic_cast[ra_ptr](self.c_obj.get()) + if agg_cast is NULL: + self._unsupported_agg_error("reduce") + return agg_cast + + cdef const scan_aggregation* view_underlying_as_scan(self) except *: + """View the underlying aggregation as a scan_aggregation.""" + cdef scan_aggregation *agg_cast = dynamic_cast[sa_ptr](self.c_obj.get()) + if agg_cast is NULL: + self._unsupported_agg_error("scan") + return agg_cast + @staticmethod cdef Aggregation from_libcudf(unique_ptr[aggregation] agg): """Create a Python Aggregation from a libcudf aggregation.""" diff --git a/python/cudf/cudf/_lib/pylibcudf/reduce.pxd b/python/cudf/cudf/_lib/pylibcudf/reduce.pxd new file mode 100644 index 00000000000..a613e877ce2 --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/reduce.pxd @@ -0,0 +1,15 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from cudf._lib.cpp.reduce cimport scan_type + +from .aggregation cimport Aggregation +from .column cimport Column +from .scalar cimport Scalar +from .types cimport DataType + + +cpdef Scalar reduce(Column col, Aggregation agg, DataType data_type) + +cpdef Column scan(Column col, Aggregation agg, scan_type inclusive) + +cpdef tuple minmax(Column col) diff --git a/python/cudf/cudf/_lib/pylibcudf/reduce.pyx b/python/cudf/cudf/_lib/pylibcudf/reduce.pyx new file mode 100644 index 00000000000..d12da712fcf --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/reduce.pyx @@ -0,0 +1,108 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from cython.operator cimport dereference +from libcpp.memory cimport unique_ptr +from libcpp.utility cimport move, pair + +from cudf._lib.cpp cimport reduce as cpp_reduce +from cudf._lib.cpp.aggregation cimport reduce_aggregation, scan_aggregation +from cudf._lib.cpp.column.column cimport column +from cudf._lib.cpp.reduce cimport scan_type +from cudf._lib.cpp.scalar.scalar cimport scalar + +from .aggregation cimport Aggregation +from .column cimport Column +from .scalar cimport Scalar +from .types cimport DataType + +from cudf._lib.cpp.reduce import scan_type as ScanType # no-cython-lint + + +cpdef Scalar reduce(Column col, Aggregation agg, DataType data_type): + """Perform a reduction on a column + + For details, see ``cudf::reduce`` documentation. + + Parameters + ---------- + col : Column + The column to perform the reduction on. + agg : Aggregation + The aggregation to perform. + data_type : DataType + The data type of the result. + + Returns + ------- + Scalar + The result of the reduction. + """ + cdef unique_ptr[scalar] result + cdef const reduce_aggregation *c_agg = agg.view_underlying_as_reduce() + with nogil: + result = move( + cpp_reduce.cpp_reduce( + col.view(), + dereference(c_agg), + data_type.c_obj + ) + ) + return Scalar.from_libcudf(move(result)) + + +cpdef Column scan(Column col, Aggregation agg, scan_type inclusive): + """Perform a scan on a column + + For details, see ``cudf::scan`` documentation. + + Parameters + ---------- + col : Column + The column to perform the scan on. + agg : Aggregation + The aggregation to perform. + inclusive : scan_type + The type of scan to perform. + + Returns + ------- + Column + The result of the scan. + """ + cdef unique_ptr[column] result + cdef const scan_aggregation *c_agg = agg.view_underlying_as_scan() + with nogil: + result = move( + cpp_reduce.cpp_scan( + col.view(), + dereference(c_agg), + inclusive, + ) + ) + return Column.from_libcudf(move(result)) + + +cpdef tuple minmax(Column col): + """Compute the minimum and maximum of a column + + For details, see ``cudf::minmax`` documentation. + + Parameters + ---------- + col : Column + The column to compute the minimum and maximum of. + + Returns + ------- + tuple + A tuple of two Scalars, the first being the minimum and the second + being the maximum. + """ + cdef pair[unique_ptr[scalar], unique_ptr[scalar]] result + with nogil: + result = move(cpp_reduce.cpp_minmax(col.view())) + + return ( + Scalar.from_libcudf(move(result.first)), + Scalar.from_libcudf(move(result.second)), + ) diff --git a/python/cudf/cudf/_lib/reduce.pyx b/python/cudf/cudf/_lib/reduce.pyx index f11bacd5d1e..5767cc8eee1 100644 --- a/python/cudf/cudf/_lib/reduce.pyx +++ b/python/cudf/cudf/_lib/reduce.pyx @@ -1,27 +1,14 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. - -from cython.operator import dereference +# Copyright (c) 2020-2024, NVIDIA CORPORATION. import cudf from cudf.core.buffer import acquire_spill_lock -from libcpp.memory cimport unique_ptr -from libcpp.utility cimport move, pair - -from cudf._lib.aggregation cimport ( - ReduceAggregation, - ScanAggregation, - make_reduce_aggregation, - make_scan_aggregation, -) +from cudf._lib.aggregation cimport make_aggregation from cudf._lib.column cimport Column -from cudf._lib.cpp.column.column cimport column -from cudf._lib.cpp.column.column_view cimport column_view -from cudf._lib.cpp.reduce cimport cpp_minmax, cpp_reduce, cpp_scan, scan_type -from cudf._lib.cpp.scalar.scalar cimport scalar -from cudf._lib.cpp.types cimport data_type from cudf._lib.scalar cimport DeviceScalar -from cudf._lib.types cimport dtype_to_data_type, is_decimal_type_id +from cudf._lib.types cimport dtype_to_pylibcudf_type, is_decimal_type_id + +from cudf._lib import pylibcudf @acquire_spill_lock() @@ -45,13 +32,6 @@ def reduce(reduction_op, Column incol, dtype=None, **kwargs): else incol._reduction_result_dtype(reduction_op) ) - cdef column_view c_incol_view = incol.view() - cdef unique_ptr[scalar] c_result - cdef ReduceAggregation cython_agg = make_reduce_aggregation( - reduction_op, kwargs) - - cdef data_type c_out_dtype = dtype_to_data_type(col_dtype) - # check empty case if len(incol) <= incol.null_count: if reduction_op == 'sum' or reduction_op == 'sum_of_squares': @@ -63,22 +43,20 @@ def reduce(reduction_op, Column incol, dtype=None, **kwargs): return cudf.utils.dtypes._get_nan_for_dtype(col_dtype) - with nogil: - c_result = move(cpp_reduce( - c_incol_view, - dereference(cython_agg.c_obj), - c_out_dtype - )) + result = pylibcudf.reduce.reduce( + incol.to_pylibcudf(mode="read"), + make_aggregation(reduction_op, kwargs).c_obj, + dtype_to_pylibcudf_type(col_dtype), + ) - if is_decimal_type_id(c_result.get()[0].type().id()): - scale = -c_result.get()[0].type().scale() + if is_decimal_type_id(result.type().id()): + scale = -result.type().scale() precision = _reduce_precision(col_dtype, reduction_op, len(incol)) - py_result = DeviceScalar.from_unique_ptr( - move(c_result), dtype=col_dtype.__class__(precision, scale) - ) - else: - py_result = DeviceScalar.from_unique_ptr(move(c_result)) - return py_result.value + return DeviceScalar.from_pylibcudf( + result, + dtype=col_dtype.__class__(precision, scale), + ).value + return DeviceScalar.from_pylibcudf(result).value @acquire_spill_lock() @@ -95,22 +73,14 @@ def scan(scan_op, Column incol, inclusive, **kwargs): inclusive: bool Flag for including nulls in relevant scan """ - cdef column_view c_incol_view = incol.view() - cdef unique_ptr[column] c_result - cdef ScanAggregation cython_agg = make_scan_aggregation(scan_op, kwargs) - - cdef scan_type c_inclusive = \ - scan_type.INCLUSIVE if inclusive else scan_type.EXCLUSIVE - - with nogil: - c_result = move(cpp_scan( - c_incol_view, - dereference(cython_agg.c_obj), - c_inclusive - )) - - py_result = Column.from_unique_ptr(move(c_result)) - return py_result + return Column.from_pylibcudf( + pylibcudf.reduce.scan( + incol.to_pylibcudf(mode="read"), + make_aggregation(scan_op, kwargs).c_obj, + pylibcudf.reduce.ScanType.INCLUSIVE if inclusive + else pylibcudf.reduce.ScanType.EXCLUSIVE, + ) + ) @acquire_spill_lock() @@ -127,18 +97,10 @@ def minmax(Column incol): ------- A pair of ``(min, max)`` values of ``incol`` """ - cdef column_view c_incol_view = incol.view() - cdef pair[unique_ptr[scalar], unique_ptr[scalar]] c_result - - with nogil: - c_result = move(cpp_minmax(c_incol_view)) - - py_result_min = DeviceScalar.from_unique_ptr(move(c_result.first)) - py_result_max = DeviceScalar.from_unique_ptr(move(c_result.second)) - + min, max = pylibcudf.reduce.minmax(incol.to_pylibcudf(mode="read")) return ( - cudf.Scalar.from_device_scalar(py_result_min), - cudf.Scalar.from_device_scalar(py_result_max) + cudf.Scalar.from_device_scalar(DeviceScalar.from_pylibcudf(min)), + cudf.Scalar.from_device_scalar(DeviceScalar.from_pylibcudf(max)), ) From 8d2b0ed71b16818cbb32fc90fb750aaec61eefed Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Tue, 6 Feb 2024 16:40:21 -0800 Subject: [PATCH 211/384] Fix the bounce buffer size in ORC writer (#14947) Closes #14932 ORC writer uses uncompressed stream sizes when allocating the bounce buffer. This can lead to issues when all uncompressed streams are larger than the GDS threshold, but compressed size is not. In this scenario, the bounce buffer is not allocated, and writing the compressed stream through the bounce buffer causes a crash. This PR moves the computation of the bounce buffer size until after compression, so it works with correct stream sizes. Authors: - Vukasin Milovanovic (https://github.com/vuule) Approvers: - Nghia Truong (https://github.com/ttnghia) - Bradley Dice (https://github.com/bdice) --- cpp/src/io/orc/writer_impl.cu | 47 +++++++++++++++++++---------------- cpp/tests/io/orc_test.cpp | 16 ++++++++++++ 2 files changed, 41 insertions(+), 22 deletions(-) diff --git a/cpp/src/io/orc/writer_impl.cu b/cpp/src/io/orc/writer_impl.cu index edc40391bfa..b0702d93d34 100644 --- a/cpp/src/io/orc/writer_impl.cu +++ b/cpp/src/io/orc/writer_impl.cu @@ -2346,29 +2346,20 @@ auto convert_table_to_orc_data(table_view const& input, auto const padded_block_header_size = util::round_up_unsafe(block_header_size, compressed_block_align); - auto bounce_buffer = [&]() { - size_t max_stream_size = 0; - bool all_device_write = true; - - for (auto& ss : strm_descs.host_view().flat_view()) { - if (!out_sink.is_device_write_preferred(ss.stream_size)) { all_device_write = false; } - size_t stream_size = ss.stream_size; - if (compression_kind != NONE) { - ss.first_block = num_compressed_blocks; - ss.bfr_offset = compressed_bfr_size; - - auto num_blocks = - std::max((stream_size + compression_blocksize - 1) / compression_blocksize, 1); - stream_size += num_blocks * block_header_size; - num_compressed_blocks += num_blocks; - compressed_bfr_size += - (padded_block_header_size + padded_max_compressed_block_size) * num_blocks; - } - max_stream_size = std::max(max_stream_size, stream_size); + for (auto& ss : strm_descs.host_view().flat_view()) { + size_t stream_size = ss.stream_size; + if (compression_kind != NONE) { + ss.first_block = num_compressed_blocks; + ss.bfr_offset = compressed_bfr_size; + + auto num_blocks = + std::max((stream_size + compression_blocksize - 1) / compression_blocksize, 1); + stream_size += num_blocks * block_header_size; + num_compressed_blocks += num_blocks; + compressed_bfr_size += + (padded_block_header_size + padded_max_compressed_block_size) * num_blocks; } - - return cudf::detail::pinned_host_vector(all_device_write ? 0 : max_stream_size); - }(); + } // Compress the data streams rmm::device_uvector compressed_data(compressed_bfr_size, stream); @@ -2399,6 +2390,18 @@ auto convert_table_to_orc_data(table_view const& input, comp_results.device_to_host_sync(stream); } + auto const max_out_stream_size = [&]() { + uint32_t max_stream_size = 0; + for (auto const& ss : strm_descs.host_view().flat_view()) { + if (!out_sink.is_device_write_preferred(ss.stream_size)) { + max_stream_size = std::max(max_stream_size, ss.stream_size); + } + } + return max_stream_size; + }(); + + cudf::detail::pinned_host_vector bounce_buffer(max_out_stream_size); + auto intermediate_stats = gather_statistic_blobs(stats_freq, orc_table, segmentation, stream); return std::tuple{std::move(enc_data), diff --git a/cpp/tests/io/orc_test.cpp b/cpp/tests/io/orc_test.cpp index 2ae6edc6c7d..305ec404a71 100644 --- a/cpp/tests/io/orc_test.cpp +++ b/cpp/tests/io/orc_test.cpp @@ -2055,4 +2055,20 @@ TEST_F(OrcStatisticsTest, Empty) EXPECT_EQ(ts6.count[0], 0); } +TEST_F(OrcWriterTest, BounceBufferBug) +{ + auto sequence = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i % 100; }); + + constexpr auto num_rows = 150000; + column_wrapper col(sequence, + sequence + num_rows); + table_view expected({col}); + + auto filepath = temp_env->get_temp_filepath("BounceBufferBug.orc"); + cudf::io::orc_writer_options out_opts = + cudf::io::orc_writer_options::builder(cudf::io::sink_info{filepath}, expected) + .compression(cudf::io::compression_type::ZSTD); + cudf::io::write_orc(out_opts); +} + CUDF_TEST_PROGRAM_MAIN() From c78033b7e77bebf9596d971b41714e1c4f29bb8f Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 6 Feb 2024 17:38:00 -0800 Subject: [PATCH 212/384] Exclude tests from builds (#14981) --- python/cudf/pyproject.toml | 1 + python/cudf_kafka/pyproject.toml | 1 + python/custreamz/pyproject.toml | 1 + python/dask_cudf/pyproject.toml | 3 +++ 4 files changed, 6 insertions(+) diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index ce30230398f..872b92d670d 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -168,6 +168,7 @@ build-dir = "build/{wheel_tag}" cmake.build-type = "Release" cmake.minimum-version = "3.26.4" ninja.make-fallback = true +sdist.exclude = ["*tests*"] sdist.reproducible = true wheel.packages = ["cudf"] diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index d6574c32873..8b96703df79 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -87,6 +87,7 @@ build-dir = "build/{wheel_tag}" cmake.build-type = "Release" cmake.minimum-version = "3.26.4" ninja.make-fallback = true +sdist.exclude = ["*tests*"] sdist.reproducible = true wheel.packages = ["cudf_kafka"] diff --git a/python/custreamz/pyproject.toml b/python/custreamz/pyproject.toml index 03ec079a890..e07c6c8c6a2 100644 --- a/python/custreamz/pyproject.toml +++ b/python/custreamz/pyproject.toml @@ -56,6 +56,7 @@ include = [ "custreamz", "custreamz.*", ] +exclude = ["*tests*"] [tool.isort] line_length = 79 diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index c3185bcb793..09e4accd427 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -57,6 +57,9 @@ license-files = ["LICENSE"] [tool.setuptools.dynamic] version = {file = "dask_cudf/VERSION"} +[tool.setuptools.packages.find] +exclude = ["*tests*"] + [tool.isort] line_length = 79 multi_line_output = 3 From 41b9e5e73d4481804517787001f7f8d2b92a3cb2 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Wed, 7 Feb 2024 10:08:41 -0500 Subject: [PATCH 213/384] Use int64 offset types for accessing code-points in nvtext::normalize (#14868) Changes some internal offset arrays used for managing temporary unicode code-points to int64 type. This effects the nvtext normalize and subword-tokenizer functions. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Karthikeyan (https://github.com/karthikeyann) URL: https://github.com/rapidsai/cudf/pull/14868 --- cpp/src/text/normalize.cu | 12 +++---- cpp/src/text/subword/data_normalizer.cu | 35 ++++++++++--------- .../text/subword/detail/data_normalizer.hpp | 16 ++++----- .../text/subword/detail/tokenizer_utils.cuh | 4 +-- .../subword/detail/wordpiece_tokenizer.hpp | 16 ++++----- cpp/src/text/subword/subword_tokenize.cu | 9 ++--- cpp/src/text/subword/wordpiece_tokenizer.cu | 8 ++--- 7 files changed, 42 insertions(+), 58 deletions(-) diff --git a/cpp/src/text/normalize.cu b/cpp/src/text/normalize.cu index d46ca25835f..6044689473c 100644 --- a/cpp/src/text/normalize.cu +++ b/cpp/src/text/normalize.cu @@ -107,7 +107,7 @@ constexpr uint32_t UTF8_3BYTE = 0x01'0000; struct codepoint_to_utf8_fn { cudf::column_device_view const d_strings; // input strings uint32_t const* cp_data; // full code-point array - cudf::size_type const* d_cp_offsets{}; // offsets to each string's code-point array + int64_t const* d_cp_offsets{}; // offsets to each string's code-point array cudf::size_type* d_offsets{}; // offsets for the output strings char* d_chars{}; // buffer for the output strings column @@ -207,11 +207,7 @@ std::unique_ptr normalize_characters(cudf::strings_column_view con auto const cp_metadata = get_codepoint_metadata(stream); auto const aux_table = get_aux_codepoint_data(stream); auto const normalizer = data_normalizer(cp_metadata.data(), aux_table.data(), do_lower_case); - auto const offsets = strings.offsets(); - auto const d_offsets = offsets.data() + strings.offset(); - auto const offset = cudf::detail::get_value(offsets, strings.offset(), stream); - auto const d_chars = strings.chars_begin(stream) + offset; - return normalizer.normalize(d_chars, d_offsets, strings.size(), stream); + return normalizer.normalize(strings, stream); }(); CUDF_EXPECTS( @@ -222,8 +218,8 @@ std::unique_ptr normalize_characters(cudf::strings_column_view con // convert the result into a strings column // - the cp_chars are the new 4-byte code-point values for all the characters in the output // - the cp_offsets identify which code-points go with which strings - uint32_t const* cp_chars = result.first->data(); - cudf::size_type const* cp_offsets = result.second->data(); + auto const cp_chars = result.first->data(); + auto const cp_offsets = result.second->data(); auto d_strings = cudf::column_device_view::create(strings.parent(), stream); diff --git a/cpp/src/text/subword/data_normalizer.cu b/cpp/src/text/subword/data_normalizer.cu index c83bc2e318f..a56d71cf951 100644 --- a/cpp/src/text/subword/data_normalizer.cu +++ b/cpp/src/text/subword/data_normalizer.cu @@ -17,8 +17,10 @@ #include #include +#include #include #include +#include #include #include @@ -274,20 +276,19 @@ data_normalizer::data_normalizer(codepoint_metadata_type const* cp_metadata, { } -uvector_pair data_normalizer::normalize(char const* d_strings, - cudf::size_type const* d_offsets, - cudf::size_type num_strings, +uvector_pair data_normalizer::normalize(cudf::strings_column_view const& input, rmm::cuda_stream_view stream) const { - if (num_strings == 0) { + if (input.is_empty()) { return uvector_pair{std::make_unique>(0, stream), - std::make_unique>(0, stream)}; + std::make_unique>(0, stream)}; } // copy offsets to working memory - auto const num_offsets = num_strings + 1; - auto d_strings_offsets = - std::make_unique>(num_offsets, stream); + auto const num_offsets = input.size() + 1; + auto d_strings_offsets = std::make_unique>(num_offsets, stream); + auto const d_offsets = + cudf::detail::offsetalator_factory::make_input_iterator(input.offsets(), input.offset()); thrust::transform(rmm::exec_policy(stream), thrust::counting_iterator(0), thrust::counting_iterator(num_offsets), @@ -296,20 +297,22 @@ uvector_pair data_normalizer::normalize(char const* d_strings, auto const offset = d_offsets[0]; // adjust for any offset to the offsets return d_offsets[idx] - offset; }); - auto const bytes_count = d_strings_offsets->element(num_strings, stream); + auto const bytes_count = d_strings_offsets->element(input.size(), stream); if (bytes_count == 0) { // if no bytes, nothing to do return uvector_pair{std::make_unique>(0, stream), - std::make_unique>(0, stream)}; + std::make_unique>(0, stream)}; } - cudf::detail::grid_1d const grid{bytes_count, THREADS_PER_BLOCK, 1}; - size_t const threads_on_device = grid.num_threads_per_block * grid.num_blocks; + int64_t const threads_per_block = THREADS_PER_BLOCK; + size_t const num_blocks = cudf::util::div_rounding_up_safe(bytes_count, threads_per_block); + size_t const threads_on_device = threads_per_block * num_blocks; size_t const max_new_char_total = MAX_NEW_CHARS * threads_on_device; auto d_code_points = std::make_unique>(max_new_char_total, stream); rmm::device_uvector d_chars_per_thread(threads_on_device, stream); - - kernel_data_normalizer<<>>( + auto const d_strings = input.chars_begin(stream) + cudf::strings::detail::get_offset_value( + input.offsets(), input.offset(), stream); + kernel_data_normalizer<<>>( reinterpret_cast(d_strings), bytes_count, d_cp_metadata, @@ -335,10 +338,10 @@ uvector_pair data_normalizer::normalize(char const* d_strings, thrust::for_each_n( rmm::exec_policy(stream), thrust::make_counting_iterator(1), - num_strings, + input.size(), update_strings_lengths_fn{d_chars_per_thread.data(), d_strings_offsets->data()}); - auto const num_chars = d_strings_offsets->element(num_strings, stream); + auto const num_chars = d_strings_offsets->element(input.size(), stream); d_code_points->resize(num_chars, stream); // should be smaller than original allocated size // return the normalized code points and the new offsets diff --git a/cpp/src/text/subword/detail/data_normalizer.hpp b/cpp/src/text/subword/detail/data_normalizer.hpp index fb507b88e7e..897a0f31e15 100644 --- a/cpp/src/text/subword/detail/data_normalizer.hpp +++ b/cpp/src/text/subword/detail/data_normalizer.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,13 +18,14 @@ #include +#include #include #include #include using uvector_pair = std::pair>, - std::unique_ptr>>; + std::unique_ptr>>; namespace nvtext { namespace detail { @@ -74,21 +75,16 @@ class data_normalizer { * characters in the text after running normalization. The second pointer is to the * offsets of the strings in the code point array. That is, string `i` starts at * `result.second->data()[i]`. - * This array will always be of length `num_strings + 1` since we need one entry + * This array will always be of length `input.size() + 1` since we need one entry * for each input and a last entry which has the total number of bytes. * - * @param d_strings A vector of strings which MUST be encoded in the UTF-8 format. - * @param d_offsets A vector of byte offsets to the beginning of individual strings in - * the `d_strings` parameter. - * @param num_strings The number of strings identified in `d_strings`. + * @param input Strings to normalize * @param stream CUDA stream used for device memory operations and kernel launches. * @return Two pointers to GPU data buffers. The first is a pointer * to the code points array and the second is a pointer to the offsets * used to locate the code points for each string. */ - uvector_pair normalize(char const* d_strings, - cudf::size_type const* d_offsets, - cudf::size_type num_strings, + uvector_pair normalize(cudf::strings_column_view const& input, rmm::cuda_stream_view stream) const; private: diff --git a/cpp/src/text/subword/detail/tokenizer_utils.cuh b/cpp/src/text/subword/detail/tokenizer_utils.cuh index 7cc0e7c0e24..f2317518663 100644 --- a/cpp/src/text/subword/detail/tokenizer_utils.cuh +++ b/cpp/src/text/subword/detail/tokenizer_utils.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -49,7 +49,7 @@ constexpr int THREADS_PER_BLOCK = 64; */ struct update_strings_lengths_fn { uint32_t const* d_chars_up_to_idx; - cudf::size_type* d_offsets; + int64_t* d_offsets; __device__ void operator()(cudf::size_type idx) { diff --git a/cpp/src/text/subword/detail/wordpiece_tokenizer.hpp b/cpp/src/text/subword/detail/wordpiece_tokenizer.hpp index e191890eeca..71e00c2e852 100644 --- a/cpp/src/text/subword/detail/wordpiece_tokenizer.hpp +++ b/cpp/src/text/subword/detail/wordpiece_tokenizer.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,8 @@ #include +#include + #include namespace nvtext { @@ -74,17 +76,11 @@ class wordpiece_tokenizer { * * This class is simply a wrapper around the basic and word piece tokenizers. * - * @param d_strings A vector of strings which MUST be encoded in the utf8 format. - * @param d_offsets A vector of byte offsets to the beginning of individual strings in - * the `d_strings` parameter. - * @param num_strings The number of strings in `d_strings`. - * @param stream CUDA stream used for device memory operations and kernel launches. + * @param input Strings to tokenize + * @param stream CUDA stream used for device memory operations and kernel launches * @return Pointer to token-ids and token-id offsets */ - uvector_pair tokenize(char const* d_strings, - cudf::size_type const* d_offsets, - cudf::size_type num_strings, - rmm::cuda_stream_view stream); + uvector_pair tokenize(cudf::strings_column_view const& input, rmm::cuda_stream_view stream); private: /** diff --git a/cpp/src/text/subword/subword_tokenize.cu b/cpp/src/text/subword/subword_tokenize.cu index c9592e5cc48..6d40882659a 100644 --- a/cpp/src/text/subword/subword_tokenize.cu +++ b/cpp/src/text/subword/subword_tokenize.cu @@ -59,7 +59,7 @@ namespace { CUDF_KERNEL void kernel_compute_tensor_metadata( // input uint32_t const* token_ids, - cudf::size_type const* offsets, + int64_t const* offsets, uint32_t const* row2tensor, uint32_t const* row2row_within_tensor, uint32_t max_sequence_length, @@ -183,16 +183,11 @@ tokenizer_result subword_tokenize(cudf::strings_column_view const& strings, "max_sequence_length times number of input rows exceeds the column size limit", std::overflow_error); - auto const offsets = strings.offsets(); - auto const d_offsets = offsets.data() + strings.offset(); - auto const offset = cudf::detail::get_value(offsets, strings.offset(), stream); - auto const d_chars = strings.chars_begin(stream) + offset; - // Create tokenizer wordpiece_tokenizer tokenizer( vocab_table, max_sequence_length, stride, do_truncate, do_lower_case); // Run tokenizer - auto const tokens = tokenizer.tokenize(d_chars, d_offsets, strings_count, stream); + auto const tokens = tokenizer.tokenize(strings, stream); // assign output components auto device_token_ids = tokens.first->data(); auto device_offsets = tokens.second->data(); diff --git a/cpp/src/text/subword/wordpiece_tokenizer.cu b/cpp/src/text/subword/wordpiece_tokenizer.cu index d2804af5f8b..6e0c324db7d 100644 --- a/cpp/src/text/subword/wordpiece_tokenizer.cu +++ b/cpp/src/text/subword/wordpiece_tokenizer.cu @@ -132,7 +132,7 @@ CUDF_KERNEL void init_data_and_mark_word_start_and_ends(uint32_t const* code_poi * @param num_strings The total number of strings to be processed. */ CUDF_KERNEL void mark_string_start_and_ends(uint32_t const* code_points, - cudf::size_type const* strings_offsets, + int64_t const* strings_offsets, uint32_t* start_word_indices, uint32_t* end_word_indices, uint32_t num_strings) @@ -419,12 +419,10 @@ wordpiece_tokenizer::wordpiece_tokenizer(hashed_vocabulary const& vocab_table, { } -uvector_pair wordpiece_tokenizer::tokenize(char const* d_strings, - cudf::size_type const* d_offsets, - cudf::size_type num_strings, +uvector_pair wordpiece_tokenizer::tokenize(cudf::strings_column_view const& input, rmm::cuda_stream_view stream) { - auto cps_and_offsets = normalizer.normalize(d_strings, d_offsets, num_strings, stream); + auto cps_and_offsets = normalizer.normalize(input, stream); tokenize(cps_and_offsets, stream); return uvector_pair(std::move(cps_and_offsets.first), std::move(cps_and_offsets.second)); } From 63a1c9ea8f87556de28f86c9b25f1b2b63a64e2c Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 7 Feb 2024 13:48:03 -0600 Subject: [PATCH 214/384] Ensure that `ctest` is called with `--no-tests=error`. (#14983) This PR ensures that all calls to `ctest` include the flag `--no-tests=error`. See https://github.com/rapidsai/build-planning/issues/18. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Jake Awe (https://github.com/AyodeAwe) URL: https://github.com/rapidsai/cudf/pull/14983 --- ci/test_cpp.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 30172b76f01..7119a79f4de 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. source "$(dirname "$0")/test_cpp_common.sh" @@ -12,14 +12,14 @@ export GTEST_OUTPUT=xml:${RAPIDS_TESTS_DIR}/ pushd $CONDA_PREFIX/bin/gtests/libcudf/ rapids-logger "Run libcudf gtests" -ctest -j20 --output-on-failure +ctest -j20 --output-on-failure --no-tests=error SUITEERROR=$? popd if (( ${SUITEERROR} == 0 )); then pushd $CONDA_PREFIX/bin/gtests/libcudf_kafka/ rapids-logger "Run libcudf_kafka gtests" - ctest -j20 --output-on-failure + ctest -j20 --output-on-failure --no-tests=error SUITEERROR=$? popd fi From 285b8362f391cb8babf57d0dd7b42cf90858862c Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 7 Feb 2024 15:46:02 -0600 Subject: [PATCH 215/384] Filter all `DeprecationWarning`'s by `ArrowTable.to_pandas()` (#14989) This PR filters all `DeprecationWarning`'s that are being originated by `ArrowTable.to_pandas` Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/14989 --- .pre-commit-config.yaml | 3 ++- python/cudf/cudf/tests/pytest.ini | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ccda2596031..d302543368e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -95,7 +95,8 @@ repos: # DeprecationWarning: https://github.com/pandas-dev/pandas/issues/54970 exclude: | (?x)^( - ^python/cudf/cudf/core/dtypes.py + ^python/cudf/cudf/core/dtypes.py| + ^python/cudf/cudf/tests/pytest.ini ) - id: no-programmatic-xfail name: no-programmatic-xfail diff --git a/python/cudf/cudf/tests/pytest.ini b/python/cudf/cudf/tests/pytest.ini index 1f38ffcb726..36ccb434bb2 100644 --- a/python/cudf/cudf/tests/pytest.ini +++ b/python/cudf/cudf/tests/pytest.ini @@ -8,3 +8,5 @@ filterwarnings = error ignore:::.*xdist.* ignore:::.*pytest.* + ignore:Passing a BlockManager to DataFrame is deprecated:DeprecationWarning + # Above deprecation warning comes from Pyarrow Table.to_pandas() with pandas-2.2+ From 73bac8329c659fdaf0c54ae250dca4b46f55ad8a Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 8 Feb 2024 02:02:51 -0600 Subject: [PATCH 216/384] Fix `DataFrame.sort_index` to respect `ignore_index` on all axis (#14995) This PR fixes `DataFrame.sort_index` to properly ignore indexes for all values of `axis`. This is fixed in pandas-2.2, hence xfailing the tests with a version check. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/14995 --- python/cudf/cudf/core/indexed_frame.py | 7 ++++-- python/cudf/cudf/tests/test_dataframe.py | 29 +++++++++++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 659e323c57d..aa75b0d825e 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -2608,12 +2608,15 @@ def sort_index( and self._data.multiindex ): out._set_column_names_like(self) + if ignore_index: + out = out.reset_index(drop=True) else: labels = sorted(self._data.names, reverse=not ascending) out = self[labels] + if ignore_index: + out._data.rangeindex = True + out._data.names = list(range(len(self._data.names))) - if ignore_index is True: - out = out.reset_index(drop=True) return self._mimic_inplace(out, inplace=inplace) def memory_usage(self, index=True, deep=False): diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index a0f6c4c3cfc..f9af0d10713 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -25,7 +25,12 @@ import cudf from cudf.api.extensions import no_default -from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210, PANDAS_LT_203 +from cudf.core._compat import ( + PANDAS_GE_200, + PANDAS_GE_210, + PANDAS_GE_220, + PANDAS_LT_203, +) from cudf.core.buffer.spill_manager import get_global_manager from cudf.core.column import column from cudf.errors import MixedTypeError @@ -3562,8 +3567,16 @@ def test_dataframe_empty_sort_index(): @pytest.mark.parametrize("inplace", [True, False]) @pytest.mark.parametrize("na_position", ["first", "last"]) def test_dataframe_sort_index( - index, axis, ascending, inplace, ignore_index, na_position + request, index, axis, ascending, inplace, ignore_index, na_position ): + request.applymarker( + pytest.mark.xfail( + condition=not PANDAS_GE_220 + and axis in (1, "columns") + and ignore_index, + reason="Bug fixed in pandas-2.2", + ) + ) pdf = pd.DataFrame( {"b": [1, 3, 2], "a": [1, 4, 3], "c": [4, 1, 5]}, index=index, @@ -3618,12 +3631,22 @@ def test_dataframe_mulitindex_sort_index( ): request.applymarker( pytest.mark.xfail( - condition=axis in (1, "columns") + condition=not PANDAS_GE_220 + and axis in (1, "columns") and ignore_index and not (level is None and not ascending), reason="https://github.com/pandas-dev/pandas/issues/56478", ) ) + request.applymarker( + pytest.mark.xfail( + condition=axis in (1, "columns") + and level is None + and not ascending + and ignore_index, + reason="https://github.com/pandas-dev/pandas/issues/57293", + ) + ) pdf = pd.DataFrame( { "b": [1.0, 3.0, np.nan], From 7f28f2f55253bcc6cf109242f6a2a126688cb16e Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 8 Feb 2024 03:19:19 -1000 Subject: [PATCH 217/384] Deprecate groupby fillna (#15000) Deprecated in pandas 2.2 https://github.com/pandas-dev/pandas/pull/55719 Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15000 --- python/cudf/cudf/core/groupby/groupby.py | 17 ++++++----------- python/cudf/cudf/tests/test_groupby.py | 23 +++++++++++++---------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 78593f20421..9e8d9908df2 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -2227,6 +2227,12 @@ def fillna( ------- DataFrame or Series """ + warnings.warn( + "groupby fillna is deprecated and " + "will be removed in a future version. Use groupby ffill or groupby bfill " + "for forward or backward filling instead.", + FutureWarning, + ) if inplace: raise NotImplementedError("Does not support inplace yet.") if limit is not None: @@ -2244,17 +2250,6 @@ def fillna( if method is not None: if method not in {"ffill", "bfill"}: raise ValueError("Method can only be of 'ffill', 'bfill'.") - # Do not remove until pandas 3.0 support is added. - assert ( - PANDAS_LT_300 - ), "Need to drop after pandas-3.0 support is added." - warnings.warn( - f"{type(self).__name__}.fillna with 'method' is " - "deprecated and will raise in a future version. " - "Use obj.ffill() or obj.bfill() instead.", - FutureWarning, - ) - return getattr(self, method, limit)() values = self.obj.__class__._from_data( diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index a0b86d735cc..bd48e5bfd31 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -20,7 +20,7 @@ import cudf from cudf import DataFrame, Series from cudf.api.extensions import no_default -from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210 +from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210, PANDAS_GE_220 from cudf.core.udf._ops import arith_ops, comparison_ops, unary_ops from cudf.core.udf.groupby_typing import SUPPORTED_GROUPBY_NUMPY_TYPES from cudf.core.udf.utils import UDFError, precompiled @@ -2745,10 +2745,10 @@ def test_groupby_fillna_multi_value(nelem): } # cudf can't fillna with a pandas.Timedelta type fill_values["4"] = fill_values["4"].to_numpy() - - expect = pdf.groupby(key_col).fillna(value=fill_values) - - got = gdf.groupby(key_col).fillna(value=fill_values) + with expect_warning_if(PANDAS_GE_220): + expect = pdf.groupby(key_col).fillna(value=fill_values) + with pytest.warns(FutureWarning): + got = gdf.groupby(key_col).fillna(value=fill_values) assert_groupby_results_equal(expect[value_cols], got[value_cols]) @@ -2791,11 +2791,12 @@ def test_groupby_fillna_multi_value_df(nelem): # cudf can't fillna with a pandas.Timedelta type fill_values["4"] = fill_values["4"].to_numpy() fill_values = pd.DataFrame(fill_values, index=pdf.index) - - expect = pdf.groupby(key_col).fillna(value=fill_values) + with expect_warning_if(PANDAS_GE_220): + expect = pdf.groupby(key_col).fillna(value=fill_values) fill_values = cudf.from_pandas(fill_values) - got = gdf.groupby(key_col).fillna(value=fill_values) + with pytest.warns(FutureWarning): + got = gdf.groupby(key_col).fillna(value=fill_values) assert_groupby_results_equal(expect[value_cols], got[value_cols]) @@ -2812,11 +2813,13 @@ def test_groupby_various_by_fillna(by, data, args): ps = pd.Series(data) gs = cudf.from_pandas(ps) - with expect_warning_if(PANDAS_GE_210 and "method" in args): + with expect_warning_if( + (PANDAS_GE_210 and "method" in args) or PANDAS_GE_220 + ): expect = ps.groupby(by).fillna(**args) if isinstance(by, pd.Grouper): by = cudf.Grouper(level=by.level) - with expect_warning_if("method" in args): + with pytest.warns(FutureWarning): got = gs.groupby(by).fillna(**args) assert_groupby_results_equal(expect, got, check_dtype=False) From 03f63ec842bfe6a4e4ff4b5f25698c12d5fecf5d Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 8 Feb 2024 03:41:34 -1000 Subject: [PATCH 218/384] Ensure to_* IO methods respect pandas 2.2 keyword only deprecation (#14999) This only really affected `to_hdf` Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/14999 --- python/cudf/cudf/_fuzz_testing/utils.py | 4 ++-- python/cudf/cudf/io/hdf.py | 2 +- python/cudf/cudf/tests/test_hdf.py | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/python/cudf/cudf/_fuzz_testing/utils.py b/python/cudf/cudf/_fuzz_testing/utils.py index 0c88c1aeacd..6e53195ac2d 100644 --- a/python/cudf/cudf/_fuzz_testing/utils.py +++ b/python/cudf/cudf/_fuzz_testing/utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. import random @@ -216,7 +216,7 @@ def pandas_to_avro(df, file_name=None, file_io_obj=None): schema = get_avro_schema(df) avro_schema = fastavro.parse_schema(schema) - records = df.to_dict("records") + records = df.to_dict(orient="records") records = convert_nulls_to_none(records, df) if file_name is not None: diff --git a/python/cudf/cudf/io/hdf.py b/python/cudf/cudf/io/hdf.py index 78e7df649cb..39f62a19f90 100644 --- a/python/cudf/cudf/io/hdf.py +++ b/python/cudf/cudf/io/hdf.py @@ -27,4 +27,4 @@ def to_hdf(path_or_buf, key, value, *args, **kwargs): "be GPU accelerated in the future" ) pd_value = value.to_pandas() - pd_value.to_hdf(path_or_buf, key, *args, **kwargs) + pd_value.to_hdf(path_or_buf, key=key, *args, **kwargs) diff --git a/python/cudf/cudf/tests/test_hdf.py b/python/cudf/cudf/tests/test_hdf.py index 063fffd948b..1ddd7f93c3e 100644 --- a/python/cudf/cudf/tests/test_hdf.py +++ b/python/cudf/cudf/tests/test_hdf.py @@ -63,7 +63,7 @@ def hdf_files(request, tmp_path_factory, pdf): pdf = pdf.drop("col_category", axis=1) fname_df = tmp_path_factory.mktemp("hdf") / "test_df.hdf" - pdf.to_hdf(fname_df, "hdf_df_tests", format=request.param) + pdf.to_hdf(fname_df, key="hdf_df_tests", format=request.param) fname_series = {} for column in pdf.columns: @@ -71,7 +71,7 @@ def hdf_files(request, tmp_path_factory, pdf): tmp_path_factory.mktemp("hdf") / "test_series.hdf" ) pdf[column].to_hdf( - fname_series[column], "hdf_series_tests", format=request.param + fname_series[column], key="hdf_series_tests", format=request.param ) return (fname_df, fname_series, request.param, nrows) @@ -116,8 +116,8 @@ def test_hdf_writer(tmpdir, pdf, gdf, complib, format): pdf_df_fname = tmpdir.join("pdf_df.hdf") gdf_df_fname = tmpdir.join("gdf_df.hdf") - pdf.to_hdf(pdf_df_fname, "hdf_tests", format=format, complib=complib) - gdf.to_hdf(gdf_df_fname, "hdf_tests", format=format, complib=complib) + pdf.to_hdf(pdf_df_fname, key="hdf_tests", format=format, complib=complib) + gdf.to_hdf(gdf_df_fname, key="hdf_tests", format=format, complib=complib) assert os.path.exists(pdf_df_fname) assert os.path.exists(gdf_df_fname) @@ -135,10 +135,10 @@ def test_hdf_writer(tmpdir, pdf, gdf, complib, format): gdf_series_fname = tmpdir.join(column + "_" + "gdf_series.hdf") pdf[column].to_hdf( - pdf_series_fname, "hdf_tests", format=format, complib=complib + pdf_series_fname, key="hdf_tests", format=format, complib=complib ) gdf[column].to_hdf( - gdf_series_fname, "hdf_tests", format=format, complib=complib + gdf_series_fname, key="hdf_tests", format=format, complib=complib ) assert os.path.exists(pdf_series_fname) From 47d28a0850168ddc54180d075dd51199bce85674 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Thu, 8 Feb 2024 09:03:05 -0500 Subject: [PATCH 219/384] Use offsetalator in cudf::strings::split functions (#14757) Adds offsetalator in place of hardcoded offset type arrays to the strings split functions: - `cudf::strings::split()` - `cudf::strings::rsplit()` - `cudf::strings::split_record()` - `cudf::strings::rsplit_record()` - `cudf::strings::split_re()` - `cudf::strings::rsplit_re()` - `cudf::strings::split_record_re()` - `cudf::strings::rsplit_record_re()` Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Mike Wilson (https://github.com/hyperbolic2346) URL: https://github.com/rapidsai/cudf/pull/14757 --- cpp/src/strings/split/split.cu | 18 ++--- cpp/src/strings/split/split.cuh | 102 +++++++++++++------------- cpp/src/strings/split/split_re.cu | 99 ++++++++++++------------- cpp/src/strings/split/split_record.cu | 5 +- 4 files changed, 112 insertions(+), 112 deletions(-) diff --git a/cpp/src/strings/split/split.cu b/cpp/src/strings/split/split.cu index c87c36ba3b9..fbab5220383 100644 --- a/cpp/src/strings/split/split.cu +++ b/cpp/src/strings/split/split.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -123,7 +122,7 @@ std::unique_ptr
split_fn(strings_column_view const& input, // builds the offsets and the vector of all tokens auto [offsets, tokens] = split_helper(input, tokenizer, stream, mr); - auto const d_offsets = offsets->view().template data(); + auto const d_offsets = cudf::detail::offsetalator_factory::make_input_iterator(offsets->view()); auto const d_tokens = tokens.data(); // compute the maximum number of tokens for any string @@ -132,7 +131,7 @@ std::unique_ptr
split_fn(strings_column_view const& input, thrust::make_counting_iterator(0), thrust::make_counting_iterator(input.size()), cuda::proclaim_return_type([d_offsets] __device__(auto idx) -> size_type { - return d_offsets[idx + 1] - d_offsets[idx]; + return static_cast(d_offsets[idx + 1] - d_offsets[idx]); }), 0, thrust::maximum{}); @@ -144,7 +143,7 @@ std::unique_ptr
split_fn(strings_column_view const& input, cuda::proclaim_return_type( [d_tokens, d_offsets, col] __device__(size_type idx) { auto const offset = d_offsets[idx]; - auto const token_count = d_offsets[idx + 1] - offset; + auto const token_count = static_cast(d_offsets[idx + 1] - offset); return (col < token_count) ? d_tokens[offset + col] : string_index_pair{nullptr, 0}; })); results.emplace_back(make_strings_column(itr, itr + input.size(), stream, mr)); @@ -360,12 +359,11 @@ std::unique_ptr
whitespace_split_fn(size_type strings_count, } // get the positions for every token - rmm::device_uvector tokens(columns_count * strings_count, stream); + rmm::device_uvector tokens( + static_cast(columns_count) * static_cast(strings_count), stream); string_index_pair* d_tokens = tokens.data(); - thrust::fill(rmm::exec_policy(stream), - d_tokens, - d_tokens + (columns_count * strings_count), - string_index_pair{nullptr, 0}); + thrust::fill( + rmm::exec_policy(stream), tokens.begin(), tokens.end(), string_index_pair{nullptr, 0}); thrust::for_each_n(rmm::exec_policy(stream), thrust::make_counting_iterator(0), strings_count, diff --git a/cpp/src/strings/split/split.cuh b/cpp/src/strings/split/split.cuh index c5fb44fc3dd..906c522e898 100644 --- a/cpp/src/strings/split/split.cuh +++ b/cpp/src/strings/split/split.cuh @@ -17,9 +17,9 @@ #include #include #include -#include #include #include +#include #include #include #include @@ -66,9 +66,9 @@ struct base_split_tokenizer { * @param chars_bytes Total number of characters to process * @return true if delimiter is found starting at position `idx` */ - __device__ bool is_delimiter(size_type idx, - size_type const* d_offsets, - size_type chars_bytes) const + __device__ bool is_delimiter(int64_t idx, + cudf::detail::input_offsetalator const d_offsets, + int64_t chars_bytes) const { auto const d_chars = get_base_ptr() + d_offsets[0]; if (idx + d_delimiter.size_bytes() > chars_bytes) { return false; } @@ -87,8 +87,8 @@ struct base_split_tokenizer { * @param d_delimiter_offsets Offsets per string to delimiters in d_positions */ __device__ size_type count_tokens(size_type idx, - size_type const* d_positions, - size_type const* d_delimiter_offsets) const + int64_t const* d_positions, + int64_t const* d_delimiter_offsets) const { if (!is_valid(idx)) { return 0; } @@ -96,12 +96,13 @@ struct base_split_tokenizer { auto const d_str = get_string(idx); auto const d_str_end = d_str.data() + d_str.size_bytes(); auto const base_ptr = get_base_ptr() + delim_size - 1; + auto const delimiters = - cudf::device_span(d_positions + d_delimiter_offsets[idx], - d_delimiter_offsets[idx + 1] - d_delimiter_offsets[idx]); + cudf::device_span(d_positions + d_delimiter_offsets[idx], + d_delimiter_offsets[idx + 1] - d_delimiter_offsets[idx]); size_type token_count = 1; // all strings will have at least one token - size_type last_pos = delimiters[0] - delim_size; + auto last_pos = !delimiters.empty() ? (delimiters[0] - delim_size) : 0L; for (auto d_pos : delimiters) { // delimiter must fit in string && overlapping delimiters are ignored if (((base_ptr + d_pos) < d_str_end) && ((d_pos - last_pos) >= delim_size)) { @@ -129,9 +130,9 @@ struct base_split_tokenizer { * @param d_all_tokens All output tokens for the strings column */ __device__ void get_tokens(size_type idx, - size_type const* d_tokens_offsets, - size_type const* d_positions, - size_type const* d_delimiter_offsets, + cudf::detail::input_offsetalator const d_tokens_offsets, + int64_t const* d_positions, + int64_t const* d_delimiter_offsets, string_index_pair* d_all_tokens) const { auto const d_tokens = // this string's tokens output @@ -149,8 +150,8 @@ struct base_split_tokenizer { } auto const delimiters = - cudf::device_span(d_positions + d_delimiter_offsets[idx], - d_delimiter_offsets[idx + 1] - d_delimiter_offsets[idx]); + cudf::device_span(d_positions + d_delimiter_offsets[idx], + d_delimiter_offsets[idx + 1] - d_delimiter_offsets[idx]); auto& derived = static_cast(*this); derived.process_tokens(d_str, delimiters, d_tokens); @@ -184,7 +185,7 @@ struct split_tokenizer_fn : base_split_tokenizer { * @param d_tokens Output vector to store tokens for this string */ __device__ void process_tokens(string_view const d_str, - device_span d_delimiters, + device_span d_delimiters, device_span d_tokens) const { auto const base_ptr = get_base_ptr(); // d_positions values based on this @@ -239,7 +240,7 @@ struct rsplit_tokenizer_fn : base_split_tokenizer { * @param d_tokens Output vector to store tokens for this string */ __device__ void process_tokens(string_view const d_str, - device_span d_delimiters, + device_span d_delimiters, device_span d_tokens) const { auto const base_ptr = get_base_ptr(); // d_positions values are based on this ptr @@ -290,7 +291,8 @@ struct rsplit_tokenizer_fn : base_split_tokenizer { * @param input The input column of strings to split * @param tokenizer Object used for counting and identifying delimiters and tokens * @param stream CUDA stream used for device memory operations and kernel launches - * @param mr Device memory resource used to allocate the returned objects' device memory. + * @param mr Device memory resource used to allocate the returned objects' device memory + * @return Token offsets and a vector of string indices */ template std::pair, rmm::device_uvector> split_helper( @@ -301,37 +303,38 @@ std::pair, rmm::device_uvector> split { auto const strings_count = input.size(); auto const chars_bytes = - cudf::detail::get_value(input.offsets(), input.offset() + strings_count, stream) - - cudf::detail::get_value(input.offsets(), input.offset(), stream); - - auto d_offsets = input.offsets_begin(); + get_offset_value(input.offsets(), input.offset() + strings_count, stream) - + get_offset_value(input.offsets(), input.offset(), stream); + auto const d_offsets = + cudf::detail::offsetalator_factory::make_input_iterator(input.offsets(), input.offset()); // count the number of delimiters in the entire column auto const delimiter_count = thrust::count_if(rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(chars_bytes), - [tokenizer, d_offsets, chars_bytes] __device__(size_type idx) { + thrust::counting_iterator(0), + thrust::counting_iterator(chars_bytes), + [tokenizer, d_offsets, chars_bytes] __device__(int64_t idx) { return tokenizer.is_delimiter(idx, d_offsets, chars_bytes); }); + // Create a vector of every delimiter position in the chars column. // These may include overlapping or otherwise out-of-bounds delimiters which // will be resolved during token processing. - auto delimiter_positions = rmm::device_uvector(delimiter_count, stream); + auto delimiter_positions = rmm::device_uvector(delimiter_count, stream); auto d_positions = delimiter_positions.data(); - auto const copy_end = - thrust::copy_if(rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(chars_bytes), - delimiter_positions.begin(), - [tokenizer, d_offsets, chars_bytes] __device__(size_type idx) { - return tokenizer.is_delimiter(idx, d_offsets, chars_bytes); - }); + auto const copy_end = cudf::detail::copy_if_safe( + thrust::counting_iterator(0), + thrust::counting_iterator(chars_bytes), + delimiter_positions.begin(), + [tokenizer, d_offsets, chars_bytes] __device__(int64_t idx) { + return tokenizer.is_delimiter(idx, d_offsets, chars_bytes); + }, + stream); // create a vector of offsets to each string's delimiter set within delimiter_positions auto const delimiter_offsets = [&] { // first, create a vector of string indices for each delimiter - auto string_indices = rmm::device_uvector(delimiter_count, stream); + auto string_indices = rmm::device_uvector(delimiter_count, stream); thrust::upper_bound(rmm::exec_policy(stream), d_offsets, d_offsets + strings_count, @@ -340,24 +343,24 @@ std::pair, rmm::device_uvector> split string_indices.begin()); // compute delimiter offsets per string - auto delimiter_offsets = rmm::device_uvector(strings_count + 1, stream); + auto delimiter_offsets = rmm::device_uvector(strings_count + 1, stream); auto d_delimiter_offsets = delimiter_offsets.data(); // memset to zero-out the delimiter counts for any null-entries or strings with no delimiters CUDF_CUDA_TRY(cudaMemsetAsync( - d_delimiter_offsets, 0, delimiter_offsets.size() * sizeof(size_type), stream.value())); + d_delimiter_offsets, 0, delimiter_offsets.size() * sizeof(int64_t), stream.value())); // next, count the number of delimiters per string auto d_string_indices = string_indices.data(); // identifies strings with delimiters only - thrust::for_each_n(rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - delimiter_count, - [d_string_indices, d_delimiter_offsets] __device__(size_type idx) { - auto const str_idx = d_string_indices[idx] - 1; - cuda::atomic_ref ref{ - *(d_delimiter_offsets + str_idx)}; - ref.fetch_add(1, cuda::std::memory_order_relaxed); - }); + thrust::for_each_n( + rmm::exec_policy(stream), + thrust::counting_iterator(0), + delimiter_count, + [d_string_indices, d_delimiter_offsets] __device__(int64_t idx) { + auto const str_idx = d_string_indices[idx] - 1; + cuda::atomic_ref ref{*(d_delimiter_offsets + str_idx)}; + ref.fetch_add(1L, cuda::std::memory_order_relaxed); + }); // finally, convert the delimiter counts into offsets thrust::exclusive_scan(rmm::exec_policy(stream), delimiter_offsets.begin(), @@ -379,11 +382,10 @@ std::pair, rmm::device_uvector> split }); // create offsets from the counts for return to the caller - auto offsets = std::get<0>( - cudf::detail::make_offsets_child_column(token_counts.begin(), token_counts.end(), stream, mr)); - auto const total_tokens = - cudf::detail::get_value(offsets->view(), strings_count, stream); - auto const d_tokens_offsets = offsets->view().data(); + auto [offsets, total_tokens] = cudf::strings::detail::make_offsets_child_column( + token_counts.begin(), token_counts.end(), stream, mr); + auto const d_tokens_offsets = + cudf::detail::offsetalator_factory::make_input_iterator(offsets->view()); // build a vector of all the token positions for all the strings auto tokens = rmm::device_uvector(total_tokens, stream); diff --git a/cpp/src/strings/split/split_re.cu b/cpp/src/strings/split/split_re.cu index 045aac279e6..d8385549840 100644 --- a/cpp/src/strings/split/split_re.cu +++ b/cpp/src/strings/split/split_re.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include @@ -36,7 +35,6 @@ #include #include #include -#include namespace cudf { namespace strings { @@ -60,7 +58,7 @@ enum class split_direction { struct token_reader_fn { column_device_view const d_strings; split_direction const direction; - size_type const* d_token_offsets; + cudf::detail::input_offsetalator const d_token_offsets; string_index_pair* d_tokens; __device__ void operator()(size_type const idx, reprog_device const prog, int32_t const prog_idx) @@ -73,9 +71,9 @@ struct token_reader_fn { auto const token_count = d_token_offsets[idx + 1] - token_offset; auto const d_result = d_tokens + token_offset; // store tokens here - size_type token_idx = 0; - auto itr = d_str.begin(); - auto last_pos = itr; + int64_t token_idx = 0; + auto itr = d_str.begin(); + auto last_pos = itr; while (itr.position() <= nchars) { auto const match = prog.find(prog_idx, d_str, itr); if (!match) { break; } @@ -90,7 +88,7 @@ struct token_reader_fn { d_result[token_idx++] = token; } else { if (direction == split_direction::FORWARD) { break; } // we are done - for (auto l = 0; l < token_idx - 1; ++l) { + for (auto l = 0L; l < token_idx - 1; ++l) { d_result[l] = d_result[l + 1]; // shift left } d_result[token_idx - 1] = token; @@ -120,50 +118,45 @@ struct token_reader_fn { /** * @brief Call regex to split each input string into tokens. * - * This will also convert the `offsets` values from counts to offsets. - * * @param d_strings Strings to split * @param d_prog Regex to evaluate against each string * @param direction Whether tokens are generated forwards or backwards. * @param max_tokens The maximum number of tokens for each split. - * @param offsets The number of matches on input. - * The offsets for each token in each string on output. + * @param counts The number of tokens in each string * @param stream CUDA stream used for kernel launches. */ -rmm::device_uvector generate_tokens(column_device_view const& d_strings, - reprog_device& d_prog, - split_direction direction, - size_type maxsplit, - mutable_column_view& offsets, - rmm::cuda_stream_view stream) +std::pair, std::unique_ptr> generate_tokens( + column_device_view const& d_strings, + reprog_device& d_prog, + split_direction direction, + size_type maxsplit, + column_view const& counts, + rmm::cuda_stream_view stream) { auto const strings_count = d_strings.size(); - - auto const max_tokens = maxsplit > 0 ? maxsplit : std::numeric_limits::max(); - - auto const begin = thrust::make_counting_iterator(0); - auto const end = thrust::make_counting_iterator(strings_count); - auto const d_offsets = offsets.data(); + auto const max_tokens = maxsplit > 0 ? maxsplit : std::numeric_limits::max(); + auto const d_counts = counts.data(); // convert match counts to token offsets - auto map_fn = [d_strings, d_offsets, max_tokens] __device__(auto idx) { - return d_strings.is_null(idx) ? 0 : std::min(d_offsets[idx], max_tokens) + 1; - }; - thrust::transform_exclusive_scan( - rmm::exec_policy(stream), begin, end + 1, d_offsets, map_fn, 0, thrust::plus{}); + auto map_fn = cuda::proclaim_return_type( + [d_strings, d_counts, max_tokens] __device__(auto idx) -> size_type { + return d_strings.is_null(idx) ? 0 : std::min(d_counts[idx], max_tokens) + 1; + }); - // the last offset entry is the total number of tokens to be generated - auto const total_tokens = cudf::detail::get_value(offsets, strings_count, stream); + auto const begin = cudf::detail::make_counting_transform_iterator(0, map_fn); + auto const end = begin + strings_count; - rmm::device_uvector tokens(total_tokens, stream); - if (total_tokens == 0) { return tokens; } - - launch_for_each_kernel(token_reader_fn{d_strings, direction, d_offsets, tokens.data()}, - d_prog, - d_strings.size(), - stream); + auto [offsets, total_tokens] = cudf::strings::detail::make_offsets_child_column( + begin, end, stream, rmm::mr::get_current_device_resource()); + auto const d_offsets = cudf::detail::offsetalator_factory::make_input_iterator(offsets->view()); - return tokens; + // build a vector of tokens + rmm::device_uvector tokens(total_tokens, stream); + if (total_tokens > 0) { + auto tr_fn = token_reader_fn{d_strings, direction, d_offsets, tokens.data()}; + launch_for_each_kernel(tr_fn, d_prog, d_strings.size(), stream); + } + return std::pair(std::move(tokens), std::move(offsets)); } /** @@ -176,13 +169,13 @@ rmm::device_uvector generate_tokens(column_device_view const& struct tokens_transform_fn { column_device_view const d_strings; string_index_pair const* d_tokens; - size_type const* d_token_offsets; + cudf::detail::input_offsetalator const d_token_offsets; size_type const column_index; __device__ string_index_pair operator()(size_type idx) const { auto const offset = d_token_offsets[idx]; - auto const token_count = d_token_offsets[idx + 1] - offset; + auto const token_count = static_cast(d_token_offsets[idx + 1] - offset); return (column_index >= token_count) || d_strings.is_null(idx) ? string_index_pair{nullptr, 0} : d_tokens[offset + column_index]; @@ -212,13 +205,13 @@ std::unique_ptr
split_re(strings_column_view const& input, auto d_strings = column_device_view::create(input.parent(), stream); // count the number of delimiters matched in each string - auto offsets = count_matches( - *d_strings, *d_prog, strings_count + 1, stream, rmm::mr::get_current_device_resource()); - auto offsets_view = offsets->mutable_view(); - auto d_offsets = offsets_view.data(); + auto const counts = count_matches( + *d_strings, *d_prog, strings_count, stream, rmm::mr::get_current_device_resource()); // get the split tokens from the input column; this also converts the counts into offsets - auto tokens = generate_tokens(*d_strings, *d_prog, direction, maxsplit, offsets_view, stream); + auto [tokens, offsets] = + generate_tokens(*d_strings, *d_prog, direction, maxsplit, counts->view(), stream); + auto const d_offsets = cudf::detail::offsetalator_factory::make_input_iterator(offsets->view()); // the output column count is the maximum number of tokens generated for any input string auto const columns_count = thrust::transform_reduce( @@ -226,7 +219,7 @@ std::unique_ptr
split_re(strings_column_view const& input, thrust::make_counting_iterator(0), thrust::make_counting_iterator(strings_count), [d_offsets] __device__(auto const idx) -> size_type { - return d_offsets[idx + 1] - d_offsets[idx]; + return static_cast(d_offsets[idx + 1] - d_offsets[idx]); }, 0, thrust::maximum{}); @@ -243,10 +236,11 @@ std::unique_ptr
split_re(strings_column_view const& input, } // convert the tokens into multiple strings columns + auto d_tokens = tokens.data(); auto make_strings_lambda = [&](size_type column_index) { // returns appropriate token for each row/column auto indices_itr = cudf::detail::make_counting_transform_iterator( - 0, tokens_transform_fn{*d_strings, tokens.data(), d_offsets, column_index}); + 0, tokens_transform_fn{*d_strings, d_tokens, d_offsets, column_index}); return make_strings_column(indices_itr, indices_itr + strings_count, stream, mr); }; // build a vector of columns @@ -276,11 +270,14 @@ std::unique_ptr split_record_re(strings_column_view const& input, auto d_strings = column_device_view::create(input.parent(), stream); // count the number of delimiters matched in each string - auto offsets = count_matches(*d_strings, *d_prog, strings_count + 1, stream, mr); - auto offsets_view = offsets->mutable_view(); + auto counts = count_matches(*d_strings, *d_prog, strings_count, stream, mr); // get the split tokens from the input column; this also converts the counts into offsets - auto tokens = generate_tokens(*d_strings, *d_prog, direction, maxsplit, offsets_view, stream); + auto [tokens, offsets] = + generate_tokens(*d_strings, *d_prog, direction, maxsplit, counts->view(), stream); + CUDF_EXPECTS(tokens.size() < static_cast(std::numeric_limits::max()), + "Size of output exceeds the column size limit", + std::overflow_error); // convert the tokens into one big strings column auto strings_output = make_strings_column(tokens.begin(), tokens.end(), stream, mr); diff --git a/cpp/src/strings/split/split_record.cu b/cpp/src/strings/split/split_record.cu index 64061aba4fd..c9ed7b0ed26 100644 --- a/cpp/src/strings/split/split_record.cu +++ b/cpp/src/strings/split/split_record.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -66,6 +66,9 @@ std::unique_ptr split_record_fn(strings_column_view const& input, // builds the offsets and the vector of all tokens auto [offsets, tokens] = split_helper(input, tokenizer, stream, mr); + CUDF_EXPECTS(tokens.size() < static_cast(std::numeric_limits::max()), + "Size of output exceeds the column size limit", + std::overflow_error); // build a strings column from the tokens auto strings_child = make_strings_column(tokens.begin(), tokens.end(), stream, mr); From 49c7d2cd1683575fc562ce284c7402d275e44212 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 8 Feb 2024 05:20:46 -1000 Subject: [PATCH 220/384] Deprecate parameters similar to pandas 2.2 (#14984) For comparison: https://github.com/pandas-dev/pandas/pull/55856 https://github.com/pandas-dev/pandas/pull/55895 https://github.com/pandas-dev/pandas/issues/55499 The `errors="ignore"` parameter is the only one that is implemented so just added a test for that deprecation Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/14984 --- python/cudf/cudf/core/index.py | 5 +++++ python/cudf/cudf/core/indexed_frame.py | 6 ++++++ python/cudf/cudf/core/tools/datetimes.py | 8 ++++++++ python/cudf/cudf/core/tools/numeric.py | 7 +++++++ python/cudf/cudf/tests/test_datetime.py | 5 +++++ python/cudf/cudf/tests/test_numerical.py | 9 ++++++--- 6 files changed, 37 insertions(+), 3 deletions(-) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index c05d89e7279..ea8ba154922 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -2402,6 +2402,11 @@ def __init__( raise NotImplementedError("freq is not yet supported") if unit is not None: + warnings.warn( + "The 'unit' keyword is " + "deprecated and will be removed in a future version. ", + FutureWarning, + ) raise NotImplementedError( "unit is not yet supported, alternatively " "dtype parameter is supported" diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index aa75b0d825e..bc24216cade 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -3921,6 +3921,12 @@ def resample( """ import cudf.core.resample + if kind is not None: + warnings.warn( + "The 'kind' keyword in is " + "deprecated and will be removed in a future version. ", + FutureWarning, + ) if (axis, convention, kind, loffset, base, origin, offset) != ( 0, "start", diff --git a/python/cudf/cudf/core/tools/datetimes.py b/python/cudf/cudf/core/tools/datetimes.py index 928154e10fd..529296da6a2 100644 --- a/python/cudf/cudf/core/tools/datetimes.py +++ b/python/cudf/cudf/core/tools/datetimes.py @@ -162,6 +162,14 @@ def to_datetime( f"{errors=} is not implemented when arg is not scalar-like" ) + if errors == "ignore": + warnings.warn( + "errors='ignore' is deprecated and will raise in a future version. " + "Use to_datetime without passing `errors` and catch exceptions " + "explicitly instead", + FutureWarning, + ) + if infer_datetime_format in {None, False}: warnings.warn( "`infer_datetime_format` is deprecated and will " diff --git a/python/cudf/cudf/core/tools/numeric.py b/python/cudf/cudf/core/tools/numeric.py index 8991fbe1c13..e1424459c8f 100644 --- a/python/cudf/cudf/core/tools/numeric.py +++ b/python/cudf/cudf/core/tools/numeric.py @@ -95,6 +95,13 @@ def to_numeric(arg, errors="raise", downcast=None): if errors not in {"raise", "ignore", "coerce"}: raise ValueError("invalid error value specified") + elif errors == "ignore": + warnings.warn( + "errors='ignore' is deprecated and will raise in a future version. " + "Use to_numeric without passing `errors` and catch exceptions " + "explicitly instead", + FutureWarning, + ) if downcast not in {None, "integer", "signed", "unsigned", "float"}: raise ValueError("invalid downcasting method provided") diff --git a/python/cudf/cudf/tests/test_datetime.py b/python/cudf/cudf/tests/test_datetime.py index 1f24337d28b..5596be30cfa 100644 --- a/python/cudf/cudf/tests/test_datetime.py +++ b/python/cudf/cudf/tests/test_datetime.py @@ -2468,3 +2468,8 @@ def test_datetime_raise_warning(freqstr): ) with pytest.warns(FutureWarning): t.dt.ceil(freqstr) + + +def test_to_datetime_errors_ignore_deprecated(): + with pytest.warns(FutureWarning): + cudf.to_datetime("2001-01-01 00:04:45", errors="ignore") diff --git a/python/cudf/cudf/tests/test_numerical.py b/python/cudf/cudf/tests/test_numerical.py index 2139e7b9860..fb1bc580aa4 100644 --- a/python/cudf/cudf/tests/test_numerical.py +++ b/python/cudf/cudf/tests/test_numerical.py @@ -5,7 +5,8 @@ import pytest import cudf -from cudf.testing._utils import NUMERIC_TYPES, assert_eq +from cudf.core._compat import PANDAS_GE_220 +from cudf.testing._utils import NUMERIC_TYPES, assert_eq, expect_warning_if from cudf.utils.dtypes import np_dtypes_to_pandas_dtypes @@ -372,8 +373,10 @@ def test_to_numeric_error(data, errors): ): cudf.to_numeric(data, errors=errors) else: - expect = pd.to_numeric(data, errors=errors) - got = cudf.to_numeric(data, errors=errors) + with expect_warning_if(PANDAS_GE_220 and errors == "ignore"): + expect = pd.to_numeric(data, errors=errors) + with expect_warning_if(errors == "ignore"): + got = cudf.to_numeric(data, errors=errors) assert_eq(expect, got) From d855d0e8ff52d822462b8667b6219968b20edfef Mon Sep 17 00:00:00 2001 From: Ashwin Srinath <3190405+shwina@users.noreply.github.com> Date: Thu, 8 Feb 2024 11:49:49 -0500 Subject: [PATCH 221/384] Fix handling of values=None in pylibcudf GroupBy.get_groups (#14998) A small bug in our previous implementation leads to a segfault when calling `.get_groups()` with no `values`. Thankfully, the cuDF Python API always calls this function with a value, but it's possible `pylibcudf` consumers will not. Authors: - Ashwin Srinath (https://github.com/shwina) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/14998 --- python/cudf/cudf/_lib/groupby.pyx | 15 ++++++----- python/cudf/cudf/_lib/pylibcudf/groupby.pxd | 1 + python/cudf/cudf/_lib/pylibcudf/groupby.pyx | 30 +++++++++++++-------- python/cudf/cudf/core/groupby/groupby.py | 6 ++--- python/cudf/cudf/tests/test_groupby.py | 8 ++++++ 5 files changed, 40 insertions(+), 20 deletions(-) diff --git a/python/cudf/cudf/_lib/groupby.pyx b/python/cudf/cudf/_lib/groupby.pyx index eb0f784de17..8384d5231b7 100644 --- a/python/cudf/cudf/_lib/groupby.pyx +++ b/python/cudf/cudf/_lib/groupby.pyx @@ -120,23 +120,26 @@ cdef class GroupBy: Returns ------- + offsets: list of integers + Integer offsets such that offsets[i+1] - offsets[i] + represents the size of group `i`. grouped_keys: list of Columns The grouped key columns grouped_values: list of Columns The grouped value columns - offsets: list of integers - Integer offsets such that offsets[i+1] - offsets[i] - represents the size of group `i`. """ - grouped_keys, grouped_values, offsets = self._groupby.get_groups( + offsets, grouped_keys, grouped_values = self._groupby.get_groups( pylibcudf.table.Table([c.to_pylibcudf(mode="read") for c in values]) if values else None ) return ( - columns_from_pylibcudf_table(grouped_keys), - columns_from_pylibcudf_table(grouped_values), offsets, + columns_from_pylibcudf_table(grouped_keys), + ( + columns_from_pylibcudf_table(grouped_values) + if grouped_values is not None else [] + ), ) def aggregate(self, values, aggregations): diff --git a/python/cudf/cudf/_lib/pylibcudf/groupby.pxd b/python/cudf/cudf/_lib/pylibcudf/groupby.pxd index d06959b3c31..f1b7a25d5f9 100644 --- a/python/cudf/cudf/_lib/pylibcudf/groupby.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/groupby.pxd @@ -37,6 +37,7 @@ cdef class GroupByRequest: cdef class GroupBy: cdef unique_ptr[groupby] c_obj + cdef Table _keys cpdef tuple aggregate(self, list requests) cpdef tuple scan(self, list requests) cpdef tuple shift(self, Table values, list offset, list fill_values) diff --git a/python/cudf/cudf/_lib/pylibcudf/groupby.pyx b/python/cudf/cudf/_lib/pylibcudf/groupby.pyx index b8cc59eed09..3b800abf266 100644 --- a/python/cudf/cudf/_lib/pylibcudf/groupby.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/groupby.pyx @@ -98,6 +98,9 @@ cdef class GroupBy: sorted keys_are_sorted=sorted.NO ): self.c_obj.reset(new groupby(keys.view(), null_handling, keys_are_sorted)) + # keep a reference to the keys table so it doesn't get + # deallocated from under us: + self._keys = keys @staticmethod cdef tuple _parse_outputs( @@ -253,26 +256,31 @@ cdef class GroupBy: Parameters ---------- values : Table, optional - The columns to get group labels for. If not specified, the group - labels for the group keys are returned. + The columns to get group labels for. If not specified, + `None` is returned for the group values. Returns ------- - Tuple[Table, Table, List[int]] + Tuple[List[int], Table, Table]] A tuple of tables containing three items: + - A list of integer offsets into the group keys/values - A table of group keys - - A table of group values - - A list of integer offsets into the tables + - A table of group values or None """ cdef groups c_groups if values: c_groups = dereference(self.c_obj).get_groups(values.view()) + return ( + c_groups.offsets, + Table.from_libcudf(move(c_groups.keys)), + Table.from_libcudf(move(c_groups.values)), + ) else: + # c_groups.values is nullptr c_groups = dereference(self.c_obj).get_groups() - - return ( - Table.from_libcudf(move(c_groups.keys)), - Table.from_libcudf(move(c_groups.values)), - c_groups.offsets, - ) + return ( + c_groups.offsets, + Table.from_libcudf(move(c_groups.keys)), + None, + ) diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 9e8d9908df2..12bba3838f3 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -790,7 +790,7 @@ def _head_tail(self, n, *, take_head: bool, preserve_order: bool): # Can't use _mimic_pandas_order because we need to # subsample the gather map from the full input ordering, # rather than permuting the gather map of the output. - _, (ordering,), _ = self._groupby.groups( + _, _, (ordering,) = self._groupby.groups( [as_column(range(0, len(self.obj)))] ) # Invert permutation from original order to groups on the @@ -1179,7 +1179,7 @@ def deserialize(cls, header, frames): return cls(obj, grouping, **kwargs) def _grouped(self): - grouped_key_cols, grouped_value_cols, offsets = self._groupby.groups( + offsets, grouped_key_cols, grouped_value_cols = self._groupby.groups( [*self.obj._index._columns, *self.obj._columns] ) grouped_keys = cudf.core.index._index_from_columns(grouped_key_cols) @@ -2578,7 +2578,7 @@ def _mimic_pandas_order( # result coming back from libcudf has null_count few rows than # the input, so we must produce an ordering from the full # input range. - _, (ordering,), _ = self._groupby.groups( + _, _, (ordering,) = self._groupby.groups( [as_column(range(0, len(self.obj)))] ) if self._dropna and any( diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index bd48e5bfd31..6514053afa7 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -3759,3 +3759,11 @@ def test_group_by_value_counts_with_count_column(): df = cudf.DataFrame({"a": [1, 5, 3], "count": [2, 5, 2]}) with pytest.raises(ValueError): df.groupby("a", as_index=False).value_counts() + + +def test_groupby_internal_groups_empty(gdf): + # test that we don't segfault when calling the internal + # .groups() method with an empty list: + gb = gdf.groupby("y")._groupby + _, _, grouped_vals = gb.groups([]) + assert grouped_vals == [] From b2164c2b432f42aa07130fbfc63115f2fb303b02 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 8 Feb 2024 09:05:39 -0800 Subject: [PATCH 222/384] Implement rolling in pylibcudf (#14982) Contributes to #13921 Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/14982 --- .../user_guide/api_docs/pylibcudf/index.rst | 1 + .../user_guide/api_docs/pylibcudf/rolling.rst | 6 + python/cudf/cudf/_lib/aggregation.pxd | 16 - python/cudf/cudf/_lib/aggregation.pyx | 327 +++--------------- python/cudf/cudf/_lib/cpp/aggregation.pxd | 2 - python/cudf/cudf/_lib/cpp/rolling.pxd | 6 +- python/cudf/cudf/_lib/groupby.pyx | 2 +- .../cudf/cudf/_lib/pylibcudf/CMakeLists.txt | 2 +- python/cudf/cudf/_lib/pylibcudf/__init__.pxd | 3 + python/cudf/cudf/_lib/pylibcudf/__init__.py | 3 + .../cudf/cudf/_lib/pylibcudf/aggregation.pxd | 3 + .../cudf/cudf/_lib/pylibcudf/aggregation.pyx | 8 + python/cudf/cudf/_lib/pylibcudf/rolling.pxd | 19 + python/cudf/cudf/_lib/pylibcudf/rolling.pyx | 73 ++++ python/cudf/cudf/_lib/reduce.pyx | 2 +- python/cudf/cudf/_lib/rolling.pyx | 71 ++-- python/cudf/cudf/_lib/sort.pyx | 15 +- python/cudf/cudf/core/indexed_frame.py | 2 +- 18 files changed, 187 insertions(+), 374 deletions(-) create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/rolling.rst delete mode 100644 python/cudf/cudf/_lib/aggregation.pxd create mode 100644 python/cudf/cudf/_lib/pylibcudf/rolling.pxd create mode 100644 python/cudf/cudf/_lib/pylibcudf/rolling.pyx diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst index 4772d654a3c..91b84d29ddf 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst @@ -16,6 +16,7 @@ This page provides API documentation for pylibcudf. groupby join reduce + rolling scalar table types diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/rolling.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/rolling.rst new file mode 100644 index 00000000000..0817d117a94 --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/rolling.rst @@ -0,0 +1,6 @@ +======= +rolling +======= + +.. automodule:: cudf._lib.pylibcudf.rolling + :members: diff --git a/python/cudf/cudf/_lib/aggregation.pxd b/python/cudf/cudf/_lib/aggregation.pxd deleted file mode 100644 index 7a2a2b022fb..00000000000 --- a/python/cudf/cudf/_lib/aggregation.pxd +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2020-2024, NVIDIA CORPORATION. - -from libcpp.memory cimport unique_ptr - -from cudf._lib cimport pylibcudf -from cudf._lib.cpp.aggregation cimport rolling_aggregation - - -cdef class RollingAggregation: - cdef unique_ptr[rolling_aggregation] c_obj - -cdef class Aggregation: - cdef pylibcudf.aggregation.Aggregation c_obj - -cdef RollingAggregation make_rolling_aggregation(op, kwargs=*) -cdef Aggregation make_aggregation(op, kwargs=*) diff --git a/python/cudf/cudf/_lib/aggregation.pyx b/python/cudf/cudf/_lib/aggregation.pyx index 036c922e128..de3cbb07c37 100644 --- a/python/cudf/cudf/_lib/aggregation.pyx +++ b/python/cudf/cudf/_lib/aggregation.pyx @@ -1,253 +1,31 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. -from enum import Enum, IntEnum - import pandas as pd - -from libcpp.string cimport string -from libcpp.utility cimport move - -from cudf._lib.types import SUPPORTED_NUMPY_TO_LIBCUDF_TYPES, NullHandling -from cudf.utils import cudautils - -from cudf._lib.types cimport ( - underlying_type_t_null_policy, - underlying_type_t_type_id, -) - from numba.np import numpy_support -cimport cudf._lib.cpp.aggregation as libcudf_aggregation -cimport cudf._lib.cpp.types as libcudf_types -from cudf._lib.cpp.aggregation cimport underlying_type_t_correlation_type - import cudf - -from cudf._lib cimport pylibcudf - from cudf._lib import pylibcudf +from cudf._lib.types import SUPPORTED_NUMPY_TO_PYLIBCUDF_TYPES +from cudf.utils import cudautils +_agg_name_map = { + "COUNT_VALID": "COUNT", + "COUNT_ALL": "SIZE", + "VARIANCE": "VAR", + "NTH_ELEMENT": "NTH", + "COLLECT_LIST": "COLLECT", + "COLLECT_SET": "UNIQUE", +} -class AggregationKind(Enum): - SUM = libcudf_aggregation.aggregation.Kind.SUM - PRODUCT = libcudf_aggregation.aggregation.Kind.PRODUCT - MIN = libcudf_aggregation.aggregation.Kind.MIN - MAX = libcudf_aggregation.aggregation.Kind.MAX - COUNT = libcudf_aggregation.aggregation.Kind.COUNT_VALID - SIZE = libcudf_aggregation.aggregation.Kind.COUNT_ALL - ANY = libcudf_aggregation.aggregation.Kind.ANY - ALL = libcudf_aggregation.aggregation.Kind.ALL - SUM_OF_SQUARES = libcudf_aggregation.aggregation.Kind.SUM_OF_SQUARES - MEAN = libcudf_aggregation.aggregation.Kind.MEAN - VAR = libcudf_aggregation.aggregation.Kind.VARIANCE - STD = libcudf_aggregation.aggregation.Kind.STD - MEDIAN = libcudf_aggregation.aggregation.Kind.MEDIAN - QUANTILE = libcudf_aggregation.aggregation.Kind.QUANTILE - ARGMAX = libcudf_aggregation.aggregation.Kind.ARGMAX - ARGMIN = libcudf_aggregation.aggregation.Kind.ARGMIN - NUNIQUE = libcudf_aggregation.aggregation.Kind.NUNIQUE - NTH = libcudf_aggregation.aggregation.Kind.NTH_ELEMENT - RANK = libcudf_aggregation.aggregation.Kind.RANK - COLLECT = libcudf_aggregation.aggregation.Kind.COLLECT_LIST - UNIQUE = libcudf_aggregation.aggregation.Kind.COLLECT_SET - PTX = libcudf_aggregation.aggregation.Kind.PTX - CUDA = libcudf_aggregation.aggregation.Kind.CUDA - CORRELATION = libcudf_aggregation.aggregation.Kind.CORRELATION - COVARIANCE = libcudf_aggregation.aggregation.Kind.COVARIANCE - - -class CorrelationType(IntEnum): - PEARSON = ( - - libcudf_aggregation.correlation_type.PEARSON - ) - KENDALL = ( - - libcudf_aggregation.correlation_type.KENDALL - ) - SPEARMAN = ( - - libcudf_aggregation.correlation_type.SPEARMAN - ) - - -class RankMethod(IntEnum): - FIRST = libcudf_aggregation.rank_method.FIRST - AVERAGE = libcudf_aggregation.rank_method.AVERAGE - MIN = libcudf_aggregation.rank_method.MIN - MAX = libcudf_aggregation.rank_method.MAX - DENSE = libcudf_aggregation.rank_method.DENSE - - -cdef class RollingAggregation: - """A Cython wrapper for rolling window aggregations. - - **This class should never be instantiated using a standard constructor, - only using one of its many factories.** These factories handle mapping - different cudf operations to their libcudf analogs, e.g. - `cudf.DataFrame.idxmin` -> `libcudf.argmin`. Additionally, they perform - any additional configuration needed to translate Python arguments into - their corresponding C++ types (for instance, C++ enumerations used for - flag arguments). The factory approach is necessary to support operations - like `df.agg(lambda x: x.sum())`; such functions are called with this - class as an argument to generation the desired aggregation. - """ - @property - def kind(self): - return AggregationKind(self.c_obj.get()[0].kind).name - - @classmethod - def sum(cls): - cdef RollingAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_sum_aggregation[rolling_aggregation]()) - return agg - - @classmethod - def min(cls): - cdef RollingAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_min_aggregation[rolling_aggregation]()) - return agg - - @classmethod - def max(cls): - cdef RollingAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_max_aggregation[rolling_aggregation]()) - return agg - - @classmethod - def idxmin(cls): - cdef RollingAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_argmin_aggregation[ - rolling_aggregation]()) - return agg - - @classmethod - def idxmax(cls): - cdef RollingAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_argmax_aggregation[ - rolling_aggregation]()) - return agg - - @classmethod - def mean(cls): - cdef RollingAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_mean_aggregation[rolling_aggregation]()) - return agg - - @classmethod - def var(cls, ddof=1): - cdef RollingAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_variance_aggregation[rolling_aggregation]( - ddof - ) - ) - return agg - - @classmethod - def std(cls, ddof=1): - cdef RollingAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_std_aggregation[rolling_aggregation](ddof) - ) - return agg - - @classmethod - def count(cls, dropna=True): - cdef libcudf_types.null_policy c_null_handling - if dropna: - c_null_handling = libcudf_types.null_policy.EXCLUDE - else: - c_null_handling = libcudf_types.null_policy.INCLUDE - - cdef RollingAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_count_aggregation[rolling_aggregation]( - c_null_handling - )) - return agg - - @classmethod - def size(cls): - cdef RollingAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_count_aggregation[rolling_aggregation]( - ( - NullHandling.INCLUDE) - )) - return agg - - @classmethod - def collect(cls): - cdef RollingAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_collect_list_aggregation[ - rolling_aggregation](libcudf_types.null_policy.INCLUDE)) - return agg - - @classmethod - def from_udf(cls, op, *args, **kwargs): - cdef RollingAggregation agg = cls() - - cdef libcudf_types.type_id tid - cdef libcudf_types.data_type out_dtype - cdef string cpp_str - - # Handling UDF type - nb_type = numpy_support.from_dtype(kwargs['dtype']) - type_signature = (nb_type[:],) - compiled_op = cudautils.compile_udf(op, type_signature) - output_np_dtype = cudf.dtype(compiled_op[1]) - cpp_str = compiled_op[0].encode('UTF-8') - if output_np_dtype not in SUPPORTED_NUMPY_TO_LIBCUDF_TYPES: - raise TypeError( - "Result of window function has unsupported dtype {}" - .format(op[1]) - ) - tid = ( - ( - ( - SUPPORTED_NUMPY_TO_LIBCUDF_TYPES[output_np_dtype] - ) - ) - ) - out_dtype = libcudf_types.data_type(tid) - - agg.c_obj = move( - libcudf_aggregation.make_udf_aggregation[rolling_aggregation]( - libcudf_aggregation.udf_type.PTX, cpp_str, out_dtype - )) - return agg - - # scan aggregations - # TODO: update this after adding per algorithm aggregation derived types - # https://github.com/rapidsai/cudf/issues/7106 - cumsum = sum - cummin = min - cummax = max - @classmethod - def cumcount(cls): - cdef RollingAggregation agg = cls() - agg.c_obj = move( - libcudf_aggregation.make_count_aggregation[rolling_aggregation]( - libcudf_types.null_policy.INCLUDE - )) - return agg - -cdef class Aggregation: - def __init__(self, pylibcudf.aggregation.Aggregation agg): +class Aggregation: + def __init__(self, agg): self.c_obj = agg @property def kind(self): - return AggregationKind(int(self.c_obj.kind())).name + name = self.c_obj.kind().name + return _agg_name_map.get(name, name) @classmethod def sum(cls): @@ -295,7 +73,7 @@ cdef class Aggregation: return cls(pylibcudf.aggregation.nunique(pylibcudf.types.NullPolicy.EXCLUDE)) @classmethod - def nth(cls, libcudf_types.size_type size): + def nth(cls, size): return cls(pylibcudf.aggregation.nth_element(size)) @classmethod @@ -350,7 +128,7 @@ cdef class Aggregation: ) @classmethod - def corr(cls, method, libcudf_types.size_type min_periods): + def corr(cls, method, min_periods): return cls(pylibcudf.aggregation.correlation( pylibcudf.aggregation.CorrelationType[method.upper()], min_periods @@ -358,11 +136,7 @@ cdef class Aggregation: )) @classmethod - def cov( - cls, - libcudf_types.size_type min_periods, - libcudf_types.size_type ddof=1 - ): + def cov(cls, min_periods, ddof=1): return cls(pylibcudf.aggregation.covariance( min_periods, ddof @@ -403,46 +177,26 @@ cdef class Aggregation: def all(cls): return cls(pylibcudf.aggregation.all()) + # Rolling aggregations + @classmethod + def from_udf(cls, op, *args, **kwargs): + # Handling UDF type + nb_type = numpy_support.from_dtype(kwargs['dtype']) + type_signature = (nb_type[:],) + ptx_code, output_dtype = cudautils.compile_udf(op, type_signature) + output_np_dtype = cudf.dtype(output_dtype) + if output_np_dtype not in SUPPORTED_NUMPY_TO_PYLIBCUDF_TYPES: + raise TypeError(f"Result of window function has unsupported dtype {op[1]}") -cdef RollingAggregation make_rolling_aggregation(op, kwargs=None): - r""" - Parameters - ---------- - op : str or callable - If callable, must meet one of the following requirements: - - * Is of the form lambda x: x.agg(*args, **kwargs), where - `agg` is the name of a supported aggregation. Used to - to specify aggregations that take arguments, e.g., - `lambda x: x.quantile(0.5)`. - * Is a user defined aggregation function that operates on - group values. In this case, the output dtype must be - specified in the `kwargs` dictionary. - \*\*kwargs : dict, optional - Any keyword arguments to be passed to the op. - - Returns - ------- - RollingAggregation - """ - if kwargs is None: - kwargs = {} + return cls( + pylibcudf.aggregation.udf( + ptx_code, + pylibcudf.DataType(SUPPORTED_NUMPY_TO_PYLIBCUDF_TYPES[output_np_dtype]), + ) + ) - cdef RollingAggregation agg - if isinstance(op, str): - agg = getattr(RollingAggregation, op)(**kwargs) - elif callable(op): - if op is list: - agg = RollingAggregation.collect() - elif "dtype" in kwargs: - agg = RollingAggregation.from_udf(op, **kwargs) - else: - agg = op(RollingAggregation) - else: - raise TypeError(f"Unknown aggregation {op}") - return agg -cdef Aggregation make_aggregation(op, kwargs=None): +def make_aggregation(op, kwargs=None): r""" Parameters ---------- @@ -466,16 +220,13 @@ cdef Aggregation make_aggregation(op, kwargs=None): if kwargs is None: kwargs = {} - cdef Aggregation agg if isinstance(op, str): - agg = getattr(Aggregation, op)(**kwargs) + return getattr(Aggregation, op)(**kwargs) elif callable(op): if op is list: - agg = Aggregation.collect() + return Aggregation.collect() elif "dtype" in kwargs: - agg = Aggregation.from_udf(op, **kwargs) + return Aggregation.from_udf(op, **kwargs) else: - agg = op(Aggregation) - else: - raise TypeError(f"Unknown aggregation {op}") - return agg + return op(Aggregation) + raise TypeError(f"Unknown aggregation {op}") diff --git a/python/cudf/cudf/_lib/cpp/aggregation.pxd b/python/cudf/cudf/_lib/cpp/aggregation.pxd index 16f48b30a50..91b9d7d024f 100644 --- a/python/cudf/cudf/_lib/cpp/aggregation.pxd +++ b/python/cudf/cudf/_lib/cpp/aggregation.pxd @@ -16,8 +16,6 @@ from cudf._lib.cpp.types cimport ( size_type, ) -ctypedef int32_t underlying_type_t_correlation_type -ctypedef int32_t underlying_type_t_rank_method cdef extern from "cudf/aggregation.hpp" namespace "cudf" nogil: diff --git a/python/cudf/cudf/_lib/cpp/rolling.pxd b/python/cudf/cudf/_lib/cpp/rolling.pxd index df2e833edc2..6b620e3a4c0 100644 --- a/python/cudf/cudf/_lib/cpp/rolling.pxd +++ b/python/cudf/cudf/_lib/cpp/rolling.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr @@ -16,11 +16,11 @@ cdef extern from "cudf/rolling.hpp" namespace "cudf" nogil: column_view preceding_window, column_view following_window, size_type min_periods, - rolling_aggregation agg) except + + rolling_aggregation& agg) except + cdef unique_ptr[column] rolling_window( column_view source, size_type preceding_window, size_type following_window, size_type min_periods, - rolling_aggregation agg) except + + rolling_aggregation& agg) except + diff --git a/python/cudf/cudf/_lib/groupby.pyx b/python/cudf/cudf/_lib/groupby.pyx index 8384d5231b7..05300a41009 100644 --- a/python/cudf/cudf/_lib/groupby.pyx +++ b/python/cudf/cudf/_lib/groupby.pyx @@ -18,11 +18,11 @@ from cudf._lib.utils cimport columns_from_pylibcudf_table from cudf._lib.scalar import as_device_scalar -from cudf._lib.aggregation cimport make_aggregation from cudf._lib.cpp.replace cimport replace_policy from cudf._lib.cpp.scalar.scalar cimport scalar from cudf._lib import pylibcudf +from cudf._lib.aggregation import make_aggregation # The sets below define the possible aggregations that can be performed on # different dtypes. These strings must be elements of the AggregationKind enum. diff --git a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt index 6144fd07ac0..5eb0e5cdf82 100644 --- a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt +++ b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt @@ -14,7 +14,7 @@ set(cython_sources aggregation.pyx binaryop.pyx column.pyx copying.pyx gpumemoryview.pyx groupby.pyx interop.pyx - join.pyx reduce.pyx scalar.pyx table.pyx types.pyx unary.pyx utils.pyx + join.pyx reduce.pyx rolling.pyx scalar.pyx table.pyx types.pyx unary.pyx utils.pyx ) set(linked_libraries cudf::cudf) rapids_cython_create_modules( diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd index 74afa2dbacd..df65e893b68 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd @@ -9,6 +9,8 @@ from . cimport ( interop, join, reduce, + rolling, + types, unary, ) from .column cimport Column @@ -33,5 +35,6 @@ __all__ = [ "join", "unary", "reduce", + "rolling", "types", ] diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.py b/python/cudf/cudf/_lib/pylibcudf/__init__.py index 96663d365a8..52dded12071 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.py +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.py @@ -8,6 +8,8 @@ interop, join, reduce, + rolling, + types, unary, ) from .column import Column @@ -31,5 +33,6 @@ "join", "unary", "reduce", + "rolling", "types", ] diff --git a/python/cudf/cudf/_lib/pylibcudf/aggregation.pxd b/python/cudf/cudf/_lib/pylibcudf/aggregation.pxd index 1b7da5a5532..a9491793b88 100644 --- a/python/cudf/cudf/_lib/pylibcudf/aggregation.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/aggregation.pxd @@ -11,6 +11,7 @@ from cudf._lib.cpp.aggregation cimport ( rank_method, rank_percentage, reduce_aggregation, + rolling_aggregation, scan_aggregation, ) from cudf._lib.cpp.types cimport ( @@ -30,6 +31,7 @@ ctypedef groupby_aggregation * gba_ptr ctypedef groupby_scan_aggregation * gbsa_ptr ctypedef reduce_aggregation * ra_ptr ctypedef scan_aggregation * sa_ptr +ctypedef rolling_aggregation * roa_ptr cdef class Aggregation: @@ -42,6 +44,7 @@ cdef class Aggregation: ) except * cdef const reduce_aggregation* view_underlying_as_reduce(self) except * cdef const scan_aggregation* view_underlying_as_scan(self) except * + cdef const rolling_aggregation* view_underlying_as_rolling(self) except * @staticmethod cdef Aggregation from_libcudf(unique_ptr[aggregation] agg) diff --git a/python/cudf/cudf/_lib/pylibcudf/aggregation.pyx b/python/cudf/cudf/_lib/pylibcudf/aggregation.pyx index 0020a0c681d..fe7daea38bf 100644 --- a/python/cudf/cudf/_lib/pylibcudf/aggregation.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/aggregation.pyx @@ -36,6 +36,7 @@ from cudf._lib.cpp.aggregation cimport ( rank_method, rank_percentage, reduce_aggregation, + rolling_aggregation, scan_aggregation, ) from cudf._lib.cpp.types cimport ( @@ -124,6 +125,13 @@ cdef class Aggregation: self._unsupported_agg_error("scan") return agg_cast + cdef const rolling_aggregation* view_underlying_as_rolling(self) except *: + """View the underlying aggregation as a rolling_aggregation.""" + cdef rolling_aggregation *agg_cast = dynamic_cast[roa_ptr](self.c_obj.get()) + if agg_cast is NULL: + self._unsupported_agg_error("rolling") + return agg_cast + @staticmethod cdef Aggregation from_libcudf(unique_ptr[aggregation] agg): """Create a Python Aggregation from a libcudf aggregation.""" diff --git a/python/cudf/cudf/_lib/pylibcudf/rolling.pxd b/python/cudf/cudf/_lib/pylibcudf/rolling.pxd new file mode 100644 index 00000000000..88d683c0c35 --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/rolling.pxd @@ -0,0 +1,19 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from cudf._lib.cpp.types cimport size_type + +from .aggregation cimport Aggregation +from .column cimport Column + +ctypedef fused WindowType: + Column + size_type + + +cpdef Column rolling_window( + Column source, + WindowType preceding_window, + WindowType following_window, + size_type min_periods, + Aggregation agg, +) diff --git a/python/cudf/cudf/_lib/pylibcudf/rolling.pyx b/python/cudf/cudf/_lib/pylibcudf/rolling.pyx new file mode 100644 index 00000000000..8a1d83911ca --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/rolling.pyx @@ -0,0 +1,73 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from cython.operator cimport dereference +from libcpp.memory cimport unique_ptr +from libcpp.utility cimport move + +from cudf._lib.cpp cimport rolling as cpp_rolling +from cudf._lib.cpp.aggregation cimport rolling_aggregation +from cudf._lib.cpp.column.column cimport column +from cudf._lib.cpp.types cimport size_type + +from .aggregation cimport Aggregation +from .column cimport Column + + +cpdef Column rolling_window( + Column source, + WindowType preceding_window, + WindowType following_window, + size_type min_periods, + Aggregation agg, +): + """Perform a rolling window operation on a column + + For details, see ``cudf::rolling_window`` documentation. + + Parameters + ---------- + source : Column + The column to perform the rolling window operation on. + preceding_window : Union[Column, size_type] + The column containing the preceding window sizes or a scalar value + indicating the sizes of all windows. + following_window : Union[Column, size_type] + The column containing the following window sizes or a scalar value + indicating the sizes of all windows. + min_periods : int + The minimum number of periods to include in the result. + agg : Aggregation + The aggregation to perform. + + Returns + ------- + Column + The result of the rolling window operation. + """ + cdef unique_ptr[column] result + # TODO: Consider making all the conversion functions nogil functions that + # reclaim the GIL internally for just the necessary scope like column.view() + cdef const rolling_aggregation *c_agg = agg.view_underlying_as_rolling() + if WindowType is Column: + with nogil: + result = move( + cpp_rolling.rolling_window( + source.view(), + preceding_window.view(), + following_window.view(), + min_periods, + dereference(c_agg), + ) + ) + else: + with nogil: + result = move( + cpp_rolling.rolling_window( + source.view(), + preceding_window, + following_window, + min_periods, + dereference(c_agg), + ) + ) + return Column.from_libcudf(move(result)) diff --git a/python/cudf/cudf/_lib/reduce.pyx b/python/cudf/cudf/_lib/reduce.pyx index 5767cc8eee1..56bfa0ba332 100644 --- a/python/cudf/cudf/_lib/reduce.pyx +++ b/python/cudf/cudf/_lib/reduce.pyx @@ -3,12 +3,12 @@ import cudf from cudf.core.buffer import acquire_spill_lock -from cudf._lib.aggregation cimport make_aggregation from cudf._lib.column cimport Column from cudf._lib.scalar cimport DeviceScalar from cudf._lib.types cimport dtype_to_pylibcudf_type, is_decimal_type_id from cudf._lib import pylibcudf +from cudf._lib.aggregation import make_aggregation @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/rolling.pyx b/python/cudf/cudf/_lib/rolling.pyx index 8c4751e3084..5439e70fdce 100644 --- a/python/cudf/cudf/_lib/rolling.pyx +++ b/python/cudf/cudf/_lib/rolling.pyx @@ -1,16 +1,11 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from cudf.core.buffer import acquire_spill_lock -from libcpp.memory cimport unique_ptr -from libcpp.utility cimport move - -from cudf._lib.aggregation cimport RollingAggregation, make_rolling_aggregation from cudf._lib.column cimport Column -from cudf._lib.cpp.column.column cimport column -from cudf._lib.cpp.column.column_view cimport column_view -from cudf._lib.cpp.rolling cimport rolling_window as cpp_rolling_window -from cudf._lib.cpp.types cimport size_type + +from cudf._lib import pylibcudf +from cudf._lib.aggregation import make_aggregation @acquire_spill_lock() @@ -41,20 +36,6 @@ def rolling(Column source_column, ------- A Column with rolling calculations """ - cdef size_type c_min_periods = min_periods - cdef size_type c_window = 0 - cdef size_type c_forward_window = 0 - cdef unique_ptr[column] c_result - cdef column_view source_column_view = source_column.view() - cdef column_view pre_column_window_view - cdef column_view fwd_column_window_view - cdef RollingAggregation cython_agg - - if callable(op): - cython_agg = make_rolling_aggregation( - op, {'dtype': source_column.dtype}) - else: - cython_agg = make_rolling_aggregation(op, agg_params) if window is None: if center: @@ -62,34 +43,24 @@ def rolling(Column source_column, raise NotImplementedError( "center is not implemented for offset-based windows" ) - pre_column_window_view = pre_column_window.view() - fwd_column_window_view = fwd_column_window.view() - with nogil: - c_result = move( - cpp_rolling_window( - source_column_view, - pre_column_window_view, - fwd_column_window_view, - c_min_periods, - cython_agg.c_obj.get()[0]) - ) + pre = pre_column_window.to_pylibcudf(mode="read") + fwd = fwd_column_window.to_pylibcudf(mode="read") else: - c_min_periods = min_periods if center: - c_window = (window // 2) + 1 - c_forward_window = window - (c_window) + pre = (window // 2) + 1 + fwd = window - (pre) else: - c_window = window - c_forward_window = 0 - - with nogil: - c_result = move( - cpp_rolling_window( - source_column_view, - c_window, - c_forward_window, - c_min_periods, - cython_agg.c_obj.get()[0]) - ) + pre = window + fwd = 0 - return Column.from_unique_ptr(move(c_result)) + return Column.from_pylibcudf( + pylibcudf.rolling.rolling_window( + source_column.to_pylibcudf(mode="read"), + pre, + fwd, + min_periods, + make_aggregation( + op, {'dtype': source_column.dtype} if callable(op) else agg_params + ).c_obj, + ) + ) diff --git a/python/cudf/cudf/_lib/sort.pyx b/python/cudf/cudf/_lib/sort.pyx index b80ea9c7fdc..e230dffbf3c 100644 --- a/python/cudf/cudf/_lib/sort.pyx +++ b/python/cudf/cudf/_lib/sort.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from itertools import repeat @@ -10,10 +10,7 @@ from libcpp.utility cimport move, pair from libcpp.vector cimport vector from cudf._lib.column cimport Column -from cudf._lib.cpp.aggregation cimport ( - rank_method, - underlying_type_t_rank_method, -) +from cudf._lib.cpp.aggregation cimport rank_method from cudf._lib.cpp.column.column cimport column from cudf._lib.cpp.column.column_view cimport column_view from cudf._lib.cpp.search cimport lower_bound, upper_bound @@ -414,16 +411,12 @@ def digitize(list source_columns, list bins, bool right=False): @acquire_spill_lock() -def rank_columns(list source_columns, object method, str na_option, +def rank_columns(list source_columns, rank_method method, str na_option, bool ascending, bool pct ): """ Compute numerical data ranks (1 through n) of each column in the dataframe """ - cdef rank_method c_rank_method = < rank_method > ( - < underlying_type_t_rank_method > method - ) - cdef cpp_order column_order = ( cpp_order.ASCENDING if ascending @@ -464,7 +457,7 @@ def rank_columns(list source_columns, object method, str na_option, c_results.push_back(move( rank( c_view, - c_rank_method, + method, column_order, c_null_handling, null_precedence, diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index bc24216cade..8e43000d0a8 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -6113,7 +6113,7 @@ def rank( if method not in {"average", "min", "max", "first", "dense"}: raise KeyError(method) - method_enum = libcudf.aggregation.RankMethod[method.upper()] + method_enum = libcudf.pylibcudf.aggregation.RankMethod[method.upper()] if na_option not in {"keep", "top", "bottom"}: raise ValueError( "na_option must be one of 'keep', 'top', or 'bottom'" From 8503b31c9aff066e620f184883105b7ee6f8551c Mon Sep 17 00:00:00 2001 From: Yunsong Wang Date: Thu, 8 Feb 2024 09:43:26 -0800 Subject: [PATCH 223/384] Clean up detail sequence header inclusion (#15007) A small fix avoiding the detail sequence header including itself. Authors: - Yunsong Wang (https://github.com/PointKernel) Approvers: - Bradley Dice (https://github.com/bdice) - David Wendt (https://github.com/davidwendt) URL: https://github.com/rapidsai/cudf/pull/15007 --- cpp/include/cudf/detail/sequence.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/include/cudf/detail/sequence.hpp b/cpp/include/cudf/detail/sequence.hpp index 3c3d1d0ed9e..6f2a43b54de 100644 --- a/cpp/include/cudf/detail/sequence.hpp +++ b/cpp/include/cudf/detail/sequence.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,6 @@ #pragma once -#include #include #include #include From 306c47ca1ef17f7bc62a249693a96aab8c48d608 Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Thu, 8 Feb 2024 12:00:24 -0600 Subject: [PATCH 224/384] JNI JSON read with DataSource and infered schema, along with basic java nested Schema JSON reads (#14954) This adds in support for some more JSON reading functionality. It allows us to infer the JSON schema using a DataSource as the input. It also adds in support for using a nested Schema when parsing JSON. Authors: - Robert (Bobby) Evans (https://github.com/revans2) Approvers: - Jason Lowe (https://github.com/jlowe) URL: https://github.com/rapidsai/cudf/pull/14954 --- java/src/main/java/ai/rapids/cudf/Schema.java | 269 +++++++++++++++-- java/src/main/java/ai/rapids/cudf/Table.java | 205 ++++++++++++- .../java/ai/rapids/cudf/TableWithMeta.java | 97 +++++- java/src/main/native/src/TableJni.cpp | 277 ++++++++++++------ .../test/java/ai/rapids/cudf/TableTest.java | 144 ++++++++- 5 files changed, 845 insertions(+), 147 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/Schema.java b/java/src/main/java/ai/rapids/cudf/Schema.java index 79e66cb608e..c8571dd841c 100644 --- a/java/src/main/java/ai/rapids/cudf/Schema.java +++ b/java/src/main/java/ai/rapids/cudf/Schema.java @@ -1,6 +1,6 @@ /* * - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,78 +26,285 @@ */ public class Schema { public static final Schema INFERRED = new Schema(); - private final List names; - private final List types; - private Schema(List names, List types) { - this.names = new ArrayList<>(names); - this.types = new ArrayList<>(types); + private final DType topLevelType; + private final List childNames; + private final List childSchemas; + private boolean flattened = false; + private String[] flattenedNames; + private DType[] flattenedTypes; + private int[] flattenedCounts; + + private Schema(DType topLevelType, + List childNames, + List childSchemas) { + this.topLevelType = topLevelType; + this.childNames = childNames; + this.childSchemas = childSchemas; } /** * Inferred schema. */ private Schema() { - names = null; - types = null; + topLevelType = null; + childNames = null; + childSchemas = null; + } + + /** + * Get the schema of a child element. Note that an inferred schema will have no children. + * @param i the index of the child to read. + * @return the new Schema + * @throws IndexOutOfBoundsException if the index is not in the range of children. + */ + public Schema getChild(int i) { + if (childSchemas == null) { + throw new IndexOutOfBoundsException("There are 0 children in this schema"); + } + return childSchemas.get(i); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(topLevelType); + if (topLevelType == DType.STRUCT) { + sb.append("{"); + if (childNames != null) { + for (int i = 0; i < childNames.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(childNames.get(i)); + sb.append(": "); + sb.append(childSchemas.get(i)); + } + } + sb.append("}"); + } else if (topLevelType == DType.LIST) { + sb.append("["); + if (childNames != null) { + for (int i = 0; i < childNames.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(childSchemas.get(i)); + } + } + sb.append("]"); + } + return sb.toString(); + } + + private void flattenIfNeeded() { + if (!flattened) { + int flatLen = flattenedLength(0); + if (flatLen == 0) { + flattenedNames = null; + flattenedTypes = null; + flattenedCounts = null; + } else { + String[] names = new String[flatLen]; + DType[] types = new DType[flatLen]; + int[] counts = new int[flatLen]; + collectFlattened(names, types, counts, 0); + flattenedNames = names; + flattenedTypes = types; + flattenedCounts = counts; + } + flattened = true; + } + } + + private int flattenedLength(int startingLength) { + if (childSchemas != null) { + for (Schema child: childSchemas) { + startingLength++; + startingLength = child.flattenedLength(startingLength); + } + } + return startingLength; + } + + private int collectFlattened(String[] names, DType[] types, int[] counts, int offset) { + if (childSchemas != null) { + for (int i = 0; i < childSchemas.size(); i++) { + Schema child = childSchemas.get(i); + names[offset] = childNames.get(i); + types[offset] = child.topLevelType; + if (child.childNames != null) { + counts[offset] = child.childNames.size(); + } else { + counts[offset] = 0; + } + offset++; + offset = this.childSchemas.get(i).collectFlattened(names, types, counts, offset); + } + } + return offset; } public static Builder builder() { - return new Builder(); + return new Builder(DType.STRUCT); + } + + public String[] getFlattenedColumnNames() { + flattenIfNeeded(); + return flattenedNames; } public String[] getColumnNames() { - if (names == null) { + if (childNames == null) { return null; } - return names.toArray(new String[names.size()]); + return childNames.toArray(new String[childNames.size()]); + } + + public boolean isNested() { + return childSchemas != null && childSchemas.size() > 0; + } + + /** + * This is really for a top level struct schema where it is nested, but + * for things like CSV we care that it does not have any children that are also + * nested. + */ + public boolean hasNestedChildren() { + if (childSchemas != null) { + for (Schema child: childSchemas) { + if (child.isNested()) { + return true; + } + } + } + return false; } - int[] getTypeIds() { - if (types == null) { + int[] getFlattenedTypeIds() { + flattenIfNeeded(); + if (flattenedTypes == null) { return null; } - int[] ret = new int[types.size()]; - for (int i = 0; i < types.size(); i++) { - ret[i] = types.get(i).getTypeId().nativeId; + int[] ret = new int[flattenedTypes.length]; + for (int i = 0; i < flattenedTypes.length; i++) { + ret[i] = flattenedTypes[i].getTypeId().nativeId; } return ret; } - int[] getTypeScales() { - if (types == null) { + int[] getFlattenedTypeScales() { + flattenIfNeeded(); + if (flattenedTypes == null) { return null; } - int[] ret = new int[types.size()]; - for (int i = 0; i < types.size(); i++) { - ret[i] = types.get(i).getScale(); + int[] ret = new int[flattenedTypes.length]; + for (int i = 0; i < flattenedTypes.length; i++) { + ret[i] = flattenedTypes[i].getScale(); } return ret; } - DType[] getTypes() { - if (types == null) { + DType[] getFlattenedTypes() { + flattenIfNeeded(); + return flattenedTypes; + } + + public DType[] getChildTypes() { + if (childSchemas == null) { return null; } - DType[] ret = new DType[types.size()]; - for (int i = 0; i < types.size(); i++) { - ret[i] = types.get(i); + DType[] ret = new DType[childSchemas.size()]; + for (int i = 0; i < ret.length; i++) { + ret[i] = childSchemas.get(i).topLevelType; } return ret; } + int[] getFlattenedNumChildren() { + flattenIfNeeded(); + return flattenedCounts; + } + + public DType getType() { + return topLevelType; + } + + /** + * Check to see if the schema includes a struct at all. + * @return true if this or any one of its descendants contains a struct, else false. + */ + public boolean isStructOrHasStructDescendant() { + if (DType.STRUCT == topLevelType) { + return true; + } else if (DType.LIST == topLevelType) { + return childSchemas.stream().anyMatch(Schema::isStructOrHasStructDescendant); + } + return false; + } + public static class Builder { - private final List names = new ArrayList<>(); - private final List types = new ArrayList<>(); + private final DType topLevelType; + private final List names; + private final List types; - public Builder column(DType type, String name) { - types.add(type); + private Builder(DType topLevelType) { + this.topLevelType = topLevelType; + if (topLevelType == DType.STRUCT || topLevelType == DType.LIST) { + // There can be children + names = new ArrayList<>(); + types = new ArrayList<>(); + } else { + names = null; + types = null; + } + } + + /** + * Add a new column + * @param type the type of column to add + * @param name the name of the column to add (Ignored for list types) + * @return the builder for the new column. This should really only be used when the type + * passed in is a LIST or a STRUCT. + */ + public Builder addColumn(DType type, String name) { + if (names == null) { + throw new IllegalStateException("A column of type " + topLevelType + + " cannot have children"); + } + if (topLevelType == DType.LIST && names.size() > 0) { + throw new IllegalStateException("A LIST column can only have one child"); + } + if (names.contains(name)) { + throw new IllegalStateException("Cannot add duplicate names to a schema"); + } + Builder ret = new Builder(type); + types.add(ret); names.add(name); + return ret; + } + + /** + * Adds a single column to the current schema. addColumn is preferred as it can be used + * to support nested types. + * @param type the type of the column. + * @param name the name of the column. + * @return this for chaining. + */ + public Builder column(DType type, String name) { + addColumn(type, name); return this; } public Schema build() { - return new Schema(names, types); + List children = null; + if (types != null) { + children = new ArrayList<>(types.size()); + for (Builder b: types) { + children.add(b.build()); + } + } + return new Schema(topLevelType, names, children); } } } diff --git a/java/src/main/java/ai/rapids/cudf/Table.java b/java/src/main/java/ai/rapids/cudf/Table.java index ecf2e860351..9a790c8518b 100644 --- a/java/src/main/java/ai/rapids/cudf/Table.java +++ b/java/src/main/java/ai/rapids/cudf/Table.java @@ -246,7 +246,7 @@ private static native long[] readCSVFromDataSource(String[] columnNames, /** * read JSON data and return a pointer to a TableWithMeta object. */ - private static native long readJSON(String[] columnNames, + private static native long readJSON(int[] numChildren, String[] columnNames, int[] dTypeIds, int[] dTypeScales, String filePath, long address, long length, boolean dayFirst, boolean lines, @@ -254,7 +254,7 @@ private static native long readJSON(String[] columnNames, boolean normalizeSingleQuotes, boolean mixedTypesAsStrings) throws CudfException; - private static native long readJSONFromDataSource(String[] columnNames, + private static native long readJSONFromDataSource(int[] numChildren, String[] columnNames, int[] dTypeIds, int[] dTypeScales, boolean dayFirst, boolean lines, boolean recoverWithNulls, @@ -262,6 +262,11 @@ private static native long readJSONFromDataSource(String[] columnNames, boolean mixedTypesAsStrings, long dsHandle) throws CudfException; + private static native long readAndInferJSONFromDataSource(boolean dayFirst, boolean lines, + boolean recoverWithNulls, + boolean normalizeSingleQuotes, + boolean mixedTypesAsStrings, + long dsHandle) throws CudfException; private static native long readAndInferJSON(long address, long length, boolean dayFirst, boolean lines, boolean recoverWithNulls, boolean normalizeSingleQuotes, boolean mixedTypesAsStrings) throws CudfException; @@ -808,8 +813,11 @@ public static Table readCSV(Schema schema, File path) { * @return the file parsed as a table on the GPU. */ public static Table readCSV(Schema schema, CSVOptions opts, File path) { + if (schema.hasNestedChildren()) { + throw new IllegalArgumentException("CSV does not support nested types"); + } return new Table( - readCSV(schema.getColumnNames(), schema.getTypeIds(), schema.getTypeScales(), + readCSV(schema.getFlattenedColumnNames(), schema.getFlattenedTypeIds(), schema.getFlattenedTypeScales(), opts.getIncludeColumnNames(), path.getAbsolutePath(), 0, 0, opts.getHeaderRow(), @@ -890,7 +898,10 @@ public static Table readCSV(Schema schema, CSVOptions opts, HostMemoryBuffer buf assert len > 0; assert len <= buffer.getLength() - offset; assert offset >= 0 && offset < buffer.length; - return new Table(readCSV(schema.getColumnNames(), schema.getTypeIds(), schema.getTypeScales(), + if (schema.hasNestedChildren()) { + throw new IllegalArgumentException("CSV does not support nested types"); + } + return new Table(readCSV(schema.getFlattenedColumnNames(), schema.getFlattenedTypeIds(), schema.getFlattenedTypeScales(), opts.getIncludeColumnNames(), null, buffer.getAddress() + offset, len, opts.getHeaderRow(), @@ -906,9 +917,12 @@ public static Table readCSV(Schema schema, CSVOptions opts, HostMemoryBuffer buf public static Table readCSV(Schema schema, CSVOptions opts, DataSource ds) { long dsHandle = DataSourceHelper.createWrapperDataSource(ds); try { - return new Table(readCSVFromDataSource(schema.getColumnNames(), - schema.getTypeIds(), - schema.getTypeScales(), + if (schema.hasNestedChildren()) { + throw new IllegalArgumentException("CSV does not support nested types"); + } + return new Table(readCSVFromDataSource(schema.getFlattenedColumnNames(), + schema.getFlattenedTypeIds(), + schema.getFlattenedTypeScales(), opts.getIncludeColumnNames(), opts.getHeaderRow(), opts.getDelim(), @@ -1043,6 +1057,134 @@ public static Table readJSON(Schema schema, JSONOptions opts, byte[] buffer) { return readJSON(schema, opts, buffer, 0, buffer.length); } + private static class DidViewChange { + ColumnVector changeWasNeeded = null; + boolean noChangeNeeded = false; + + public static DidViewChange yes(ColumnVector cv) { + DidViewChange ret = new DidViewChange(); + ret.changeWasNeeded = cv; + return ret; + } + + public static DidViewChange no() { + DidViewChange ret = new DidViewChange(); + ret.noChangeNeeded = true; + return ret; + } + } + + private static DidViewChange gatherJSONColumns(Schema schema, TableWithMeta.NestedChildren children, + ColumnView cv) { + // We need to do this recursively to be sure it all matches as expected. + // If we run into problems where the data types don't match, we are not + // going to fix up the data types. We are only going to reorder the columns. + if (schema.getType() == DType.STRUCT) { + if (cv.getType() != DType.STRUCT) { + // The types don't match so just return the input unchanged... + return DidViewChange.no(); + } else { + String[] foundNames = children.getNames(); + HashMap indices = new HashMap<>(); + for (int i = 0; i < foundNames.length; i++) { + indices.put(foundNames[i], i); + } + // We might need to rearrange the columns to match what we want. + DType[] types = schema.getChildTypes(); + String[] neededNames = schema.getColumnNames(); + ColumnView[] columns = new ColumnView[neededNames.length]; + try { + boolean somethingChanged = false; + if (columns.length != foundNames.length) { + somethingChanged = true; + } + for (int i = 0; i < columns.length; i++) { + String neededColumnName = neededNames[i]; + Integer index = indices.get(neededColumnName); + if (index != null) { + if (schema.getChild(i).isStructOrHasStructDescendant()) { + ColumnView child = cv.getChildColumnView(index); + boolean shouldCloseChild = true; + try { + if (index != i) { + somethingChanged = true; + } + DidViewChange childResult = gatherJSONColumns(schema.getChild(i), + children.getChild(index), child); + if (childResult.noChangeNeeded) { + shouldCloseChild = false; + columns[i] = child; + } else { + somethingChanged = true; + columns[i] = childResult.changeWasNeeded; + } + } finally { + if (shouldCloseChild) { + child.close(); + } + } + } else { + if (index != i) { + somethingChanged = true; + } + columns[i] = cv.getChildColumnView(index); + } + } else { + somethingChanged = true; + try (Scalar s = Scalar.fromNull(types[i])) { + columns[i] = ColumnVector.fromScalar(s, (int) cv.getRowCount()); + } + } + } + if (somethingChanged) { + try (ColumnView ret = new ColumnView(cv.type, cv.rows, Optional.of(cv.nullCount), + cv.getValid(), null, columns)) { + return DidViewChange.yes(ret.copyToColumnVector()); + } + } else { + return DidViewChange.no(); + } + } finally { + for (ColumnView c: columns) { + if (c != null) { + c.close(); + } + } + } + } + } else if (schema.getType() == DType.LIST && cv.getType() == DType.LIST) { + if (schema.isStructOrHasStructDescendant()) { + String [] childNames = children.getNames(); + if (childNames.length == 2 && + "offsets".equals(childNames[0]) && + "element".equals(childNames[1])) { + try (ColumnView child = cv.getChildColumnView(0)){ + DidViewChange listResult = gatherJSONColumns(schema.getChild(0), + children.getChild(1), child); + if (listResult.noChangeNeeded) { + return DidViewChange.no(); + } else { + try (ColumnView listView = new ColumnView(cv.type, cv.rows, + Optional.of(cv.nullCount), cv.getValid(), cv.getOffsets(), + new ColumnView[]{listResult.changeWasNeeded})) { + return DidViewChange.yes(listView.copyToColumnVector()); + } finally { + listResult.changeWasNeeded.close(); + } + } + } + } + } + // Nothing to change so just return the input, but we need to inc a ref count to really + // make it work, so for now we are going to turn it into a ColumnVector. + return DidViewChange.no(); + } else { + // Nothing to change so just return the input, but we need to inc a ref count to really + // make it work, so for now we are going to turn it into a ColumnVector. + return DidViewChange.no(); + } + } + private static Table gatherJSONColumns(Schema schema, TableWithMeta twm) { String[] neededColumns = schema.getColumnNames(); if (neededColumns == null || neededColumns.length == 0) { @@ -1054,14 +1196,24 @@ private static Table gatherJSONColumns(Schema schema, TableWithMeta twm) { indices.put(foundNames[i], i); } // We might need to rearrange the columns to match what we want. - DType[] types = schema.getTypes(); + DType[] types = schema.getChildTypes(); ColumnVector[] columns = new ColumnVector[neededColumns.length]; try (Table tbl = twm.releaseTable()) { for (int i = 0; i < columns.length; i++) { String neededColumnName = neededColumns[i]; Integer index = indices.get(neededColumnName); if (index != null) { - columns[i] = tbl.getColumn(index).incRefCount(); + if (schema.getChild(i).isStructOrHasStructDescendant()) { + DidViewChange gathered = gatherJSONColumns(schema.getChild(i), twm.getChild(index), + tbl.getColumn(index)); + if (gathered.noChangeNeeded) { + columns[i] = tbl.getColumn(index).incRefCount(); + } else { + columns[i] = gathered.changeWasNeeded; + } + } else { + columns[i] = tbl.getColumn(index).incRefCount(); + } } else { try (Scalar s = Scalar.fromNull(types[i])) { columns[i] = ColumnVector.fromScalar(s, (int)tbl.getRowCount()); @@ -1088,7 +1240,8 @@ private static Table gatherJSONColumns(Schema schema, TableWithMeta twm) { */ public static Table readJSON(Schema schema, JSONOptions opts, File path) { try (TableWithMeta twm = new TableWithMeta( - readJSON(schema.getColumnNames(), schema.getTypeIds(), schema.getTypeScales(), + readJSON(schema.getFlattenedNumChildren(), schema.getFlattenedColumnNames(), + schema.getFlattenedTypeIds(), schema.getFlattenedTypeScales(), path.getAbsolutePath(), 0, 0, opts.isDayFirst(), opts.isLines(), opts.isRecoverWithNull(), @@ -1150,6 +1303,26 @@ public static TableWithMeta readJSON(JSONOptions opts, HostMemoryBuffer buffer, opts.isMixedTypesAsStrings())); } + /** + * Read JSON formatted data and infer the column names and schema. + * @param opts various JSON parsing options. + * @return the data parsed as a table on the GPU and the metadata for the table returned. + */ + public static TableWithMeta readAndInferJSON(JSONOptions opts, DataSource ds) { + long dsHandle = DataSourceHelper.createWrapperDataSource(ds); + try { + TableWithMeta twm = new TableWithMeta(readAndInferJSONFromDataSource(opts.isDayFirst(), + opts.isLines(), + opts.isRecoverWithNull(), + opts.isNormalizeSingleQuotes(), + opts.isMixedTypesAsStrings(), + dsHandle)); + return twm; + } finally { + DataSourceHelper.destroyWrapperDataSource(dsHandle); + } + } + /** * Read JSON formatted data. * @param schema the schema of the data. You may use Schema.INFERRED to infer the schema. @@ -1167,8 +1340,9 @@ public static Table readJSON(Schema schema, JSONOptions opts, HostMemoryBuffer b assert len > 0; assert len <= buffer.length - offset; assert offset >= 0 && offset < buffer.length; - try (TableWithMeta twm = new TableWithMeta(readJSON(schema.getColumnNames(), - schema.getTypeIds(), schema.getTypeScales(), null, + try (TableWithMeta twm = new TableWithMeta(readJSON( + schema.getFlattenedNumChildren(), schema.getFlattenedColumnNames(), + schema.getFlattenedTypeIds(), schema.getFlattenedTypeScales(), null, buffer.getAddress() + offset, len, opts.isDayFirst(), opts.isLines(), opts.isRecoverWithNull(), opts.isNormalizeSingleQuotes(), opts.isMixedTypesAsStrings()))) { @@ -1185,9 +1359,10 @@ public static Table readJSON(Schema schema, JSONOptions opts, HostMemoryBuffer b */ public static Table readJSON(Schema schema, JSONOptions opts, DataSource ds) { long dsHandle = DataSourceHelper.createWrapperDataSource(ds); - try (TableWithMeta twm = new TableWithMeta(readJSONFromDataSource(schema.getColumnNames(), - schema.getTypeIds(), schema.getTypeScales(), opts.isDayFirst(), opts.isLines(), - opts.isRecoverWithNull(), opts.isNormalizeSingleQuotes(), opts.isMixedTypesAsStrings(), dsHandle))) { + try (TableWithMeta twm = new TableWithMeta(readJSONFromDataSource(schema.getFlattenedNumChildren(), + schema.getFlattenedColumnNames(), schema.getFlattenedTypeIds(), schema.getFlattenedTypeScales(), opts.isDayFirst(), + opts.isLines(), opts.isRecoverWithNull(), opts.isNormalizeSingleQuotes(), + opts.isMixedTypesAsStrings(), dsHandle))) { return gatherJSONColumns(schema, twm); } finally { DataSourceHelper.destroyWrapperDataSource(dsHandle); diff --git a/java/src/main/java/ai/rapids/cudf/TableWithMeta.java b/java/src/main/java/ai/rapids/cudf/TableWithMeta.java index b6b8ad6bc28..040fa68f01e 100644 --- a/java/src/main/java/ai/rapids/cudf/TableWithMeta.java +++ b/java/src/main/java/ai/rapids/cudf/TableWithMeta.java @@ -1,6 +1,6 @@ /* * - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,12 +19,56 @@ package ai.rapids.cudf; +import java.util.Arrays; + /** * A table along with some metadata about the table. This is typically returned when * reading data from an input file where the metadata can be important. */ public class TableWithMeta implements AutoCloseable { private long handle; + private NestedChildren children = null; + + public static class NestedChildren { + private final String[] names; + private final NestedChildren[] children; + + private NestedChildren(String[] names, NestedChildren[] children) { + this.names = names; + this.children = children; + } + + public String[] getNames() { + return names; + } + + public NestedChildren getChild(int i) { + return children[i]; + } + public boolean isChildNested(int i) { + return (getChild(i) != null); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("{"); + if (names != null) { + for (int i = 0; i < names.length; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(names[i]); + sb.append(": "); + if (children != null) { + sb.append(children[i]); + } + } + } + sb.append("}"); + return sb.toString(); + } + } TableWithMeta(long handle) { this.handle = handle; @@ -43,12 +87,57 @@ public Table releaseTable() { } } + private static class ChildAndOffset { + public NestedChildren child; + public int newOffset; + } + + private ChildAndOffset unflatten(int startOffset, String[] flatNames, int[] flatCounts) { + ChildAndOffset ret = new ChildAndOffset(); + int length = flatCounts[startOffset]; + if (length == 0) { + ret.newOffset = startOffset + 1; + return ret; + } else { + String[] names = new String[length]; + NestedChildren[] children = new NestedChildren[length]; + int currentOffset = startOffset + 1; + for (int i = 0; i < length; i++) { + names[i] = flatNames[currentOffset]; + ChildAndOffset tmp = unflatten(currentOffset, flatNames, flatCounts); + children[i] = tmp.child; + currentOffset = tmp.newOffset; + } + ret.newOffset = currentOffset; + ret.child = new NestedChildren(names, children); + return ret; + } + } + + NestedChildren getChildren() { + if (children == null) { + int[] flatCount = getFlattenedChildCounts(handle); + String[] flatNames = getFlattenedColumnNames(handle); + ChildAndOffset tmp = unflatten(0, flatNames, flatCount); + children = tmp.child; + } + return children; + } + /** * Get the names of the top level columns. In the future new APIs can be added to get * names of child columns. */ public String[] getColumnNames() { - return getColumnNames(handle); + return getChildren().getNames(); + } + + public NestedChildren getChild(int i) { + return getChildren().getChild(i); + } + + public boolean isChildNested(int i) { + return getChildren().isChildNested(i); } @Override @@ -63,5 +152,7 @@ public void close() { private static native long[] releaseTable(long handle); - private static native String[] getColumnNames(long handle); + private static native String[] getFlattenedColumnNames(long handle); + + private static native int[] getFlattenedChildCounts(long handle); } diff --git a/java/src/main/native/src/TableJni.cpp b/java/src/main/native/src/TableJni.cpp index cef18b245e7..1d6f1332b06 100644 --- a/java/src/main/native/src/TableJni.cpp +++ b/java/src/main/native/src/TableJni.cpp @@ -925,6 +925,49 @@ cudf::table_view remove_validity_if_needed(cudf::table_view *input_table_view) { return cudf::table_view(views); } +cudf::io::schema_element read_schema_element(int &index, + cudf::jni::native_jintArray const &children, + cudf::jni::native_jstringArray const &names, + cudf::jni::native_jintArray const &types, + cudf::jni::native_jintArray const &scales) { + auto d_type = cudf::data_type{static_cast(types[index]), scales[index]}; + if (d_type.id() == cudf::type_id::STRUCT || d_type.id() == cudf::type_id::LIST) { + std::map child_elems; + int num_children = children[index]; + // go to the next entry, so recursion can parse it. + index++; + for (int i = 0; i < num_children; i++) { + child_elems.insert( + std::pair{names.get(index).get(), + cudf::jni::read_schema_element(index, children, names, types, scales)}); + } + return cudf::io::schema_element{d_type, std::move(child_elems)}; + } else { + if (children[index] != 0) { + throw std::invalid_argument("found children for a type that should have none"); + } + // go to the next entry before returning... + index++; + return cudf::io::schema_element{d_type, {}}; + } +} + +void append_flattened_child_counts(cudf::io::column_name_info const &info, + std::vector &counts) { + counts.push_back(info.children.size()); + for (cudf::io::column_name_info const &child : info.children) { + append_flattened_child_counts(child, counts); + } +} + +void append_flattened_child_names(cudf::io::column_name_info const &info, + std::vector &names) { + names.push_back(info.name); + for (cudf::io::column_name_info const &child : info.children) { + append_flattened_child_names(child, names); + } +} + } // namespace } // namespace jni @@ -1148,14 +1191,12 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_Table_readCSVFromDataSource( cudf::jni::native_jintArray n_types(env, j_types); cudf::jni::native_jintArray n_scales(env, j_scales); if (n_types.is_null() != n_scales.is_null()) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "types and scales must match null", - NULL); + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "types and scales must match null", NULL); } std::vector data_types; if (!n_types.is_null()) { if (n_types.size() != n_scales.size()) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "types and scales must match size", - NULL); + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "types and scales must match size", NULL); } data_types.reserve(n_types.size()); std::transform(n_types.begin(), n_types.end(), n_scales.begin(), @@ -1207,11 +1248,10 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_Table_readCSV( JNI_NULL_CHECK(env, inputfilepath, "input file or buffer must be supplied", NULL); read_buffer = false; } else if (inputfilepath != NULL) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "cannot pass in both a buffer and an inputfilepath", NULL); } else if (buffer_length <= 0) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "An empty buffer is not supported", - NULL); + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "An empty buffer is not supported", NULL); } try { @@ -1220,14 +1260,12 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_Table_readCSV( cudf::jni::native_jintArray n_types(env, j_types); cudf::jni::native_jintArray n_scales(env, j_scales); if (n_types.is_null() != n_scales.is_null()) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "types and scales must match null", - NULL); + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "types and scales must match null", NULL); } std::vector data_types; if (!n_types.is_null()) { if (n_types.size() != n_scales.size()) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "types and scales must match size", - NULL); + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "types and scales must match size", NULL); } data_types.reserve(n_types.size()); std::transform(n_types.begin(), n_types.end(), n_scales.begin(), @@ -1238,8 +1276,7 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_Table_readCSV( cudf::jni::native_jstring filename(env, inputfilepath); if (!read_buffer && filename.is_empty()) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "inputfilepath can't be empty", - NULL); + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "inputfilepath can't be empty", NULL); } cudf::jni::native_jstringArray n_null_values(env, null_values); @@ -1390,13 +1427,43 @@ JNIEXPORT void JNICALL Java_ai_rapids_cudf_Table_endWriteCSVToBuffer(JNIEnv *env CATCH_STD(env, ); } +JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readAndInferJSONFromDataSource( + JNIEnv *env, jclass, jboolean day_first, jboolean lines, jboolean recover_with_null, + jboolean normalize_single_quotes, jboolean mixed_types_as_string, jlong ds_handle) { + + JNI_NULL_CHECK(env, ds_handle, "no data source handle given", 0); + + try { + cudf::jni::auto_set_device(env); + auto ds = reinterpret_cast(ds_handle); + cudf::io::source_info source{ds}; + + auto const recovery_mode = recover_with_null ? + cudf::io::json_recovery_mode_t::RECOVER_WITH_NULL : + cudf::io::json_recovery_mode_t::FAIL; + cudf::io::json_reader_options_builder opts = + cudf::io::json_reader_options::builder(source) + .dayfirst(static_cast(day_first)) + .lines(static_cast(lines)) + .recovery_mode(recovery_mode) + .normalize_single_quotes(static_cast(normalize_single_quotes)) + .mixed_types_as_string(mixed_types_as_string); + + auto result = + std::make_unique(cudf::io::read_json(opts.build())); + + return reinterpret_cast(result.release()); + } + CATCH_STD(env, 0); +} + JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readAndInferJSON( JNIEnv *env, jclass, jlong buffer, jlong buffer_length, jboolean day_first, jboolean lines, jboolean recover_with_null, jboolean normalize_single_quotes, jboolean mixed_types_as_string) { JNI_NULL_CHECK(env, buffer, "buffer cannot be null", 0); if (buffer_length <= 0) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "An empty buffer is not supported", 0); + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "An empty buffer is not supported", 0); } try { @@ -1434,19 +1501,48 @@ JNIEXPORT void JNICALL Java_ai_rapids_cudf_TableWithMeta_close(JNIEnv *env, jcla CATCH_STD(env, ); } -JNIEXPORT jobjectArray JNICALL Java_ai_rapids_cudf_TableWithMeta_getColumnNames(JNIEnv *env, jclass, - jlong handle) { +JNIEXPORT jintArray JNICALL +Java_ai_rapids_cudf_TableWithMeta_getFlattenedChildCounts(JNIEnv *env, jclass, jlong handle) { JNI_NULL_CHECK(env, handle, "handle is null", nullptr); try { cudf::jni::auto_set_device(env); auto ptr = reinterpret_cast(handle); - auto length = ptr->metadata.schema_info.size(); + std::vector counts; + counts.push_back(ptr->metadata.schema_info.size()); + for (cudf::io::column_name_info const &child : ptr->metadata.schema_info) { + cudf::jni::append_flattened_child_counts(child, counts); + } + + auto length = counts.size(); + cudf::jni::native_jintArray ret(env, length); + for (size_t i = 0; i < length; i++) { + ret[i] = counts[i]; + } + ret.commit(); + return ret.get_jArray(); + } + CATCH_STD(env, nullptr); +} + +JNIEXPORT jobjectArray JNICALL +Java_ai_rapids_cudf_TableWithMeta_getFlattenedColumnNames(JNIEnv *env, jclass, jlong handle) { + JNI_NULL_CHECK(env, handle, "handle is null", nullptr); + + try { + cudf::jni::auto_set_device(env); + auto ptr = reinterpret_cast(handle); + std::vector names; + names.push_back("ROOT"); + for (cudf::io::column_name_info const &child : ptr->metadata.schema_info) { + cudf::jni::append_flattened_child_names(child, names); + } + + auto length = names.size(); auto ret = static_cast( env->NewObjectArray(length, env->FindClass("java/lang/String"), nullptr)); for (size_t i = 0; i < length; i++) { - env->SetObjectArrayElement(ret, i, - env->NewStringUTF(ptr->metadata.schema_info[i].name.c_str())); + env->SetObjectArrayElement(ret, i, env->NewStringUTF(names[i].c_str())); } return ret; @@ -1471,8 +1567,8 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_TableWithMeta_releaseTable(JNIE } JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readJSONFromDataSource( - JNIEnv *env, jclass, jobjectArray col_names, jintArray j_types, jintArray j_scales, - jboolean day_first, jboolean lines, jboolean recover_with_null, + JNIEnv *env, jclass, jintArray j_num_children, jobjectArray col_names, jintArray j_types, + jintArray j_scales, jboolean day_first, jboolean lines, jboolean recover_with_null, jboolean normalize_single_quotes, jboolean mixed_types_as_string, jlong ds_handle) { JNI_NULL_CHECK(env, ds_handle, "no data source handle given", 0); @@ -1482,21 +1578,15 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readJSONFromDataSource( cudf::jni::native_jstringArray n_col_names(env, col_names); cudf::jni::native_jintArray n_types(env, j_types); cudf::jni::native_jintArray n_scales(env, j_scales); + cudf::jni::native_jintArray n_children(env, j_num_children); if (n_types.is_null() != n_scales.is_null()) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "types and scales must match null", - 0); + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "types and scales must match null", 0); } - std::vector data_types; - if (!n_types.is_null()) { - if (n_types.size() != n_scales.size()) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "types and scales must match size", - 0); - } - data_types.reserve(n_types.size()); - std::transform(n_types.begin(), n_types.end(), n_scales.begin(), - std::back_inserter(data_types), [](auto const &type, auto const &scale) { - return cudf::data_type{static_cast(type), scale}; - }); + if (n_types.is_null() != n_col_names.is_null()) { + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "types and names must match null", 0); + } + if (n_types.is_null() != n_children.is_null()) { + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "types and num children must match null", 0); } auto ds = reinterpret_cast(ds_handle); @@ -1513,20 +1603,26 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readJSONFromDataSource( .normalize_single_quotes(static_cast(normalize_single_quotes)) .mixed_types_as_string(mixed_types_as_string); - if (!n_col_names.is_null() && data_types.size() > 0) { + if (!n_types.is_null()) { + if (n_types.size() != n_scales.size()) { + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "types and scales must match size", 0); + } if (n_col_names.size() != n_types.size()) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", - "types and column names must match size", 0); + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "types and column names must match size", + 0); + } + if (n_children.size() != n_types.size()) { + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "types and num children must match size", + 0); } - std::map map; - - auto col_names_vec = n_col_names.as_cpp_vector(); - std::transform(col_names_vec.begin(), col_names_vec.end(), data_types.begin(), - std::inserter(map, map.end()), - [](std::string a, cudf::data_type b) { return std::make_pair(a, b); }); - opts.dtypes(map); - } else if (data_types.size() > 0) { + std::map data_types; + int at = 0; + while (at < n_types.size()) { + data_types.insert(std::pair{ + n_col_names.get(at).get(), + cudf::jni::read_schema_element(at, n_children, n_col_names, n_types, n_scales)}); + } opts.dtypes(data_types); } else { // should infer the types @@ -1541,19 +1637,20 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readJSONFromDataSource( } JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readJSON( - JNIEnv *env, jclass, jobjectArray col_names, jintArray j_types, jintArray j_scales, - jstring inputfilepath, jlong buffer, jlong buffer_length, jboolean day_first, jboolean lines, - jboolean recover_with_null, jboolean normalize_single_quotes, jboolean mixed_types_as_string) { + JNIEnv *env, jclass, jintArray j_num_children, jobjectArray col_names, jintArray j_types, + jintArray j_scales, jstring inputfilepath, jlong buffer, jlong buffer_length, + jboolean day_first, jboolean lines, jboolean recover_with_null, + jboolean normalize_single_quotes, jboolean mixed_types_as_string) { bool read_buffer = true; if (buffer == 0) { JNI_NULL_CHECK(env, inputfilepath, "input file or buffer must be supplied", 0); read_buffer = false; } else if (inputfilepath != NULL) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "cannot pass in both a buffer and an inputfilepath", 0); } else if (buffer_length <= 0) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "An empty buffer is not supported", 0); + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "An empty buffer is not supported", 0); } try { @@ -1561,26 +1658,20 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readJSON( cudf::jni::native_jstringArray n_col_names(env, col_names); cudf::jni::native_jintArray n_types(env, j_types); cudf::jni::native_jintArray n_scales(env, j_scales); + cudf::jni::native_jintArray n_children(env, j_num_children); if (n_types.is_null() != n_scales.is_null()) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "types and scales must match null", - 0); + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "types and scales must match null", 0); } - std::vector data_types; - if (!n_types.is_null()) { - if (n_types.size() != n_scales.size()) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "types and scales must match size", - 0); - } - data_types.reserve(n_types.size()); - std::transform(n_types.begin(), n_types.end(), n_scales.begin(), - std::back_inserter(data_types), [](auto const &type, auto const &scale) { - return cudf::data_type{static_cast(type), scale}; - }); + if (n_types.is_null() != n_col_names.is_null()) { + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "types and names must match null", 0); + } + if (n_types.is_null() != n_children.is_null()) { + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "types and num children must match null", 0); } cudf::jni::native_jstring filename(env, inputfilepath); if (!read_buffer && filename.is_empty()) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "inputfilepath can't be empty", 0); + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "inputfilepath can't be empty", 0); } auto source = read_buffer ? cudf::io::source_info{reinterpret_cast(buffer), @@ -1598,20 +1689,26 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readJSON( .normalize_single_quotes(static_cast(normalize_single_quotes)) .mixed_types_as_string(mixed_types_as_string); - if (!n_col_names.is_null() && data_types.size() > 0) { + if (!n_types.is_null()) { + if (n_types.size() != n_scales.size()) { + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "types and scales must match size", 0); + } if (n_col_names.size() != n_types.size()) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", - "types and column names must match size", 0); + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "types and column names must match size", + 0); + } + if (n_children.size() != n_types.size()) { + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "types and num children must match size", + 0); } - std::map map; - - auto col_names_vec = n_col_names.as_cpp_vector(); - std::transform(col_names_vec.begin(), col_names_vec.end(), data_types.begin(), - std::inserter(map, map.end()), - [](std::string a, cudf::data_type b) { return std::make_pair(a, b); }); - opts.dtypes(map); - } else if (data_types.size() > 0) { + std::map data_types; + int at = 0; + while (at < n_types.size()) { + data_types.insert(std::pair{ + n_col_names.get(at).get(), + cudf::jni::read_schema_element(at, n_children, n_col_names, n_types, n_scales)}); + } opts.dtypes(data_types); } else { // should infer the types @@ -1665,19 +1762,17 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_Table_readParquet( JNI_NULL_CHECK(env, inputfilepath, "input file or buffer must be supplied", NULL); read_buffer = false; } else if (inputfilepath != NULL) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "cannot pass in both a buffer and an inputfilepath", NULL); } else if (buffer_length <= 0) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "An empty buffer is not supported", - NULL); + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "An empty buffer is not supported", NULL); } try { cudf::jni::auto_set_device(env); cudf::jni::native_jstring filename(env, inputfilepath); if (!read_buffer && filename.is_empty()) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "inputfilepath can't be empty", - NULL); + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "inputfilepath can't be empty", NULL); } cudf::jni::native_jstringArray n_filter_col_names(env, filter_col_names); @@ -1731,19 +1826,17 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_Table_readAvro(JNIEnv *env, jcl if (!read_buffer) { JNI_NULL_CHECK(env, inputfilepath, "input file or buffer must be supplied", NULL); } else if (inputfilepath != NULL) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "cannot pass in both a buffer and an inputfilepath", NULL); } else if (buffer_length <= 0) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "An empty buffer is not supported", - NULL); + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "An empty buffer is not supported", NULL); } try { cudf::jni::auto_set_device(env); cudf::jni::native_jstring filename(env, inputfilepath); if (!read_buffer && filename.is_empty()) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "inputfilepath can't be empty", - NULL); + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "inputfilepath can't be empty", NULL); } cudf::jni::native_jstringArray n_filter_col_names(env, filter_col_names); @@ -1942,19 +2035,17 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_Table_readORC( JNI_NULL_CHECK(env, inputfilepath, "input file or buffer must be supplied", NULL); read_buffer = false; } else if (inputfilepath != NULL) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "cannot pass in both a buffer and an inputfilepath", NULL); } else if (buffer_length <= 0) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "An empty buffer is not supported", - NULL); + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "An empty buffer is not supported", NULL); } try { cudf::jni::auto_set_device(env); cudf::jni::native_jstring filename(env, inputfilepath); if (!read_buffer && filename.is_empty()) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "inputfilepath can't be empty", - NULL); + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "inputfilepath can't be empty", NULL); } cudf::jni::native_jstringArray n_filter_col_names(env, filter_col_names); @@ -3187,7 +3278,7 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_Table_dropDuplicates(JNIEnv *en case 2: return cudf::duplicate_keep_option::KEEP_LAST; case 3: return cudf::duplicate_keep_option::KEEP_NONE; default: - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "Invalid `keep` option", + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "Invalid `keep` option", cudf::duplicate_keep_option::KEEP_ANY); } }(); @@ -3384,7 +3475,7 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_Table_rollingWindowAggregate( cudf::jni::native_jbooleanArray unbounded_following{env, j_unbounded_following}; if (not valid_window_parameters(values, agg_instances, min_periods, preceding, following)) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "Number of aggregation columns must match number of agg ops, and window-specs", nullptr); } @@ -3459,7 +3550,7 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_Table_rangeRollingWindowAggrega cudf::jni::native_jpointerArray following(env, j_following); if (not valid_window_parameters(values, agg_instances, min_periods, preceding, following)) { - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", + JNI_THROW_NEW(env, cudf::jni::ILLEGAL_ARG_CLASS, "Number of aggregation columns must match number of agg ops, and window-specs", nullptr); } diff --git a/java/src/test/java/ai/rapids/cudf/TableTest.java b/java/src/test/java/ai/rapids/cudf/TableTest.java index f1c4d0803a3..76f127eae77 100644 --- a/java/src/test/java/ai/rapids/cudf/TableTest.java +++ b/java/src/test/java/ai/rapids/cudf/TableTest.java @@ -33,6 +33,7 @@ import com.google.common.base.Charsets; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.avro.SchemaBuilder; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.parquet.hadoop.ParquetFileReader; @@ -53,7 +54,6 @@ import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.IntStream; import static ai.rapids.cudf.AssertUtils.assertColumnsAreEqual; import static ai.rapids.cudf.AssertUtils.assertPartialColumnsAreEqual; @@ -75,6 +75,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; public class TableTest extends CudfTestBase { + private static final HostMemoryAllocator hostMemoryAllocator = DefaultHostMemoryAllocator.get(); private static final File TEST_PARQUET_FILE = TestUtils.getResourceAsFile("acq.parquet"); @@ -348,6 +349,139 @@ void testReadSingleQuotesJSONFile() throws IOException { } } + private static final byte[] NESTED_JSON_DATA_BUFFER = ("{\"a\":{\"c\":\"C1\"}}\n" + + "{\"a\":{\"c\":\"C2\", \"b\":\"B2\"}}\n" + + "{\"d\":[1,2,3]}\n" + + "{\"e\": [{\"g\": 1}, {\"f\": 2}, {\"f\": 3, \"g\": 4}], \"d\":[]}").getBytes(StandardCharsets.UTF_8); + + @Test + void testReadJSONNestedTypes() { + Schema.Builder root = Schema.builder(); + Schema.Builder a = root.addColumn(DType.STRUCT, "a"); + a.addColumn(DType.STRING, "b"); + a.addColumn(DType.STRING, "c"); + a.addColumn(DType.STRING, "missing"); + Schema.Builder d = root.addColumn(DType.LIST, "d"); + d.addColumn(DType.INT64, "ignored"); + root.addColumn(DType.INT64, "also_missing"); + Schema.Builder e = root.addColumn(DType.LIST, "e"); + Schema.Builder eChild = e.addColumn(DType.STRUCT, "ignored"); + eChild.addColumn(DType.INT64, "f"); + eChild.addColumn(DType.STRING, "missing_in_list"); + eChild.addColumn(DType.INT64, "g"); + Schema schema = root.build(); + JSONOptions opts = JSONOptions.builder() + .withLines(true) + .build(); + StructType aStruct = new StructType(true, + new BasicType(true, DType.STRING), + new BasicType(true, DType.STRING), + new BasicType(true, DType.STRING)); + ListType dList = new ListType(true, new BasicType(true, DType.INT64)); + StructType eChildStruct = new StructType(true, + new BasicType(true, DType.INT64), + new BasicType(true, DType.STRING), + new BasicType(true, DType.INT64)); + ListType eList = new ListType(true, eChildStruct); + try (Table expected = new Table.TestBuilder() + .column(aStruct, + new StructData(null, "C1", null), + new StructData("B2", "C2", null), + null, + null) + .column(dList, + null, + null, + Arrays.asList(1L,2L,3L), + new ArrayList()) + .column((Long)null, null, null, null) // also_missing + .column(eList, + null, + null, + null, + Arrays.asList(new StructData(null, null, 1L), new StructData(2L, null, null), new StructData(3L, null, 4L))) + .build(); + Table table = Table.readJSON(schema, opts, NESTED_JSON_DATA_BUFFER)) { + assertTablesAreEqual(expected, table); + } + } + + @Test + void testReadJSONNestedTypesVerySmallChanges() { + Schema.Builder root = Schema.builder(); + Schema.Builder e = root.addColumn(DType.LIST, "e"); + Schema.Builder eChild = e.addColumn(DType.STRUCT, "ignored"); + eChild.addColumn(DType.INT64, "g"); + eChild.addColumn(DType.INT64, "f"); + Schema schema = root.build(); + JSONOptions opts = JSONOptions.builder() + .withLines(true) + .build(); + StructType eChildStruct = new StructType(true, + new BasicType(true, DType.INT64), + new BasicType(true, DType.INT64)); + ListType eList = new ListType(true, eChildStruct); + try (Table expected = new Table.TestBuilder() + .column(eList, + null, + null, + null, + Arrays.asList(new StructData(1L, null), new StructData(null, 2L), new StructData(4L, 3L))) + .build(); + Table table = Table.readJSON(schema, opts, NESTED_JSON_DATA_BUFFER)) { + assertTablesAreEqual(expected, table); + } + } + + @Test + void testReadJSONNestedTypesDataSource() { + Schema.Builder root = Schema.builder(); + Schema.Builder a = root.addColumn(DType.STRUCT, "a"); + a.addColumn(DType.STRING, "b"); + a.addColumn(DType.STRING, "c"); + a.addColumn(DType.STRING, "missing"); + Schema.Builder d = root.addColumn(DType.LIST, "d"); + d.addColumn(DType.INT64, "ignored"); + root.addColumn(DType.INT64, "also_missing"); + Schema.Builder e = root.addColumn(DType.LIST, "e"); + Schema.Builder eChild = e.addColumn(DType.STRUCT, "ignored"); + eChild.addColumn(DType.INT64, "g"); + Schema schema = root.build(); + JSONOptions opts = JSONOptions.builder() + .withLines(true) + .build(); + StructType aStruct = new StructType(true, + new BasicType(true, DType.STRING), + new BasicType(true, DType.STRING), + new BasicType(true, DType.STRING)); + ListType dList = new ListType(true, new BasicType(true, DType.INT64)); + StructType eChildStruct = new StructType(true, + new BasicType(true, DType.INT64)); + ListType eList = new ListType(true, eChildStruct); + try (Table expected = new Table.TestBuilder() + .column(aStruct, + new StructData(null, "C1", null), + new StructData("B2", "C2", null), + null, + null) + .column(dList, + null, + null, + Arrays.asList(1L,2L,3L), + new ArrayList()) + .column((Long)null, null, null, null) // also_missing + .column(eList, + null, + null, + null, + Arrays.asList(new StructData(1L), new StructData((Long)null), new StructData(4L))) + .build(); + MultiBufferDataSource source = sourceFrom(NESTED_JSON_DATA_BUFFER); + Table table = Table.readJSON(schema, opts, source)) { + assertTablesAreEqual(expected, table); + } + } + void testReadMixedType2JSONFileFeatureDisabled() { Schema schema = Schema.builder() .column(DType.STRING, "a") @@ -870,7 +1004,7 @@ private void testWriteCSVToFileImpl(char fieldDelim, boolean includeHeader, .column(DType.STRING, "str") .build(); CSVWriterOptions writeOptions = CSVWriterOptions.builder() - .withColumnNames(schema.getColumnNames()) + .withColumnNames(schema.getFlattenedColumnNames()) .withIncludeHeader(includeHeader) .withFieldDelimiter((byte)fieldDelim) .withRowDelimiter("\n") @@ -922,7 +1056,7 @@ private void testWriteUnquotedCSVToFileImpl(char fieldDelim) throws IOException .column(DType.STRING, "str") .build(); CSVWriterOptions writeOptions = CSVWriterOptions.builder() - .withColumnNames(schema.getColumnNames()) + .withColumnNames(schema.getFlattenedColumnNames()) .withIncludeHeader(false) .withFieldDelimiter((byte)fieldDelim) .withRowDelimiter("\n") @@ -966,7 +1100,7 @@ private void testChunkedCSVWriterUnquotedImpl(char fieldDelim) throws IOExceptio .column(DType.STRING, "str") .build(); CSVWriterOptions writeOptions = CSVWriterOptions.builder() - .withColumnNames(schema.getColumnNames()) + .withColumnNames(schema.getFlattenedColumnNames()) .withIncludeHeader(false) .withFieldDelimiter((byte)fieldDelim) .withRowDelimiter("\n") @@ -1020,7 +1154,7 @@ private void testChunkedCSVWriterImpl(char fieldDelim, boolean includeHeader, .column(DType.STRING, "str") .build(); CSVWriterOptions writeOptions = CSVWriterOptions.builder() - .withColumnNames(schema.getColumnNames()) + .withColumnNames(schema.getFlattenedColumnNames()) .withIncludeHeader(includeHeader) .withFieldDelimiter((byte)fieldDelim) .withRowDelimiter("\n") From 3f8cb74e067eb5126eeae26d09a47a4d14bcb9c4 Mon Sep 17 00:00:00 2001 From: Shruti Shivakumar Date: Thu, 8 Feb 2024 10:03:14 -0800 Subject: [PATCH 225/384] POC for whitespace removal in input JSON data using FST (#14931) This PR provides a proof-of-concept for the usage of FST in removing unquoted spaces and tabs in JSON strings. This is a useful feature in the cases where we want to cast a hierarchical JSON object to a string, and overcomes the challenge of processing mixed types using Spark. [#14865](https://github.com/rapidsai/cudf/issues/14865) The FST assumes that the single quotes in the input data have already been normalized (possibly using [`normalize_single_quotes`](https://github.com/rapidsai/cudf/pull/14729)). Authors: - Shruti Shivakumar (https://github.com/shrshi) Approvers: - Vukasin Milovanovic (https://github.com/vuule) - Elias Stehle (https://github.com/elstehle) - Bradley Dice (https://github.com/bdice) - Karthikeyan (https://github.com/karthikeyann) URL: https://github.com/rapidsai/cudf/pull/14931 --- cpp/tests/CMakeLists.txt | 1 + .../io/json_whitespace_normalization_test.cu | 262 ++++++++++++++++++ 2 files changed, 263 insertions(+) create mode 100644 cpp/tests/io/json_whitespace_normalization_test.cu diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 8b0e625fecf..4c07970714d 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -315,6 +315,7 @@ ConfigureTest(NESTED_JSON_TEST io/nested_json_test.cpp io/json_tree.cpp) ConfigureTest(ARROW_IO_SOURCE_TEST io/arrow_io_source_test.cpp) ConfigureTest(MULTIBYTE_SPLIT_TEST io/text/multibyte_split_test.cpp) ConfigureTest(JSON_QUOTE_NORMALIZATION io/json_quote_normalization_test.cpp) +ConfigureTest(JSON_WHITESPACE_NORMALIZATION io/json_whitespace_normalization_test.cu) ConfigureTest( DATA_CHUNK_SOURCE_TEST io/text/data_chunk_source_test.cpp GPUS 1 diff --git a/cpp/tests/io/json_whitespace_normalization_test.cu b/cpp/tests/io/json_whitespace_normalization_test.cu new file mode 100644 index 00000000000..ef4172b0ff7 --- /dev/null +++ b/cpp/tests/io/json_whitespace_normalization_test.cu @@ -0,0 +1,262 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include + +#include +#include + +namespace { +// Type used to represent the atomic symbol type used within the finite-state machine +using SymbolT = char; +using StateT = char; + +// Type sufficiently large to index symbols within the input and output (may be unsigned) +using SymbolOffsetT = uint32_t; + +enum class dfa_symbol_group_id : uint32_t { + DOUBLE_QUOTE_CHAR, ///< Quote character SG: " + ESCAPE_CHAR, ///< Escape character SG: '\\' + NEWLINE_CHAR, ///< Newline character SG: '\n' + WHITESPACE_SYMBOLS, ///< Whitespace characters SG: '\t' or ' ' + OTHER_SYMBOLS, ///< SG implicitly matching all other characters + NUM_SYMBOL_GROUPS ///< Total number of symbol groups +}; +// Alias for readability of symbol group ids +constexpr auto NUM_SYMBOL_GROUPS = static_cast(dfa_symbol_group_id::NUM_SYMBOL_GROUPS); +// The i-th string representing all the characters of a symbol group +std::array, NUM_SYMBOL_GROUPS - 1> const wna_sgs{ + {{'"'}, {'\\'}, {'\n'}, {' ', '\t'}}}; + +/** + * -------- FST states --------- + * ----------------------------- + * TT_OOS | Out-of-string state handling whitespace and non-whitespace chars outside double + * | quotes as well as any other character not enclosed by a string. Also handles + * | newline character present within a string + * TT_DQS | Double-quoted string state handling all characters within double quotes except + * | newline character + * TT_DEC | State handling escaped characters inside double-quoted string. Note that this + * | state is necessary to process escaped double-quote characters. Without this + * | state, whitespaces following escaped double quotes inside strings may be removed. + * + * NOTE: An important case NOT handled by this FST is that of whitespace following newline + * characters within a string. Consider the following example + * Input: {"a":"x\n y"} + * FST output: {"a":"x\ny"} + * Expected output: {"a":"x\n y"} + */ +enum class dfa_states : StateT { TT_OOS = 0U, TT_DQS, TT_DEC, TT_NUM_STATES }; +// Aliases for readability of the transition table +constexpr auto TT_OOS = dfa_states::TT_OOS; +constexpr auto TT_DQS = dfa_states::TT_DQS; +constexpr auto TT_DEC = dfa_states::TT_DEC; +constexpr auto TT_NUM_STATES = static_cast(dfa_states::TT_NUM_STATES); + +// Transition table +std::array, TT_NUM_STATES> const wna_state_tt{ + {/* IN_STATE " \ \n OTHER */ + /* TT_OOS */ {{TT_DQS, TT_OOS, TT_OOS, TT_OOS, TT_OOS}}, + /* TT_DQS */ {{TT_OOS, TT_DEC, TT_OOS, TT_DQS, TT_DQS}}, + /* TT_DEC */ {{TT_DQS, TT_DQS, TT_DQS, TT_DQS, TT_DQS}}}}; + +// The DFA's starting state +constexpr StateT start_state = static_cast(TT_OOS); + +struct TransduceToNormalizedWS { + /** + * @brief Returns the -th output symbol on the transition (state_id, match_id). + */ + template + constexpr CUDF_HOST_DEVICE SymbolT operator()(StateT const state_id, + SymbolGroupT const match_id, + RelativeOffsetT const relative_offset, + SymbolT const read_symbol) const + { + // -------- TRANSLATION TABLE ------------ + // Let the alphabet set be Sigma + // --------------------------------------- + // ---------- NON-SPECIAL CASES: ---------- + // Output symbol same as input symbol + // state | read_symbol -> output_symbol + // DQS | Sigma -> Sigma + // OOS | Sigma\{,\t} -> Sigma\{,\t} + // DEC | Sigma -> Sigma + // ---------- SPECIAL CASES: -------------- + // Input symbol translates to output symbol + // OOS | {} -> + // OOS | {\t} -> + + // Case when read symbol is a space or tab but is unquoted + // This will be the same condition as in `operator()(state_id, match_id, read_symbol)` function + // However, since there is no output in this case i.e. the count returned by + // operator()(state_id, match_id, read_symbol) is zero, this function is never called. + // So skipping the check for this case. + + // In all other cases, we have an output symbol for the input symbol. + // We simply output the input symbol + return read_symbol; + } + + /** + * @brief Returns the number of output characters for a given transition. + * During whitespace normalization, we always emit one output character i.e., the input + * character, except when we need to remove the space/tab character + */ + template + constexpr CUDF_HOST_DEVICE uint32_t operator()(StateT const state_id, + SymbolGroupT const match_id, + SymbolT const read_symbol) const + { + // Case when read symbol is a space or tab but is unquoted + if (match_id == static_cast(dfa_symbol_group_id::WHITESPACE_SYMBOLS) && + state_id == static_cast(dfa_states::TT_OOS)) { + return 0; + } + return 1; + } +}; +} // namespace + +// Base test fixture for tests +struct JsonWSNormalizationTest : public cudf::test::BaseFixture {}; + +void run_test(std::string const& input, std::string const& output) +{ + auto parser = cudf::io::fst::detail::make_fst( + cudf::io::fst::detail::make_symbol_group_lut(wna_sgs), + cudf::io::fst::detail::make_transition_table(wna_state_tt), + cudf::io::fst::detail::make_translation_functor(TransduceToNormalizedWS{}), + cudf::test::get_default_stream()); + + auto d_input_scalar = cudf::make_string_scalar(input, cudf::test::get_default_stream()); + auto& d_input = static_cast&>(*d_input_scalar); + + // Prepare input & output buffers + constexpr std::size_t single_item = 1; + cudf::detail::hostdevice_vector output_gpu(input.size(), + cudf::test::get_default_stream()); + cudf::detail::hostdevice_vector output_gpu_size(single_item, + cudf::test::get_default_stream()); + + // Allocate device-side temporary storage & run algorithm + parser.Transduce(d_input.data(), + static_cast(d_input.size()), + output_gpu.device_ptr(), + thrust::make_discard_iterator(), + output_gpu_size.device_ptr(), + start_state, + cudf::test::get_default_stream()); + + // Async copy results from device to host + output_gpu.device_to_host_async(cudf::test::get_default_stream()); + output_gpu_size.device_to_host_async(cudf::test::get_default_stream()); + + // Make sure results have been copied back to host + cudf::test::get_default_stream().synchronize(); + + // Verify results + ASSERT_EQ(output_gpu_size[0], output.size()); + CUDF_TEST_EXPECT_VECTOR_EQUAL(output_gpu, output, output.size()); +} + +TEST_F(JsonWSNormalizationTest, GroundTruth_Spaces) +{ + std::string input = R"({ "A" : "TEST" })"; + std::string output = R"({"A":"TEST"})"; + run_test(input, output); +} + +TEST_F(JsonWSNormalizationTest, GroundTruth_MoreSpaces) +{ + std::string input = R"({"a": [1, 2, 3, 4, 5, 6, 7, 8], "b": {"c": "d"}})"; + std::string output = R"({"a":[1,2,3,4,5,6,7,8],"b":{"c":"d"}})"; + run_test(input, output); +} + +TEST_F(JsonWSNormalizationTest, GroundTruth_SpacesInString) +{ + std::string input = R"({" a ":50})"; + std::string output = R"({" a ":50})"; + run_test(input, output); +} + +TEST_F(JsonWSNormalizationTest, GroundTruth_NewlineInString) +{ + std::string input = "{\"a\" : \"x\ny\"}\n{\"a\" : \"x\\ny\"}"; + std::string output = "{\"a\":\"x\ny\"}\n{\"a\":\"x\\ny\"}"; + run_test(input, output); +} + +TEST_F(JsonWSNormalizationTest, GroundTruth_Tabs) +{ + std::string input = "{\"a\":\t\"b\"}"; + std::string output = R"({"a":"b"})"; + run_test(input, output); +} + +TEST_F(JsonWSNormalizationTest, GroundTruth_SpacesAndTabs) +{ + std::string input = "{\"A\" : \t\"TEST\" }"; + std::string output = R"({"A":"TEST"})"; + run_test(input, output); +} + +TEST_F(JsonWSNormalizationTest, GroundTruth_MultilineJSONWithSpacesAndTabs) +{ + std::string input = + "{ \"foo rapids\": [1,2,3], \"bar\trapids\": 123 }\n\t{ \"foo rapids\": { \"a\": 1 }, " + "\"bar\trapids\": 456 }"; + std::string output = + "{\"foo rapids\":[1,2,3],\"bar\trapids\":123}\n{\"foo rapids\":{\"a\":1},\"bar\trapids\":456}"; + run_test(input, output); +} + +TEST_F(JsonWSNormalizationTest, GroundTruth_PureJSONExample) +{ + std::string input = R"([{"a":50}, {"a" : 60}])"; + std::string output = R"([{"a":50},{"a":60}])"; + run_test(input, output); +} + +TEST_F(JsonWSNormalizationTest, GroundTruth_NoNormalizationRequired) +{ + std::string input = R"({"a\\n\r\a":50})"; + std::string output = R"({"a\\n\r\a":50})"; + run_test(input, output); +} + +TEST_F(JsonWSNormalizationTest, GroundTruth_InvalidInput) +{ + std::string input = "{\"a\" : \"b }\n{ \"c \" :\t\"d\"}"; + std::string output = "{\"a\":\"b }\n{\"c \":\"d\"}"; + run_test(input, output); +} + +CUDF_TEST_PROGRAM_MAIN() From a25f267c12e224f7675c17bd40ae0c601b3dd37e Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 8 Feb 2024 08:06:26 -1000 Subject: [PATCH 226/384] Raise for pyarrow array that is tz-aware (#14980) Similar to the where pandas inputs that are tz-aware raise a `NotImplementedError` Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/14980 --- python/cudf/cudf/core/column/column.py | 7 +++++++ python/cudf/cudf/tests/test_datetime.py | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 2bb0ac7bf12..f665d83964c 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -1926,6 +1926,13 @@ def as_column( "yet supported in pyarrow, see: " "https://github.com/apache/arrow/issues/20213" ) + elif ( + pa.types.is_timestamp(arbitrary.type) + and arbitrary.type.tz is not None + ): + raise NotImplementedError( + "cuDF does not yet support timezone-aware datetimes" + ) elif (nan_as_null is None or nan_as_null) and pa.types.is_floating( arbitrary.type ): diff --git a/python/cudf/cudf/tests/test_datetime.py b/python/cudf/cudf/tests/test_datetime.py index 5596be30cfa..513123a65d3 100644 --- a/python/cudf/cudf/tests/test_datetime.py +++ b/python/cudf/cudf/tests/test_datetime.py @@ -2470,6 +2470,15 @@ def test_datetime_raise_warning(freqstr): t.dt.ceil(freqstr) +def test_timezone_array_notimplemented(): + pa_array = pa.array( + [datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc)], + type=pa.timestamp("ns", "UTC"), + ) + with pytest.raises(NotImplementedError): + cudf.Series(pa_array) + + def test_to_datetime_errors_ignore_deprecated(): with pytest.warns(FutureWarning): cudf.to_datetime("2001-01-01 00:04:45", errors="ignore") From c3cf7c6587e069d032ac79c605c0c3d2a80673af Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Thu, 8 Feb 2024 11:47:51 -0800 Subject: [PATCH 227/384] Reduce execution time of Python ORC tests (#14776) Reduced size of the excessively large tests, making sure to keep the code coverage. Also fixed a few tests to provide better coverage (original intent unclear). Authors: - Vukasin Milovanovic (https://github.com/vuule) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/14776 --- python/cudf/cudf/tests/test_orc.py | 47 +++++++++++++++--------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/python/cudf/cudf/tests/test_orc.py b/python/cudf/cudf/tests/test_orc.py index 4f293c9860e..868543cd1f0 100644 --- a/python/cudf/cudf/tests/test_orc.py +++ b/python/cudf/cudf/tests/test_orc.py @@ -604,13 +604,13 @@ def normalized_equals(value1, value2): @pytest.mark.parametrize("stats_freq", ["STRIPE", "ROWGROUP"]) -@pytest.mark.parametrize("nrows", [1, 100, 6000000]) +@pytest.mark.parametrize("nrows", [1, 100, 100000]) def test_orc_write_statistics(tmpdir, datadir, nrows, stats_freq): from pyarrow import orc supported_stat_types = supported_numpy_dtypes + ["str"] - # Can't write random bool columns until issue #6763 is fixed - if nrows == 6000000: + # Writing bool columns to multiple row groups is disabled until #6763 is fixed + if nrows == 100000: supported_stat_types.remove("bool") # Make a dataframe @@ -623,7 +623,7 @@ def test_orc_write_statistics(tmpdir, datadir, nrows, stats_freq): fname = tmpdir.join("gdf.orc") # Write said dataframe to ORC with cuDF - gdf.to_orc(fname.strpath, statistics=stats_freq) + gdf.to_orc(fname.strpath, statistics=stats_freq, stripe_size_rows=30000) # Read back written ORC's statistics orc_file = orc.ORCFile(fname) @@ -678,20 +678,22 @@ def test_orc_write_statistics(tmpdir, datadir, nrows, stats_freq): @pytest.mark.parametrize("stats_freq", ["STRIPE", "ROWGROUP"]) -@pytest.mark.parametrize("nrows", [2, 100, 6000000]) +@pytest.mark.parametrize("nrows", [2, 100, 200000]) def test_orc_chunked_write_statistics(tmpdir, datadir, nrows, stats_freq): from pyarrow import orc np.random.seed(0) supported_stat_types = supported_numpy_dtypes + ["str"] - # Can't write random bool columns until issue #6763 is fixed - if nrows == 6000000: + # Writing bool columns to multiple row groups is disabled until #6763 is fixed + if nrows == 200000: supported_stat_types.remove("bool") gdf_fname = tmpdir.join("chunked_stats.orc") - writer = ORCWriter(gdf_fname) + writer = ORCWriter( + gdf_fname, statistics=stats_freq, stripe_size_rows=30000 + ) - max_char_length = 1000 if nrows < 10000 else 100 + max_char_length = 100 if nrows < 10000 else 10 # Make a dataframe gdf = cudf.DataFrame( @@ -699,7 +701,7 @@ def test_orc_chunked_write_statistics(tmpdir, datadir, nrows, stats_freq): "col_" + str(dtype): gen_rand_series( dtype, - int(nrows / 2), + nrows // 2, has_nulls=True, low=0, high=max_char_length, @@ -718,7 +720,7 @@ def test_orc_chunked_write_statistics(tmpdir, datadir, nrows, stats_freq): "col_" + str(dtype): gen_rand_series( dtype, - int(nrows / 2), + nrows // 2, has_nulls=True, low=0, high=max_char_length, @@ -785,7 +787,7 @@ def test_orc_chunked_write_statistics(tmpdir, datadir, nrows, stats_freq): assert stats_num_vals == actual_num_vals -@pytest.mark.parametrize("nrows", [1, 100, 6000000]) +@pytest.mark.parametrize("nrows", [1, 100, 100000]) def test_orc_write_bool_statistics(tmpdir, datadir, nrows): from pyarrow import orc @@ -794,7 +796,7 @@ def test_orc_write_bool_statistics(tmpdir, datadir, nrows): fname = tmpdir.join("gdf.orc") # Write said dataframe to ORC with cuDF - gdf.to_orc(fname.strpath) + gdf.to_orc(fname.strpath, stripe_size_rows=30000) # Read back written ORC's statistics orc_file = orc.ORCFile(fname) @@ -848,21 +850,20 @@ def test_orc_bool_encode_fail(): np.random.seed(0) buffer = BytesIO() - # Generate a boolean column longer than a single stripe - fail_df = cudf.DataFrame({"col": gen_rand_series("bool", 600000)}) - # Invalidate the first row in the second stripe to break encoding - fail_df["col"][500000] = None + # Generate a boolean column longer than a single row group + fail_df = cudf.DataFrame({"col": gen_rand_series("bool", 20000)}) + # Invalidate a row in the first row group + fail_df["col"][5000] = None # Should throw instead of generating a file that is incompatible # with other readers (see issue #6763) with pytest.raises(RuntimeError): fail_df.to_orc(buffer) - # Generate a boolean column that fits into a single stripe - okay_df = cudf.DataFrame({"col": gen_rand_series("bool", 500000)}) - okay_df["col"][500000 - 1] = None - # Invalid row is in the last row group of the stripe; - # encoding is assumed to be correct + # Generate a boolean column longer than a single row group + okay_df = cudf.DataFrame({"col": gen_rand_series("bool", 20000)}) + okay_df["col"][15000] = None + # Invalid row is in the last row group; encoding is assumed to be correct okay_df.to_orc(buffer) # Also validate data @@ -1130,7 +1131,7 @@ def test_pyspark_struct(datadir): assert_eq(pdf, gdf) -def gen_map_buff(size=10000): +def gen_map_buff(size): from string import ascii_letters as al from pyarrow import orc From 72942806516934cc45ed71f5333d4aa75c7fd12e Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 8 Feb 2024 14:55:26 -0800 Subject: [PATCH 228/384] Implement replace in pylibcudf (#15005) Contributes to #13921 Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/15005 --- .../user_guide/api_docs/pylibcudf/index.rst | 1 + .../user_guide/api_docs/pylibcudf/replace.rst | 6 + python/cudf/cudf/_lib/cpp/CMakeLists.txt | 4 +- python/cudf/cudf/_lib/cpp/replace.pxd | 3 +- python/cudf/cudf/_lib/cpp/replace.pyx | 0 .../cudf/cudf/_lib/pylibcudf/CMakeLists.txt | 2 +- python/cudf/cudf/_lib/pylibcudf/__init__.pxd | 2 + python/cudf/cudf/_lib/pylibcudf/__init__.py | 2 + python/cudf/cudf/_lib/pylibcudf/replace.pxd | 36 +++ python/cudf/cudf/_lib/pylibcudf/replace.pyx | 208 ++++++++++++++++++ python/cudf/cudf/_lib/replace.pyx | 157 ++++--------- 11 files changed, 304 insertions(+), 117 deletions(-) create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/replace.rst create mode 100644 python/cudf/cudf/_lib/cpp/replace.pyx create mode 100644 python/cudf/cudf/_lib/pylibcudf/replace.pxd create mode 100644 python/cudf/cudf/_lib/pylibcudf/replace.pyx diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst index 91b84d29ddf..834cd46dc16 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst @@ -18,6 +18,7 @@ This page provides API documentation for pylibcudf. reduce rolling scalar + replace table types unary diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/replace.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/replace.rst new file mode 100644 index 00000000000..7f846872fca --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/replace.rst @@ -0,0 +1,6 @@ +======= +replace +======= + +.. automodule:: cudf._lib.pylibcudf.replace + :members: diff --git a/python/cudf/cudf/_lib/cpp/CMakeLists.txt b/python/cudf/cudf/_lib/cpp/CMakeLists.txt index da06cf225e9..21c38652362 100644 --- a/python/cudf/cudf/_lib/cpp/CMakeLists.txt +++ b/python/cudf/cudf/_lib/cpp/CMakeLists.txt @@ -12,7 +12,9 @@ # the License. # ============================================================================= -set(cython_sources aggregation.pyx binaryop.pyx copying.pyx reduce.pxd types.pyx unary.pyx) +set(cython_sources aggregation.pyx binaryop.pyx copying.pyx replace.pyx reduce.pxd types.pyx + unary.pyx +) set(linked_libraries cudf::cudf) diff --git a/python/cudf/cudf/_lib/cpp/replace.pxd b/python/cudf/cudf/_lib/cpp/replace.pxd index 74bc9c2bb4c..5d57f01b816 100644 --- a/python/cudf/cudf/_lib/cpp/replace.pxd +++ b/python/cudf/cudf/_lib/cpp/replace.pxd @@ -12,7 +12,7 @@ from cudf._lib.cpp.scalar.scalar cimport scalar cdef extern from "cudf/replace.hpp" namespace "cudf" nogil: - cdef enum class replace_policy(bool): + cpdef enum class replace_policy(bool): PRECEDING FOLLOWING @@ -42,7 +42,6 @@ cdef extern from "cudf/replace.hpp" namespace "cudf" nogil: column_view source_column, scalar lo, scalar hi) except + -cdef extern from "cudf/replace.hpp" namespace "cudf" nogil: cdef unique_ptr[column] normalize_nans_and_zeros( column_view source_column) except + diff --git a/python/cudf/cudf/_lib/cpp/replace.pyx b/python/cudf/cudf/_lib/cpp/replace.pyx new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt index 5eb0e5cdf82..248b9afaa21 100644 --- a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt +++ b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt @@ -14,7 +14,7 @@ set(cython_sources aggregation.pyx binaryop.pyx column.pyx copying.pyx gpumemoryview.pyx groupby.pyx interop.pyx - join.pyx reduce.pyx rolling.pyx scalar.pyx table.pyx types.pyx unary.pyx utils.pyx + join.pyx reduce.pyx replace.pyx rolling.pyx scalar.pyx table.pyx types.pyx unary.pyx utils.pyx ) set(linked_libraries cudf::cudf) rapids_cython_create_modules( diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd index df65e893b68..316a47eebf0 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd @@ -9,6 +9,7 @@ from . cimport ( interop, join, reduce, + replace, rolling, types, unary, @@ -35,6 +36,7 @@ __all__ = [ "join", "unary", "reduce", + "replace", "rolling", "types", ] diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.py b/python/cudf/cudf/_lib/pylibcudf/__init__.py index 52dded12071..642c3c18920 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.py +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.py @@ -8,6 +8,7 @@ interop, join, reduce, + replace, rolling, types, unary, @@ -33,6 +34,7 @@ "join", "unary", "reduce", + "replace", "rolling", "types", ] diff --git a/python/cudf/cudf/_lib/pylibcudf/replace.pxd b/python/cudf/cudf/_lib/pylibcudf/replace.pxd new file mode 100644 index 00000000000..fc42b985c8e --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/replace.pxd @@ -0,0 +1,36 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +from libcpp cimport bool + +from cudf._lib.cpp.replace cimport replace_policy + +from .column cimport Column +from .scalar cimport Scalar + +ctypedef fused ReplacementType: + Column + Scalar + replace_policy + # Allowing object is a workaround for + # https://github.com/cython/cython/issues/5984. See the implementation of + # replace_nulls for details. + object + + +cpdef Column replace_nulls(Column source_column, ReplacementType replacement) + +cpdef Column find_and_replace_all( + Column source_column, + Column values_to_replace, + Column replacement_values, +) + +cpdef Column clamp( + Column source_column, + Scalar lo, + Scalar hi, + Scalar lo_replace=*, + Scalar hi_replace=*, +) + +cpdef Column normalize_nans_and_zeros(Column source_column, bool inplace=*) diff --git a/python/cudf/cudf/_lib/pylibcudf/replace.pyx b/python/cudf/cudf/_lib/pylibcudf/replace.pyx new file mode 100644 index 00000000000..dd3a733ee3a --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/replace.pyx @@ -0,0 +1,208 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + + +from cython.operator import dereference + +from libcpp cimport bool +from libcpp.memory cimport unique_ptr +from libcpp.utility cimport move + +from cudf._lib.cpp cimport replace as cpp_replace +from cudf._lib.cpp.column.column cimport column + +from cudf._lib.cpp.replace import \ + replace_policy as ReplacePolicy # no-cython-lint + +from .column cimport Column +from .scalar cimport Scalar + + +cpdef Column replace_nulls(Column source_column, ReplacementType replacement): + """Replace nulls in source_column. + + The values used to replace nulls depends on the type of replacement: + - If replacement is a Column, the corresponding value from replacement + is used. + - If replacement is a Scalar, the same value is used for all nulls. + - If replacement is a replace_policy, the policy is used to determine + the replacement value: + + - PRECEDING: The first non-null value that precedes the null is used. + - FOLLOWING: The first non-null value that follows the null is used. + + For more details, see :cpp:func:`replace_nulls`. + + Parameters + ---------- + source_column : Column + The column in which to replace nulls. + replacement_column : Union[Column, Scalar, replace_policy] + If a Column, the values to use as replacements. If a Scalar, the value + to use as a replacement. If a replace_policy, the policy to use to + determine the replacement value. + + Returns + ------- + Column + A copy of source_column with nulls replaced by values from + replacement_column. + """ + cdef unique_ptr[column] c_result + cdef replace_policy policy + # Due to https://github.com/cython/cython/issues/5984, if this function is + # called as a Python function (i.e. without typed inputs, which is always + # true in pure Python files), the type of `replacement` will be `object` + # instead of `replace_policy`. This is a workaround to handle that case. + if ReplacementType is object: + if isinstance(replacement, ReplacePolicy): + policy = replacement + with nogil: + c_result = move( + cpp_replace.replace_nulls(source_column.view(), policy) + ) + return Column.from_libcudf(move(c_result)) + else: + raise TypeError("replacement must be a Column, Scalar, or replace_policy") + + with nogil: + if ReplacementType is Column: + c_result = move( + cpp_replace.replace_nulls(source_column.view(), replacement.view()) + ) + elif ReplacementType is Scalar: + c_result = move( + cpp_replace.replace_nulls( + source_column.view(), dereference(replacement.c_obj) + ) + ) + elif ReplacementType is replace_policy: + c_result = move( + cpp_replace.replace_nulls(source_column.view(), replacement) + ) + else: + assert False, "Internal error. Please contact pylibcudf developers" + return Column.from_libcudf(move(c_result)) + + +cpdef Column find_and_replace_all( + Column source_column, + Column values_to_replace, + Column replacement_values, +): + """Replace all occurrences of values_to_replace with replacement_values. + + For details, see :cpp:func:`find_and_replace_all`. + + Parameters + ---------- + source_column : Column + The column in which to replace values. + values_to_replace : Column + The column containing values to replace. + replacement_values : Column + The column containing replacement values. + + Returns + ------- + Column + A copy of source_column with all occurrences of values_to_replace + replaced by replacement_values. + """ + cdef unique_ptr[column] c_result + with nogil: + c_result = move( + cpp_replace.find_and_replace_all( + source_column.view(), + values_to_replace.view(), + replacement_values.view(), + ) + ) + return Column.from_libcudf(move(c_result)) + + +cpdef Column clamp( + Column source_column, + Scalar lo, + Scalar hi, + Scalar lo_replace=None, + Scalar hi_replace=None, +): + """Clamp the values in source_column to the range [lo, hi]. + + For details, see :cpp:func:`clamp`. + + Parameters + ---------- + source_column : Column + The column to clamp. + lo : Scalar + The lower bound of the clamp range. + hi : Scalar + The upper bound of the clamp range. + lo_replace : Scalar, optional + The value to use for elements that are less than lo. If not specified, + the value of lo is used. + hi_replace : Scalar, optional + The value to use for elements that are greater than hi. If not + specified, the value of hi is used. + + Returns + ------- + Column + A copy of source_column with values clamped to the range [lo, hi]. + """ + if (lo_replace is None) != (hi_replace is None): + raise ValueError("lo_replace and hi_replace must be specified together") + + cdef unique_ptr[column] c_result + with nogil: + if lo_replace is None: + c_result = move( + cpp_replace.clamp( + source_column.view(), + dereference(lo.c_obj), + dereference(hi.c_obj), + ) + ) + else: + c_result = move( + cpp_replace.clamp( + source_column.view(), + dereference(lo.c_obj), + dereference(hi.c_obj), + dereference(lo_replace.c_obj), + dereference(hi_replace.c_obj), + ) + ) + return Column.from_libcudf(move(c_result)) + + +cpdef Column normalize_nans_and_zeros(Column source_column, bool inplace=False): + """Normalize NaNs and zeros in source_column. + + For details, see :cpp:func:`normalize_nans_and_zeros`. + + Parameters + ---------- + source_column : Column + The column to normalize. + inplace : bool, optional + If True, normalize source_column in place. If False, return a new + column with the normalized values. + + Returns + ------- + Column + A copy of source_column with NaNs and zeros normalized. + """ + cdef unique_ptr[column] c_result + with nogil: + if inplace: + cpp_replace.normalize_nans_and_zeros(source_column.mutable_view()) + else: + c_result = move( + cpp_replace.normalize_nans_and_zeros(source_column.view()) + ) + + if not inplace: + return Column.from_libcudf(move(c_result)) diff --git a/python/cudf/cudf/_lib/replace.pyx b/python/cudf/cudf/_lib/replace.pyx index c763a86d6e5..2b5f32c7675 100644 --- a/python/cudf/cudf/_lib/replace.pyx +++ b/python/cudf/cudf/_lib/replace.pyx @@ -1,27 +1,14 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. - -from libcpp.memory cimport unique_ptr -from libcpp.utility cimport move +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from cudf.api.types import is_scalar from cudf.core.buffer import acquire_spill_lock from cudf._lib.column cimport Column +from cudf._lib.scalar cimport DeviceScalar +from cudf._lib import pylibcudf from cudf._lib.scalar import as_device_scalar -from cudf._lib.cpp.column.column cimport column -from cudf._lib.cpp.column.column_view cimport column_view, mutable_column_view -from cudf._lib.cpp.replace cimport ( - clamp as cpp_clamp, - find_and_replace_all as cpp_find_and_replace_all, - normalize_nans_and_zeros as cpp_normalize_nans_and_zeros, - replace_nulls as cpp_replace_nulls, - replace_policy as cpp_replace_policy, -) -from cudf._lib.cpp.scalar.scalar cimport scalar -from cudf._lib.scalar cimport DeviceScalar - @acquire_spill_lock() def replace(Column input_col, Column values_to_replace, @@ -37,17 +24,13 @@ def replace(Column input_col, Column values_to_replace, replacement_values : Column with values which will replace """ - cdef column_view input_col_view = input_col.view() - cdef column_view values_to_replace_view = values_to_replace.view() - cdef column_view replacement_values_view = replacement_values.view() - - cdef unique_ptr[column] c_result - with nogil: - c_result = move(cpp_find_and_replace_all(input_col_view, - values_to_replace_view, - replacement_values_view)) - - return Column.from_unique_ptr(move(c_result)) + return Column.from_pylibcudf( + pylibcudf.replace.find_and_replace_all( + input_col.to_pylibcudf(mode="read"), + values_to_replace.to_pylibcudf(mode="read"), + replacement_values.to_pylibcudf(mode="read"), + ) + ) @acquire_spill_lock() @@ -61,16 +44,12 @@ def replace_nulls_column(Column input_col, Column replacement_values): input_col : Column whose value will be updated replacement_values : Column with values which will replace nulls """ - - cdef column_view input_col_view = input_col.view() - cdef column_view replacement_values_view = replacement_values.view() - - cdef unique_ptr[column] c_result - with nogil: - c_result = move(cpp_replace_nulls(input_col_view, - replacement_values_view)) - - return Column.from_unique_ptr(move(c_result)) + return Column.from_pylibcudf( + pylibcudf.replace.replace_nulls( + input_col.to_pylibcudf(mode="read"), + replacement_values.to_pylibcudf(mode="read"), + ) + ) @acquire_spill_lock() @@ -83,17 +62,12 @@ def replace_nulls_scalar(Column input_col, DeviceScalar replacement_value): input_col : Column whose value will be updated replacement_value : DeviceScalar with value which will replace nulls """ - - cdef column_view input_col_view = input_col.view() - cdef const scalar* replacement_value_scalar = replacement_value\ - .get_raw_ptr() - - cdef unique_ptr[column] c_result - with nogil: - c_result = move(cpp_replace_nulls(input_col_view, - replacement_value_scalar[0])) - - return Column.from_unique_ptr(move(c_result)) + return Column.from_pylibcudf( + pylibcudf.replace.replace_nulls( + input_col.to_pylibcudf(mode="read"), + replacement_value.c_value, + ) + ) @acquire_spill_lock() @@ -106,21 +80,15 @@ def replace_nulls_fill(Column input_col, object method): input_col : Column whose value will be updated method : 'ffill' or 'bfill' """ - - cdef column_view input_col_view = input_col.view() - - cdef unique_ptr[column] c_result - cdef cpp_replace_policy policy = ( - cpp_replace_policy.PRECEDING - if method == 'ffill' - else cpp_replace_policy.FOLLOWING + return Column.from_pylibcudf( + pylibcudf.replace.replace_nulls( + input_col.to_pylibcudf(mode="read"), + pylibcudf.replace.ReplacePolicy.PRECEDING + if method == 'ffill' + else pylibcudf.replace.ReplacePolicy.FOLLOWING, + ) ) - with nogil: - c_result = move(cpp_replace_nulls(input_col_view, policy)) - - return Column.from_unique_ptr(move(c_result)) - def replace_nulls( Column input_col, @@ -150,37 +118,6 @@ def replace_nulls( return replace_nulls_column(input_col, replacement) -@acquire_spill_lock() -def clamp(Column input_col, DeviceScalar lo, DeviceScalar lo_replace, - DeviceScalar hi, DeviceScalar hi_replace): - """ - Clip the input_col such that values < lo will be replaced by lo_replace - and > hi will be replaced by hi_replace - - Parameters - ---------- - input_col : Column whose value will be updated - lo : DeviceScalar value for clipping lower values - lo_replace : DeviceScalar value which will replace clipped with lo - hi : DeviceScalar value for clipping upper values - lo_replace : DeviceScalar value which will replace clipped with hi - """ - - cdef column_view input_col_view = input_col.view() - cdef const scalar* lo_value = lo.get_raw_ptr() - cdef const scalar* lo_replace_value = lo_replace.get_raw_ptr() - cdef const scalar* hi_value = hi.get_raw_ptr() - cdef const scalar* hi_replace_value = hi_replace.get_raw_ptr() - - cdef unique_ptr[column] c_result - with nogil: - c_result = move(cpp_clamp( - input_col_view, lo_value[0], - lo_replace_value[0], hi_value[0], hi_replace_value[0])) - - return Column.from_unique_ptr(move(c_result)) - - @acquire_spill_lock() def clamp(Column input_col, DeviceScalar lo, DeviceScalar hi): """ @@ -193,16 +130,13 @@ def clamp(Column input_col, DeviceScalar lo, DeviceScalar hi): lo : DeviceScalar value for clipping lower values hi : DeviceScalar value for clipping upper values """ - - cdef column_view input_col_view = input_col.view() - cdef const scalar* lo_value = lo.get_raw_ptr() - cdef const scalar* hi_value = hi.get_raw_ptr() - - cdef unique_ptr[column] c_result - with nogil: - c_result = move(cpp_clamp(input_col_view, lo_value[0], hi_value[0])) - - return Column.from_unique_ptr(move(c_result)) + return Column.from_pylibcudf( + pylibcudf.replace.clamp( + input_col.to_pylibcudf(mode="read"), + lo.c_value, + hi.c_value, + ) + ) @acquire_spill_lock() @@ -223,10 +157,9 @@ def normalize_nans_and_zeros_inplace(Column input_col): """ Inplace normalizing """ - - cdef mutable_column_view input_col_view = input_col.mutable_view() - with nogil: - cpp_normalize_nans_and_zeros(input_col_view) + pylibcudf.replace.normalize_nans_and_zeros( + input_col.to_pylibcudf(mode="write"), inplace=True + ) @acquire_spill_lock() @@ -234,13 +167,11 @@ def normalize_nans_and_zeros_column(Column input_col): """ Returns a new normalized Column """ - - cdef column_view input_col_view = input_col.view() - cdef unique_ptr[column] c_result - with nogil: - c_result = move(cpp_normalize_nans_and_zeros(input_col_view)) - - return Column.from_unique_ptr(move(c_result)) + return Column.from_pylibcudf( + pylibcudf.replace.normalize_nans_and_zeros( + input_col.to_pylibcudf(mode="read") + ) + ) def normalize_nans_and_zeros(Column input_col, in_place=False): From fbb1f899d6fdf44272c822037b2c8e9b62256668 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 8 Feb 2024 15:43:03 -1000 Subject: [PATCH 229/384] Deprecate replace with categorical columns (#14988) Matches pandas 2.2 behavior: https://github.com/pandas-dev/pandas/pull/56385 Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Michael Wang (https://github.com/isVoid) URL: https://github.com/rapidsai/cudf/pull/14988 --- python/cudf/cudf/core/column/categorical.py | 13 +++- python/cudf/cudf/tests/test_replace.py | 66 +++++++++++++++------ 2 files changed, 60 insertions(+), 19 deletions(-) diff --git a/python/cudf/cudf/core/column/categorical.py b/python/cudf/cudf/core/column/categorical.py index bbff72722ab..9ecd461cf99 100644 --- a/python/cudf/cudf/core/column/categorical.py +++ b/python/cudf/cudf/core/column/categorical.py @@ -2,6 +2,7 @@ from __future__ import annotations +import warnings from collections import abc from functools import cached_property from typing import TYPE_CHECKING, Any, Mapping, Optional, Sequence, Tuple, cast @@ -990,7 +991,7 @@ def find_and_replace( replaced, to_replace_col, replacement_col ) - return column.build_categorical_column( + result = column.build_categorical_column( categories=new_cats["cats"], codes=column.build_column(output.base_data, dtype=output.dtype), mask=output.base_mask, @@ -998,6 +999,16 @@ def find_and_replace( size=output.size, ordered=self.dtype.ordered, ) + if result.dtype != self.dtype: + warnings.warn( + "The behavior of replace with " + "CategoricalDtype is deprecated. In a future version, replace " + "will only be used for cases that preserve the categories. " + "To change the categories, use ser.cat.rename_categories " + "instead.", + FutureWarning, + ) + return result def isnull(self) -> ColumnBase: """ diff --git a/python/cudf/cudf/tests/test_replace.py b/python/cudf/cudf/tests/test_replace.py index 0f8f8de36a1..0b57f9fe846 100644 --- a/python/cudf/cudf/tests/test_replace.py +++ b/python/cudf/cudf/tests/test_replace.py @@ -57,13 +57,24 @@ def test_series_replace_all(gsr, to_replace, value): else: pd_value = value - actual = gsr.replace(to_replace=gd_to_replace, value=gd_value) - if pd_value is None: - # TODO: Remove this workaround once cudf - # introduces `no_default` values - expected = psr.replace(to_replace=pd_to_replace) - else: - expected = psr.replace(to_replace=pd_to_replace, value=pd_value) + with expect_warning_if( + isinstance(gsr.dtype, cudf.CategoricalDtype) + and isinstance(gd_to_replace, str) + and gd_to_replace == "one" + ): + actual = gsr.replace(to_replace=gd_to_replace, value=gd_value) + with expect_warning_if( + PANDAS_GE_220 + and isinstance(gsr.dtype, cudf.CategoricalDtype) + and isinstance(gd_to_replace, str) + and gd_to_replace == "one" + ): + if pd_value is None: + # TODO: Remove this workaround once cudf + # introduces `no_default` values + expected = psr.replace(to_replace=pd_to_replace) + else: + expected = psr.replace(to_replace=pd_to_replace, value=pd_value) assert_eq( expected.sort_values().reset_index(drop=True), @@ -82,16 +93,19 @@ def test_series_replace(): # Categorical psr3 = pd.Series(["one", "two", "three"], dtype="category") - psr4 = psr3.replace("one", "two") + with expect_warning_if(PANDAS_GE_220): + psr4 = psr3.replace("one", "two") sr3 = cudf.from_pandas(psr3) - sr4 = sr3.replace("one", "two") + with pytest.warns(FutureWarning): + sr4 = sr3.replace("one", "two") assert_eq( psr4.sort_values().reset_index(drop=True), sr4.sort_values().reset_index(drop=True), ) - - psr5 = psr3.replace("one", "five") - sr5 = sr3.replace("one", "five") + with expect_warning_if(PANDAS_GE_220): + psr5 = psr3.replace("one", "five") + with pytest.warns(FutureWarning): + sr5 = sr3.replace("one", "five") assert_eq(psr5, sr5) @@ -236,11 +250,26 @@ def test_dataframe_replace(df, to_replace, value): else: gd_to_replace = to_replace - if pd_value is None: - expected = pdf.replace(to_replace=pd_to_replace) - else: - expected = pdf.replace(to_replace=pd_to_replace, value=pd_value) - actual = gdf.replace(to_replace=gd_to_replace, value=gd_value) + with expect_warning_if( + PANDAS_GE_220 + and isinstance(df["a"].dtype, cudf.CategoricalDtype) + and isinstance(to_replace, str) + and to_replace == "two" + and isinstance(value, str) + and value == "three" + ): + if pd_value is None: + expected = pdf.replace(to_replace=pd_to_replace) + else: + expected = pdf.replace(to_replace=pd_to_replace, value=pd_value) + with expect_warning_if( + isinstance(df["a"].dtype, cudf.CategoricalDtype) + and isinstance(to_replace, str) + and to_replace == "two" + and isinstance(value, str) + and value == "three" + ): + actual = gdf.replace(to_replace=gd_to_replace, value=gd_value) expected_sorted = expected.sort_values(by=list(expected.columns), axis=0) actual_sorted = actual.sort_values(by=list(actual.columns), axis=0) @@ -1342,7 +1371,8 @@ def test_series_replace_errors(): ], ) def test_replace_nulls(gsr, old, new, expected): - actual = gsr.replace(old, new) + with expect_warning_if(isinstance(gsr.dtype, cudf.CategoricalDtype)): + actual = gsr.replace(old, new) assert_eq( expected.sort_values().reset_index(drop=True), actual.sort_values().reset_index(drop=True), From 6638b5248fdf8cfcdff29f8209799f02abf77de1 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 9 Feb 2024 08:30:38 -0600 Subject: [PATCH 230/384] Fix CI workflows for pandas-tests and add test summary. (#14847) This PR fixes issues with the `pandas-tests` job that were introduced during the pandas 2 migration. It also closes #14846 by adding GitHub Actions summaries for all wheel test jobs, including `cudf.pandas`. Depends on https://github.com/rapidsai/shared-workflows/pull/173. Authors: - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) - Ashwin Srinath (https://github.com/shwina) Approvers: - Ashwin Srinath (https://github.com/shwina) - Vyas Ramasubramani (https://github.com/vyasr) - Jake Awe (https://github.com/AyodeAwe) URL: https://github.com/rapidsai/cudf/pull/14847 --- .github/workflows/pr.yaml | 2 ++ ci/cudf_pandas_scripts/pandas-tests/run.sh | 14 +++++++++----- ci/test_wheel_cudf.sh | 17 +++++++++++++++-- ci/test_wheel_dask_cudf.sh | 14 ++++++++++++-- .../cudf/pandas/scripts/run-pandas-tests.sh | 10 ++++++---- 5 files changed, 44 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 14a74618413..1dc31da8e80 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -164,6 +164,8 @@ jobs: matrix_filter: map(select(.ARCH == "amd64")) | max_by(.CUDA_VER) | [.] build_type: pull-request script: ci/cudf_pandas_scripts/pandas-tests/run.sh pr + # Hide test failures because they exceed the GITHUB_STEP_SUMMARY output limit. + test_summary_show: "none" #pandas-tests-diff: # # diff the results of running the Pandas unit tests and publish a job summary # needs: [pandas-tests-main, pandas-tests-pr] diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index be5705a9548..482af42201f 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -1,12 +1,14 @@ #!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. # All rights reserved. # SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + PANDAS_TESTS_BRANCH=${1} rapids-logger "Running Pandas tests using $PANDAS_TESTS_BRANCH branch" -rapids-logger "PR number: $RAPIDS_REF_NAME" +rapids-logger "PR number: ${RAPIDS_REF_NAME:-"unknown"}" # Set the manylinux version used for downloading the wheels so that we test the # newer ABI wheels on the newer images that support their installation. @@ -25,14 +27,16 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="cudf_${manylinux}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep python -m pip install $(ls ./local-cudf-dep/cudf*.whl)[test,pandas-tests] -git checkout $COMMIT +RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} +RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ +mkdir -p "${RAPIDS_TESTS_DIR}" bash python/cudf/cudf/pandas/scripts/run-pandas-tests.sh \ -n 10 \ --tb=line \ - --skip-slow \ + -m "not slow" \ --max-worker-restart=3 \ - --import-mode=importlib \ + --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-pandas.xml" \ --report-log=${PANDAS_TESTS_BRANCH}.json 2>&1 # summarize the results and save them to artifacts: diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index 8c42651e299..b7e8f862ed5 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. set -eou pipefail @@ -22,9 +22,22 @@ RAPIDS_PY_WHEEL_NAME="cudf_${manylinux}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-downloa # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/cudf*.whl)[test] +RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} +RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ +mkdir -p "${RAPIDS_TESTS_DIR}" + # Run smoke tests for aarch64 pull requests if [[ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then + rapids-logger "Run smoke tests for cudf" python ./ci/wheel_smoke_test_cudf.py else - python -m pytest -n 8 ./python/cudf/cudf/tests + rapids-logger "pytest cudf" + pushd python/cudf/cudf/tests + python -m pytest \ + --cache-clear \ + --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf.xml" \ + --numprocesses=8 \ + --dist=loadscope \ + . + popd fi diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index e9162b816aa..74fcb43ddca 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. set -eou pipefail @@ -26,5 +26,15 @@ python -m pip install --no-deps ./local-cudf-dep/cudf*.whl # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/dask_cudf*.whl)[test] +RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} +RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ +mkdir -p "${RAPIDS_TESTS_DIR}" + # Run tests in dask_cudf/tests and dask_cudf/io/tests -python -m pytest -n 8 ./python/dask_cudf/dask_cudf/ +rapids-logger "pytest dask_cudf" +pushd python/dask_cudf/dask_cudf +python -m pytest \ + --junitxml="${RAPIDS_TESTS_DIR}/junit-dask-cudf.xml" \ + --numprocesses=8 \ + . +popd diff --git a/python/cudf/cudf/pandas/scripts/run-pandas-tests.sh b/python/cudf/cudf/pandas/scripts/run-pandas-tests.sh index 4fe152cc493..319e5ba80fc 100755 --- a/python/cudf/cudf/pandas/scripts/run-pandas-tests.sh +++ b/python/cudf/cudf/pandas/scripts/run-pandas-tests.sh @@ -16,12 +16,13 @@ # # This script creates a `pandas-testing` directory if it doesn't exist +set -euo pipefail # Grab the Pandas source corresponding to the version # of Pandas installed. PANDAS_VERSION=$(python -c "import pandas; print(pandas.__version__)") -PYTEST_IGNORES="--ignore=tests/io/test_user_agent.py" +PYTEST_IGNORES="--ignore=tests/io/test_user_agent.py --ignore=tests/interchange/test_impl.py" mkdir -p pandas-testing cd pandas-testing @@ -92,7 +93,7 @@ cd pandas-tests/ # test_overwrite_warns unsafely patchs over Series.mean affecting other tests when run in parallel # test_complex_series_frame_alignment randomly selects a DataFrames and axis to test but particular random selection(s) always fails # test_numpy_ufuncs_basic compares floating point values to unbounded precision, sometimes leading to failures -TEST_NUMPY_UFUNCS_BASIC_FLAKY="test_numpy_ufuncs_basic[float-exp] \ +TEST_NUMPY_UFUNCS_BASIC_FLAKY="not test_numpy_ufuncs_basic[float-exp] \ and not test_numpy_ufuncs_basic[float-exp2] \ and not test_numpy_ufuncs_basic[float-expm1] \ and not test_numpy_ufuncs_basic[float-log] \ @@ -183,11 +184,12 @@ and not test_numpy_ufuncs_basic[nullable_float-rad2deg]" PANDAS_CI="1" python -m pytest -p cudf.pandas \ -m "not single_cpu and not db" \ - -k "not test_overwrite_warns and not test_complex_series_frame_alignment and not $TEST_NUMPY_UFUNCS_BASIC_FLAKY" \ + -k "not test_overwrite_warns and not test_complex_series_frame_alignment and $TEST_NUMPY_UFUNCS_BASIC_FLAKY" \ --durations=50 \ --import-mode=importlib \ -o xfail_strict=True \ - ${PYTEST_IGNORES} $@ + ${PYTEST_IGNORES} \ + "$@" || [ $? = 1 ] # Exit success if exit code was 1 (permit test failures but not other errors) mv *.json .. cd .. From e36718b146bac35069e388e4b4748291c4ff6049 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 9 Feb 2024 09:01:03 -1000 Subject: [PATCH 231/384] Fix is_string_dtype test for pandas 2.2 (#15012) Fixed in pandas 2.2: https://github.com/pandas-dev/pandas/issues/54661 Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15012 --- python/cudf/cudf/tests/test_api_types.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/cudf/cudf/tests/test_api_types.py b/python/cudf/cudf/tests/test_api_types.py index 7780f9853a2..6cb267ae0e8 100644 --- a/python/cudf/cudf/tests/test_api_types.py +++ b/python/cudf/cudf/tests/test_api_types.py @@ -7,7 +7,7 @@ import cudf from cudf.api import types -from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210, PANDAS_GE_214 +from cudf.core._compat import PANDAS_GE_210, PANDAS_GE_214, PANDAS_GE_220 from cudf.testing._utils import expect_warning_if @@ -499,8 +499,8 @@ def test_is_integer(obj, expect): (pd.Series(dtype="int"), False), (pd.Series(dtype="float"), False), (pd.Series(dtype="complex"), False), - (pd.Series(dtype="str"), not PANDAS_GE_200), - (pd.Series(dtype="unicode"), not PANDAS_GE_200), + (pd.Series(dtype="str"), PANDAS_GE_220), + (pd.Series(dtype="unicode"), PANDAS_GE_220), (pd.Series(dtype="datetime64[s]"), False), (pd.Series(dtype="timedelta64[s]"), False), (pd.Series(dtype="category"), False), From e596480c9fd60baef23352fa9ca755b50b77cda6 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Fri, 9 Feb 2024 15:45:08 -0500 Subject: [PATCH 232/384] Use offsetalator in cudf::strings::reverse (#15001) Updates `cudf::strings::reverse` to use the offsetalator instead of hardcoded int32 type for offsets column data. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - MithunR (https://github.com/mythrocks) - Vukasin Milovanovic (https://github.com/vuule) URL: https://github.com/rapidsai/cudf/pull/15001 --- cpp/src/strings/reverse.cu | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cpp/src/strings/reverse.cu b/cpp/src/strings/reverse.cu index aecb029f25f..f9aec41b5e3 100644 --- a/cpp/src/strings/reverse.cu +++ b/cpp/src/strings/reverse.cu @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -37,7 +38,7 @@ namespace { */ struct reverse_characters_fn { column_device_view const d_strings; - size_type const* d_offsets; + cudf::detail::input_offsetalator d_offsets; char* d_chars; __device__ void operator()(size_type idx) @@ -62,10 +63,10 @@ std::unique_ptr reverse(strings_column_view const& input, if (input.is_empty()) { return make_empty_column(type_id::STRING); } // copy the column; replace data in the chars column - auto result = std::make_unique(input.parent(), stream, mr); - auto const d_offsets = - result->view().child(strings_column_view::offsets_column_index).data(); - auto d_chars = result->mutable_view().head(); + auto result = std::make_unique(input.parent(), stream, mr); + auto sv = strings_column_view(result->view()); + auto const d_offsets = cudf::detail::offsetalator_factory::make_input_iterator(sv.offsets()); + auto d_chars = result->mutable_view().head(); auto const d_column = column_device_view::create(input.parent(), stream); thrust::for_each_n(rmm::exec_policy(stream), From 0c0c7e6c82820ea223ee2a4abf63923e3eae2e25 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 9 Feb 2024 18:23:12 -0600 Subject: [PATCH 233/384] Add `future_stack` to `DataFrame.stack` (#15015) This PR introduces `future_stack` to `stack` API. This also means deprecating `dropna`. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/15015 --- python/cudf/cudf/core/dataframe.py | 47 +++++++++++++++++++++----- python/cudf/cudf/core/reshape.py | 2 +- python/cudf/cudf/tests/test_reshape.py | 33 +++++++++++++++--- 3 files changed, 69 insertions(+), 13 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 727d5135297..1a6376d1c00 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -6711,7 +6711,7 @@ def to_orc( ) @_cudf_nvtx_annotate - def stack(self, level=-1, dropna=True): + def stack(self, level=-1, dropna=no_default, future_stack=False): """Stack the prescribed level(s) from columns to index Return a reshaped DataFrame or Series having a multi-level @@ -6843,6 +6843,23 @@ def stack(self, level=-1, dropna=True): weight kg 3.0 dtype: float64 """ + if future_stack: + if dropna is not no_default: + raise ValueError( + "dropna must be unspecified with future_stack=True as the new " + "implementation does not introduce rows of NA values. This " + "argument will be removed in a future version of cudf." + ) + else: + if dropna is not no_default or self._data.nlevels > 1: + warnings.warn( + "The previous implementation of stack is deprecated and will be " + "removed in a future version of cudf. Specify future_stack=True " + "to adopt the new implementation and silence this warning.", + FutureWarning, + ) + if dropna is no_default: + dropna = True if isinstance(level, (int, str)): level = [level] @@ -6858,7 +6875,7 @@ def stack(self, level=-1, dropna=True): level = [level] if not isinstance(level, list) else level - if len(level) > 1 and not dropna: + if not future_stack and len(level) > 1 and not dropna: raise NotImplementedError( "When stacking multiple levels, setting `dropna` to False " "will generate new column combination that does not exist " @@ -6900,7 +6917,9 @@ def stack(self, level=-1, dropna=True): # Since `level` may only specify a subset of all levels, `unique()` is # required to remove duplicates. In pandas, the order of the keys in # the specified levels are always sorted. - unique_named_levels = named_levels.unique().sort_values() + unique_named_levels = named_levels.unique() + if not future_stack: + unique_named_levels = unique_named_levels.sort_values() # Each index from the original dataframe should repeat by the number # of unique values in the named_levels @@ -6949,11 +6968,19 @@ def unnamed_group_generator(): # `unique_named_levels` assigns -1 to these key # combinations, representing an all-null column that # is used in the subsequent libcudf call. - yield grpdf.reindex( - unique_named_levels, axis=0, fill_value=-1 - ).sort_index().values + if future_stack: + yield grpdf.reindex( + unique_named_levels, axis=0, fill_value=-1 + ).values + else: + yield grpdf.reindex( + unique_named_levels, axis=0, fill_value=-1 + ).sort_index().values else: - yield column_idx_df.sort_index().values + if future_stack: + yield column_idx_df.values + else: + yield column_idx_df.sort_index().values column_indices = list(unnamed_group_generator()) @@ -7004,6 +7031,10 @@ def unnamed_group_generator(): [ stacked[i] for i in unnamed_level_values.argsort().argsort() + ] + if not future_stack + else [ + stacked[i] for i in unnamed_level_values.argsort() ], ) ), @@ -7013,7 +7044,7 @@ def unnamed_group_generator(): result = DataFrame._from_data(data, index=new_index) - if dropna: + if not future_stack and dropna: return result.dropna(how="all") else: return result diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index 2ea538d66a1..656db855253 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -1120,7 +1120,7 @@ def unstack(df, level, fill_value=None): "Calling unstack() on single index dataframe" " with different column datatype is not supported." ) - res = df.T.stack(dropna=False) + res = df.T.stack(future_stack=False) # Result's index is a multiindex res.index.names = ( tuple(df._data.to_pandas_index().names) + df.index.names diff --git a/python/cudf/cudf/tests/test_reshape.py b/python/cudf/cudf/tests/test_reshape.py index b49a921e812..59c5a0662be 100644 --- a/python/cudf/cudf/tests/test_reshape.py +++ b/python/cudf/cudf/tests/test_reshape.py @@ -9,12 +9,14 @@ import cudf from cudf import melt as cudf_melt +from cudf.core._compat import PANDAS_GE_210, PANDAS_GE_220 from cudf.core.buffer.spill_manager import get_global_manager from cudf.testing._utils import ( ALL_TYPES, DATETIME_TYPES, NUMERIC_TYPES, assert_eq, + expect_warning_if, ) pytest_xfail = pytest.mark.xfail @@ -153,6 +155,10 @@ def test_df_stack_reset_index(): assert_eq(expected, actual) +@pytest.mark.skipif( + not PANDAS_GE_210, + reason="Need pandas-2.1.0+ to match `stack` api", +) @pytest.mark.parametrize( "columns", [ @@ -206,8 +212,15 @@ def test_df_stack_multiindex_column_axis(columns, index, level, dropna): ) gdf = cudf.from_pandas(pdf) - got = gdf.stack(level=level, dropna=dropna) - expect = pdf.stack(level=level, dropna=dropna) + with pytest.warns(FutureWarning): + got = gdf.stack(level=level, dropna=dropna, future_stack=False) + with expect_warning_if(PANDAS_GE_220): + expect = pdf.stack(level=level, dropna=dropna, future_stack=False) + + assert_eq(expect, got, check_dtype=False) + + got = gdf.stack(level=level, future_stack=True) + expect = pdf.stack(level=level, future_stack=True) assert_eq(expect, got, check_dtype=False) @@ -228,6 +241,10 @@ def test_df_stack_mixed_dtypes(): assert_eq(expect, got, check_dtype=False) +@pytest.mark.skipif( + not PANDAS_GE_210, + reason="Need pandas-2.1.0+ to match `stack` api", +) @pytest.mark.parametrize("level", [["animal", "hair_length"], [1, 2]]) def test_df_stack_multiindex_column_axis_pd_example(level): columns = pd.MultiIndex.from_tuples( @@ -242,8 +259,16 @@ def test_df_stack_multiindex_column_axis_pd_example(level): df = pd.DataFrame(np.random.randn(4, 4), columns=columns) - expect = df.stack(level=level) - got = cudf.from_pandas(df).stack(level=level) + with expect_warning_if(PANDAS_GE_220): + expect = df.stack(level=level, future_stack=False) + gdf = cudf.from_pandas(df) + with pytest.warns(FutureWarning): + got = gdf.stack(level=level, future_stack=False) + + assert_eq(expect, got) + + expect = df.stack(level=level, future_stack=True) + got = gdf.stack(level=level, future_stack=True) assert_eq(expect, got) From 8edbeca2242985176f0f23dfd9a2dbd54b4360ae Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 9 Feb 2024 20:01:15 -0600 Subject: [PATCH 234/384] Fix `Index.difference` to handle duplicate values when one of the inputs is empty (#15016) This PR removes duplicate values in two short-circuit code-paths of `Index.difference` which is already fixed in `pandas-2.2` Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/15016 --- python/cudf/cudf/core/_base_index.py | 4 ++-- python/cudf/cudf/tests/test_index.py | 21 +++++++++++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index babead9ca97..58e2241e810 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -1096,12 +1096,12 @@ def difference(self, other, sort=None): other = cudf.Index(other, name=getattr(other, "name", self.name)) if not len(other): - res = self._get_reconciled_name_object(other) + res = self._get_reconciled_name_object(other).unique() if sort: return res.sort_values() return res elif self.equals(other): - res = self[:0]._get_reconciled_name_object(other) + res = self[:0]._get_reconciled_name_object(other).unique() if sort: return res.sort_values() return res diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index 7a190fb428a..3cbfea8063f 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -15,7 +15,7 @@ import cudf from cudf.api.extensions import no_default from cudf.api.types import is_bool_dtype -from cudf.core._compat import PANDAS_GE_200 +from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_220 from cudf.core.index import ( CategoricalIndex, DatetimeIndex, @@ -797,9 +797,26 @@ def test_index_to_series(data): "name_data,name_other", [("abc", "c"), (None, "abc"), ("abc", pd.NA), ("abc", "abc")], ) -def test_index_difference(data, other, sort, name_data, name_other): +def test_index_difference(request, data, other, sort, name_data, name_other): pd_data = pd.Index(data, name=name_data) pd_other = pd.Index(other, name=name_other) + request.applymarker( + pytest.mark.xfail( + condition=PANDAS_GE_220 + and isinstance(pd_data.dtype, pd.CategoricalDtype) + and not isinstance(pd_other.dtype, pd.CategoricalDtype) + and pd_other.isnull().any(), + reason="https://github.com/pandas-dev/pandas/issues/57318", + ) + ) + request.applymarker( + pytest.mark.xfail( + condition=not PANDAS_GE_220 + and len(pd_other) == 0 + and len(pd_data) != len(pd_data.unique()), + reason="Bug fixed in pandas-2.2+", + ) + ) gd_data = cudf.from_pandas(pd_data) gd_other = cudf.from_pandas(pd_other) From 630c885001b679cb16ee997c0249b9c69212f4d1 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 10 Feb 2024 00:29:43 -0600 Subject: [PATCH 235/384] Support CUDA 12.2 (#14712) * switches to CUDA 12.2.2 for building conda packages and wheels * adds new tests running against CUDA 12.2.2 ### Notes for Reviewers This is part of ongoing work to build and test packages against CUDA 12.2.2 across all of RAPIDS. For more details see: * https://github.com/rapidsai/build-planning/issues/7 * https://github.com/rapidsai/shared-workflows/pull/166 * adds some `dependencies.yaml` simplifications missed in #14733 Planning a second round of PRs to revert these references back to a proper `branch-24.{nn}` release branch of `shared-workflows` once https://github.com/rapidsai/shared-workflows/pull/166 is merged. *(created with `rapids-reviser`)* Authors: - James Lamb (https://github.com/jameslamb) - Bradley Dice (https://github.com/bdice) - https://github.com/jakirkham Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Bradley Dice (https://github.com/bdice) - Jake Awe (https://github.com/AyodeAwe) URL: https://github.com/rapidsai/cudf/pull/14712 --- .github/workflows/build.yaml | 18 ++++---- .github/workflows/pr.yaml | 42 +++++++++---------- .github/workflows/test.yaml | 22 +++++----- ..._64.yaml => all_cuda-122_arch-x86_64.yaml} | 4 +- conda/recipes/cudf/meta.yaml | 6 +++ conda/recipes/cudf_kafka/meta.yaml | 10 ++++- conda/recipes/libcudf/meta.yaml | 26 +++++++++--- dependencies.yaml | 30 ++++--------- 8 files changed, 87 insertions(+), 71 deletions(-) rename conda/environments/{all_cuda-120_arch-x86_64.yaml => all_cuda-122_arch-x86_64.yaml} (97%) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c663f52f548..b92e0a53b46 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@test-cuda-12.2 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -37,7 +37,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@test-cuda-12.2 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,7 +46,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@test-cuda-12.2 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -57,7 +57,7 @@ jobs: if: github.ref_type == 'branch' needs: python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@test-cuda-12.2 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-cudf: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@test-cuda-12.2 with: build_type: ${{ inputs.build_type || 'branch' }} build-2_28-wheels: "true" @@ -80,7 +80,7 @@ jobs: wheel-publish-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@test-cuda-12.2 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -90,9 +90,9 @@ jobs: wheel-build-dask-cudf: needs: wheel-publish-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@test-cuda-12.2 with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} @@ -101,7 +101,7 @@ jobs: wheel-publish-dask-cudf: needs: wheel-build-dask-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@test-cuda-12.2 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1dc31da8e80..57923dca5d9 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -32,16 +32,16 @@ jobs: #- pandas-tests-diff #- pandas-tests-diff-comment secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@test-cuda-12.2 checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@test-cuda-12.2 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@test-cuda-12.2 with: build_type: pull-request conda-cpp-checks: @@ -54,19 +54,19 @@ jobs: conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@test-cuda-12.2 with: build_type: pull-request conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@test-cuda-12.2 with: build_type: pull-request conda-python-cudf-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@test-cuda-12.2 with: build_type: pull-request test_script: "ci/test_python_cudf.sh" @@ -74,14 +74,14 @@ jobs: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@test-cuda-12.2 with: build_type: pull-request test_script: "ci/test_python_other.sh" conda-java-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@test-cuda-12.2 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -91,7 +91,7 @@ jobs: conda-notebook-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@test-cuda-12.2 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -101,7 +101,7 @@ jobs: docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@test-cuda-12.2 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -111,7 +111,7 @@ jobs: wheel-build-cudf: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@test-cuda-12.2 with: build_type: pull-request build-2_28-wheels: "true" @@ -119,29 +119,29 @@ jobs: wheel-tests-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@test-cuda-12.2 with: build_type: pull-request script: ci/test_wheel_cudf.sh wheel-build-dask-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@test-cuda-12.2 with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) build_type: pull-request script: "ci/build_wheel_dask_cudf.sh" wheel-tests-dask-cudf: needs: wheel-build-dask-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@test-cuda-12.2 with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) build_type: pull-request script: ci/test_wheel_dask_cudf.sh devcontainer: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.04 + uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@test-cuda-12.2 with: build_command: | sccache -z; @@ -150,16 +150,16 @@ jobs: unit-tests-cudf-pandas: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@test-cuda-12.2 with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) build_type: pull-request script: ci/cudf_pandas_scripts/run_tests.sh pandas-tests: # run the Pandas unit tests using PR branch needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@test-cuda-12.2 with: matrix_filter: map(select(.ARCH == "amd64")) | max_by(.CUDA_VER) | [.] build_type: pull-request @@ -171,7 +171,7 @@ jobs: # needs: [pandas-tests-main, pandas-tests-pr] # secrets: inherit # # This branch exports a `job_output` output that the downstream job reads. - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@test-cuda-12.2 # with: # node_type: cpu4 # build_type: pull-request diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e044d69c6d8..e7eef4de1b3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -25,7 +25,7 @@ jobs: enable_check_symbols: true conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@test-cuda-12.2 with: build_type: nightly branch: ${{ inputs.branch }} @@ -33,7 +33,7 @@ jobs: sha: ${{ inputs.sha }} conda-cpp-memcheck-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@test-cuda-12.2 with: build_type: nightly branch: ${{ inputs.branch }} @@ -45,7 +45,7 @@ jobs: run_script: "ci/test_cpp_memcheck.sh" conda-python-cudf-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@test-cuda-12.2 with: build_type: nightly branch: ${{ inputs.branch }} @@ -55,7 +55,7 @@ jobs: conda-python-other-tests: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@test-cuda-12.2 with: build_type: nightly branch: ${{ inputs.branch }} @@ -64,7 +64,7 @@ jobs: test_script: "ci/test_python_other.sh" conda-java-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@test-cuda-12.2 with: build_type: nightly branch: ${{ inputs.branch }} @@ -76,7 +76,7 @@ jobs: run_script: "ci/test_java.sh" conda-notebook-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@test-cuda-12.2 with: build_type: nightly branch: ${{ inputs.branch }} @@ -88,7 +88,7 @@ jobs: run_script: "ci/test_notebooks.sh" wheel-tests-cudf: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@test-cuda-12.2 with: build_type: nightly branch: ${{ inputs.branch }} @@ -97,9 +97,9 @@ jobs: script: ci/test_wheel_cudf.sh wheel-tests-dask-cudf: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@test-cuda-12.2 with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} @@ -107,7 +107,7 @@ jobs: script: ci/test_wheel_dask_cudf.sh unit-tests-cudf-pandas: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@test-cuda-12.2 with: build_type: nightly branch: ${{ inputs.branch }} @@ -117,7 +117,7 @@ jobs: pandas-tests: # run the Pandas unit tests secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@test-cuda-12.2 with: matrix_filter: map(select(.ARCH == "amd64")) | max_by(.CUDA_VER) | [.] build_type: nightly diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml similarity index 97% rename from conda/environments/all_cuda-120_arch-x86_64.yaml rename to conda/environments/all_cuda-122_arch-x86_64.yaml index a8be9d65c43..c0950c7da98 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -25,7 +25,7 @@ dependencies: - cuda-nvtx-dev - cuda-python>=12.0,<13.0a0 - cuda-sanitizer-api -- cuda-version=12.0 +- cuda-version=12.2 - cupy>=12.0.0 - cxx-compiler - cython>=3.0.3 @@ -101,4 +101,4 @@ dependencies: - zlib>=1.2.13 - pip: - git+https://github.com/python-streamz/streamz.git@master -name: all_cuda-120_arch-x86_64 +name: all_cuda-122_arch-x86_64 diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 0dffdc10421..85eff55b2c6 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -39,6 +39,10 @@ build: ignore_run_exports_from: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} + {% else %} + - {{ compiler('cuda') }} + - cuda-cudart-dev + - libcufile-dev # [linux64] {% endif %} requirements: @@ -91,6 +95,8 @@ requirements: - cubinlinker # CUDA enhanced compatibility. - cuda-python >=11.7.1,<12.0a0 {% else %} + - cuda-cudart + - libcufile # [linux64] # Needed by Numba for CUDA support - cuda-nvcc-impl # TODO: Add nvjitlink here diff --git a/conda/recipes/cudf_kafka/meta.yaml b/conda/recipes/cudf_kafka/meta.yaml index 872324d3f73..45e41bf8de7 100644 --- a/conda/recipes/cudf_kafka/meta.yaml +++ b/conda/recipes/cudf_kafka/meta.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. {% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} @@ -36,6 +36,9 @@ build: ignore_run_exports_from: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} + {% else %} + - {{ compiler('cuda') }} + - cuda-cudart-dev {% endif %} requirements: @@ -59,7 +62,7 @@ requirements: - libcudf_kafka ={{ version }} - scikit-build-core >=0.7.0 - setuptools - {% if cuda_major == "12" %} + {% if cuda_major != "11" %} - cuda-cudart-dev {% endif %} run: @@ -67,6 +70,9 @@ requirements: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - libcudf_kafka ={{ version }} - cudf ={{ version }} + {% if cuda_major != "11" %} + - cuda-cudart + {% endif %} test: requires: diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml index 0459908fd00..63eb83084dd 100644 --- a/conda/recipes/libcudf/meta.yaml +++ b/conda/recipes/libcudf/meta.yaml @@ -1,10 +1,9 @@ -# Copyright (c) 2018-2023, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. {% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} {% set cuda_major = cuda_version.split('.')[0] %} -{% set cuda_spec = ">=" + cuda_major ~ ",<" + (cuda_major | int + 1) ~ ".0a0" %} # i.e. >=11,<12.0a0 {% set date_string = environ['RAPIDS_DATE_STRING'] %} package: @@ -87,13 +86,17 @@ outputs: ignore_run_exports_from: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} + {% else %} + - {{ compiler('cuda') }} {% endif %} requirements: build: - cmake {{ cmake_version }} host: + - cuda-version ={{ cuda_version }} - libarrow {{ libarrow_version }} run: + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - cudatoolkit - libcufile {{ cuda11_libcufile_run_version }} # [linux64] @@ -101,7 +104,6 @@ outputs: - cuda-nvrtc - libcufile # [linux64] {% endif %} - - cuda-version {{ cuda_spec }} - nvcomp {{ nvcomp_version }} - librmm ={{ minor_version }} - libkvikio ={{ minor_version }} @@ -127,6 +129,8 @@ outputs: ignore_run_exports_from: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} + {% else %} + - {{ compiler('cuda') }} {% endif %} requirements: build: @@ -155,6 +159,9 @@ outputs: ignore_run_exports_from: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} + {% else %} + - {{ compiler('cuda') }} + - cuda-nvtx-dev {% endif %} requirements: build: @@ -179,6 +186,10 @@ outputs: - cuda-version ={{ cuda_version }} run: - {{ pin_subpackage('libcudf', exact=True) }} + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} + {% if cuda_major != "11" %} + - cuda-nvtx + {% endif %} about: home: https://rapids.ai/ license: Apache-2.0 @@ -194,6 +205,9 @@ outputs: ignore_run_exports_from: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} + {% else %} + - {{ compiler('cuda') }} + - libcurand-dev {% endif %} requirements: build: @@ -201,7 +215,7 @@ outputs: host: - {{ pin_subpackage('libcudf', exact=True) }} - {{ pin_subpackage('libcudf_kafka', exact=True) }} - - cuda-version {{ cuda_spec }} + - cuda-version ={{ cuda_version }} {% if cuda_major == "11" %} - libcurand {{ cuda11_libcurand_run_version }} {% else %} @@ -211,11 +225,13 @@ outputs: - gtest {{ gtest_version }} - gmock {{ gtest_version }} run: + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - {{ pin_subpackage('libcudf', exact=True) }} - {{ pin_subpackage('libcudf_kafka', exact=True) }} - - cuda-version {{ cuda_spec }} {% if cuda_major == "11" %} - libcurand {{ cuda11_libcurand_run_version }} + {% else %} + - libcurand {% endif %} - benchmark {{ gbench_version }} - gtest {{ gtest_version }} diff --git a/dependencies.yaml b/dependencies.yaml index 90b0527479a..c4c2cd3c764 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -3,7 +3,7 @@ files: all: output: conda matrix: - cuda: ["11.8", "12.0"] + cuda: ["11.8", "12.2"] arch: [x86_64] includes: - build_all @@ -231,14 +231,6 @@ dependencies: cuda: "11.8" packages: - nvcc_linux-aarch64=11.8 - - output_types: conda - matrices: - - matrix: - cuda: "12.0" - packages: - - cuda-version=12.0 - - matrix: # Fallback for CUDA 11 or no matrix - packages: build_cpp: common: - output_types: conda @@ -359,6 +351,10 @@ dependencies: cuda: "12.0" packages: - cuda-version=12.0 + - matrix: + cuda: "12.2" + packages: + - cuda-version=12.2 cuda: specific: - output_types: conda @@ -405,6 +401,9 @@ dependencies: - *libcurand114 - output_types: conda matrices: + - matrix: + arch: aarch64 + packages: - matrix: cuda: "12.*" arch: x86_64 @@ -436,9 +435,6 @@ dependencies: # so 11.2 uses 11.4 packages (the oldest available). - *libcufile_114 - *libcufile_dev114 - # Fallback matrix for aarch64, which doesn't support libcufile. - - matrix: - packages: develop: common: - output_types: [conda, requirements] @@ -587,19 +583,11 @@ dependencies: cuda: "12.*" packages: - cuda-sanitizer-api - - matrix: # Fallback for CUDA 11 or no matrix - packages: - - output_types: conda - matrices: - - matrix: - cuda: "12.0" - packages: - - cuda-version=12.0 - matrix: cuda: "11.8" packages: - cuda-sanitizer-api=11.8.86 - - matrix: + - matrix: # Fallback for CUDA 11 or no matrix packages: test_java: common: From daa63d2e09247549a0ba62300cb669d870af20f1 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Mon, 12 Feb 2024 10:46:13 -0500 Subject: [PATCH 236/384] Use offsetalator in cudf::strings::wrap() (#15002) Updates `cudf::strings::wrap()` to use the offsetalator instead of hardcoded int32 type for offsets column data. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Nghia Truong (https://github.com/ttnghia) - Karthikeyan (https://github.com/karthikeyann) URL: https://github.com/rapidsai/cudf/pull/15002 --- cpp/src/strings/wrap.cu | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cpp/src/strings/wrap.cu b/cpp/src/strings/wrap.cu index 19f1ac55bb0..0b3b6e78f82 100644 --- a/cpp/src/strings/wrap.cu +++ b/cpp/src/strings/wrap.cu @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -41,7 +42,7 @@ namespace { // anonym. // struct execute_wrap { execute_wrap(column_device_view const d_column, - int32_t const* d_offsets, + cudf::detail::input_offsetalator d_offsets, char* d_chars, size_type width) : d_column_(d_column), d_offsets_(d_offsets), d_chars_(d_chars), width_(width) @@ -83,7 +84,7 @@ struct execute_wrap { private: column_device_view const d_column_; - int32_t const* d_offsets_; + cudf::detail::input_offsetalator d_offsets_; char* d_chars_; size_type width_; }; @@ -110,7 +111,8 @@ std::unique_ptr wrap(strings_column_view const& strings, // build offsets column auto offsets_column = std::make_unique(strings.offsets(), stream, mr); // makes a copy - auto d_new_offsets = offsets_column->view().template data(); + auto d_new_offsets = + cudf::detail::offsetalator_factory::make_input_iterator(offsets_column->view()); auto chars_buffer = rmm::device_buffer{strings.chars_begin(stream), static_cast(strings.chars_size(stream)), From 49c2995b1b861b12d3b25ad997adec9c50ed872f Mon Sep 17 00:00:00 2001 From: Suraj Aralihalli Date: Mon, 12 Feb 2024 09:46:56 -0800 Subject: [PATCH 237/384] Introduce `GetJsonObjectOptions` in `getJSONObject` Java API (#14956) Resolves [10219](https://github.com/NVIDIA/spark-rapids/issues/10219) This PR introduces a new class named `GetJsonObjectOptions` that holds the configurations to control the behavior of the underlying `cudf::get_json_object` function. It incorporates this new class into the `getJSONObject` JAVA API as an additional argument but also keeps the previous API to maintain backwards compatibility. It also includes a test case, `testGetJSONObjectWithSingleQuotes`, validating the behavior of `getJSONObject` when single quotes are enabled. Authors: - Suraj Aralihalli (https://github.com/SurajAralihalli) Approvers: - Robert (Bobby) Evans (https://github.com/revans2) - MithunR (https://github.com/mythrocks) - Karthikeyan (https://github.com/karthikeyann) URL: https://github.com/rapidsai/cudf/pull/14956 --- .../main/java/ai/rapids/cudf/ColumnView.java | 22 +++++- .../ai/rapids/cudf/GetJsonObjectOptions.java | 75 +++++++++++++++++++ java/src/main/native/src/ColumnViewJni.cpp | 12 ++- .../java/ai/rapids/cudf/ColumnVectorTest.java | 16 ++++ 4 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 java/src/main/java/ai/rapids/cudf/GetJsonObjectOptions.java diff --git a/java/src/main/java/ai/rapids/cudf/ColumnView.java b/java/src/main/java/ai/rapids/cudf/ColumnView.java index 8eabed7f364..997ff77bae3 100644 --- a/java/src/main/java/ai/rapids/cudf/ColumnView.java +++ b/java/src/main/java/ai/rapids/cudf/ColumnView.java @@ -2978,6 +2978,24 @@ public final ColumnVector repeatStrings(ColumnView repeatTimes) { repeatTimes.getNativeView())); } + /** + * Apply a JSONPath string to all rows in an input strings column. + * + * Applies a JSONPath string to an incoming strings column where each row in the column + * is a valid json string. The output is returned by row as a strings column. + * + * For reference, https://tools.ietf.org/id/draft-goessner-dispatch-jsonpath-00.html + * Note: Only implements the operators: $ . [] * + * + * @param path The JSONPath string to be applied to each row + * @param path The GetJsonObjectOptions to control get_json_object behaviour + * @return new strings ColumnVector containing the retrieved json object strings + */ + public final ColumnVector getJSONObject(Scalar path, GetJsonObjectOptions options) { + assert(type.equals(DType.STRING)) : "column type must be a String"; + return new ColumnVector(getJSONObject(getNativeView(), path.getScalarHandle(), options.isAllowSingleQuotes(), options.isStripQuotesFromSingleStrings(), options.isMissingFieldsAsNulls())); + } + /** * Apply a JSONPath string to all rows in an input strings column. * @@ -2992,7 +3010,7 @@ public final ColumnVector repeatStrings(ColumnView repeatTimes) { */ public final ColumnVector getJSONObject(Scalar path) { assert(type.equals(DType.STRING)) : "column type must be a String"; - return new ColumnVector(getJSONObject(getNativeView(), path.getScalarHandle())); + return getJSONObject(path, GetJsonObjectOptions.DEFAULT); } /** @@ -4194,7 +4212,7 @@ private static native long repeatStringsWithColumnRepeatTimes(long stringsHandle long repeatTimesHandle); - private static native long getJSONObject(long viewHandle, long scalarHandle) throws CudfException; + private static native long getJSONObject(long viewHandle, long scalarHandle, boolean allowSingleQuotes, boolean stripQuotesFromSingleStrings, boolean missingFieldsAsNulls) throws CudfException; /** * Native method to parse and convert a timestamp column vector to string column vector. A unix diff --git a/java/src/main/java/ai/rapids/cudf/GetJsonObjectOptions.java b/java/src/main/java/ai/rapids/cudf/GetJsonObjectOptions.java new file mode 100644 index 00000000000..5f9a174b2d3 --- /dev/null +++ b/java/src/main/java/ai/rapids/cudf/GetJsonObjectOptions.java @@ -0,0 +1,75 @@ +/* + * + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ai.rapids.cudf; + +public final class GetJsonObjectOptions { + + public static GetJsonObjectOptions DEFAULT = new GetJsonObjectOptions.Builder().build(); + + private final boolean allowSingleQuotes; + private final boolean stripQuotesFromSingleStrings; + private final boolean missingFieldsAsNulls; + + private GetJsonObjectOptions(Builder builder) { + this.allowSingleQuotes = builder.allowSingleQuotes; + this.stripQuotesFromSingleStrings = builder.stripQuotesFromSingleStrings; + this.missingFieldsAsNulls = builder.missingFieldsAsNulls; + } + + public boolean isAllowSingleQuotes() { + return allowSingleQuotes; + } + + public boolean isStripQuotesFromSingleStrings() { + return stripQuotesFromSingleStrings; + } + + public boolean isMissingFieldsAsNulls() { + return missingFieldsAsNulls; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private boolean allowSingleQuotes = false; + private boolean stripQuotesFromSingleStrings = true; + private boolean missingFieldsAsNulls = false; + + public Builder allowSingleQuotes(boolean allowSingleQuotes) { + this.allowSingleQuotes = allowSingleQuotes; + return this; + } + + public Builder stripQuotesFromSingleStrings(boolean stripQuotesFromSingleStrings) { + this.stripQuotesFromSingleStrings = stripQuotesFromSingleStrings; + return this; + } + + public Builder missingFieldsAsNulls(boolean missingFieldsAsNulls) { + this.missingFieldsAsNulls = missingFieldsAsNulls; + return this; + } + + public GetJsonObjectOptions build() { + return new GetJsonObjectOptions(this); + } + } +} diff --git a/java/src/main/native/src/ColumnViewJni.cpp b/java/src/main/native/src/ColumnViewJni.cpp index 47dc802cd49..1c4eb8a83ab 100644 --- a/java/src/main/native/src/ColumnViewJni.cpp +++ b/java/src/main/native/src/ColumnViewJni.cpp @@ -2436,9 +2436,9 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_copyColumnViewToCV(JNIEnv CATCH_STD(env, 0) } -JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_getJSONObject(JNIEnv *env, jclass, - jlong j_view_handle, - jlong j_scalar_handle) { +JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_getJSONObject( + JNIEnv *env, jclass, jlong j_view_handle, jlong j_scalar_handle, jboolean allow_single_quotes, + jboolean strip_quotes_from_single_strings, jboolean missing_fields_as_nulls) { JNI_NULL_CHECK(env, j_view_handle, "view cannot be null", 0); JNI_NULL_CHECK(env, j_scalar_handle, "path cannot be null", 0); @@ -2448,7 +2448,11 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_getJSONObject(JNIEnv *env cudf::column_view *n_column_view = reinterpret_cast(j_view_handle); cudf::strings_column_view n_strings_col_view(*n_column_view); cudf::string_scalar *n_scalar_path = reinterpret_cast(j_scalar_handle); - return release_as_jlong(cudf::get_json_object(n_strings_col_view, *n_scalar_path)); + auto options = cudf::get_json_object_options{}; + options.set_allow_single_quotes(allow_single_quotes); + options.set_strip_quotes_from_single_strings(strip_quotes_from_single_strings); + options.set_missing_fields_as_nulls(missing_fields_as_nulls); + return release_as_jlong(cudf::get_json_object(n_strings_col_view, *n_scalar_path, options)); } CATCH_STD(env, 0) } diff --git a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java index dfead3716ee..75573046af2 100644 --- a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java +++ b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java @@ -6379,6 +6379,7 @@ void testGetJSONObject() { " }\n" + "}"; + try (ColumnVector json = ColumnVector.fromStrings(jsonString, jsonString); ColumnVector expectedAuthors = ColumnVector.fromStrings("[\"Nigel Rees\",\"Evelyn " + "Waugh\",\"Herman Melville\",\"J. R. R. Tolkien\"]", "[\"Nigel Rees\",\"Evelyn " + @@ -6389,6 +6390,21 @@ void testGetJSONObject() { } } + @Test + void testGetJSONObjectWithSingleQuotes() { + String jsonString = "{" + + "\'a\': \'A\"\'" + + "}"; + + GetJsonObjectOptions options = GetJsonObjectOptions.builder().allowSingleQuotes(true).build(); + try (ColumnVector json = ColumnVector.fromStrings(jsonString, jsonString); + ColumnVector expectedAuthors = ColumnVector.fromStrings("A\"", "A\""); + Scalar path = Scalar.fromString("$.a"); + ColumnVector gotAuthors = json.getJSONObject(path, options)) { + assertColumnsAreEqual(expectedAuthors, gotAuthors); + } +} + @Test void testMakeStructEmpty() { final int numRows = 10; From 82f6a5356aa10fd22c13f6aa85d1770c4c1a1c1b Mon Sep 17 00:00:00 2001 From: Ray Douglass Date: Mon, 12 Feb 2024 15:44:48 -0500 Subject: [PATCH 238/384] Update Changelog [skip ci] --- CHANGELOG.md | 225 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 225 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cb6caa25ee..bce764f59e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,228 @@ +# cuDF 24.02.00 (12 Feb 2024) + +## 🚨 Breaking Changes + +- Remove **kwargs from astype ([#14765](https://github.com/rapidsai/cudf/pull/14765)) [@mroeschke](https://github.com/mroeschke) +- Remove mimesis as a testing dependency ([#14723](https://github.com/rapidsai/cudf/pull/14723)) [@mroeschke](https://github.com/mroeschke) +- Update to Dask's `shuffle_method` kwarg ([#14708](https://github.com/rapidsai/cudf/pull/14708)) [@pentschev](https://github.com/pentschev) +- Drop Pascal GPU support. ([#14630](https://github.com/rapidsai/cudf/pull/14630)) [@bdice](https://github.com/bdice) +- Update to CCCL 2.2.0. ([#14576](https://github.com/rapidsai/cudf/pull/14576)) [@bdice](https://github.com/bdice) +- Expunge as_frame conversions in Column algorithms ([#14491](https://github.com/rapidsai/cudf/pull/14491)) [@wence-](https://github.com/wence-) +- Deprecate cudf::make_strings_column accepting typed offsets ([#14461](https://github.com/rapidsai/cudf/pull/14461)) [@davidwendt](https://github.com/davidwendt) +- Remove deprecated nvtext::load_merge_pairs_file ([#14460](https://github.com/rapidsai/cudf/pull/14460)) [@davidwendt](https://github.com/davidwendt) +- Include writer code and writerVersion in ORC files ([#14458](https://github.com/rapidsai/cudf/pull/14458)) [@vuule](https://github.com/vuule) +- Remove null mask for zero nulls in json readers ([#14451](https://github.com/rapidsai/cudf/pull/14451)) [@karthikeyann](https://github.com/karthikeyann) +- REF: Remove **kwargs from to_pandas, raise if nullable is not implemented ([#14438](https://github.com/rapidsai/cudf/pull/14438)) [@mroeschke](https://github.com/mroeschke) +- Consolidate 1D pandas object handling in as_column ([#14394](https://github.com/rapidsai/cudf/pull/14394)) [@mroeschke](https://github.com/mroeschke) +- Move chars column to parent data buffer in strings column ([#14202](https://github.com/rapidsai/cudf/pull/14202)) [@karthikeyann](https://github.com/karthikeyann) +- Switch to scikit-build-core ([#13531](https://github.com/rapidsai/cudf/pull/13531)) [@vyasr](https://github.com/vyasr) + +## 🐛 Bug Fixes + +- Exclude tests from builds ([#14981](https://github.com/rapidsai/cudf/pull/14981)) [@vyasr](https://github.com/vyasr) +- Fix the bounce buffer size in ORC writer ([#14947](https://github.com/rapidsai/cudf/pull/14947)) [@vuule](https://github.com/vuule) +- Revert sum/product aggregation to always produce `int64_t` type ([#14907](https://github.com/rapidsai/cudf/pull/14907)) [@SurajAralihalli](https://github.com/SurajAralihalli) +- Fixed an issue with output chunking computation stemming from input chunking. ([#14889](https://github.com/rapidsai/cudf/pull/14889)) [@nvdbaranec](https://github.com/nvdbaranec) +- Fix total_byte_size in Parquet row group metadata ([#14802](https://github.com/rapidsai/cudf/pull/14802)) [@etseidl](https://github.com/etseidl) +- Fix index difference to follow the pandas format ([#14789](https://github.com/rapidsai/cudf/pull/14789)) [@amiralimi](https://github.com/amiralimi) +- Fix shared-workflows repo name ([#14784](https://github.com/rapidsai/cudf/pull/14784)) [@raydouglass](https://github.com/raydouglass) +- Remove unparseable attributes from all nodes ([#14780](https://github.com/rapidsai/cudf/pull/14780)) [@vyasr](https://github.com/vyasr) +- Refactor and add validation to IntervalIndex.__init__ ([#14778](https://github.com/rapidsai/cudf/pull/14778)) [@mroeschke](https://github.com/mroeschke) +- Work around incompatibilities between V2 page header handling and zStandard compression in Parquet writer ([#14772](https://github.com/rapidsai/cudf/pull/14772)) [@etseidl](https://github.com/etseidl) +- Fix calls to deprecated strings factory API ([#14771](https://github.com/rapidsai/cudf/pull/14771)) [@davidwendt](https://github.com/davidwendt) +- Fix ptx file discovery in editable installs ([#14767](https://github.com/rapidsai/cudf/pull/14767)) [@vyasr](https://github.com/vyasr) +- Revise ``shuffle`` deprecation to align with dask/dask ([#14762](https://github.com/rapidsai/cudf/pull/14762)) [@rjzamora](https://github.com/rjzamora) +- Enable intermediate proxies to be picklable ([#14752](https://github.com/rapidsai/cudf/pull/14752)) [@shwina](https://github.com/shwina) +- Add CUDF_TEST_PROGRAM_MAIN macro to tests lacking it ([#14751](https://github.com/rapidsai/cudf/pull/14751)) [@etseidl](https://github.com/etseidl) +- Fix CMake args ([#14746](https://github.com/rapidsai/cudf/pull/14746)) [@vyasr](https://github.com/vyasr) +- Fix logic bug introduced in #14730 ([#14742](https://github.com/rapidsai/cudf/pull/14742)) [@wence-](https://github.com/wence-) +- [Java] Choose The Correct RoundingMode For Checking Decimal OutOfBounds ([#14731](https://github.com/rapidsai/cudf/pull/14731)) [@razajafri](https://github.com/razajafri) +- Fix ``Groupby.get_group`` ([#14728](https://github.com/rapidsai/cudf/pull/14728)) [@rjzamora](https://github.com/rjzamora) +- Ensure that all CUDA kernels in cudf have hidden visibility. ([#14726](https://github.com/rapidsai/cudf/pull/14726)) [@robertmaynard](https://github.com/robertmaynard) +- Split cuda versions for notebook testing ([#14722](https://github.com/rapidsai/cudf/pull/14722)) [@raydouglass](https://github.com/raydouglass) +- Fix to_numeric not preserving Series index and name ([#14718](https://github.com/rapidsai/cudf/pull/14718)) [@mroeschke](https://github.com/mroeschke) +- Update dask-cudf wheel name ([#14713](https://github.com/rapidsai/cudf/pull/14713)) [@raydouglass](https://github.com/raydouglass) +- Fix strings::contains matching end of string target ([#14711](https://github.com/rapidsai/cudf/pull/14711)) [@davidwendt](https://github.com/davidwendt) +- Update to Dask's `shuffle_method` kwarg ([#14708](https://github.com/rapidsai/cudf/pull/14708)) [@pentschev](https://github.com/pentschev) +- Write file-level statistics when writing ORC files with zero rows ([#14707](https://github.com/rapidsai/cudf/pull/14707)) [@vuule](https://github.com/vuule) +- Potential fix for peformance regression in #14415 ([#14706](https://github.com/rapidsai/cudf/pull/14706)) [@etseidl](https://github.com/etseidl) +- Ensure DataFrame column types are preserved during serialization ([#14705](https://github.com/rapidsai/cudf/pull/14705)) [@mroeschke](https://github.com/mroeschke) +- Skip numba test that fails on ARM ([#14702](https://github.com/rapidsai/cudf/pull/14702)) [@brandon-b-miller](https://github.com/brandon-b-miller) +- Allow Z in datetime string parsing in non pandas compat mode ([#14701](https://github.com/rapidsai/cudf/pull/14701)) [@mroeschke](https://github.com/mroeschke) +- Fix nan_as_null not being respected when passing arrow object ([#14688](https://github.com/rapidsai/cudf/pull/14688)) [@mroeschke](https://github.com/mroeschke) +- Fix constructing Series/Index from arrow array and dtype ([#14686](https://github.com/rapidsai/cudf/pull/14686)) [@mroeschke](https://github.com/mroeschke) +- Fix Aggregation Type Promotion: Ensure Unsigned Input Types Result in Unsigned Output for Sum and Multiply ([#14679](https://github.com/rapidsai/cudf/pull/14679)) [@SurajAralihalli](https://github.com/SurajAralihalli) +- Add BaseOffset as a final proxy type to pass instancechecks for offsets against `BaseOffset` ([#14678](https://github.com/rapidsai/cudf/pull/14678)) [@shwina](https://github.com/shwina) +- Add row conversion code from spark-rapids-jni ([#14664](https://github.com/rapidsai/cudf/pull/14664)) [@ttnghia](https://github.com/ttnghia) +- Unconditionally export the CCCL path ([#14656](https://github.com/rapidsai/cudf/pull/14656)) [@vyasr](https://github.com/vyasr) +- Ensure libcudf searches for our patched version of CCCL first ([#14655](https://github.com/rapidsai/cudf/pull/14655)) [@robertmaynard](https://github.com/robertmaynard) +- Constrain CUDA in notebook testing to prevent CUDA 12.1 usage until we have pynvjitlink ([#14648](https://github.com/rapidsai/cudf/pull/14648)) [@vyasr](https://github.com/vyasr) +- Fix invalid memory access in Parquet reader ([#14637](https://github.com/rapidsai/cudf/pull/14637)) [@etseidl](https://github.com/etseidl) +- Use column_empty over as_column([]) ([#14632](https://github.com/rapidsai/cudf/pull/14632)) [@mroeschke](https://github.com/mroeschke) +- Add (implicit) handling for torch tensors in is_scalar ([#14623](https://github.com/rapidsai/cudf/pull/14623)) [@wence-](https://github.com/wence-) +- Fix astype/fillna not maintaining column subclass and types ([#14615](https://github.com/rapidsai/cudf/pull/14615)) [@mroeschke](https://github.com/mroeschke) +- Remove non-empty nulls in cudf::get_json_object ([#14609](https://github.com/rapidsai/cudf/pull/14609)) [@davidwendt](https://github.com/davidwendt) +- Remove `cuda::proclaim_return_type` from nested lambda ([#14607](https://github.com/rapidsai/cudf/pull/14607)) [@ttnghia](https://github.com/ttnghia) +- Fix DataFrame.reindex when column reindexing to MultiIndex/RangeIndex ([#14605](https://github.com/rapidsai/cudf/pull/14605)) [@mroeschke](https://github.com/mroeschke) +- Address potential race conditions in Parquet reader ([#14602](https://github.com/rapidsai/cudf/pull/14602)) [@etseidl](https://github.com/etseidl) +- Fix DataFrame.reindex removing column name ([#14601](https://github.com/rapidsai/cudf/pull/14601)) [@mroeschke](https://github.com/mroeschke) +- Remove unsanitized input test data from copy gtests ([#14600](https://github.com/rapidsai/cudf/pull/14600)) [@davidwendt](https://github.com/davidwendt) +- Fix race detected in Parquet writer ([#14598](https://github.com/rapidsai/cudf/pull/14598)) [@etseidl](https://github.com/etseidl) +- Correct invalid or missing return types ([#14587](https://github.com/rapidsai/cudf/pull/14587)) [@robertmaynard](https://github.com/robertmaynard) +- Fix unsanitized nulls from strings segmented-reduce ([#14586](https://github.com/rapidsai/cudf/pull/14586)) [@davidwendt](https://github.com/davidwendt) +- Upgrade to nvCOMP 3.0.5 ([#14581](https://github.com/rapidsai/cudf/pull/14581)) [@davidwendt](https://github.com/davidwendt) +- Fix unsanitized nulls produced by `cudf::clamp` APIs ([#14580](https://github.com/rapidsai/cudf/pull/14580)) [@davidwendt](https://github.com/davidwendt) +- Fix unsanitized nulls produced by libcudf dictionary decode ([#14578](https://github.com/rapidsai/cudf/pull/14578)) [@davidwendt](https://github.com/davidwendt) +- Fixes a symbol group lookup table issue ([#14561](https://github.com/rapidsai/cudf/pull/14561)) [@elstehle](https://github.com/elstehle) +- Drop llvm16 from cuda118-conda devcontainer image ([#14526](https://github.com/rapidsai/cudf/pull/14526)) [@charlesbluca](https://github.com/charlesbluca) +- REF: Make DataFrame.from_pandas process by column ([#14483](https://github.com/rapidsai/cudf/pull/14483)) [@mroeschke](https://github.com/mroeschke) +- Improve memory footprint of isin by using contains ([#14478](https://github.com/rapidsai/cudf/pull/14478)) [@wence-](https://github.com/wence-) +- Move creation of env.yaml outside the current directory ([#14476](https://github.com/rapidsai/cudf/pull/14476)) [@davidwendt](https://github.com/davidwendt) +- Enable `pd.Timestamp` objects to be picklable when `cudf.pandas` is active ([#14474](https://github.com/rapidsai/cudf/pull/14474)) [@shwina](https://github.com/shwina) +- Correct dtype of count aggregations on empty dataframes ([#14473](https://github.com/rapidsai/cudf/pull/14473)) [@wence-](https://github.com/wence-) +- Avoid DataFrame conversion in `MultiIndex.from_pandas` ([#14470](https://github.com/rapidsai/cudf/pull/14470)) [@mroeschke](https://github.com/mroeschke) +- JSON writer: avoid default stream use in `string_scalar` constructors ([#14444](https://github.com/rapidsai/cudf/pull/14444)) [@vuule](https://github.com/vuule) +- Fix default stream use in the CSV reader ([#14443](https://github.com/rapidsai/cudf/pull/14443)) [@vuule](https://github.com/vuule) +- Preserve DataFrame(columns=).columns dtype during empty-like construction ([#14381](https://github.com/rapidsai/cudf/pull/14381)) [@mroeschke](https://github.com/mroeschke) +- Defer PTX file load to runtime ([#13690](https://github.com/rapidsai/cudf/pull/13690)) [@brandon-b-miller](https://github.com/brandon-b-miller) + +## 📖 Documentation + +- Disable parallel build ([#14796](https://github.com/rapidsai/cudf/pull/14796)) [@vyasr](https://github.com/vyasr) +- Add pylibcudf to the docs ([#14791](https://github.com/rapidsai/cudf/pull/14791)) [@vyasr](https://github.com/vyasr) +- Describe unpickling expectations when cudf.pandas is enabled ([#14693](https://github.com/rapidsai/cudf/pull/14693)) [@shwina](https://github.com/shwina) +- Update CONTRIBUTING for pyproject-only builds ([#14653](https://github.com/rapidsai/cudf/pull/14653)) [@vyasr](https://github.com/vyasr) +- More doxygen fixes ([#14639](https://github.com/rapidsai/cudf/pull/14639)) [@vyasr](https://github.com/vyasr) +- Enable doxygen XML generation and fix issues ([#14477](https://github.com/rapidsai/cudf/pull/14477)) [@vyasr](https://github.com/vyasr) +- Some doxygen improvements ([#14469](https://github.com/rapidsai/cudf/pull/14469)) [@vyasr](https://github.com/vyasr) +- Remove warning in dask-cudf docs ([#14454](https://github.com/rapidsai/cudf/pull/14454)) [@wence-](https://github.com/wence-) +- Update README links with redirects. ([#14378](https://github.com/rapidsai/cudf/pull/14378)) [@bdice](https://github.com/bdice) +- Add pip install instructions to README ([#13677](https://github.com/rapidsai/cudf/pull/13677)) [@shwina](https://github.com/shwina) + +## 🚀 New Features + +- Add ci check for external kernels ([#14768](https://github.com/rapidsai/cudf/pull/14768)) [@robertmaynard](https://github.com/robertmaynard) +- JSON single quote normalization API ([#14729](https://github.com/rapidsai/cudf/pull/14729)) [@shrshi](https://github.com/shrshi) +- Write cuDF version in Parquet "created_by" metadata field ([#14721](https://github.com/rapidsai/cudf/pull/14721)) [@etseidl](https://github.com/etseidl) +- Implement remaining copying APIs in pylibcudf along with required helper functions ([#14640](https://github.com/rapidsai/cudf/pull/14640)) [@vyasr](https://github.com/vyasr) +- Don't constrain `numba<0.58` ([#14616](https://github.com/rapidsai/cudf/pull/14616)) [@brandon-b-miller](https://github.com/brandon-b-miller) +- Add DELTA_LENGTH_BYTE_ARRAY encoder and decoder for Parquet ([#14590](https://github.com/rapidsai/cudf/pull/14590)) [@etseidl](https://github.com/etseidl) +- JSON - Parse mixed types as string in JSON reader ([#14572](https://github.com/rapidsai/cudf/pull/14572)) [@karthikeyann](https://github.com/karthikeyann) +- JSON quote normalization ([#14545](https://github.com/rapidsai/cudf/pull/14545)) [@shrshi](https://github.com/shrshi) +- Make DefaultHostMemoryAllocator settable ([#14523](https://github.com/rapidsai/cudf/pull/14523)) [@gerashegalov](https://github.com/gerashegalov) +- Implement more copying APIs in pylibcudf ([#14508](https://github.com/rapidsai/cudf/pull/14508)) [@vyasr](https://github.com/vyasr) +- Include writer code and writerVersion in ORC files ([#14458](https://github.com/rapidsai/cudf/pull/14458)) [@vuule](https://github.com/vuule) +- Parquet sub-rowgroup reading. ([#14360](https://github.com/rapidsai/cudf/pull/14360)) [@nvdbaranec](https://github.com/nvdbaranec) +- Move chars column to parent data buffer in strings column ([#14202](https://github.com/rapidsai/cudf/pull/14202)) [@karthikeyann](https://github.com/karthikeyann) +- PARQUET-2261 Size Statistics ([#14000](https://github.com/rapidsai/cudf/pull/14000)) [@etseidl](https://github.com/etseidl) +- Improve GroupBy JIT error handling ([#13854](https://github.com/rapidsai/cudf/pull/13854)) [@brandon-b-miller](https://github.com/brandon-b-miller) +- Generate unified Python/C++ docs ([#13846](https://github.com/rapidsai/cudf/pull/13846)) [@vyasr](https://github.com/vyasr) +- Expand JIT groupby test suite ([#13813](https://github.com/rapidsai/cudf/pull/13813)) [@brandon-b-miller](https://github.com/brandon-b-miller) + +## 🛠️ Improvements + +- Pin `pytest<8` ([#14920](https://github.com/rapidsai/cudf/pull/14920)) [@galipremsagar](https://github.com/galipremsagar) +- Move cudf::char_utf8 definition from detail to public header ([#14779](https://github.com/rapidsai/cudf/pull/14779)) [@davidwendt](https://github.com/davidwendt) +- Clean up `TimedeltaIndex.__init__` constructor ([#14775](https://github.com/rapidsai/cudf/pull/14775)) [@mroeschke](https://github.com/mroeschke) +- Clean up `DatetimeIndex.__init__` constructor ([#14774](https://github.com/rapidsai/cudf/pull/14774)) [@mroeschke](https://github.com/mroeschke) +- Some `frame.py` typing, move seldom used methods in `frame.py` ([#14766](https://github.com/rapidsai/cudf/pull/14766)) [@mroeschke](https://github.com/mroeschke) +- Remove **kwargs from astype ([#14765](https://github.com/rapidsai/cudf/pull/14765)) [@mroeschke](https://github.com/mroeschke) +- fix benchmarks compatibility with newer pytest-cases ([#14764](https://github.com/rapidsai/cudf/pull/14764)) [@jameslamb](https://github.com/jameslamb) +- Add `pynvjitlink` as a dependency ([#14763](https://github.com/rapidsai/cudf/pull/14763)) [@brandon-b-miller](https://github.com/brandon-b-miller) +- Resolve degenerate performance in `create_structs_data` ([#14761](https://github.com/rapidsai/cudf/pull/14761)) [@SurajAralihalli](https://github.com/SurajAralihalli) +- Simplify ColumnAccessor methods; avoid unnecessary validations ([#14758](https://github.com/rapidsai/cudf/pull/14758)) [@mroeschke](https://github.com/mroeschke) +- Pin pytest-cases<3.8.2 ([#14756](https://github.com/rapidsai/cudf/pull/14756)) [@mroeschke](https://github.com/mroeschke) +- Use _from_data instead of _from_columns for initialzing Frame ([#14755](https://github.com/rapidsai/cudf/pull/14755)) [@mroeschke](https://github.com/mroeschke) +- Consolidate cudf object handling in as_column ([#14754](https://github.com/rapidsai/cudf/pull/14754)) [@mroeschke](https://github.com/mroeschke) +- Reduce execution time of Parquet C++ tests ([#14750](https://github.com/rapidsai/cudf/pull/14750)) [@vuule](https://github.com/vuule) +- Implement to_datetime(..., utc=True) ([#14749](https://github.com/rapidsai/cudf/pull/14749)) [@mroeschke](https://github.com/mroeschke) +- Remove usages of rapids-env-update ([#14748](https://github.com/rapidsai/cudf/pull/14748)) [@KyleFromNVIDIA](https://github.com/KyleFromNVIDIA) +- Provide explicit pool size and avoid RMM detail APIs ([#14741](https://github.com/rapidsai/cudf/pull/14741)) [@harrism](https://github.com/harrism) +- Implement `cudf.MultiIndex.from_arrays` ([#14740](https://github.com/rapidsai/cudf/pull/14740)) [@mroeschke](https://github.com/mroeschke) +- Remove unused/single use methods ([#14739](https://github.com/rapidsai/cudf/pull/14739)) [@mroeschke](https://github.com/mroeschke) +- refactor CUDA versions in dependencies.yaml ([#14733](https://github.com/rapidsai/cudf/pull/14733)) [@jameslamb](https://github.com/jameslamb) +- Remove unneeded methods in Column ([#14730](https://github.com/rapidsai/cudf/pull/14730)) [@mroeschke](https://github.com/mroeschke) +- Clean up base column methods ([#14725](https://github.com/rapidsai/cudf/pull/14725)) [@mroeschke](https://github.com/mroeschke) +- Ensure column.fillna signatures are consistent ([#14724](https://github.com/rapidsai/cudf/pull/14724)) [@mroeschke](https://github.com/mroeschke) +- Remove mimesis as a testing dependency ([#14723](https://github.com/rapidsai/cudf/pull/14723)) [@mroeschke](https://github.com/mroeschke) +- Replace as_numerical with as_numerical_column/codes ([#14719](https://github.com/rapidsai/cudf/pull/14719)) [@mroeschke](https://github.com/mroeschke) +- Use offsetalator in gather_chars ([#14700](https://github.com/rapidsai/cudf/pull/14700)) [@davidwendt](https://github.com/davidwendt) +- Use make_strings_children for fill() specialization logic ([#14697](https://github.com/rapidsai/cudf/pull/14697)) [@davidwendt](https://github.com/davidwendt) +- Change `io::detail::orc` namespace into `io::orc::detail` ([#14696](https://github.com/rapidsai/cudf/pull/14696)) [@ttnghia](https://github.com/ttnghia) +- Fix call to deprecated factory function ([#14695](https://github.com/rapidsai/cudf/pull/14695)) [@davidwendt](https://github.com/davidwendt) +- Use as_column instead of arange for range like inputs ([#14689](https://github.com/rapidsai/cudf/pull/14689)) [@mroeschke](https://github.com/mroeschke) +- Reorganize ORC reader into multiple files and perform some small fixes to cuIO code ([#14665](https://github.com/rapidsai/cudf/pull/14665)) [@ttnghia](https://github.com/ttnghia) +- Split parquet test into multiple files ([#14663](https://github.com/rapidsai/cudf/pull/14663)) [@etseidl](https://github.com/etseidl) +- Custom error messages for IO with nonexistent files ([#14662](https://github.com/rapidsai/cudf/pull/14662)) [@vuule](https://github.com/vuule) +- Explicitly pass .dtype into is_foo_dtype functions ([#14657](https://github.com/rapidsai/cudf/pull/14657)) [@mroeschke](https://github.com/mroeschke) +- Basic validation in reader benchmarks ([#14647](https://github.com/rapidsai/cudf/pull/14647)) [@vuule](https://github.com/vuule) +- Update dependencies.yaml to support CUDA 12.*. ([#14644](https://github.com/rapidsai/cudf/pull/14644)) [@bdice](https://github.com/bdice) +- Consolidate memoryview handling in as_column ([#14643](https://github.com/rapidsai/cudf/pull/14643)) [@mroeschke](https://github.com/mroeschke) +- Convert `FieldType` to scoped enum ([#14642](https://github.com/rapidsai/cudf/pull/14642)) [@vuule](https://github.com/vuule) +- Use instance over is_foo_dtype ([#14641](https://github.com/rapidsai/cudf/pull/14641)) [@mroeschke](https://github.com/mroeschke) +- Use isinstance over is_foo_dtype internally ([#14638](https://github.com/rapidsai/cudf/pull/14638)) [@mroeschke](https://github.com/mroeschke) +- Remove unnecessary **kwargs in function signatures ([#14635](https://github.com/rapidsai/cudf/pull/14635)) [@mroeschke](https://github.com/mroeschke) +- Drop nvbench patch for nvml. ([#14631](https://github.com/rapidsai/cudf/pull/14631)) [@bdice](https://github.com/bdice) +- Drop Pascal GPU support. ([#14630](https://github.com/rapidsai/cudf/pull/14630)) [@bdice](https://github.com/bdice) +- Add cpp/doxygen/xml to .gitignore ([#14613](https://github.com/rapidsai/cudf/pull/14613)) [@davidwendt](https://github.com/davidwendt) +- Create strings-specific make_offsets_child_column for multiple offset types ([#14612](https://github.com/rapidsai/cudf/pull/14612)) [@davidwendt](https://github.com/davidwendt) +- Use the offsetalator in cudf::concatenate for strings ([#14611](https://github.com/rapidsai/cudf/pull/14611)) [@davidwendt](https://github.com/davidwendt) +- Make Parquet ColumnIndex null_counts optional ([#14596](https://github.com/rapidsai/cudf/pull/14596)) [@etseidl](https://github.com/etseidl) +- Support `freq` in DatetimeIndex ([#14593](https://github.com/rapidsai/cudf/pull/14593)) [@shwina](https://github.com/shwina) +- Remove legacy benchmarks for cuDF-python ([#14591](https://github.com/rapidsai/cudf/pull/14591)) [@osidekyle](https://github.com/osidekyle) +- Remove WORKSPACE env var from cudf_test temp_directory class ([#14588](https://github.com/rapidsai/cudf/pull/14588)) [@davidwendt](https://github.com/davidwendt) +- Use exceptions instead of return values to handle errors in `CompactProtocolReader` ([#14582](https://github.com/rapidsai/cudf/pull/14582)) [@vuule](https://github.com/vuule) +- Use cuda::proclaim_return_type on device lambdas. ([#14577](https://github.com/rapidsai/cudf/pull/14577)) [@bdice](https://github.com/bdice) +- Update to CCCL 2.2.0. ([#14576](https://github.com/rapidsai/cudf/pull/14576)) [@bdice](https://github.com/bdice) +- Update dependencies.yaml to new pip index ([#14575](https://github.com/rapidsai/cudf/pull/14575)) [@vyasr](https://github.com/vyasr) +- Simplify Python CMake ([#14565](https://github.com/rapidsai/cudf/pull/14565)) [@vyasr](https://github.com/vyasr) +- Java expose parquet pass_read_limit ([#14564](https://github.com/rapidsai/cudf/pull/14564)) [@revans2](https://github.com/revans2) +- Add column sanitization checks in `CUDF_TEST_EXPECT_COLUMN_*` macros ([#14559](https://github.com/rapidsai/cudf/pull/14559)) [@SurajAralihalli](https://github.com/SurajAralihalli) +- Use cudf_test temp_directory class for nvtext::subword_tokenize gbenchmark ([#14558](https://github.com/rapidsai/cudf/pull/14558)) [@davidwendt](https://github.com/davidwendt) +- Fix return type of prefix increment overloads ([#14544](https://github.com/rapidsai/cudf/pull/14544)) [@vuule](https://github.com/vuule) +- Make bpe_merge_pairs_impl member private ([#14543](https://github.com/rapidsai/cudf/pull/14543)) [@davidwendt](https://github.com/davidwendt) +- Small clean up in `io::statistics` ([#14542](https://github.com/rapidsai/cudf/pull/14542)) [@vuule](https://github.com/vuule) +- Change json gtest environment variable to compile-time definition ([#14541](https://github.com/rapidsai/cudf/pull/14541)) [@davidwendt](https://github.com/davidwendt) +- Remove extra total chars size calculation from cudf::concatenate ([#14540](https://github.com/rapidsai/cudf/pull/14540)) [@davidwendt](https://github.com/davidwendt) +- Refactor IndexedFrame.hash_values to use cudf::hashing functions, add xxhash64 to cudf Python. ([#14538](https://github.com/rapidsai/cudf/pull/14538)) [@bdice](https://github.com/bdice) +- Move non-templated inline function definitions from table_view.hpp to table_view.cpp ([#14535](https://github.com/rapidsai/cudf/pull/14535)) [@davidwendt](https://github.com/davidwendt) +- Add JNI for strings::code_points ([#14533](https://github.com/rapidsai/cudf/pull/14533)) [@thirtiseven](https://github.com/thirtiseven) +- Add a test for issue 12773 ([#14529](https://github.com/rapidsai/cudf/pull/14529)) [@vyasr](https://github.com/vyasr) +- Split libarrow build dependencies. ([#14506](https://github.com/rapidsai/cudf/pull/14506)) [@bdice](https://github.com/bdice) +- Implement `IndexedFrame.duplicated` with `distinct_indices` + `scatter` ([#14493](https://github.com/rapidsai/cudf/pull/14493)) [@wence-](https://github.com/wence-) +- Expunge as_frame conversions in Column algorithms ([#14491](https://github.com/rapidsai/cudf/pull/14491)) [@wence-](https://github.com/wence-) +- Remove unsanitized null from input strings column in rank_tests.cpp ([#14475](https://github.com/rapidsai/cudf/pull/14475)) [@davidwendt](https://github.com/davidwendt) +- Refactor Parquet kernel_error ([#14464](https://github.com/rapidsai/cudf/pull/14464)) [@etseidl](https://github.com/etseidl) +- Deprecate cudf::make_strings_column accepting typed offsets ([#14461](https://github.com/rapidsai/cudf/pull/14461)) [@davidwendt](https://github.com/davidwendt) +- Remove deprecated nvtext::load_merge_pairs_file ([#14460](https://github.com/rapidsai/cudf/pull/14460)) [@davidwendt](https://github.com/davidwendt) +- Introduce Comprehensive Pathological Unit Tests for Issue #14409 ([#14459](https://github.com/rapidsai/cudf/pull/14459)) [@aocsa](https://github.com/aocsa) +- Expose stream parameter in public nvtext APIs ([#14456](https://github.com/rapidsai/cudf/pull/14456)) [@davidwendt](https://github.com/davidwendt) +- Include encode type in the error message when unsupported Parquet encoding is detected ([#14453](https://github.com/rapidsai/cudf/pull/14453)) [@ZelboK](https://github.com/ZelboK) +- Remove null mask for zero nulls in json readers ([#14451](https://github.com/rapidsai/cudf/pull/14451)) [@karthikeyann](https://github.com/karthikeyann) +- Refactor cudf.Series.__init__ ([#14450](https://github.com/rapidsai/cudf/pull/14450)) [@mroeschke](https://github.com/mroeschke) +- Remove the use of `volatile` in Parquet ([#14448](https://github.com/rapidsai/cudf/pull/14448)) [@vuule](https://github.com/vuule) +- REF: Remove **kwargs from to_pandas, raise if nullable is not implemented ([#14438](https://github.com/rapidsai/cudf/pull/14438)) [@mroeschke](https://github.com/mroeschke) +- Testing stream pool implementation ([#14437](https://github.com/rapidsai/cudf/pull/14437)) [@shrshi](https://github.com/shrshi) +- Match pandas join ordering obligations in pandas-compatible mode ([#14428](https://github.com/rapidsai/cudf/pull/14428)) [@wence-](https://github.com/wence-) +- Forward-merge branch-23.12 to branch-24.02 ([#14426](https://github.com/rapidsai/cudf/pull/14426)) [@bdice](https://github.com/bdice) +- Use isinstance(..., cudf.IntervalDtype) instead of is_interval_dtype ([#14424](https://github.com/rapidsai/cudf/pull/14424)) [@mroeschke](https://github.com/mroeschke) +- Use isinstance(..., cudf.CategoricalDtype) instead of is_categorical_dtype ([#14423](https://github.com/rapidsai/cudf/pull/14423)) [@mroeschke](https://github.com/mroeschke) +- Forward-merge branch-23.12 to branch-24.02 ([#14422](https://github.com/rapidsai/cudf/pull/14422)) [@bdice](https://github.com/bdice) +- REF: Remove instances of pd.core ([#14421](https://github.com/rapidsai/cudf/pull/14421)) [@mroeschke](https://github.com/mroeschke) +- Expose streams in public filling APIs for label_bins ([#14401](https://github.com/rapidsai/cudf/pull/14401)) [@ZelboK](https://github.com/ZelboK) +- Consolidate 1D pandas object handling in as_column ([#14394](https://github.com/rapidsai/cudf/pull/14394)) [@mroeschke](https://github.com/mroeschke) +- Limit DELTA_BINARY_PACKED encoder to the same number of bits as the physical type being encoded ([#14392](https://github.com/rapidsai/cudf/pull/14392)) [@etseidl](https://github.com/etseidl) +- Add SHA-1 and SHA-2 hash functions. ([#14391](https://github.com/rapidsai/cudf/pull/14391)) [@bdice](https://github.com/bdice) +- Expose streams in Parquet reader and writer APIs ([#14359](https://github.com/rapidsai/cudf/pull/14359)) [@shrshi](https://github.com/shrshi) +- Update to fmt 10.1.1 and spdlog 1.12.0. ([#14355](https://github.com/rapidsai/cudf/pull/14355)) [@bdice](https://github.com/bdice) +- Replace default stream for scalars and column factories usages (because of defaulted arguments) ([#14354](https://github.com/rapidsai/cudf/pull/14354)) [@karthikeyann](https://github.com/karthikeyann) +- Expose streams in ORC reader and writer APIs ([#14350](https://github.com/rapidsai/cudf/pull/14350)) [@shrshi](https://github.com/shrshi) +- Convert compression and io to string axis type in IO benchmarks ([#14347](https://github.com/rapidsai/cudf/pull/14347)) [@SurajAralihalli](https://github.com/SurajAralihalli) +- Add cuDF devcontainers ([#14015](https://github.com/rapidsai/cudf/pull/14015)) [@trxcllnt](https://github.com/trxcllnt) +- Refactoring of Buffers (last step towards unifying COW and Spilling) ([#13801](https://github.com/rapidsai/cudf/pull/13801)) [@madsbk](https://github.com/madsbk) +- Switch to scikit-build-core ([#13531](https://github.com/rapidsai/cudf/pull/13531)) [@vyasr](https://github.com/vyasr) +- Simplify null count checking in column equality comparator ([#13312](https://github.com/rapidsai/cudf/pull/13312)) [@vyasr](https://github.com/vyasr) + # cuDF 23.12.00 (6 Dec 2023) ## 🚨 Breaking Changes From ac438c456f7f492fd1bc59603de4e76387f86bb0 Mon Sep 17 00:00:00 2001 From: Ray Douglass <3107146+raydouglass@users.noreply.github.com> Date: Tue, 13 Feb 2024 09:22:45 -0500 Subject: [PATCH 239/384] Unpin numba<0.58 (#15031) I think unpinning `numba` in the conda recipe was just missed in #14616. I discovered this issue [trying to build the `24.02` release](https://github.com/rapidsai/cudf/actions/runs/7878153691/job/21496377912#step:7:1674). PRs & nightly builds are working because the `rapidsai-nightly` channel has an older version of `pynvjitlink` that supported `numba>=0.57` whereas the `rapidsai` channel only has the latest version which pins to `numba>=0.58`. Authors: - Ray Douglass (https://github.com/raydouglass) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Bradley Dice (https://github.com/bdice) --- conda/recipes/cudf/meta.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 4f39a9fe452..d46d9263864 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -78,8 +78,7 @@ requirements: - typing_extensions >=4.0.0 - pandas >=1.3,<1.6.0dev0 - cupy >=12.0.0 - # TODO: Pin to numba<0.58 until #14160 is resolved - - numba >=0.57,<0.58 + - numba >=0.57 # TODO: Pin to numpy<1.25 until cudf requires pandas 2 - numpy >=1.21,<1.25 - {{ pin_compatible('pyarrow', max_pin='x') }} From d6902b083f1b74d508b92ba90e099e55f8ec0954 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 13 Feb 2024 10:10:04 -0500 Subject: [PATCH 240/384] Change copy_if_safe to call thrust instead of the overload function (#15018) Found while working on large strings where copy-if is called. In places where `copy_if_safe` utility is called the non-stencil overload calls the stencil-ed function by forwarding the `first` iterator as the `stencil` parameter. This works logically because both values will return the same result. Unfortunately, this can be a performance issue if the iterator is complex/slow transform iterator since it would be called twice (an inlined twice). Changing the non-stencil version to call `thrust::copy_if` directly fixes the potential issue. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Yunsong Wang (https://github.com/PointKernel) - Mike Wilson (https://github.com/hyperbolic2346) URL: https://github.com/rapidsai/cudf/pull/15018 --- cpp/include/cudf/detail/utilities/algorithm.cuh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cpp/include/cudf/detail/utilities/algorithm.cuh b/cpp/include/cudf/detail/utilities/algorithm.cuh index fab86172704..757ed0dd551 100644 --- a/cpp/include/cudf/detail/utilities/algorithm.cuh +++ b/cpp/include/cudf/detail/utilities/algorithm.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -89,7 +89,17 @@ OutputIterator copy_if_safe(InputIterator first, Predicate pred, rmm::cuda_stream_view stream) { - return copy_if_safe(first, last, first, result, pred, stream); + auto const copy_size = std::min(static_cast(std::distance(first, last)), + static_cast(std::numeric_limits::max())); + + auto itr = first; + while (itr != last) { + auto const copy_end = + static_cast(std::distance(itr, last)) <= copy_size ? last : itr + copy_size; + result = thrust::copy_if(rmm::exec_policy(stream), itr, copy_end, result, pred); + itr = copy_end; + } + return result; } } // namespace cudf::detail From ac4debdf47d64c1cec9e689e18c738b5b6714e71 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 13 Feb 2024 12:16:09 -1000 Subject: [PATCH 241/384] Deprecate delim_whitespace in read_csv for pandas 2.2 (#14986) Toward pandas 2.2 compat: Deprecated in pandas in https://github.com/pandas-dev/pandas/pull/56557 Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Ashwin Srinath (https://github.com/shwina) URL: https://github.com/rapidsai/cudf/pull/14986 --- python/cudf/cudf/io/csv.py | 10 +++++++++- python/cudf/cudf/tests/test_csv.py | 28 ++++++++++++++++++++-------- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/python/cudf/cudf/io/csv.py b/python/cudf/cudf/io/csv.py index 764885dd7b6..3eeeac405b3 100644 --- a/python/cudf/cudf/io/csv.py +++ b/python/cudf/cudf/io/csv.py @@ -1,5 +1,6 @@ -# Copyright (c) 2018-2023, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. +import warnings from collections import abc from io import BytesIO, StringIO @@ -55,6 +56,13 @@ def read_csv( ): """{docstring}""" + if delim_whitespace is not False: + warnings.warn( + "The 'delim_whitespace' keyword in pd.read_csv is deprecated and " + "will be removed in a future version. Use ``sep='\\s+'`` instead", + FutureWarning, + ) + if use_python_file_object and bytes_per_thread is not None: raise ValueError( "bytes_per_thread is only supported when " diff --git a/python/cudf/cudf/tests/test_csv.py b/python/cudf/cudf/tests/test_csv.py index 8171f3a1872..9b08ef30545 100644 --- a/python/cudf/cudf/tests/test_csv.py +++ b/python/cudf/cudf/tests/test_csv.py @@ -17,8 +17,12 @@ import cudf from cudf import read_csv -from cudf.core._compat import PANDAS_GE_200 -from cudf.testing._utils import assert_eq, assert_exceptions_equal +from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_220 +from cudf.testing._utils import ( + assert_eq, + assert_exceptions_equal, + expect_warning_if, +) def make_numeric_dataframe(nrows, dtype): @@ -1263,20 +1267,28 @@ def test_csv_reader_delim_whitespace(): buffer = "1 2 3\n4 5 6" # with header row - cu_df = read_csv(StringIO(buffer), delim_whitespace=True) - pd_df = pd.read_csv(StringIO(buffer), delim_whitespace=True) + with pytest.warns(FutureWarning): + cu_df = read_csv(StringIO(buffer), delim_whitespace=True) + with expect_warning_if(PANDAS_GE_220): + pd_df = pd.read_csv(StringIO(buffer), delim_whitespace=True) assert_eq(pd_df, cu_df) # without header row - cu_df = read_csv(StringIO(buffer), delim_whitespace=True, header=None) - pd_df = pd.read_csv(StringIO(buffer), delim_whitespace=True, header=None) + with pytest.warns(FutureWarning): + cu_df = read_csv(StringIO(buffer), delim_whitespace=True, header=None) + with expect_warning_if(PANDAS_GE_220): + pd_df = pd.read_csv( + StringIO(buffer), delim_whitespace=True, header=None + ) assert pd_df.shape == cu_df.shape # should raise an error if used with delimiter or sep with pytest.raises(ValueError): - read_csv(StringIO(buffer), delim_whitespace=True, delimiter=" ") + with pytest.warns(FutureWarning): + read_csv(StringIO(buffer), delim_whitespace=True, delimiter=" ") with pytest.raises(ValueError): - read_csv(StringIO(buffer), delim_whitespace=True, sep=" ") + with pytest.warns(FutureWarning): + read_csv(StringIO(buffer), delim_whitespace=True, sep=" ") def test_csv_reader_unnamed_cols(): From 3547d412ee43ab8aaa9329df9dc1cc24e8cc260c Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Wed, 14 Feb 2024 10:09:17 -0500 Subject: [PATCH 242/384] Use offsetalator in cudf::get_json_object() (#15009) Updates `cudf::get_json_object()` to use the offsetalator to build the output strings column. It adds a sizes vector to hold the output row lengths which is then converted to offsets using the new `make_offsets_child_column()` utitlity. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Mark Harris (https://github.com/harrism) - Vukasin Milovanovic (https://github.com/vuule) URL: https://github.com/rapidsai/cudf/pull/15009 --- cpp/src/json/json_path.cu | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/cpp/src/json/json_path.cu b/cpp/src/json/json_path.cu index 146b54c0d87..2be5798098d 100644 --- a/cpp/src/json/json_path.cu +++ b/cpp/src/json/json_path.cu @@ -19,10 +19,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -903,7 +905,8 @@ template __launch_bounds__(block_size) CUDF_KERNEL void get_json_object_kernel(column_device_view col, path_operator const* const commands, - size_type* output_offsets, + size_type* d_sizes, + cudf::detail::input_offsetalator output_offsets, thrust::optional out_buf, thrust::optional out_validity, thrust::optional out_valid_count, @@ -934,7 +937,7 @@ __launch_bounds__(block_size) CUDF_KERNEL // filled in only during the precompute step. during the compute step, the offsets // are fed back in so we do -not- want to write them out - if (!out_buf.has_value()) { output_offsets[tid] = static_cast(output_size); } + if (!out_buf.has_value()) { d_sizes[tid] = output_size; } // validity filled in only during the output step if (out_validity.has_value()) { @@ -971,11 +974,6 @@ std::unique_ptr get_json_object(cudf::strings_column_view const& c if (col.is_empty()) return make_empty_column(type_id::STRING); - // allocate output offsets buffer. - auto offsets = cudf::make_fixed_width_column( - data_type{type_id::INT32}, col.size() + 1, mask_state::UNALLOCATED, stream, mr); - cudf::mutable_column_view offsets_view(*offsets); - // if the query is empty, return a string column containing all nulls if (!std::get<0>(preprocess).has_value()) { return std::make_unique( @@ -986,6 +984,11 @@ std::unique_ptr get_json_object(cudf::strings_column_view const& c col.size()); // null count } + // compute output sizes + auto sizes = + rmm::device_uvector(col.size(), stream, rmm::mr::get_current_device_resource()); + auto d_offsets = cudf::detail::offsetalator_factory::make_input_iterator(col.offsets()); + constexpr int block_size = 512; cudf::detail::grid_1d const grid{col.size(), block_size}; auto cdv = column_device_view::create(col.parent(), stream); @@ -994,20 +997,17 @@ std::unique_ptr get_json_object(cudf::strings_column_view const& c <<>>( *cdv, std::get<0>(preprocess).value().data(), - offsets_view.head(), + sizes.data(), + d_offsets, thrust::nullopt, thrust::nullopt, thrust::nullopt, options); // convert sizes to offsets - thrust::exclusive_scan(rmm::exec_policy(stream), - offsets_view.head(), - offsets_view.head() + col.size() + 1, - offsets_view.head(), - 0); - size_type const output_size = - cudf::detail::get_value(offsets_view, col.size(), stream); + auto [offsets, output_size] = + cudf::strings::detail::make_offsets_child_column(sizes.begin(), sizes.end(), stream, mr); + d_offsets = cudf::detail::offsetalator_factory::make_input_iterator(offsets->view()); // allocate output string column rmm::device_uvector chars(output_size, stream, mr); @@ -1024,7 +1024,8 @@ std::unique_ptr get_json_object(cudf::strings_column_view const& c <<>>( *cdv, std::get<0>(preprocess).value().data(), - offsets_view.head(), + sizes.data(), + d_offsets, chars.data(), static_cast(validity.data()), d_valid_count.data(), From dd131dc83ea05e1bda99b228823d7e0c3c0fd676 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Wed, 14 Feb 2024 10:12:21 -0500 Subject: [PATCH 243/384] Use offsetalator in cudf::interleave_columns() (#15004) Updates `cudf::interleave_columns()` to use the new `make_offsets_child_column` utility and the offsetalator to build the output strings column. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Nghia Truong (https://github.com/ttnghia) - Vukasin Milovanovic (https://github.com/vuule) URL: https://github.com/rapidsai/cudf/pull/15004 --- cpp/src/reshape/interleave_columns.cu | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/src/reshape/interleave_columns.cu b/cpp/src/reshape/interleave_columns.cu index 22b45fe7a58..6aa322d4d78 100644 --- a/cpp/src/reshape/interleave_columns.cu +++ b/cpp/src/reshape/interleave_columns.cu @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -188,9 +189,10 @@ struct interleave_columns_impl(0), offsets_transformer); - auto [offsets_column, bytes] = cudf::detail::make_offsets_child_column( + auto [offsets_column, bytes] = cudf::strings::detail::make_offsets_child_column( offsets_transformer_itr, offsets_transformer_itr + num_strings, stream, mr); - auto d_results_offsets = offsets_column->view().template data(); + auto d_results_offsets = + cudf::detail::offsetalator_factory::make_input_iterator(offsets_column->view()); // Create the chars column rmm::device_uvector chars(bytes, stream, mr); From ee1c76897ebe9a1c9796619de2c80a1fac7bc268 Mon Sep 17 00:00:00 2001 From: Sanjana Gajendran Date: Wed, 14 Feb 2024 09:50:21 -0800 Subject: [PATCH 244/384] Fix broken link for developer guide (#15025) Closes #14991 Authors: - Sanjana Gajendran (https://github.com/sanjana098) Approvers: - Bradley Dice (https://github.com/bdice) - Mark Harris (https://github.com/harrism) URL: https://github.com/rapidsai/cudf/pull/15025 --- docs/cudf/source/developer_guide/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cudf/source/developer_guide/index.md b/docs/cudf/source/developer_guide/index.md index 27e05ce6459..5cafa8f784c 100644 --- a/docs/cudf/source/developer_guide/index.md +++ b/docs/cudf/source/developer_guide/index.md @@ -9,7 +9,7 @@ cuDF is a GPU-accelerated, [Pandas-like](https://pandas.pydata.org/) DataFrame l Under the hood, all of cuDF's functionality relies on the CUDA-accelerated `libcudf` C++ library. Thus, cuDF's internals are designed to efficiently and robustly map pandas APIs to `libcudf` functions. For more information about the `libcudf` library, a good starting point is the -[developer guide](https://github.com/rapidsai/cudf/blob/main/cpp/docs/DEVELOPER_GUIDE.md). +[developer guide](https://docs.rapids.ai/api/libcudf/stable/developer_guide). This document assumes familiarity with the [overall contributing guide](https://github.com/rapidsai/cudf/blob/main/CONTRIBUTING.md). From 825d30c172e7a2742d62099387d6081a8e8bc531 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Wed, 14 Feb 2024 15:33:42 -0500 Subject: [PATCH 245/384] Requesting a clean build directory also clears Jitify cache (#15052) Developers expect that 'cleaning' a build directory will remove all forms of cached files ( objects, libraries, jit cache, etc ). To ensure that happens consistenly we also need to remove the jitify cache objects for cudf. Authors: - Robert Maynard (https://github.com/robertmaynard) Approvers: - Bradley Dice (https://github.com/bdice) - Mark Harris (https://github.com/harrism) URL: https://github.com/rapidsai/cudf/pull/15052 --- cpp/cmake/Modules/JitifyPreprocessKernels.cmake | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cpp/cmake/Modules/JitifyPreprocessKernels.cmake b/cpp/cmake/Modules/JitifyPreprocessKernels.cmake index 8a40be1dc94..8c4e2b47fca 100644 --- a/cpp/cmake/Modules/JitifyPreprocessKernels.cmake +++ b/cpp/cmake/Modules/JitifyPreprocessKernels.cmake @@ -69,3 +69,18 @@ add_custom_target( DEPENDS ${JIT_PREPROCESSED_FILES} COMMENT "Target representing jitified files." ) + +# when a user requests CMake to clean the build directory +# +# * `cmake --build --target clean` +# * `cmake --build --clean-first` +# * ninja clean +# +# We also remove the jitify2 program cache as well. This ensures that we don't keep older versions +# of the programs in cache +set(cache_path "$ENV{HOME}/.cudf") +if(ENV{LIBCUDF_KERNEL_CACHE_PATH}) + set(cache_path "$ENV{LIBCUDF_KERNEL_CACHE_PATH}") +endif() +cmake_path(APPEND cache_path "${CUDF_VERSION}/") +set_target_properties(jitify_preprocess_run PROPERTIES ADDITIONAL_CLEAN_FILES "${cache_path}") From f43f7c56e1879d2888710c7c52e7969c7e5c9291 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Wed, 14 Feb 2024 15:42:15 -0500 Subject: [PATCH 246/384] Improve performance of copy_if_else for long strings (#15017) Reworks the `cudf::strings::detail::copy_if_else()` to improve performance for long strings. The rework builds a vector of rows to pass to the `make_strings_column` factory that uses the optimized `gather_chars` function. Also includes a benchmark for copy_if_else specifically for strings columns. Closes #15014 Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Vukasin Milovanovic (https://github.com/vuule) URL: https://github.com/rapidsai/cudf/pull/15017 --- cpp/benchmarks/CMakeLists.txt | 1 + cpp/benchmarks/string/copy_if_else.cpp | 62 ++++++++++++++++++ .../cudf/strings/detail/copy_if_else.cuh | 63 +++++++------------ 3 files changed, 84 insertions(+), 42 deletions(-) create mode 100644 cpp/benchmarks/string/copy_if_else.cpp diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 35b03fa33d0..6ddc5a6b8de 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -311,6 +311,7 @@ ConfigureNVBench( string/case.cpp string/char_types.cpp string/contains.cpp + string/copy_if_else.cpp string/count.cpp string/extract.cpp string/gather.cpp diff --git a/cpp/benchmarks/string/copy_if_else.cpp b/cpp/benchmarks/string/copy_if_else.cpp new file mode 100644 index 00000000000..e06cca497c2 --- /dev/null +++ b/cpp/benchmarks/string/copy_if_else.cpp @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include + +#include + +static void bench_copy(nvbench::state& state) +{ + auto const num_rows = static_cast(state.get_int64("num_rows")); + auto const row_width = static_cast(state.get_int64("row_width")); + + if (static_cast(num_rows) * static_cast(row_width) >= + static_cast(std::numeric_limits::max())) { + state.skip("Skip benchmarks greater than size_type limit"); + } + + data_profile const str_profile = data_profile_builder().distribution( + cudf::type_id::STRING, distribution_id::NORMAL, 0, row_width); + auto const source_table = + create_random_table({cudf::type_id::STRING}, row_count{num_rows}, str_profile); + auto const target_table = + create_random_table({cudf::type_id::STRING}, row_count{num_rows}, str_profile); + data_profile const bool_profile = data_profile_builder().no_validity(); + auto const booleans = + create_random_table({cudf::type_id::BOOL8}, row_count{num_rows}, bool_profile); + + auto const source = source_table->view().column(0); + auto const target = target_table->view().column(0); + auto const left_right = booleans->view().column(0); + + state.set_cuda_stream(nvbench::make_cuda_stream_view(cudf::get_default_stream().value())); + auto chars_size = cudf::strings_column_view(target).chars_size(cudf::get_default_stream()); + state.add_global_memory_reads(chars_size); // all bytes are read; + state.add_global_memory_writes(chars_size); // both columns are similar size + + state.exec(nvbench::exec_tag::sync, [&](nvbench::launch& launch) { + [[maybe_unused]] auto result = cudf::copy_if_else(source, target, left_right); + }); +} + +NVBENCH_BENCH(bench_copy) + .set_name("copy_if_else") + .add_int64_axis("row_width", {32, 64, 128, 256, 512, 1024, 2048, 4096}) + .add_int64_axis("num_rows", {4096, 32768, 262144, 2097152, 16777216}); diff --git a/cpp/include/cudf/strings/detail/copy_if_else.cuh b/cpp/include/cudf/strings/detail/copy_if_else.cuh index 64e14dcc549..e1ef97b7803 100644 --- a/cpp/include/cudf/strings/detail/copy_if_else.cuh +++ b/cpp/include/cudf/strings/detail/copy_if_else.cuh @@ -16,18 +16,16 @@ #pragma once #include -#include #include -#include -#include +#include #include +#include #include -#include #include -#include #include +#include #include @@ -65,10 +63,10 @@ std::unique_ptr copy_if_else(StringIterLeft lhs_begin, rmm::mr::device_memory_resource* mr) { auto strings_count = std::distance(lhs_begin, lhs_end); - if (strings_count == 0) return make_empty_column(type_id::STRING); + if (strings_count == 0) { return make_empty_column(type_id::STRING); } // create null mask - auto valid_mask = cudf::detail::valid_if( + auto [null_mask, null_count] = cudf::detail::valid_if( thrust::make_counting_iterator(0), thrust::make_counting_iterator(strings_count), [lhs_begin, rhs_begin, filter_fn] __device__(size_type idx) { @@ -76,44 +74,25 @@ std::unique_ptr copy_if_else(StringIterLeft lhs_begin, }, stream, mr); - size_type null_count = valid_mask.second; - auto null_mask = (null_count > 0) ? std::move(valid_mask.first) : rmm::device_buffer{}; + if (null_count == 0) { null_mask = rmm::device_buffer{}; } - // build offsets column - auto offsets_transformer = cuda::proclaim_return_type( - [lhs_begin, rhs_begin, filter_fn] __device__(size_type idx) { - auto const result = filter_fn(idx) ? lhs_begin[idx] : rhs_begin[idx]; - return result.has_value() ? result->size_bytes() : 0; - }); - - auto offsets_transformer_itr = thrust::make_transform_iterator( - thrust::make_counting_iterator(0), offsets_transformer); - auto [offsets_column, bytes] = cudf::detail::make_offsets_child_column( - offsets_transformer_itr, offsets_transformer_itr + strings_count, stream, mr); - auto d_offsets = offsets_column->view().template data(); + // build vector of strings + rmm::device_uvector indices(strings_count, stream); + thrust::transform(rmm::exec_policy_nosync(stream), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(strings_count), + indices.begin(), + [lhs_begin, rhs_begin, filter_fn] __device__(size_type idx) { + auto const result = filter_fn(idx) ? lhs_begin[idx] : rhs_begin[idx]; + auto const d_str = result.has_value() ? *result : string_view{"", 0}; + return string_index_pair{d_str.data(), d_str.size_bytes()}; + }); - // build chars column - auto chars_column = create_chars_child_column(bytes, stream, mr); - auto d_chars = chars_column->mutable_view().template data(); - // fill in chars - thrust::for_each_n( - rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - strings_count, - [lhs_begin, rhs_begin, filter_fn, d_offsets, d_chars] __device__(size_type idx) { - auto const result = filter_fn(idx) ? lhs_begin[idx] : rhs_begin[idx]; - if (!result.has_value()) return; - auto const d_str = *result; - memcpy(d_chars + d_offsets[idx], d_str.data(), d_str.size_bytes()); - }); - - return make_strings_column(strings_count, - std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), - null_count, - std::move(null_mask)); + // convert vector into strings column + auto result = make_strings_column(indices.begin(), indices.end(), stream, mr); + result->set_null_mask(std::move(null_mask), null_count); + return result; } - } // namespace detail } // namespace strings } // namespace cudf From 82d17722d7684aa204f09ffc77059497d886de66 Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Wed, 14 Feb 2024 13:04:40 -0800 Subject: [PATCH 247/384] Support for LZ4 compression in ORC and Parquet (#14906) Closes https://github.com/rapidsai/cudf/issues/14495 Adds support for reading and writing ORC and Parquet files with LZ4 compression. Also adds the new value to the Python API. Included basic C++ and Python tests so that the option is exercised in CI. Authors: - Vukasin Milovanovic (https://github.com/vuule) Approvers: - Bradley Dice (https://github.com/bdice) - Shruti Shivakumar (https://github.com/shrshi) - MithunR (https://github.com/mythrocks) - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/14906 --- cpp/src/io/comp/nvcomp_adapter.cpp | 50 ++++++++++++++++++++++ cpp/src/io/comp/nvcomp_adapter.hpp | 2 +- cpp/src/io/orc/reader_impl_preprocess.cu | 13 ++++++ cpp/src/io/orc/stripe_enc.cu | 6 +++ cpp/src/io/orc/writer_impl.cu | 2 + cpp/src/io/parquet/parquet_common.hpp | 3 +- cpp/src/io/parquet/reader_impl_chunking.cu | 21 ++++++++- cpp/src/io/parquet/writer_impl.cu | 14 +++++- cpp/tests/io/orc_test.cpp | 39 +++++++++++++++++ cpp/tests/io/parquet_misc_test.cpp | 40 +++++++++++++++++ python/cudf/cudf/_lib/orc.pyx | 6 ++- python/cudf/cudf/_lib/parquet.pyx | 6 ++- python/cudf/cudf/tests/test_orc.py | 25 +++++++++++ python/cudf/cudf/tests/test_parquet.py | 20 +++++++++ python/cudf/cudf/utils/ioutils.py | 10 +++-- 15 files changed, 247 insertions(+), 10 deletions(-) diff --git a/cpp/src/io/comp/nvcomp_adapter.cpp b/cpp/src/io/comp/nvcomp_adapter.cpp index 40ed7677603..7d98e047c7c 100644 --- a/cpp/src/io/comp/nvcomp_adapter.cpp +++ b/cpp/src/io/comp/nvcomp_adapter.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -65,6 +66,8 @@ std::optional batched_decompress_get_temp_size_ex(compression_ty #else return std::nullopt; #endif + case compression_type::LZ4: + return nvcompBatchedLZ4DecompressGetTempSizeEx(std::forward(args)...); case compression_type::DEFLATE: [[fallthrough]]; default: return std::nullopt; } @@ -93,6 +96,8 @@ auto batched_decompress_get_temp_size(compression_type compression, Args&&... ar CUDF_FAIL("Decompression error: " + nvcomp::is_decompression_disabled(nvcomp::compression_type::DEFLATE).value()); #endif + case compression_type::LZ4: + return nvcompBatchedLZ4DecompressGetTempSize(std::forward(args)...); default: CUDF_FAIL("Unsupported compression type"); } } @@ -118,6 +123,7 @@ auto batched_decompress_async(compression_type compression, Args&&... args) CUDF_FAIL("Decompression error: " + nvcomp::is_decompression_disabled(nvcomp::compression_type::DEFLATE).value()); #endif + case compression_type::LZ4: return nvcompBatchedLZ4DecompressAsync(std::forward(args)...); default: CUDF_FAIL("Unsupported compression type"); } } @@ -128,6 +134,7 @@ std::string compression_type_name(compression_type compression) case compression_type::SNAPPY: return "Snappy"; case compression_type::ZSTD: return "Zstandard"; case compression_type::DEFLATE: return "Deflate"; + case compression_type::LZ4: return "LZ4"; } return "compression_type(" + std::to_string(static_cast(compression)) + ")"; } @@ -217,6 +224,10 @@ auto batched_compress_get_temp_size(compression_type compression, CUDF_FAIL("Compression error: " + nvcomp::is_compression_disabled(nvcomp::compression_type::ZSTD).value()); #endif + case compression_type::LZ4: + nvcomp_status = nvcompBatchedLZ4CompressGetTempSize( + batch_size, max_uncompressed_chunk_bytes, nvcompBatchedLZ4DefaultOpts, &temp_size); + break; default: CUDF_FAIL("Unsupported compression type"); } @@ -256,6 +267,13 @@ auto batched_compress_get_temp_size_ex(compression_type compression, &temp_size, max_total_uncompressed_bytes); break; + case compression_type::LZ4: + nvcomp_status = nvcompBatchedLZ4CompressGetTempSizeEx(batch_size, + max_uncompressed_chunk_bytes, + nvcompBatchedLZ4DefaultOpts, + &temp_size, + max_total_uncompressed_bytes); + break; default: CUDF_FAIL("Unsupported compression type"); } @@ -317,6 +335,10 @@ size_t compress_max_output_chunk_size(compression_type compression, CUDF_FAIL("Compression error: " + nvcomp::is_compression_disabled(nvcomp::compression_type::ZSTD).value()); #endif + case compression_type::LZ4: + status = nvcompBatchedLZ4CompressGetMaxOutputChunkSize( + capped_uncomp_bytes, nvcompBatchedLZ4DefaultOpts, &max_comp_chunk_size); + break; default: CUDF_FAIL("Unsupported compression type"); } @@ -385,6 +407,18 @@ static void batched_compress_async(compression_type compression, CUDF_FAIL("Compression error: " + nvcomp::is_compression_disabled(nvcomp::compression_type::ZSTD).value()); #endif + case compression_type::LZ4: + nvcomp_status = nvcompBatchedLZ4CompressAsync(device_uncompressed_ptrs, + device_uncompressed_bytes, + max_uncompressed_chunk_bytes, + batch_size, + device_temp_ptr, + temp_bytes, + device_compressed_ptrs, + device_compressed_bytes, + nvcompBatchedLZ4DefaultOpts, + stream.value()); + break; default: CUDF_FAIL("Unsupported compression type"); } CUDF_EXPECTS(nvcomp_status == nvcompStatus_t::nvcompSuccess, "Error in compression"); @@ -494,6 +528,12 @@ std::optional is_compression_disabled_impl(compression_type compres } return std::nullopt; } + case compression_type::LZ4: + if (not params.are_stable_integrations_enabled) { + return "LZ4 compression has been disabled through the `LIBCUDF_NVCOMP_POLICY` " + "environment variable."; + } + return std::nullopt; default: return "Unsupported compression type"; } return "Unsupported compression type"; @@ -572,6 +612,13 @@ std::optional is_decompression_disabled_impl(compression_type compr return std::nullopt; } case compression_type::ZSTD: return is_zstd_decomp_disabled(params); + case compression_type::LZ4: { + if (not params.are_stable_integrations_enabled) { + return "LZ4 decompression has been disabled through the `LIBCUDF_NVCOMP_POLICY` " + "environment variable."; + } + return std::nullopt; + } default: return "Unsupported compression type"; } return "Unsupported compression type"; @@ -612,6 +659,7 @@ size_t compress_input_alignment_bits(compression_type compression) case compression_type::DEFLATE: return 0; case compression_type::SNAPPY: return 0; case compression_type::ZSTD: return 2; + case compression_type::LZ4: return 2; default: CUDF_FAIL("Unsupported compression type"); } } @@ -622,6 +670,7 @@ size_t compress_output_alignment_bits(compression_type compression) case compression_type::DEFLATE: return 3; case compression_type::SNAPPY: return 0; case compression_type::ZSTD: return 0; + case compression_type::LZ4: return 2; default: CUDF_FAIL("Unsupported compression type"); } } @@ -638,6 +687,7 @@ std::optional compress_max_allowed_chunk_size(compression_type compressi CUDF_FAIL("Compression error: " + nvcomp::is_compression_disabled(nvcomp::compression_type::ZSTD).value()); #endif + case compression_type::LZ4: return 16 * 1024 * 1024; default: return std::nullopt; } } diff --git a/cpp/src/io/comp/nvcomp_adapter.hpp b/cpp/src/io/comp/nvcomp_adapter.hpp index 69a278757ce..ebaec617c10 100644 --- a/cpp/src/io/comp/nvcomp_adapter.hpp +++ b/cpp/src/io/comp/nvcomp_adapter.hpp @@ -29,7 +29,7 @@ namespace cudf::io::nvcomp { -enum class compression_type { SNAPPY, ZSTD, DEFLATE }; +enum class compression_type { SNAPPY, ZSTD, DEFLATE, LZ4 }; /** * @brief Set of parameters that impact whether the use nvCOMP features is enabled. diff --git a/cpp/src/io/orc/reader_impl_preprocess.cu b/cpp/src/io/orc/reader_impl_preprocess.cu index 179afa12bd5..08f5adb0729 100644 --- a/cpp/src/io/orc/reader_impl_preprocess.cu +++ b/cpp/src/io/orc/reader_impl_preprocess.cu @@ -304,6 +304,19 @@ rmm::device_buffer decompress_stripe_data( total_decomp_size, stream); break; + case compression_type::LZ4: + if (auto const reason = nvcomp::is_decompression_disabled(nvcomp::compression_type::LZ4); + reason) { + CUDF_FAIL("Decompression error: " + reason.value()); + } + nvcomp::batched_decompress(nvcomp::compression_type::LZ4, + inflate_in_view, + inflate_out_view, + inflate_res, + max_uncomp_block_size, + total_decomp_size, + stream); + break; default: CUDF_FAIL("Unexpected decompression dispatch"); break; } diff --git a/cpp/src/io/orc/stripe_enc.cu b/cpp/src/io/orc/stripe_enc.cu index b7dd0ea9ec3..516922219d1 100644 --- a/cpp/src/io/orc/stripe_enc.cu +++ b/cpp/src/io/orc/stripe_enc.cu @@ -1390,6 +1390,12 @@ std::optional CompressOrcDataStreams( CUDF_FAIL("Compression error: " + reason.value()); } nvcomp::batched_compress(nvcomp::compression_type::ZSTD, comp_in, comp_out, comp_res, stream); + } else if (compression == LZ4) { + if (auto const reason = nvcomp::is_compression_disabled(nvcomp::compression_type::LZ4); + reason) { + CUDF_FAIL("Compression error: " + reason.value()); + } + nvcomp::batched_compress(nvcomp::compression_type::LZ4, comp_in, comp_out, comp_res, stream); } else if (compression != NONE) { CUDF_FAIL("Unsupported compression type"); } diff --git a/cpp/src/io/orc/writer_impl.cu b/cpp/src/io/orc/writer_impl.cu index b0702d93d34..cc1a18c9173 100644 --- a/cpp/src/io/orc/writer_impl.cu +++ b/cpp/src/io/orc/writer_impl.cu @@ -98,6 +98,7 @@ auto to_nvcomp_compression_type(CompressionKind compression_kind) if (compression_kind == SNAPPY) return nvcomp::compression_type::SNAPPY; if (compression_kind == ZLIB) return nvcomp::compression_type::DEFLATE; if (compression_kind == ZSTD) return nvcomp::compression_type::ZSTD; + if (compression_kind == LZ4) return nvcomp::compression_type::LZ4; CUDF_FAIL("Unsupported compression type"); } @@ -111,6 +112,7 @@ orc::CompressionKind to_orc_compression(compression_type compression) case compression_type::SNAPPY: return orc::CompressionKind::SNAPPY; case compression_type::ZLIB: return orc::CompressionKind::ZLIB; case compression_type::ZSTD: return orc::CompressionKind::ZSTD; + case compression_type::LZ4: return orc::CompressionKind::LZ4; case compression_type::NONE: return orc::CompressionKind::NONE; default: CUDF_FAIL("Unsupported compression type"); } diff --git a/cpp/src/io/parquet/parquet_common.hpp b/cpp/src/io/parquet/parquet_common.hpp index a680e44f360..8507eca047e 100644 --- a/cpp/src/io/parquet/parquet_common.hpp +++ b/cpp/src/io/parquet/parquet_common.hpp @@ -103,8 +103,9 @@ enum Compression { GZIP = 2, LZO = 3, BROTLI = 4, // Added in 2.3.2 - LZ4 = 5, // Added in 2.3.2 + LZ4 = 5, // deprecated; based on LZ4, but with an additional undocumented framing scheme ZSTD = 6, // Added in 2.3.2 + LZ4_RAW = 7, // "standard" LZ4 block format }; /** diff --git a/cpp/src/io/parquet/reader_impl_chunking.cu b/cpp/src/io/parquet/reader_impl_chunking.cu index e0cb2fbb4f4..69141faa7fc 100644 --- a/cpp/src/io/parquet/reader_impl_chunking.cu +++ b/cpp/src/io/parquet/reader_impl_chunking.cu @@ -706,7 +706,11 @@ std::vector compute_page_splits_by_row(device_span compute_page_splits_by_row(device_span& chunks, CUDF_FAIL("Compression error: " + reason.value()); } nvcomp::batched_compress(nvcomp::compression_type::ZSTD, comp_in, comp_out, comp_res, stream); - + break; + } + case Compression::LZ4_RAW: { + if (auto const reason = nvcomp::is_compression_disabled(nvcomp::compression_type::LZ4); + reason) { + CUDF_FAIL("Compression error: " + reason.value()); + } + nvcomp::batched_compress(nvcomp::compression_type::LZ4, comp_in, comp_out, comp_res, stream); break; } case Compression::UNCOMPRESSED: break; diff --git a/cpp/tests/io/orc_test.cpp b/cpp/tests/io/orc_test.cpp index 305ec404a71..f1a397f1747 100644 --- a/cpp/tests/io/orc_test.cpp +++ b/cpp/tests/io/orc_test.cpp @@ -138,6 +138,9 @@ struct OrcStatisticsTest : public cudf::test::BaseFixture {}; // Test fixture for metadata tests struct OrcMetadataReaderTest : public cudf::test::BaseFixture {}; +struct OrcCompressionTest : public cudf::test::BaseFixture, + public ::testing::WithParamInterface {}; + namespace { // Generates a vector of uniform random values of type T template @@ -2055,6 +2058,42 @@ TEST_F(OrcStatisticsTest, Empty) EXPECT_EQ(ts6.count[0], 0); } +TEST_P(OrcCompressionTest, Basic) +{ + constexpr auto num_rows = 12000; + auto const compression_type = GetParam(); + + // Generate compressible data + auto int_sequence = + cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i % 100; }); + auto float_sequence = + cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i / 32; }); + + int32_col int_col(int_sequence, int_sequence + num_rows); + float32_col float_col(float_sequence, float_sequence + num_rows); + + table_view expected({int_col, float_col}); + + std::vector out_buffer; + cudf::io::orc_writer_options out_opts = + cudf::io::orc_writer_options::builder(cudf::io::sink_info{&out_buffer}, expected) + .compression(compression_type); + cudf::io::write_orc(out_opts); + + cudf::io::orc_reader_options in_opts = cudf::io::orc_reader_options::builder( + cudf::io::source_info{out_buffer.data(), out_buffer.size()}); + auto result = cudf::io::read_orc(in_opts); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result.tbl->view()); +} + +INSTANTIATE_TEST_CASE_P(OrcCompressionTest, + OrcCompressionTest, + ::testing::Values(cudf::io::compression_type::NONE, + cudf::io::compression_type::SNAPPY, + cudf::io::compression_type::LZ4, + cudf::io::compression_type::ZSTD)); + TEST_F(OrcWriterTest, BounceBufferBug) { auto sequence = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i % 100; }); diff --git a/cpp/tests/io/parquet_misc_test.cpp b/cpp/tests/io/parquet_misc_test.cpp index aa5a1cad96a..01027d04658 100644 --- a/cpp/tests/io/parquet_misc_test.cpp +++ b/cpp/tests/io/parquet_misc_test.cpp @@ -30,6 +30,9 @@ template struct ParquetWriterDeltaTest : public ParquetWriterTest {}; +struct ParquetCompressionTest : public cudf::test::BaseFixture, + public ::testing::WithParamInterface {}; + TYPED_TEST_SUITE(ParquetWriterDeltaTest, SupportedDeltaTestTypes); TYPED_TEST(ParquetWriterDeltaTest, SupportedDeltaTestTypes) @@ -232,3 +235,40 @@ TYPED_TEST(ParquetWriterComparableTypeTest, ThreeColumnSorted) EXPECT_EQ(ci.boundary_order, expected_orders[i]); } } + +TEST_P(ParquetCompressionTest, Basic) +{ + constexpr auto num_rows = 12000; + auto const compression_type = GetParam(); + + // Generate compressible data + auto int_sequence = + cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i % 100; }); + auto float_sequence = + cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i / 32; }); + + cudf::test::fixed_width_column_wrapper int_col(int_sequence, int_sequence + num_rows); + cudf::test::fixed_width_column_wrapper float_col(float_sequence, + float_sequence + num_rows); + + table_view expected({int_col, float_col}); + + std::vector out_buffer; + cudf::io::parquet_writer_options out_opts = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&out_buffer}, expected) + .compression(compression_type); + cudf::io::write_parquet(out_opts); + + cudf::io::parquet_reader_options in_opts = cudf::io::parquet_reader_options::builder( + cudf::io::source_info{out_buffer.data(), out_buffer.size()}); + auto result = cudf::io::read_parquet(in_opts); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result.tbl->view()); +} + +INSTANTIATE_TEST_CASE_P(ParquetCompressionTest, + ParquetCompressionTest, + ::testing::Values(cudf::io::compression_type::NONE, + cudf::io::compression_type::SNAPPY, + cudf::io::compression_type::LZ4, + cudf::io::compression_type::ZSTD)); diff --git a/python/cudf/cudf/_lib/orc.pyx b/python/cudf/cudf/_lib/orc.pyx index af2759e16f9..16feccc12d0 100644 --- a/python/cudf/cudf/_lib/orc.pyx +++ b/python/cudf/cudf/_lib/orc.pyx @@ -157,12 +157,16 @@ cpdef read_orc(object filepaths_or_buffers, cdef compression_type _get_comp_type(object compression): if compression is None or compression is False: return compression_type.NONE - elif compression == "snappy": + + compression = str(compression).upper() + if compression == "SNAPPY": return compression_type.SNAPPY elif compression == "ZLIB": return compression_type.ZLIB elif compression == "ZSTD": return compression_type.ZSTD + elif compression == "LZ4": + return compression_type.LZ4 else: raise ValueError(f"Unsupported `compression` type {compression}") diff --git a/python/cudf/cudf/_lib/parquet.pyx b/python/cudf/cudf/_lib/parquet.pyx index fab7d76c3c2..226733f8e67 100644 --- a/python/cudf/cudf/_lib/parquet.pyx +++ b/python/cudf/cudf/_lib/parquet.pyx @@ -693,10 +693,14 @@ cdef cudf_io_types.statistics_freq _get_stat_freq(object statistics): cdef cudf_io_types.compression_type _get_comp_type(object compression): if compression is None: return cudf_io_types.compression_type.NONE - elif compression == "snappy": + + compression = str(compression).upper() + if compression == "SNAPPY": return cudf_io_types.compression_type.SNAPPY elif compression == "ZSTD": return cudf_io_types.compression_type.ZSTD + elif compression == "LZ4": + return cudf_io_types.compression_type.LZ4 else: raise ValueError("Unsupported `compression` type") diff --git a/python/cudf/cudf/tests/test_orc.py b/python/cudf/cudf/tests/test_orc.py index 868543cd1f0..cf2fd29d41e 100644 --- a/python/cudf/cudf/tests/test_orc.py +++ b/python/cudf/cudf/tests/test_orc.py @@ -1927,3 +1927,28 @@ def test_orc_chunked_writer_stripe_size(datadir): orc_file = orc.ORCFile(buffer) assert_eq(orc_file.nstripes, 5) + + +def test_reader_lz4(): + from pyarrow import orc + + pdf = pd.DataFrame({"ints": [1, 2] * 5001}) + pa_table = pa.Table.from_pandas(pdf) + + buffer = BytesIO() + writer = orc.ORCWriter(buffer, compression="LZ4") + writer.write(pa_table) + writer.close() + + got = cudf.read_orc(buffer) + assert_eq(pdf, got) + + +def test_writer_lz4(): + gdf = cudf.DataFrame({"ints": [1, 2] * 5001}) + + buffer = BytesIO() + gdf.to_orc(buffer, compression="LZ4") + + got = pd.read_orc(buffer) + assert_eq(gdf, got) diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index b4e24bd1617..851f0c30dc8 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -3124,3 +3124,23 @@ def test_parquet_reader_multiindex(): def test_parquet_reader_engine_error(): with pytest.raises(ValueError): cudf.read_parquet(BytesIO(), engine="abc") + + +def test_reader_lz4(): + pdf = pd.DataFrame({"ints": [1, 2] * 5001}) + + buffer = BytesIO() + pdf.to_parquet(buffer, compression="LZ4") + + got = cudf.read_parquet(buffer) + assert_eq(pdf, got) + + +def test_writer_lz4(): + gdf = cudf.DataFrame({"ints": [1, 2] * 5001}) + + buffer = BytesIO() + gdf.to_parquet(buffer, compression="LZ4") + + got = pd.read_parquet(buffer) + assert_eq(gdf, got) diff --git a/python/cudf/cudf/utils/ioutils.py b/python/cudf/cudf/utils/ioutils.py index feb02bac60d..925fd24e6c8 100644 --- a/python/cudf/cudf/utils/ioutils.py +++ b/python/cudf/cudf/utils/ioutils.py @@ -229,8 +229,9 @@ File path or Root Directory path. Will be used as Root Directory path while writing a partitioned dataset. Use list of str with partition_offsets to write parts of the dataframe to different files. -compression : {{'snappy', 'ZSTD', None}}, default 'snappy' - Name of the compression to use. Use ``None`` for no compression. +compression : {{'snappy', 'ZSTD', 'LZ4', None}}, default 'snappy' + Name of the compression to use; case insensitive. + Use ``None`` for no compression. index : bool, default None If ``True``, include the dataframe's index(es) in the file output. If ``False``, they will not be written to the file. @@ -491,8 +492,9 @@ ---------- fname : str File path or object where the ORC dataset will be stored. -compression : {{ 'snappy', 'ZSTD', None }}, default 'snappy' - Name of the compression to use. Use None for no compression. +compression : {{ 'snappy', 'ZSTD', 'ZLIB', 'LZ4', None }}, default 'snappy' + Name of the compression to use; case insensitive. + Use ``None`` for no compression. statistics: str {{ "ROWGROUP", "STRIPE", None }}, default "ROWGROUP" The granularity with which column statistics must be written to the file. From e57afddcb52c9c91c37b88733efc5a0880904454 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 14 Feb 2024 16:16:32 -1000 Subject: [PATCH 248/384] Adjust tests in test_dataframe.py for pandas 2.2 (#15023) * Removed an unnecessary `replace` that causes a deprecated down casting `test_all` * Updated the tests cases in `test_update_for_dataframes` to do replacement with equivalent types as an `update` that upcasts/downcasts is deprecated in pandas 2.2 Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15023 --- python/cudf/cudf/tests/test_dataframe.py | 109 +++++++++-------------- 1 file changed, 42 insertions(+), 67 deletions(-) diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index f9af0d10713..565b9b09001 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -4174,10 +4174,8 @@ def test_dataframe_round_dict_decimal_validation(): def test_all(data): # Provide a dtype when data is empty to avoid future pandas changes. dtype = None if data else float - # Pandas treats `None` in object type columns as True for some reason, so - # replacing with `False` if np.array(data).ndim <= 1: - pdata = pd.Series(data=data, dtype=dtype).replace([None], False) + pdata = pd.Series(data=data, dtype=dtype) gdata = cudf.Series.from_pandas(pdata) got = gdata.all() expected = pdata.all() @@ -9257,78 +9255,55 @@ def test_agg_for_dataframe_with_string_columns(aggs): @pytest_unmark_spilling +@pytest.mark.parametrize("overwrite", [True, False]) @pytest.mark.parametrize( - "join", - ["left"], -) -@pytest.mark.parametrize( - "overwrite", - [True, False], -) -@pytest.mark.parametrize( - "errors", - ["ignore"], -) -@pytest.mark.parametrize( - "data", + "left_keys,right_keys", [ - {"a": [1, 2, 3], "b": [3, 4, 5]}, - {"e": [1.0, 2.0, 3.0], "d": [3.0, 4.0, 5.0]}, - {"c": [True, False, False], "d": [False, True, True]}, - {"g": [2.0, np.nan, 4.0], "n": [np.nan, np.nan, np.nan]}, - {"d": [np.nan, np.nan, np.nan], "e": [np.nan, np.nan, np.nan]}, - {"a": [1.0, 2, 3], "b": pd.Series([4.0, 8.0, 3.0], index=[1, 2, 3])}, - { - "d": [1.0, 2.0, 3.0], - "c": pd.Series([np.nan, np.nan, np.nan], index=[1, 2, 3]), - }, - { - "a": [False, True, False], - "b": pd.Series([1.0, 2.0, np.nan], index=[1, 2, 3]), - }, - { - "a": [np.nan, np.nan, np.nan], - "e": pd.Series([np.nan, np.nan, np.nan], index=[1, 2, 3]), - }, + [("a", "b"), ("a", "b")], + [("a", "b"), ("a", "c")], + [("a", "b"), ("d", "e")], ], ) @pytest.mark.parametrize( - "data2", + "data_left,data_right", [ - {"b": [3, 5, 6], "e": [8, 2, 1]}, - {"c": [True, False, True], "d": [3.0, 4.0, 5.0]}, - {"e": [False, False, True], "g": [True, True, False]}, - {"g": [np.nan, np.nan, np.nan], "c": [np.nan, np.nan, np.nan]}, - {"a": [7, 5, 8], "b": pd.Series([2.0, 7.0, 9.0], index=[0, 1, 2])}, - { - "b": [np.nan, 2.0, np.nan], - "c": pd.Series([2, np.nan, 5.0], index=[2, 3, 4]), - }, - { - "a": pd.Series([True, None, True], dtype=pd.BooleanDtype()), - "d": pd.Series( - [False, True, None], index=[0, 1, 3], dtype=pd.BooleanDtype() - ), - }, + [([1, 2, 3], [3, 4, 5]), ([1, 2, 3], [3, 4, 5])], + [ + ([1.0, 2.0, 3.0], [3.0, 4.0, 5.0]), + ([1.0, 2.0, 3.0], [3.0, 4.0, 5.0]), + ], + [ + ([True, False, True], [False, False, False]), + ([True, False, True], [False, False, False]), + ], + [ + ([np.nan, np.nan, np.nan], [np.nan, np.nan, np.nan]), + ([np.nan, np.nan, np.nan], [np.nan, np.nan, np.nan]), + ], + [([1, 2, 3], [3, 4, 5]), ([1, 2, 4], [30, 40, 50])], + [ + ([1.0, 2.0, 3.0], [3.0, 4.0, 5.0]), + ([1.0, 2.0, 4.0], [30.0, 40.0, 50.0]), + ], + [([1, 2, 3], [3, 4, 5]), ([10, 20, 40], [30, 40, 50])], + [ + ([1.0, 2.0, 3.0], [3.0, 4.0, 5.0]), + ([10.0, 20.0, 40.0], [30.0, 40.0, 50.0]), + ], ], ) -def test_update_for_dataframes(request, data, data2, join, overwrite, errors): - request.applymarker( - pytest.mark.xfail( - condition=request.node.name - in { - "test_update_for_dataframes[data21-data2-ignore-True-left]", - "test_update_for_dataframes[data24-data7-ignore-True-left]", - "test_update_for_dataframes[data25-data2-ignore-True-left]", - }, - reason="mixing of bools & non-bools is not allowed.", - ) - ) - pdf = pd.DataFrame(data) - gdf = cudf.DataFrame(data, nan_as_null=False) - - other_pd = pd.DataFrame(data2) - other_gd = cudf.DataFrame(data2, nan_as_null=False) +def test_update_for_dataframes( + left_keys, right_keys, data_left, data_right, overwrite +): + errors = "ignore" + join = "left" + left = dict(zip(left_keys, data_left)) + right = dict(zip(right_keys, data_right)) + pdf = pd.DataFrame(left) + gdf = cudf.DataFrame(left, nan_as_null=False) + + other_pd = pd.DataFrame(right) + other_gd = cudf.DataFrame(right, nan_as_null=False) pdf.update(other=other_pd, join=join, overwrite=overwrite, errors=errors) gdf.update(other=other_gd, join=join, overwrite=overwrite, errors=errors) From 99ed8b9977cf52a5188637959bce9ca5b1f00ab9 Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Thu, 15 Feb 2024 12:11:46 +0000 Subject: [PATCH 249/384] Expose libcudf filter expression in read_parquet (#15028) libcudf's parquet reader supports filtering rows of the input dataset based on a (restricted subset of) libcudf Expression. Previously this functionality was not exposed in Python-land, do so here. Authors: - Lawrence Mitchell (https://github.com/wence-) Approvers: - Ashwin Srinath (https://github.com/shwina) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/15028 --- python/cudf/cudf/_lib/cpp/io/parquet.pxd | 5 +++- python/cudf/cudf/_lib/expressions.pxd | 9 +++++-- python/cudf/cudf/_lib/expressions.pyx | 30 +++++++++++++++++++++++- python/cudf/cudf/_lib/parquet.pyx | 22 +++++++++++++---- 4 files changed, 57 insertions(+), 9 deletions(-) diff --git a/python/cudf/cudf/_lib/cpp/io/parquet.pxd b/python/cudf/cudf/_lib/cpp/io/parquet.pxd index cdd1bde0274..8de16d06a9d 100644 --- a/python/cudf/cudf/_lib/cpp/io/parquet.pxd +++ b/python/cudf/cudf/_lib/cpp/io/parquet.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from libc.stdint cimport uint8_t from libcpp cimport bool @@ -53,6 +53,9 @@ cdef extern from "cudf/io/parquet.hpp" namespace "cudf::io" nogil: parquet_reader_options_builder& timestamp_type( data_type type ) except + + parquet_reader_options_builder& filter( + const expression & f + ) except + parquet_reader_options build() except + cdef cudf_io_types.table_with_metadata read_parquet( diff --git a/python/cudf/cudf/_lib/expressions.pxd b/python/cudf/cudf/_lib/expressions.pxd index fc69dc13bb2..c2ee504c626 100644 --- a/python/cudf/cudf/_lib/expressions.pxd +++ b/python/cudf/cudf/_lib/expressions.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. from libc.stdint cimport int32_t, int64_t from libcpp.memory cimport unique_ptr @@ -9,7 +9,12 @@ from cudf._lib.cpp.expressions cimport ( literal, operation, ) -from cudf._lib.cpp.scalar.scalar cimport numeric_scalar, scalar, string_scalar +from cudf._lib.cpp.scalar.scalar cimport ( + numeric_scalar, + scalar, + string_scalar, + timestamp_scalar, +) cdef class Expression: diff --git a/python/cudf/cudf/_lib/expressions.pyx b/python/cudf/cudf/_lib/expressions.pyx index 01a080f635f..a3b07075507 100644 --- a/python/cudf/cudf/_lib/expressions.pyx +++ b/python/cudf/cudf/_lib/expressions.pyx @@ -1,7 +1,9 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. from enum import Enum +import numpy as np + from cython.operator cimport dereference from libc.stdint cimport int64_t from libcpp.memory cimport make_unique, unique_ptr @@ -10,6 +12,7 @@ from libcpp.utility cimport move from cudf._lib.cpp cimport expressions as libcudf_exp from cudf._lib.cpp.types cimport size_type +from cudf._lib.cpp.wrappers.timestamps cimport timestamp_ms, timestamp_us # Necessary for proper casting, see below. ctypedef int32_t underlying_type_ast_operator @@ -95,6 +98,31 @@ cdef class Literal(Expression): self.c_obj = move(make_unique[libcudf_exp.literal]( dereference(self.c_scalar) )) + elif isinstance(value, np.datetime64): + scale, _ = np.datetime_data(value.dtype) + int_value = value.astype(np.int64) + if scale == "ms": + self.c_scalar.reset(new timestamp_scalar[timestamp_ms]( + int_value, True) + ) + self.c_obj = move(make_unique[libcudf_exp.literal]( + dereference(self.c_scalar) + )) + elif scale == "us": + self.c_scalar.reset(new timestamp_scalar[timestamp_us]( + int_value, True) + ) + self.c_obj = move(make_unique[libcudf_exp.literal]( + dereference(self.c_scalar) + )) + else: + raise NotImplementedError( + f"Unhandled datetime scale {scale=}" + ) + else: + raise NotImplementedError( + f"Don't know how to make literal with type {type(value)}" + ) cdef class ColumnReference(Expression): diff --git a/python/cudf/cudf/_lib/parquet.pyx b/python/cudf/cudf/_lib/parquet.pyx index 226733f8e67..d3f5b423373 100644 --- a/python/cudf/cudf/_lib/parquet.pyx +++ b/python/cudf/cudf/_lib/parquet.pyx @@ -37,11 +37,13 @@ cimport cudf._lib.cpp.io.data_sink as cudf_io_data_sink cimport cudf._lib.cpp.io.types as cudf_io_types cimport cudf._lib.cpp.types as cudf_types from cudf._lib.column cimport Column +from cudf._lib.cpp.expressions cimport expression from cudf._lib.cpp.io.parquet cimport ( chunked_parquet_writer_options, merge_row_group_metadata as parquet_merge_metadata, parquet_chunked_writer as cpp_parquet_chunked_writer, parquet_reader_options, + parquet_reader_options_builder, parquet_writer_options, read_parquet as parquet_reader, write_parquet as parquet_writer, @@ -49,6 +51,7 @@ from cudf._lib.cpp.io.parquet cimport ( from cudf._lib.cpp.io.types cimport column_in_metadata, table_input_metadata from cudf._lib.cpp.table.table_view cimport table_view from cudf._lib.cpp.types cimport data_type, size_type +from cudf._lib.expressions cimport Expression from cudf._lib.io.datasource cimport NativeFileDatasource from cudf._lib.io.utils cimport ( make_sinks_info, @@ -119,10 +122,14 @@ def _parse_metadata(meta): cpdef read_parquet(filepaths_or_buffers, columns=None, row_groups=None, - use_pandas_metadata=True): + use_pandas_metadata=True, + Expression filters=None): """ Cython function to call into libcudf API, see `read_parquet`. + filters, if not None, should be an Expression that evaluates to a + boolean predicate as a function of columns being read. + See Also -------- cudf.io.parquet.read_parquet @@ -148,19 +155,22 @@ cpdef read_parquet(filepaths_or_buffers, columns=None, row_groups=None, cdef data_type cpp_timestamp_type = cudf_types.data_type( cudf_types.type_id.EMPTY ) - if row_groups is not None: cpp_row_groups = row_groups - cdef parquet_reader_options args # Setup parquet reader arguments - args = move( + cdef parquet_reader_options args + cdef parquet_reader_options_builder builder + builder = ( parquet_reader_options.builder(source) .row_groups(cpp_row_groups) .use_pandas_metadata(cpp_use_pandas_metadata) .timestamp_type(cpp_timestamp_type) - .build() ) + if filters is not None: + builder = builder.filter(dereference(filters.c_obj.get())) + + args = move(builder.build()) cdef vector[string] cpp_columns allow_range_index = True if columns is not None: @@ -169,6 +179,8 @@ cpdef read_parquet(filepaths_or_buffers, columns=None, row_groups=None, for col in columns: cpp_columns.push_back(str(col).encode()) args.set_columns(cpp_columns) + # Filters don't handle the range index correctly + allow_range_index &= filters is None # Read Parquet cdef cudf_io_types.table_with_metadata c_result From 65d9c5e94fed53d92989074451dcfc7a21b159a0 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 15 Feb 2024 08:14:24 -1000 Subject: [PATCH 250/384] Implement concatenate, lists.explode, merge, sorting, and stream compaction in pylibcudf (#15011) Contributes to #13921 Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/15011 --- .../api_docs/pylibcudf/concatenate.rst | 6 + .../user_guide/api_docs/pylibcudf/index.rst | 5 + .../user_guide/api_docs/pylibcudf/lists.rst | 6 + .../user_guide/api_docs/pylibcudf/merge.rst | 6 + .../user_guide/api_docs/pylibcudf/sorting.rst | 6 + .../api_docs/pylibcudf/stream_compaction.rst | 6 + python/cudf/cudf/_lib/concat.pyx | 66 +--- python/cudf/cudf/_lib/cpp/CMakeLists.txt | 4 +- python/cudf/cudf/_lib/cpp/concatenate.pxd | 17 +- .../cudf/cudf/_lib/cpp/stream_compaction.pxd | 15 +- .../cudf/cudf/_lib/cpp/stream_compaction.pyx | 0 python/cudf/cudf/_lib/lists.pyx | 28 +- python/cudf/cudf/_lib/merge.pyx | 63 ++-- .../cudf/cudf/_lib/pylibcudf/CMakeLists.txt | 23 +- python/cudf/cudf/_lib/pylibcudf/__init__.pxd | 12 +- python/cudf/cudf/_lib/pylibcudf/__init__.py | 12 +- .../cudf/cudf/_lib/pylibcudf/concatenate.pxd | 10 + .../cudf/cudf/_lib/pylibcudf/concatenate.pyx | 54 +++ python/cudf/cudf/_lib/pylibcudf/lists.pxd | 8 + python/cudf/cudf/_lib/pylibcudf/lists.pyx | 35 ++ python/cudf/cudf/_lib/pylibcudf/merge.pxd | 11 + python/cudf/cudf/_lib/pylibcudf/merge.pyx | 57 +++ python/cudf/cudf/_lib/pylibcudf/sorting.pxd | 61 +++ python/cudf/cudf/_lib/pylibcudf/sorting.pyx | 351 ++++++++++++++++++ .../cudf/_lib/pylibcudf/stream_compaction.pxd | 38 ++ .../cudf/_lib/pylibcudf/stream_compaction.pyx | 171 +++++++++ python/cudf/cudf/_lib/pylibcudf/table.pxd | 4 +- python/cudf/cudf/_lib/pylibcudf/types.pxd | 1 + python/cudf/cudf/_lib/pylibcudf/types.pyx | 1 + python/cudf/cudf/_lib/scalar.pxd | 4 +- python/cudf/cudf/_lib/sort.pyx | 258 +++++-------- python/cudf/cudf/_lib/stream_compaction.pyx | 193 +++------- python/cudf/cudf/_lib/utils.pxd | 5 +- python/cudf/cudf/_lib/utils.pyx | 64 ++-- 34 files changed, 1121 insertions(+), 480 deletions(-) create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/concatenate.rst create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/lists.rst create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/merge.rst create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/sorting.rst create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/stream_compaction.rst create mode 100644 python/cudf/cudf/_lib/cpp/stream_compaction.pyx create mode 100644 python/cudf/cudf/_lib/pylibcudf/concatenate.pxd create mode 100644 python/cudf/cudf/_lib/pylibcudf/concatenate.pyx create mode 100644 python/cudf/cudf/_lib/pylibcudf/lists.pxd create mode 100644 python/cudf/cudf/_lib/pylibcudf/lists.pyx create mode 100644 python/cudf/cudf/_lib/pylibcudf/merge.pxd create mode 100644 python/cudf/cudf/_lib/pylibcudf/merge.pyx create mode 100644 python/cudf/cudf/_lib/pylibcudf/sorting.pxd create mode 100644 python/cudf/cudf/_lib/pylibcudf/sorting.pyx create mode 100644 python/cudf/cudf/_lib/pylibcudf/stream_compaction.pxd create mode 100644 python/cudf/cudf/_lib/pylibcudf/stream_compaction.pyx diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/concatenate.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/concatenate.rst new file mode 100644 index 00000000000..e83739056f4 --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/concatenate.rst @@ -0,0 +1,6 @@ +=========== +concatenate +=========== + +.. automodule:: cudf._lib.pylibcudf.concatenate + :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst index 834cd46dc16..73f63ae1343 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst @@ -11,13 +11,18 @@ This page provides API documentation for pylibcudf. aggregation binaryop column + concatenate copying gpumemoryview groupby join + lists + merge reduce rolling scalar + stream_compaction + sorting replace table types diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/lists.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/lists.rst new file mode 100644 index 00000000000..a127dd6006a --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/lists.rst @@ -0,0 +1,6 @@ +===== +lists +===== + +.. automodule:: cudf._lib.pylibcudf.lists + :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/merge.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/merge.rst new file mode 100644 index 00000000000..ef1189a064a --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/merge.rst @@ -0,0 +1,6 @@ +===== +merge +===== + +.. automodule:: cudf._lib.pylibcudf.merge + :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/sorting.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/sorting.rst new file mode 100644 index 00000000000..e9441366eeb --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/sorting.rst @@ -0,0 +1,6 @@ +======= +sorting +======= + +.. automodule:: cudf._lib.pylibcudf.sorting + :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/stream_compaction.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/stream_compaction.rst new file mode 100644 index 00000000000..00b479446d8 --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/stream_compaction.rst @@ -0,0 +1,6 @@ +================= +stream_compaction +================= + +.. automodule:: cudf._lib.pylibcudf.stream_compaction + :members: diff --git a/python/cudf/cudf/_lib/concat.pyx b/python/cudf/cudf/_lib/concat.pyx index 1ec4719631e..89ddcfee99e 100644 --- a/python/cudf/cudf/_lib/concat.pyx +++ b/python/cudf/cudf/_lib/concat.pyx @@ -1,62 +1,34 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp cimport bool -from libcpp.memory cimport make_unique, unique_ptr -from libcpp.utility cimport move -from libcpp.vector cimport vector from cudf._lib.column cimport Column -from cudf._lib.cpp.column.column cimport column, column_view -from cudf._lib.cpp.concatenate cimport ( - concatenate_columns as libcudf_concatenate_columns, - concatenate_masks as libcudf_concatenate_masks, - concatenate_tables as libcudf_concatenate_tables, -) -from cudf._lib.cpp.table.table cimport table, table_view -from cudf._lib.utils cimport ( - data_from_unique_ptr, - make_column_views, - table_view_from_table, -) - -from cudf.core.buffer import acquire_spill_lock, as_buffer - -from rmm._lib.device_buffer cimport DeviceBuffer, device_buffer - - -cpdef concat_masks(object columns): - cdef device_buffer c_result - cdef unique_ptr[device_buffer] c_unique_result - cdef vector[column_view] c_views = make_column_views(columns) - with nogil: - c_result = move(libcudf_concatenate_masks(c_views)) - c_unique_result = move(make_unique[device_buffer](move(c_result))) - return as_buffer( - DeviceBuffer.c_from_unique_ptr(move(c_unique_result)) - ) +from cudf._lib.utils cimport data_from_pylibcudf_table + +from cudf._lib import pylibcudf +from cudf.core.buffer import acquire_spill_lock @acquire_spill_lock() def concat_columns(object columns): - cdef unique_ptr[column] c_result - cdef vector[column_view] c_views = make_column_views(columns) - with nogil: - c_result = move(libcudf_concatenate_columns(c_views)) - return Column.from_unique_ptr(move(c_result)) + return Column.from_pylibcudf( + pylibcudf.concatenate.concatenate( + [col.to_pylibcudf(mode="read") for col in columns] + ) + ) @acquire_spill_lock() def concat_tables(object tables, bool ignore_index=False): - cdef unique_ptr[table] c_result - cdef vector[table_view] c_views - c_views.reserve(len(tables)) - for tbl in tables: - c_views.push_back(table_view_from_table(tbl, ignore_index)) - with nogil: - c_result = move(libcudf_concatenate_tables(c_views)) - - return data_from_unique_ptr( - move(c_result), + plc_tables = [] + for table in tables: + cols = table._data.columns + if not ignore_index: + cols = table._index._data.columns + cols + plc_tables.append(pylibcudf.Table([c.to_pylibcudf(mode="read") for c in cols])) + + return data_from_pylibcudf_table( + pylibcudf.concatenate.concatenate(plc_tables), column_names=tables[0]._column_names, index_names=None if ignore_index else tables[0]._index_names ) diff --git a/python/cudf/cudf/_lib/cpp/CMakeLists.txt b/python/cudf/cudf/_lib/cpp/CMakeLists.txt index 21c38652362..89d3dc66f00 100644 --- a/python/cudf/cudf/_lib/cpp/CMakeLists.txt +++ b/python/cudf/cudf/_lib/cpp/CMakeLists.txt @@ -12,8 +12,8 @@ # the License. # ============================================================================= -set(cython_sources aggregation.pyx binaryop.pyx copying.pyx replace.pyx reduce.pxd types.pyx - unary.pyx +set(cython_sources aggregation.pyx binaryop.pyx copying.pyx replace.pyx reduce.pxd + stream_compaction.pyx types.pyx unary.pyx ) set(linked_libraries cudf::cudf) diff --git a/python/cudf/cudf/_lib/cpp/concatenate.pxd b/python/cudf/cudf/_lib/cpp/concatenate.pxd index 05068318962..a64c7426f5e 100644 --- a/python/cudf/cudf/_lib/cpp/concatenate.pxd +++ b/python/cudf/cudf/_lib/cpp/concatenate.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr from libcpp.vector cimport vector @@ -16,16 +16,7 @@ cdef extern from "cudf/concatenate.hpp" namespace "cudf" nogil: # constructable from a vector. In case they are needed in the future, # host_span versions can be added, e.g: # - # cdef device_buffer concatenate_masks "cudf::concatenate_masks"( - # host_span[column_view] views - # ) except + + # cdef unique_ptr[column] concatenate(host_span[column_view] columns) except + - cdef device_buffer concatenate_masks "cudf::concatenate_masks"( - const vector[column_view] views - ) except + - cdef unique_ptr[column] concatenate_columns "cudf::concatenate"( - const vector[column_view] columns - ) except + - cdef unique_ptr[table] concatenate_tables "cudf::concatenate"( - const vector[table_view] tables - ) except + + cdef unique_ptr[column] concatenate(const vector[column_view] columns) except + + cdef unique_ptr[table] concatenate(const vector[table_view] tables) except + diff --git a/python/cudf/cudf/_lib/cpp/stream_compaction.pxd b/python/cudf/cudf/_lib/cpp/stream_compaction.pxd index aef2f639d76..e8539ecb9c3 100644 --- a/python/cudf/cudf/_lib/cpp/stream_compaction.pxd +++ b/python/cudf/cudf/_lib/cpp/stream_compaction.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp cimport bool from libcpp.memory cimport unique_ptr @@ -19,13 +19,12 @@ from cudf._lib.cpp.types cimport ( ) -cdef extern from "cudf/stream_compaction.hpp" namespace "cudf" \ - nogil: - ctypedef enum duplicate_keep_option: - KEEP_ANY 'cudf::duplicate_keep_option::KEEP_ANY' - KEEP_FIRST 'cudf::duplicate_keep_option::KEEP_FIRST' - KEEP_LAST 'cudf::duplicate_keep_option::KEEP_LAST' - KEEP_NONE 'cudf::duplicate_keep_option::KEEP_NONE' +cdef extern from "cudf/stream_compaction.hpp" namespace "cudf" nogil: + cpdef enum class duplicate_keep_option: + KEEP_ANY + KEEP_FIRST + KEEP_LAST + KEEP_NONE cdef unique_ptr[table] drop_nulls(table_view source_table, vector[size_type] keys, diff --git a/python/cudf/cudf/_lib/cpp/stream_compaction.pyx b/python/cudf/cudf/_lib/cpp/stream_compaction.pyx new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/cudf/cudf/_lib/lists.pyx b/python/cudf/cudf/_lib/lists.pyx index f76d7a9a388..f4d16967300 100644 --- a/python/cudf/cudf/_lib/lists.pyx +++ b/python/cudf/cudf/_lib/lists.pyx @@ -18,13 +18,11 @@ from cudf._lib.cpp.lists.contains cimport contains, index_of as cpp_index_of from cudf._lib.cpp.lists.count_elements cimport ( count_elements as cpp_count_elements, ) -from cudf._lib.cpp.lists.explode cimport explode_outer as cpp_explode_outer from cudf._lib.cpp.lists.extract cimport extract_list_element from cudf._lib.cpp.lists.lists_column_view cimport lists_column_view from cudf._lib.cpp.lists.sorting cimport sort_lists as cpp_sort_lists from cudf._lib.cpp.lists.stream_compaction cimport distinct as cpp_distinct from cudf._lib.cpp.scalar.scalar cimport scalar -from cudf._lib.cpp.table.table cimport table from cudf._lib.cpp.table.table_view cimport table_view from cudf._lib.cpp.types cimport ( nan_equality, @@ -34,7 +32,12 @@ from cudf._lib.cpp.types cimport ( size_type, ) from cudf._lib.scalar cimport DeviceScalar -from cudf._lib.utils cimport columns_from_unique_ptr, table_view_from_columns +from cudf._lib.utils cimport ( + columns_from_pylibcudf_table, + table_view_from_columns, +) + +from cudf._lib import pylibcudf @acquire_spill_lock() @@ -55,18 +58,13 @@ def count_elements(Column col): @acquire_spill_lock() -def explode_outer( - list source_columns, int explode_column_idx -): - cdef table_view c_table_view = table_view_from_columns(source_columns) - cdef size_type c_explode_column_idx = explode_column_idx - - cdef unique_ptr[table] c_result - - with nogil: - c_result = move(cpp_explode_outer(c_table_view, c_explode_column_idx)) - - return columns_from_unique_ptr(move(c_result)) +def explode_outer(list source_columns, int explode_column_idx): + return columns_from_pylibcudf_table( + pylibcudf.lists.explode_outer( + pylibcudf.Table([c.to_pylibcudf(mode="read") for c in source_columns]), + explode_column_idx, + ) + ) @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/merge.pyx b/python/cudf/cudf/_lib/merge.pyx index 935d8c69adc..fe7f7ad2918 100644 --- a/python/cudf/cudf/_lib/merge.pyx +++ b/python/cudf/cudf/_lib/merge.pyx @@ -1,15 +1,10 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp cimport bool -from libcpp.memory cimport unique_ptr -from libcpp.utility cimport move -from libcpp.vector cimport vector -cimport cudf._lib.cpp.types as libcudf_types -from cudf._lib.cpp.merge cimport merge as cpp_merge -from cudf._lib.cpp.table.table cimport table -from cudf._lib.cpp.table.table_view cimport table_view -from cudf._lib.utils cimport columns_from_unique_ptr, table_view_from_columns +from cudf._lib.utils cimport columns_from_pylibcudf_table + +from cudf._lib import pylibcudf def merge_sorted( @@ -22,45 +17,31 @@ def merge_sorted( of sorted columns. `input_columns` is a list of lists of columns to be merged. """ - cdef vector[libcudf_types.size_type] c_column_keys = key_columns_indices - cdef vector[table_view] c_input_tables - cdef vector[libcudf_types.order] c_column_order - cdef vector[libcudf_types.null_order] c_null_precedence - - c_input_tables.reserve(len(input_columns)) - for source_columns in input_columns: - c_input_tables.push_back( - table_view_from_columns(source_columns)) + c_input_tables = [ + pylibcudf.Table( + [c.to_pylibcudf(mode="read") for c in source_columns] + ) for source_columns in input_columns + ] num_keys = len(key_columns_indices) - cdef libcudf_types.order column_order = ( - libcudf_types.order.ASCENDING if ascending - else libcudf_types.order.DESCENDING + column_order = ( + pylibcudf.types.Order.ASCENDING if ascending + else pylibcudf.types.Order.DESCENDING ) - c_column_order = vector[libcudf_types.order](num_keys, column_order) if not ascending: na_position = "last" if na_position == "first" else "first" - cdef libcudf_types.null_order null_precedence = ( - libcudf_types.null_order.BEFORE if na_position == "first" - else libcudf_types.null_order.AFTER - ) - c_null_precedence = vector[libcudf_types.null_order]( - num_keys, - null_precedence + null_precedence = ( + pylibcudf.types.NullOrder.BEFORE if na_position == "first" + else pylibcudf.types.NullOrder.AFTER ) - # Perform sorted merge operation - cdef unique_ptr[table] c_result - with nogil: - c_result = move( - cpp_merge( - c_input_tables, - c_column_keys, - c_column_order, - c_null_precedence, - ) + return columns_from_pylibcudf_table( + pylibcudf.merge.merge( + c_input_tables, + key_columns_indices, + [column_order] * num_keys, + [null_precedence] * num_keys, ) - - return columns_from_unique_ptr(move(c_result)) + ) diff --git a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt index 248b9afaa21..68e6765cc49 100644 --- a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt +++ b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt @@ -13,8 +13,27 @@ # ============================================================================= set(cython_sources - aggregation.pyx binaryop.pyx column.pyx copying.pyx gpumemoryview.pyx groupby.pyx interop.pyx - join.pyx reduce.pyx replace.pyx rolling.pyx scalar.pyx table.pyx types.pyx unary.pyx utils.pyx + aggregation.pyx + binaryop.pyx + column.pyx + concatenate.pyx + copying.pyx + gpumemoryview.pyx + groupby.pyx + interop.pyx + join.pyx + lists.pyx + merge.pyx + reduce.pyx + replace.pyx + rolling.pyx + scalar.pyx + stream_compaction.pyx + sorting.pyx + table.pyx + types.pyx + unary.pyx + utils.pyx ) set(linked_libraries cudf::cudf) rapids_cython_create_modules( diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd index 316a47eebf0..5ef10fb2ffc 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd @@ -4,13 +4,18 @@ from . cimport ( aggregation, binaryop, + concatenate, copying, groupby, interop, join, + lists, + merge, reduce, replace, rolling, + sorting, + stream_compaction, types, unary, ) @@ -29,14 +34,19 @@ __all__ = [ "Table", "aggregation", "binaryop", + "concatenate", "copying", "gpumemoryview", "groupby", "interop", "join", - "unary", + "lists", + "merge", "reduce", "replace", "rolling", + "stream_compaction", + "sorting", "types", + "unary", ] diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.py b/python/cudf/cudf/_lib/pylibcudf/__init__.py index 642c3c18920..4689c49fdb1 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.py +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.py @@ -3,13 +3,18 @@ from . import ( aggregation, binaryop, + concatenate, copying, groupby, interop, join, + lists, + merge, reduce, replace, rolling, + sorting, + stream_compaction, types, unary, ) @@ -27,14 +32,19 @@ "TypeId", "aggregation", "binaryop", + "concatenate", "copying", "gpumemoryview", "groupby", "interop", "join", - "unary", + "lists", + "merge", "reduce", "replace", "rolling", + "stream_compaction", + "sorting", "types", + "unary", ] diff --git a/python/cudf/cudf/_lib/pylibcudf/concatenate.pxd b/python/cudf/cudf/_lib/pylibcudf/concatenate.pxd new file mode 100644 index 00000000000..c506ffb93c9 --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/concatenate.pxd @@ -0,0 +1,10 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from .table cimport Table + + +# There is no way to define a fused type that is a list of other objects, so we cannot +# unify the column and table paths without using runtime dispatch instead. In this case +# we choose to prioritize API consistency over performance, so we use the same function +# with a bit of runtime dispatch overhead. +cpdef concatenate(list objects) diff --git a/python/cudf/cudf/_lib/pylibcudf/concatenate.pyx b/python/cudf/cudf/_lib/pylibcudf/concatenate.pyx new file mode 100644 index 00000000000..ce7ef84e20e --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/concatenate.pyx @@ -0,0 +1,54 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from libcpp.memory cimport unique_ptr +from libcpp.utility cimport move +from libcpp.vector cimport vector + +from cudf._lib.cpp cimport concatenate as cpp_concatenate +from cudf._lib.cpp.column.column cimport column +from cudf._lib.cpp.column.column_view cimport column_view +from cudf._lib.cpp.table.table cimport table +from cudf._lib.cpp.table.table_view cimport table_view + +from .column cimport Column +from .table cimport Table + + +cpdef concatenate(list objects): + """Concatenate columns or tables. + + Parameters + ---------- + objects : Union[List[Column], List[Table]] + The list of Columns or Tables to concatenate. + + Returns + ------- + Union[Column, Table] + The concatenated Column or Table. + """ + if len(objects) == 0: + raise ValueError("input list may not be empty") + + cdef vector[column_view] c_columns + cdef vector[table_view] c_tables + + cdef unique_ptr[column] c_col_result + cdef unique_ptr[table] c_tbl_result + + if isinstance(objects[0], Table): + for tbl in objects: + c_tables.push_back((tbl).view()) + + with nogil: + c_tbl_result = move(cpp_concatenate.concatenate(c_tables)) + return Table.from_libcudf(move(c_tbl_result)) + elif isinstance(objects[0], Column): + for column in objects: + c_columns.push_back((column).view()) + + with nogil: + c_col_result = move(cpp_concatenate.concatenate(c_columns)) + return Column.from_libcudf(move(c_col_result)) + else: + raise ValueError("input must be a list of Columns or Tables") diff --git a/python/cudf/cudf/_lib/pylibcudf/lists.pxd b/python/cudf/cudf/_lib/pylibcudf/lists.pxd new file mode 100644 index 00000000000..cf96dfcb81e --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/lists.pxd @@ -0,0 +1,8 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from cudf._lib.cpp.types cimport size_type + +from .table cimport Table + + +cpdef Table explode_outer(Table, size_type explode_column_idx) diff --git a/python/cudf/cudf/_lib/pylibcudf/lists.pyx b/python/cudf/cudf/_lib/pylibcudf/lists.pyx new file mode 100644 index 00000000000..faeca56286e --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/lists.pyx @@ -0,0 +1,35 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from libcpp.memory cimport unique_ptr +from libcpp.utility cimport move + +from cudf._lib.cpp.lists cimport explode as cpp_explode +from cudf._lib.cpp.table.table cimport table +from cudf._lib.cpp.types cimport size_type + +from .table cimport Table + + +cpdef Table explode_outer(Table input, size_type explode_column_idx): + """Explode a column of lists into rows. + + All other columns will be duplicated for each element in the list. + + Parameters + ---------- + input : Table + The input table + explode_column_idx : int + The index of the column to explode + + Returns + ------- + Table + A new table with the exploded column + """ + cdef unique_ptr[table] c_result + + with nogil: + c_result = move(cpp_explode.explode_outer(input.view(), explode_column_idx)) + + return Table.from_libcudf(move(c_result)) diff --git a/python/cudf/cudf/_lib/pylibcudf/merge.pxd b/python/cudf/cudf/_lib/pylibcudf/merge.pxd new file mode 100644 index 00000000000..4b598aa8f4f --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/merge.pxd @@ -0,0 +1,11 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from .table cimport Table + + +cpdef Table merge ( + list tables_to_merge, + list key_cols, + list column_order, + list null_precedence, +) diff --git a/python/cudf/cudf/_lib/pylibcudf/merge.pyx b/python/cudf/cudf/_lib/pylibcudf/merge.pyx new file mode 100644 index 00000000000..91b2b0ea65b --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/merge.pyx @@ -0,0 +1,57 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from libcpp.memory cimport unique_ptr +from libcpp.utility cimport move +from libcpp.vector cimport vector + +from cudf._lib.cpp cimport merge as cpp_merge +from cudf._lib.cpp.table.table cimport table +from cudf._lib.cpp.table.table_view cimport table_view +from cudf._lib.cpp.types cimport null_order, order, size_type + +from .table cimport Table + + +cpdef Table merge ( + list tables_to_merge, + list key_cols, + list column_order, + list null_precedence, +): + """Merge a set of sorted tables. + + Parameters + ---------- + tables_to_merge : list + List of tables to merge. + key_cols : list + List of column indexes to merge on. + column_order : List[ColumnOrder] + Whether each column should be sorted in ascending or descending order. + null_precedence : List[NullOrder] + Whether nulls should come before or after non-nulls. + + Returns + ------- + Table + The merged table. + """ + cdef vector[size_type] c_key_cols = key_cols + cdef vector[order] c_column_order = column_order + cdef vector[null_order] c_null_precedence = null_precedence + cdef vector[table_view] c_tables_to_merge + + for tbl in tables_to_merge: + c_tables_to_merge.push_back(( tbl).view()) + + cdef unique_ptr[table] c_result + with nogil: + c_result = move( + cpp_merge.merge( + c_tables_to_merge, + c_key_cols, + c_column_order, + c_null_precedence, + ) + ) + return Table.from_libcudf(move(c_result)) diff --git a/python/cudf/cudf/_lib/pylibcudf/sorting.pxd b/python/cudf/cudf/_lib/pylibcudf/sorting.pxd new file mode 100644 index 00000000000..fb22da0b0fd --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/sorting.pxd @@ -0,0 +1,61 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from libcpp cimport bool + +from cudf._lib.cpp.aggregation cimport rank_method +from cudf._lib.cpp.types cimport null_order, null_policy, order, size_type + +from .column cimport Column +from .table cimport Table + + +cpdef Column sorted_order(Table source_table, list column_order, list null_precedence) + +cpdef Column stable_sorted_order( + Table source_table, + list column_order, + list null_precedence, +) + +cpdef Column rank( + Column input_view, + rank_method method, + order column_order, + null_policy null_handling, + null_order null_precedence, + bool percentage, +) + +cpdef bool is_sorted(Table table, list column_order, list null_precedence) + +cpdef Table segmented_sort_by_key( + Table values, + Table keys, + Column segment_offsets, + list column_order, + list null_precedence, +) + +cpdef Table stable_segmented_sort_by_key( + Table values, + Table keys, + Column segment_offsets, + list column_order, + list null_precedence, +) + +cpdef Table sort_by_key( + Table values, + Table keys, + list column_order, + list null_precedence, +) + +cpdef Table stable_sort_by_key( + Table values, + Table keys, + list column_order, + list null_precedence, +) + +cpdef Table sort(Table source_table, list column_order, list null_precedence) diff --git a/python/cudf/cudf/_lib/pylibcudf/sorting.pyx b/python/cudf/cudf/_lib/pylibcudf/sorting.pyx new file mode 100644 index 00000000000..4e73760720a --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/sorting.pyx @@ -0,0 +1,351 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from libcpp.memory cimport unique_ptr +from libcpp.utility cimport move +from libcpp.vector cimport vector + +from cudf._lib.cpp cimport sorting as cpp_sorting +from cudf._lib.cpp.aggregation cimport rank_method +from cudf._lib.cpp.column.column cimport column +from cudf._lib.cpp.table.table cimport table +from cudf._lib.cpp.types cimport null_order, null_policy, order + +from .column cimport Column +from .table cimport Table + + +cpdef Column sorted_order(Table source_table, list column_order, list null_precedence): + """Computes the row indices required to sort the table. + + Parameters + ---------- + source_table : Table + The table to sort. + column_order : List[ColumnOrder] + Whether each column should be sorted in ascending or descending order. + null_precedence : List[NullOrder] + Whether nulls should come before or after non-nulls. + + Returns + ------- + Column + The row indices required to sort the table. + """ + cdef unique_ptr[column] c_result + cdef vector[order] c_orders = column_order + cdef vector[null_order] c_null_precedence = null_precedence + with nogil: + c_result = move( + cpp_sorting.sorted_order( + source_table.view(), + c_orders, + c_null_precedence, + ) + ) + return Column.from_libcudf(move(c_result)) + + +cpdef Column stable_sorted_order( + Table source_table, + list column_order, + list null_precedence, +): + """Computes the row indices required to sort the table, maintaining input order. + + Parameters + ---------- + source_table : Table + The table to sort. + column_order : List[ColumnOrder] + Whether each column should be sorted in ascending or descending order. + null_precedence : List[NullOrder] + Whether nulls should come before or after non-nulls. + + Returns + ------- + Column + The row indices required to sort the table. + """ + cdef unique_ptr[column] c_result + cdef vector[order] c_orders = column_order + cdef vector[null_order] c_null_precedence = null_precedence + with nogil: + c_result = move( + cpp_sorting.stable_sorted_order( + source_table.view(), + c_orders, + c_null_precedence, + ) + ) + return Column.from_libcudf(move(c_result)) + + +cpdef Column rank( + Column input_view, + rank_method method, + order column_order, + null_policy null_handling, + null_order null_precedence, + bool percentage, +): + """Computes the rank of each element in the column. + + Parameters + ---------- + input_view : Column + The column to rank. + method : rank_method + The method to use for ranking ties. + column_order : order + Whether the column should be sorted in ascending or descending order. + null_handling : null_policy + Whether or not nulls should be included in the ranking. + null_precedence : null_order + Whether nulls should come before or after non-nulls. + percentage : bool + Whether to return the rank as a percentage. + + Returns + ------- + Column + The rank of each element in the column. + """ + cdef unique_ptr[column] c_result + with nogil: + c_result = move( + cpp_sorting.rank( + input_view.view(), + method, + column_order, + null_handling, + null_precedence, + percentage, + ) + ) + return Column.from_libcudf(move(c_result)) + + +cpdef bool is_sorted(Table tbl, list column_order, list null_precedence): + """Checks if the table is sorted. + + Parameters + ---------- + tbl : Table + The table to check. + column_order : List[ColumnOrder] + Whether each column is expected to be sorted in ascending or descending order. + null_precedence : List[NullOrder] + Whether nulls are expected before or after non-nulls. + + Returns + ------- + bool + Whether the table is sorted. + """ + cdef bool c_result + cdef vector[order] c_orders = column_order + cdef vector[null_order] c_null_precedence = null_precedence + with nogil: + c_result = move( + cpp_sorting.is_sorted( + tbl.view(), + c_orders, + c_null_precedence, + ) + ) + return c_result + + +cpdef Table segmented_sort_by_key( + Table values, + Table keys, + Column segment_offsets, + list column_order, + list null_precedence, +): + """Sorts the table by key, within segments. + + Parameters + ---------- + values : Table + The table to sort. + keys : Table + The table to sort by. + segment_offsets : Column + The offsets of the segments. + column_order : List[ColumnOrder] + Whether each column should be sorted in ascending or descending order. + null_precedence : List[NullOrder] + Whether nulls should come before or after non-nulls. + + Returns + ------- + Table + The sorted table. + """ + cdef unique_ptr[table] c_result + cdef vector[order] c_orders = column_order + cdef vector[null_order] c_null_precedence = null_precedence + with nogil: + c_result = move( + cpp_sorting.segmented_sort_by_key( + values.view(), + keys.view(), + segment_offsets.view(), + c_orders, + c_null_precedence, + ) + ) + return Table.from_libcudf(move(c_result)) + + +cpdef Table stable_segmented_sort_by_key( + Table values, + Table keys, + Column segment_offsets, + list column_order, + list null_precedence, +): + """Sorts the table by key, within segments, maintaining input order. + + Parameters + ---------- + values : Table + The table to sort. + keys : Table + The table to sort by. + segment_offsets : Column + The offsets of the segments. + column_order : List[ColumnOrder] + Whether each column should be sorted in ascending or descending order. + null_precedence : List[NullOrder] + Whether nulls should come before or after non-nulls. + + Returns + ------- + Table + The sorted table. + """ + cdef unique_ptr[table] c_result + cdef vector[order] c_orders = column_order + cdef vector[null_order] c_null_precedence = null_precedence + with nogil: + c_result = move( + cpp_sorting.stable_segmented_sort_by_key( + values.view(), + keys.view(), + segment_offsets.view(), + c_orders, + c_null_precedence, + ) + ) + return Table.from_libcudf(move(c_result)) + + +cpdef Table sort_by_key( + Table values, + Table keys, + list column_order, + list null_precedence, +): + """Sorts the table by key. + + Parameters + ---------- + values : Table + The table to sort. + keys : Table + The table to sort by. + column_order : List[ColumnOrder] + Whether each column should be sorted in ascending or descending order. + null_precedence : List[NullOrder] + Whether nulls should come before or after non-nulls. + + Returns + ------- + Table + The sorted table. + """ + cdef unique_ptr[table] c_result + cdef vector[order] c_orders = column_order + cdef vector[null_order] c_null_precedence = null_precedence + with nogil: + c_result = move( + cpp_sorting.sort_by_key( + values.view(), + keys.view(), + c_orders, + c_null_precedence, + ) + ) + return Table.from_libcudf(move(c_result)) + + +cpdef Table stable_sort_by_key( + Table values, + Table keys, + list column_order, + list null_precedence, +): + """Sorts the table by key, maintaining input order. + + Parameters + ---------- + values : Table + The table to sort. + keys : Table + The table to sort by. + column_order : List[ColumnOrder] + Whether each column should be sorted in ascending or descending order. + null_precedence : List[NullOrder] + Whether nulls should come before or after non-nulls. + + Returns + ------- + Table + The sorted table. + """ + cdef unique_ptr[table] c_result + cdef vector[order] c_orders = column_order + cdef vector[null_order] c_null_precedence = null_precedence + with nogil: + c_result = move( + cpp_sorting.stable_sort_by_key( + values.view(), + keys.view(), + c_orders, + c_null_precedence, + ) + ) + return Table.from_libcudf(move(c_result)) + + +cpdef Table sort(Table source_table, list column_order, list null_precedence): + """Sorts the table. + + Parameters + ---------- + source_table : Table + The table to sort. + column_order : List[ColumnOrder] + Whether each column should be sorted in ascending or descending order. + null_precedence : List[NullOrder] + Whether nulls should come before or after non-nulls. + + Returns + ------- + Table + The sorted table. + """ + cdef unique_ptr[table] c_result + cdef vector[order] c_orders = column_order + cdef vector[null_order] c_null_precedence = null_precedence + with nogil: + c_result = move( + cpp_sorting.sort( + source_table.view(), + c_orders, + c_null_precedence, + ) + ) + return Table.from_libcudf(move(c_result)) diff --git a/python/cudf/cudf/_lib/pylibcudf/stream_compaction.pxd b/python/cudf/cudf/_lib/pylibcudf/stream_compaction.pxd new file mode 100644 index 00000000000..78adb20021c --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/stream_compaction.pxd @@ -0,0 +1,38 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from cudf._lib.cpp.stream_compaction cimport duplicate_keep_option +from cudf._lib.cpp.types cimport ( + nan_equality, + nan_policy, + null_equality, + null_policy, + size_type, +) + +from .column cimport Column +from .table cimport Table + + +cpdef Table drop_nulls(Table source_table, list keys, size_type keep_threshold) + +cpdef Table apply_boolean_mask(Table source_table, Column boolean_mask) + +cpdef size_type distinct_count( + Column source_table, + null_policy null_handling, + nan_policy nan_handling +) + +cpdef Table stable_distinct( + Table input, + list keys, + duplicate_keep_option keep, + null_equality nulls_equal, +) + +cpdef Column distinct_indices( + Table input, + duplicate_keep_option keep, + null_equality nulls_equal, + nan_equality nans_equal, +) diff --git a/python/cudf/cudf/_lib/pylibcudf/stream_compaction.pyx b/python/cudf/cudf/_lib/pylibcudf/stream_compaction.pyx new file mode 100644 index 00000000000..0357866980a --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/stream_compaction.pyx @@ -0,0 +1,171 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from libcpp.memory cimport unique_ptr +from libcpp.utility cimport move +from libcpp.vector cimport vector + +from cudf._lib.cpp cimport stream_compaction as cpp_stream_compaction +from cudf._lib.cpp.column.column cimport column +from cudf._lib.cpp.stream_compaction cimport duplicate_keep_option +from cudf._lib.cpp.table.table cimport table +from cudf._lib.cpp.types cimport ( + nan_equality, + nan_policy, + null_equality, + null_policy, + size_type, +) + +from cudf._lib.cpp.stream_compaction import \ + duplicate_keep_option as DuplicateKeepOption # no-cython-lint, isort:skip + +from .column cimport Column +from .table cimport Table + + +cpdef Table drop_nulls(Table source_table, list keys, size_type keep_threshold): + """Filters out rows from the input table based on the presence of nulls. + + Parameters + ---------- + source_table : Table + The input table to filter. + keys : List[size_type] + The list of column indexes to consider for null filtering. + keep_threshold : size_type + The minimum number of non-nulls required to keep a row. + + Returns + ------- + Table + A new table with rows removed based on the null count. + """ + cdef unique_ptr[table] c_result + cdef vector[size_type] c_keys = keys + with nogil: + c_result = move( + cpp_stream_compaction.drop_nulls( + source_table.view(), c_keys, keep_threshold + ) + ) + return Table.from_libcudf(move(c_result)) + + +cpdef Table apply_boolean_mask(Table source_table, Column boolean_mask): + """Filters out rows from the input table based on a boolean mask. + + Parameters + ---------- + source_table : Table + The input table to filter. + boolean_mask : Column + The boolean mask to apply to the input table. + + Returns + ------- + Table + A new table with rows removed based on the boolean mask. + """ + cdef unique_ptr[table] c_result + with nogil: + c_result = move( + cpp_stream_compaction.apply_boolean_mask( + source_table.view(), boolean_mask.view() + ) + ) + return Table.from_libcudf(move(c_result)) + + +cpdef size_type distinct_count( + Column source_table, + null_policy null_handling, + nan_policy nan_handling +): + """Returns the number of unique elements in the input column. + + Parameters + ---------- + source_table : Column + The input column to count the unique elements of. + null_handling : null_policy + Flag to include or exclude nulls from the count. + nan_handling : nan_policy + Flag to include or exclude NaNs from the count. + + Returns + ------- + size_type + The number of unique elements in the input column. + """ + return cpp_stream_compaction.distinct_count( + source_table.view(), null_handling, nan_handling + ) + + +cpdef Table stable_distinct( + Table input, + list keys, + duplicate_keep_option keep, + null_equality nulls_equal, +): + """Get the distinct rows from the input table, preserving input order. + + Parameters + ---------- + input : Table + The input table to filter. + keys : list + The list of column indexes to consider for distinct filtering. + keep : duplicate_keep_option + The option to specify which rows to keep in the case of duplicates. + nulls_equal : null_equality + The option to specify how nulls are handled in the comparison. + + Returns + ------- + Table + A new table with distinct rows from the input table. + """ + cdef unique_ptr[table] c_result + cdef vector[size_type] c_keys = keys + with nogil: + c_result = move( + cpp_stream_compaction.stable_distinct( + input.view(), c_keys, keep, nulls_equal + ) + ) + return Table.from_libcudf(move(c_result)) + + +cpdef Column distinct_indices( + Table input, + duplicate_keep_option keep, + null_equality nulls_equal, + nan_equality nans_equal, +): + """Get the indices of the distinct rows from the input table. + + Parameters + ---------- + input : Table + The input table to filter. + keep : duplicate_keep_option + The option to specify which rows to keep in the case of duplicates. + nulls_equal : null_equality + The option to specify how nulls are handled in the comparison. + nans_equal : nan_equality + The option to specify how NaNs are handled in the comparison. + + Returns + ------- + Column + A new column with the indices of the distinct rows from the input table. + """ + cdef unique_ptr[column] c_result + with nogil: + c_result = move( + cpp_stream_compaction.distinct_indices( + input.view(), keep, nulls_equal, nans_equal + ) + ) + return Column.from_libcudf(move(c_result)) diff --git a/python/cudf/cudf/_lib/pylibcudf/table.pxd b/python/cudf/cudf/_lib/pylibcudf/table.pxd index 6fe06f00491..2e76c811717 100644 --- a/python/cudf/cudf/_lib/pylibcudf/table.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/table.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr from pyarrow cimport lib as pa @@ -9,7 +9,7 @@ from cudf._lib.cpp.table.table_view cimport table_view cdef class Table: # List[pylibcudf.Column] - cdef list _columns + cdef public list _columns cdef table_view view(self) nogil diff --git a/python/cudf/cudf/_lib/pylibcudf/types.pxd b/python/cudf/cudf/_lib/pylibcudf/types.pxd index 1ad3d19f15c..e0f6a73fd55 100644 --- a/python/cudf/cudf/_lib/pylibcudf/types.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/types.pxd @@ -7,6 +7,7 @@ from cudf._lib.cpp.types cimport ( data_type, interpolation, nan_equality, + nan_policy, null_equality, null_order, null_policy, diff --git a/python/cudf/cudf/_lib/pylibcudf/types.pyx b/python/cudf/cudf/_lib/pylibcudf/types.pyx index 5b25e7674e2..f6ff6e5a2fc 100644 --- a/python/cudf/cudf/_lib/pylibcudf/types.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/types.pyx @@ -5,6 +5,7 @@ from libc.stdint cimport int32_t from cudf._lib.cpp.types cimport data_type, type_id from cudf._lib.cpp.types import type_id as TypeId # no-cython-lint, isort:skip +from cudf._lib.cpp.types import nan_policy as NanPolicy # no-cython-lint, isort:skip from cudf._lib.cpp.types import null_policy as NullPolicy # no-cython-lint, isort:skip from cudf._lib.cpp.types import interpolation as Interpolation # no-cython-lint, isort:skip from cudf._lib.cpp.types import nan_equality as NanEquality # no-cython-lint, isort:skip diff --git a/python/cudf/cudf/_lib/scalar.pxd b/python/cudf/cudf/_lib/scalar.pxd index b5c5a8a64a3..49f5c527aa0 100644 --- a/python/cudf/cudf/_lib/scalar.pxd +++ b/python/cudf/cudf/_lib/scalar.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp cimport bool from libcpp.memory cimport unique_ptr @@ -13,7 +13,7 @@ from cudf._lib.cpp.scalar.scalar cimport scalar cdef class DeviceScalar: - cdef pylibcudf.Scalar c_value + cdef public pylibcudf.Scalar c_value cdef object _dtype diff --git a/python/cudf/cudf/_lib/sort.pyx b/python/cudf/cudf/_lib/sort.pyx index e230dffbf3c..b2b84c17cf4 100644 --- a/python/cudf/cudf/_lib/sort.pyx +++ b/python/cudf/cudf/_lib/sort.pyx @@ -6,29 +6,21 @@ from cudf.core.buffer import acquire_spill_lock from libcpp cimport bool from libcpp.memory cimport unique_ptr -from libcpp.utility cimport move, pair +from libcpp.utility cimport move from libcpp.vector cimport vector from cudf._lib.column cimport Column from cudf._lib.cpp.aggregation cimport rank_method from cudf._lib.cpp.column.column cimport column -from cudf._lib.cpp.column.column_view cimport column_view from cudf._lib.cpp.search cimport lower_bound, upper_bound -from cudf._lib.cpp.sorting cimport ( - is_sorted as cpp_is_sorted, - rank, - segmented_sort_by_key as cpp_segmented_sort_by_key, - sort as cpp_sort, - sort_by_key as cpp_sort_by_key, - sorted_order, - stable_segmented_sort_by_key as cpp_stable_segmented_sort_by_key, - stable_sort_by_key as cpp_stable_sort_by_key, - stable_sorted_order, -) -from cudf._lib.cpp.table.table cimport table from cudf._lib.cpp.table.table_view cimport table_view -from cudf._lib.cpp.types cimport null_order, null_policy, order as cpp_order -from cudf._lib.utils cimport columns_from_unique_ptr, table_view_from_columns +from cudf._lib.cpp.types cimport null_order, order as cpp_order +from cudf._lib.utils cimport ( + columns_from_pylibcudf_table, + table_view_from_columns, +) + +from cudf._lib import pylibcudf @acquire_spill_lock() @@ -60,58 +52,42 @@ def is_sorted( ``null_position``, False otherwise. """ - cdef vector[cpp_order] column_order - cdef vector[null_order] null_precedence - if ascending is None: - column_order = vector[cpp_order]( - len(source_columns), cpp_order.ASCENDING - ) + column_order = [pylibcudf.types.Order.ASCENDING] * len(source_columns) else: if len(ascending) != len(source_columns): raise ValueError( f"Expected a list-like of length {len(source_columns)}, " f"got length {len(ascending)} for `ascending`" ) - column_order = vector[cpp_order]( - len(source_columns), cpp_order.DESCENDING - ) + column_order = [pylibcudf.types.Order.DESCENDING] * len(source_columns) for idx, val in enumerate(ascending): if val: - column_order[idx] = cpp_order.ASCENDING + column_order[idx] = pylibcudf.types.Order.ASCENDING if null_position is None: - null_precedence = vector[null_order]( - len(source_columns), null_order.AFTER - ) + null_precedence = [pylibcudf.types.NullOrder.AFTER] * len(source_columns) else: if len(null_position) != len(source_columns): raise ValueError( f"Expected a list-like of length {len(source_columns)}, " f"got length {len(null_position)} for `null_position`" ) - null_precedence = vector[null_order]( - len(source_columns), null_order.AFTER - ) + null_precedence = [pylibcudf.types.NullOrder.AFTER] * len(source_columns) for idx, val in enumerate(null_position): if val: - null_precedence[idx] = null_order.BEFORE - - cdef bool c_result - cdef table_view source_table_view = table_view_from_columns(source_columns) - with nogil: - c_result = cpp_is_sorted( - source_table_view, - column_order, - null_precedence - ) - - return c_result + null_precedence[idx] = pylibcudf.types.NullOrder.BEFORE + + return pylibcudf.sorting.is_sorted( + pylibcudf.Table( + [c.to_pylibcudf(mode="read") for c in source_columns] + ), + column_order, + null_precedence + ) -cdef pair[vector[cpp_order], vector[null_order]] ordering( - column_order, null_precedence -): +def ordering(column_order, null_precedence): """ Construct order and null order vectors @@ -128,21 +104,19 @@ cdef pair[vector[cpp_order], vector[null_order]] ordering( ------- pair of vectors (order, and null_order) """ - cdef vector[cpp_order] c_column_order - cdef vector[null_order] c_null_precedence + c_column_order = [] + c_null_precedence = [] for asc, null in zip(column_order, null_precedence): - c_column_order.push_back( - cpp_order.ASCENDING if asc else cpp_order.DESCENDING + c_column_order.append( + pylibcudf.types.Order.ASCENDING if asc else pylibcudf.types.Order.DESCENDING ) if asc ^ (null == "first"): - c_null_precedence.push_back(null_order.AFTER) + c_null_precedence.append(pylibcudf.types.NullOrder.AFTER) elif asc ^ (null == "last"): - c_null_precedence.push_back(null_order.BEFORE) + c_null_precedence.append(pylibcudf.types.NullOrder.BEFORE) else: raise ValueError(f"Invalid null precedence {null}") - return pair[vector[cpp_order], vector[null_order]]( - c_column_order, c_null_precedence - ) + return c_column_order, c_null_precedence @acquire_spill_lock() @@ -174,25 +148,18 @@ def order_by( ------- Column of indices that sorts the table """ - cdef table_view source_table_view = table_view_from_columns( - columns_from_table - ) - cdef pair[vector[cpp_order], vector[null_order]] order = ordering( - ascending, repeat(na_position) + order = ordering(ascending, repeat(na_position)) + func = getattr(pylibcudf.sorting, f"{'stable_' if stable else ''}sorted_order") + + return Column.from_pylibcudf( + func( + pylibcudf.Table( + [c.to_pylibcudf(mode="read") for c in columns_from_table], + ), + order[0], + order[1], + ) ) - cdef unique_ptr[column] c_result - if stable: - with nogil: - c_result = move(stable_sorted_order(source_table_view, - order.first, - order.second)) - else: - with nogil: - c_result = move(sorted_order(source_table_view, - order.first, - order.second)) - - return Column.from_unique_ptr(move(c_result)) @acquire_spill_lock() @@ -216,22 +183,18 @@ def sort( Sequence of "first" or "last" values (default "first") indicating the position of null values when sorting the keys. """ - cdef table_view values_view = table_view_from_columns(values) - cdef unique_ptr[table] result ncol = len(values) - cdef pair[vector[cpp_order], vector[null_order]] order = ordering( + order = ordering( column_order or repeat(True, ncol), null_precedence or repeat("first", ncol), ) - with nogil: - result = move( - cpp_sort( - values_view, - order.first, - order.second, - ) + return columns_from_pylibcudf_table( + pylibcudf.sorting.sort( + pylibcudf.Table([c.to_pylibcudf(mode="read") for c in values]), + order[0], + order[1], ) - return columns_from_unique_ptr(move(result)) + ) @acquire_spill_lock() @@ -267,26 +230,16 @@ def sort_by_key( list[Column] list of value columns sorted by keys """ - cdef table_view value_view = table_view_from_columns(values) - cdef table_view key_view = table_view_from_columns(keys) - cdef pair[vector[cpp_order], vector[null_order]] order = ordering( - ascending, na_position + order = ordering(ascending, na_position) + func = getattr(pylibcudf.sorting, f"{'stable_' if stable else ''}sort_by_key") + return columns_from_pylibcudf_table( + func( + pylibcudf.Table([c.to_pylibcudf(mode="read") for c in values]), + pylibcudf.Table([c.to_pylibcudf(mode="read") for c in keys]), + order[0], + order[1], + ) ) - cdef unique_ptr[table] c_result - if stable: - with nogil: - c_result = move(cpp_stable_sort_by_key(value_view, - key_view, - order.first, - order.second)) - else: - with nogil: - c_result = move(cpp_sort_by_key(value_view, - key_view, - order.first, - order.second)) - - return columns_from_unique_ptr(move(c_result)) @acquire_spill_lock() @@ -325,38 +278,24 @@ def segmented_sort_by_key( list[Column] list of value columns sorted by keys """ - cdef table_view values_view = table_view_from_columns(values) - cdef table_view keys_view = table_view_from_columns(keys) - cdef column_view offsets_view = segment_offsets.view() - cdef unique_ptr[table] result ncol = len(values) - cdef pair[vector[cpp_order], vector[null_order]] order = ordering( + order = ordering( column_order or repeat(True, ncol), null_precedence or repeat("first", ncol), ) - if stable: - with nogil: - result = move( - cpp_stable_segmented_sort_by_key( - values_view, - keys_view, - offsets_view, - order.first, - order.second, - ) - ) - else: - with nogil: - result = move( - cpp_segmented_sort_by_key( - values_view, - keys_view, - offsets_view, - order.first, - order.second, - ) - ) - return columns_from_unique_ptr(move(result)) + func = getattr( + pylibcudf.sorting, + f"{'stable_' if stable else ''}segmented_sort_by_key" + ) + return columns_from_pylibcudf_table( + func( + pylibcudf.Table([c.to_pylibcudf(mode="read") for c in values]), + pylibcudf.Table([c.to_pylibcudf(mode="read") for c in keys]), + segment_offsets.to_pylibcudf(mode="read"), + order[0], + order[1], + ) + ) @acquire_spill_lock() @@ -417,10 +356,10 @@ def rank_columns(list source_columns, rank_method method, str na_option, """ Compute numerical data ranks (1 through n) of each column in the dataframe """ - cdef cpp_order column_order = ( - cpp_order.ASCENDING + column_order = ( + pylibcudf.types.Order.ASCENDING if ascending - else cpp_order.DESCENDING + else pylibcudf.types.Order.DESCENDING ) # ascending # #top = na_is_smallest @@ -430,41 +369,32 @@ def rank_columns(list source_columns, rank_method method, str na_option, # #top = na_is_largest # #bottom = na_is_smallest # #keep = na_is_smallest - cdef null_order null_precedence if ascending: if na_option == 'top': - null_precedence = null_order.BEFORE + null_precedence = pylibcudf.types.NullOrder.BEFORE else: - null_precedence = null_order.AFTER + null_precedence = pylibcudf.types.NullOrder.AFTER else: if na_option == 'top': - null_precedence = null_order.AFTER + null_precedence = pylibcudf.types.NullOrder.AFTER else: - null_precedence = null_order.BEFORE - cdef null_policy c_null_handling = ( - null_policy.EXCLUDE + null_precedence = pylibcudf.types.NullOrder.BEFORE + c_null_handling = ( + pylibcudf.types.NullPolicy.EXCLUDE if na_option == 'keep' - else null_policy.INCLUDE + else pylibcudf.types.NullPolicy.INCLUDE ) - cdef bool percentage = pct - cdef vector[unique_ptr[column]] c_results - cdef column_view c_view - cdef Column col - for col in source_columns: - c_view = col.view() - with nogil: - c_results.push_back(move( - rank( - c_view, - method, - column_order, - c_null_handling, - null_precedence, - percentage - ) - )) - - return [Column.from_unique_ptr( - move(c_results[i]) - ) for i in range(c_results.size())] + return [ + Column.from_pylibcudf( + pylibcudf.sorting.rank( + col.to_pylibcudf(mode="read"), + method, + column_order, + c_null_handling, + null_precedence, + pct, + ) + ) + for col in source_columns + ] diff --git a/python/cudf/cudf/_lib/stream_compaction.pyx b/python/cudf/cudf/_lib/stream_compaction.pyx index d7725e8df94..04883eac559 100644 --- a/python/cudf/cudf/_lib/stream_compaction.pyx +++ b/python/cudf/cudf/_lib/stream_compaction.pyx @@ -3,31 +3,11 @@ from cudf.core.buffer import acquire_spill_lock from libcpp cimport bool -from libcpp.memory cimport unique_ptr -from libcpp.utility cimport move -from libcpp.vector cimport vector from cudf._lib.column cimport Column -from cudf._lib.cpp.column.column cimport column -from cudf._lib.cpp.column.column_view cimport column_view -from cudf._lib.cpp.stream_compaction cimport ( - apply_boolean_mask as cpp_apply_boolean_mask, - distinct_count as cpp_distinct_count, - distinct_indices as cpp_distinct_indices, - drop_nulls as cpp_drop_nulls, - duplicate_keep_option, - stable_distinct as cpp_stable_distinct, -) -from cudf._lib.cpp.table.table cimport table -from cudf._lib.cpp.table.table_view cimport table_view -from cudf._lib.cpp.types cimport ( - nan_equality, - nan_policy, - null_equality, - null_policy, - size_type, -) -from cudf._lib.utils cimport columns_from_unique_ptr, table_view_from_columns +from cudf._lib.utils cimport columns_from_pylibcudf_table + +from cudf._lib import pylibcudf @acquire_spill_lock() @@ -48,32 +28,26 @@ def drop_nulls(list columns, how="any", keys=None, thresh=None): ------- columns with null rows dropped """ + if how not in {"any", "all"}: + raise ValueError("how must be 'any' or 'all'") - cdef vector[size_type] cpp_keys = ( - keys if keys is not None else range(len(columns)) - ) + keys = list(keys if keys is not None else range(len(columns))) - cdef size_type c_keep_threshold = cpp_keys.size() + # Note: If how == "all" and thresh is specified this prioritizes thresh if thresh is not None: - c_keep_threshold = thresh + keep_threshold = thresh elif how == "all": - c_keep_threshold = 1 - - cdef unique_ptr[table] c_result - cdef table_view source_table_view = table_view_from_columns(columns) + keep_threshold = 1 + else: + keep_threshold = len(keys) - if how not in {"any", "all"}: - raise ValueError("how must be 'any' or 'all'") - with nogil: - c_result = move( - cpp_drop_nulls( - source_table_view, - cpp_keys, - c_keep_threshold - ) + return columns_from_pylibcudf_table( + pylibcudf.stream_compaction.drop_nulls( + pylibcudf.Table([c.to_pylibcudf(mode="read") for c in columns]), + keys, + keep_threshold, ) - - return columns_from_unique_ptr(move(c_result)) + ) @acquire_spill_lock() @@ -90,20 +64,19 @@ def apply_boolean_mask(list columns, Column boolean_mask): ------- columns obtained from applying mask """ - - cdef unique_ptr[table] c_result - cdef table_view source_table_view = table_view_from_columns(columns) - cdef column_view boolean_mask_view = boolean_mask.view() - - with nogil: - c_result = move( - cpp_apply_boolean_mask( - source_table_view, - boolean_mask_view - ) + return columns_from_pylibcudf_table( + pylibcudf.stream_compaction.apply_boolean_mask( + pylibcudf.Table([c.to_pylibcudf(mode="read") for c in columns]), + boolean_mask.to_pylibcudf(mode="read"), ) + ) + - return columns_from_unique_ptr(move(c_result)) +_keep_options = { + "first": pylibcudf.stream_compaction.DuplicateKeepOption.KEEP_FIRST, + "last": pylibcudf.stream_compaction.DuplicateKeepOption.KEEP_LAST, + False: pylibcudf.stream_compaction.DuplicateKeepOption.KEEP_NONE, +} @acquire_spill_lock() @@ -126,41 +99,18 @@ def drop_duplicates(list columns, ------- columns with duplicate dropped """ - - cdef vector[size_type] cpp_keys = ( - keys if keys is not None else range(len(columns)) - ) - cdef duplicate_keep_option cpp_keep_option - - if keep == 'first': - cpp_keep_option = duplicate_keep_option.KEEP_FIRST - elif keep == 'last': - cpp_keep_option = duplicate_keep_option.KEEP_LAST - elif keep is False: - cpp_keep_option = duplicate_keep_option.KEEP_NONE - else: + if (keep_option := _keep_options.get(keep)) is None: raise ValueError('keep must be either "first", "last" or False') - # shifting the index number by number of index columns - cdef null_equality cpp_nulls_equal = ( - null_equality.EQUAL - if nulls_are_equal - else null_equality.UNEQUAL - ) - cdef table_view source_table_view = table_view_from_columns(columns) - cdef unique_ptr[table] c_result - - with nogil: - c_result = move( - cpp_stable_distinct( - source_table_view, - cpp_keys, - cpp_keep_option, - cpp_nulls_equal - ) + return columns_from_pylibcudf_table( + pylibcudf.stream_compaction.stable_distinct( + pylibcudf.Table([c.to_pylibcudf(mode="read") for c in columns]), + list(keys if keys is not None else range(len(columns))), + keep_option, + pylibcudf.types.NullEquality.EQUAL + if nulls_are_equal else pylibcudf.types.NullEquality.UNEQUAL, ) - - return columns_from_unique_ptr(move(c_result)) + ) @acquire_spill_lock() @@ -189,40 +139,19 @@ def distinct_indices( -------- drop_duplicates """ - cdef duplicate_keep_option cpp_keep_option - - if keep == 'first': - cpp_keep_option = duplicate_keep_option.KEEP_FIRST - elif keep == 'last': - cpp_keep_option = duplicate_keep_option.KEEP_LAST - elif keep is False: - cpp_keep_option = duplicate_keep_option.KEEP_NONE - else: - raise ValueError('keep must be either "first", "last", or False') + if (keep_option := _keep_options.get(keep)) is None: + raise ValueError('keep must be either "first", "last" or False') - # shifting the index number by number of index columns - cdef null_equality cpp_nulls_equal = ( - null_equality.EQUAL - if nulls_equal - else null_equality.UNEQUAL - ) - cdef nan_equality cpp_nans_equal = ( - nan_equality.ALL_EQUAL - if nans_equal - else nan_equality.UNEQUAL - ) - cdef table_view source = table_view_from_columns(columns) - cdef unique_ptr[column] c_result - with nogil: - c_result = move( - cpp_distinct_indices( - source, - cpp_keep_option, - cpp_nulls_equal, - cpp_nans_equal, - ) + return Column.from_pylibcudf( + pylibcudf.stream_compaction.distinct_indices( + pylibcudf.Table([c.to_pylibcudf(mode="read") for c in columns]), + keep_option, + pylibcudf.types.NullEquality.EQUAL + if nulls_equal else pylibcudf.types.NullEquality.UNEQUAL, + pylibcudf.types.NanEquality.ALL_EQUAL + if nans_equal else pylibcudf.types.NanEquality.UNEQUAL, ) - return Column.from_unique_ptr(move(c_result)) + ) @acquire_spill_lock() @@ -242,24 +171,10 @@ def distinct_count(Column source_column, ignore_nulls=True, nan_as_null=False): ------- Count of number of unique rows in `source_column` """ - - cdef null_policy cpp_null_handling = ( - null_policy.EXCLUDE - if ignore_nulls - else null_policy.INCLUDE - ) - cdef nan_policy cpp_nan_handling = ( - nan_policy.NAN_IS_NULL - if nan_as_null - else nan_policy.NAN_IS_VALID + return pylibcudf.stream_compaction.distinct_count( + source_column.to_pylibcudf(mode="read"), + pylibcudf.types.NullPolicy.EXCLUDE + if ignore_nulls else pylibcudf.types.NullPolicy.INCLUDE, + pylibcudf.types.NanPolicy.NAN_IS_NULL + if nan_as_null else pylibcudf.types.NanPolicy.NAN_IS_VALID, ) - - cdef column_view source_column_view = source_column.view() - with nogil: - count = cpp_distinct_count( - source_column_view, - cpp_null_handling, - cpp_nan_handling - ) - - return count diff --git a/python/cudf/cudf/_lib/utils.pxd b/python/cudf/cudf/_lib/utils.pxd index 653fa8f2b8b..51c69bdcaf9 100644 --- a/python/cudf/cudf/_lib/utils.pxd +++ b/python/cudf/cudf/_lib/utils.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr from libcpp.string cimport string @@ -8,10 +8,9 @@ from cudf._lib.cpp.column.column cimport column_view from cudf._lib.cpp.table.table cimport table, table_view -cdef vector[column_view] make_column_views(object columns) except* -cdef vector[string] get_column_names(object table, object index) except* cdef data_from_unique_ptr( unique_ptr[table] c_tbl, column_names, index_names=*) +cdef data_from_pylibcudf_table(tbl, column_names, index_names=*) cdef data_from_table_view( table_view tv, object owner, object column_names, object index_names=*) cdef table_view table_view_from_columns(columns) except * diff --git a/python/cudf/cudf/_lib/utils.pyx b/python/cudf/cudf/_lib/utils.pyx index 7ba717a0003..896cc55b425 100644 --- a/python/cudf/cudf/_lib/utils.pyx +++ b/python/cudf/cudf/_lib/utils.pyx @@ -7,7 +7,6 @@ import cudf from cython.operator cimport dereference from libcpp.memory cimport unique_ptr -from libcpp.string cimport string from libcpp.utility cimport move from libcpp.vector cimport vector @@ -53,28 +52,6 @@ cdef table_view table_view_from_table(tbl, ignore_index=False) except*: if not ignore_index and tbl._index is not None else tbl._data.columns ) -cdef vector[column_view] make_column_views(object columns): - cdef vector[column_view] views - views.reserve(len(columns)) - for col in columns: - views.push_back(( col).view()) - return views - - -cdef vector[string] get_column_names(object tbl, object index): - cdef vector[string] column_names - if index is not False: - if isinstance(tbl._index, cudf.core.multiindex.MultiIndex): - for idx_name in tbl._index.names: - column_names.push_back(str.encode(idx_name)) - else: - if tbl._index.name is not None: - column_names.push_back(str.encode(tbl._index.name)) - - for col_name in tbl._column_names: - column_names.push_back(str.encode(col_name)) - - return column_names cpdef generate_pandas_metadata(table, index): @@ -261,14 +238,12 @@ cdef columns_from_pylibcudf_table(tbl): return [Column.from_pylibcudf(plc) for plc in tbl.columns()] -cdef data_from_unique_ptr( - unique_ptr[table] c_tbl, column_names, index_names=None -): - """Convert a libcudf table into a dict with an index. +cdef _data_from_columns(columns, column_names, index_names=None): + """Convert a list of columns into a dict with an index. This method is intended to provide the bridge between the columns returned - from calls to libcudf APIs and the cuDF Python Frame objects, which require - named columns and a separate index. + from calls to libcudf or pylibcudf APIs and the cuDF Python Frame objects, which + require named columns and a separate index. Since cuDF Python has an independent representation of a table as a collection of columns, this function simply returns a dict of columns @@ -279,8 +254,8 @@ cdef data_from_unique_ptr( Parameters ---------- - c_tbl : unique_ptr[cudf::table] - The libcudf table whose columns will be extracted + columns : list[Column] + The columns to be extracted column_names : iterable The keys associated with the columns in the output data. index_names : iterable, optional @@ -288,16 +263,7 @@ cdef data_from_unique_ptr( corresponding first set of columns into a (Multi)Index. If this argument is omitted, all columns are assumed to be part of the output table and no index is constructed. - - - Returns - ------- - tuple(Dict[str, Column], Optional[Index]) - A dict of the columns in the output table. """ - - columns = columns_from_unique_ptr(move(c_tbl)) - # First construct the index, if any index = ( # TODO: For performance, the _from_data methods of Frame types assume @@ -325,6 +291,24 @@ cdef data_from_unique_ptr( } return data, index + +cdef data_from_unique_ptr( + unique_ptr[table] c_tbl, column_names, index_names=None +): + return _data_from_columns( + columns_from_unique_ptr(move(c_tbl)), + column_names, + index_names + ) + + +cdef data_from_pylibcudf_table(tbl, column_names, index_names=None): + return _data_from_columns( + columns_from_pylibcudf_table(tbl), + column_names, + index_names + ) + cdef columns_from_table_view( table_view tv, object owners, From 3ba63c3c3cb72950adc4c9699fcfa1a72796a041 Mon Sep 17 00:00:00 2001 From: Yunsong Wang Date: Thu, 15 Feb 2024 10:50:54 -0800 Subject: [PATCH 251/384] Update cudf for compatibility with the latest cuco (#14849) Depends on https://github.com/rapidsai/rapids-cmake/pull/526 CMakes changes will be reverted once https://github.com/rapidsai/rapids-cmake/pull/526 is merged. This PR updates libcudf to make it compatible with the latest cuco. Authors: - Yunsong Wang (https://github.com/PointKernel) Approvers: - David Wendt (https://github.com/davidwendt) - Vyas Ramasubramani (https://github.com/vyasr) - Karthikeyan (https://github.com/karthikeyann) URL: https://github.com/rapidsai/cudf/pull/14849 --- .../cudf/detail/hash_reduce_by_row.cuh | 4 +- cpp/include/cudf/detail/join.hpp | 2 +- cpp/src/io/json/json_tree.cu | 32 ++++++++------- cpp/src/io/orc/orc_gpu.hpp | 4 +- cpp/src/io/parquet/parquet_gpu.cuh | 4 +- cpp/src/join/join_common_utils.hpp | 13 ++++--- cpp/src/search/contains_table.cu | 19 ++++----- cpp/src/stream_compaction/distinct_count.cu | 15 +++---- .../stream_compaction_common.hpp | 4 +- cpp/src/text/bpe/byte_pair_encoding.cuh | 39 ++++++++++--------- cpp/src/text/bpe/load_merge_pairs.cu | 4 ++ cpp/src/text/vocabulary_tokenize.cu | 22 ++++++----- 12 files changed, 89 insertions(+), 73 deletions(-) diff --git a/cpp/include/cudf/detail/hash_reduce_by_row.cuh b/cpp/include/cudf/detail/hash_reduce_by_row.cuh index 006cb5142c9..a740b5c4e93 100644 --- a/cpp/include/cudf/detail/hash_reduce_by_row.cuh +++ b/cpp/include/cudf/detail/hash_reduce_by_row.cuh @@ -31,8 +31,8 @@ namespace cudf::detail { -using hash_map_type = - cuco::static_map; +using hash_map_type = cuco::legacy:: + static_map; /** * @brief The base struct for customized reduction functor to perform reduce-by-key with keys are diff --git a/cpp/include/cudf/detail/join.hpp b/cpp/include/cudf/detail/join.hpp index ad6269dae30..27d14874bce 100644 --- a/cpp/include/cudf/detail/join.hpp +++ b/cpp/include/cudf/detail/join.hpp @@ -62,7 +62,7 @@ struct hash_join { cudf::size_type, cuda::thread_scope_device, cudf::detail::cuco_allocator, - cuco::double_hashing>; + cuco::legacy::double_hashing>; hash_join() = delete; ~hash_join() = default; diff --git a/cpp/src/io/json/json_tree.cu b/cpp/src/io/json/json_tree.cu index db9daf28c06..148aeb5ec7a 100644 --- a/cpp/src/io/json/json_tree.cu +++ b/cpp/src/io/json/json_tree.cu @@ -548,13 +548,14 @@ rmm::device_uvector hash_node_type_with_field_name(device_span{d_hasher}, - cudf::detail::cuco_allocator{stream}, - stream.value()}; + cuco::static_set{cuco::extent{compute_hash_table_size(num_fields, 40)}, // 40% occupancy + cuco::empty_key{empty_node_index_sentinel}, + d_equal, + cuco::linear_probing<1, hasher_type>{d_hasher}, + {}, + {}, + cudf::detail::cuco_allocator{stream}, + stream.value()}; key_set.insert_if_async(iter, iter + num_nodes, thrust::counting_iterator(0), // stencil @@ -562,7 +563,7 @@ rmm::device_uvector hash_node_type_with_field_name(device_span size_type { + [key_set = key_set.ref(cuco::op::find)] __device__(auto node_id) -> size_type { auto const it = key_set.find(node_id); return (it == key_set.end()) ? size_type{0} : *it; }; @@ -735,13 +736,14 @@ std::pair, rmm::device_uvector> hash_n constexpr size_type empty_node_index_sentinel = -1; using hasher_type = decltype(d_hashed_cache); - auto key_set = cuco::experimental::static_set{ - cuco::experimental::extent{compute_hash_table_size(num_nodes)}, - cuco::empty_key{empty_node_index_sentinel}, - d_equal, - cuco::experimental::linear_probing<1, hasher_type>{d_hashed_cache}, - cudf::detail::cuco_allocator{stream}, - stream.value()}; + auto key_set = cuco::static_set{cuco::extent{compute_hash_table_size(num_nodes)}, + cuco::empty_key{empty_node_index_sentinel}, + d_equal, + cuco::linear_probing<1, hasher_type>{d_hashed_cache}, + {}, + {}, + cudf::detail::cuco_allocator{stream}, + stream.value()}; // insert and convert node ids to unique set ids auto nodes_itr = thrust::make_counting_iterator(0); diff --git a/cpp/src/io/orc/orc_gpu.hpp b/cpp/src/io/orc/orc_gpu.hpp index 243704b65d4..c2570d71c24 100644 --- a/cpp/src/io/orc/orc_gpu.hpp +++ b/cpp/src/io/orc/orc_gpu.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,7 +44,7 @@ using cudf::detail::host_2dspan; auto constexpr KEY_SENTINEL = size_type{-1}; auto constexpr VALUE_SENTINEL = size_type{-1}; -using map_type = cuco::static_map; +using map_type = cuco::legacy::static_map; /** * @brief The alias of `map_type::pair_atomic_type` class. diff --git a/cpp/src/io/parquet/parquet_gpu.cuh b/cpp/src/io/parquet/parquet_gpu.cuh index 10e12ebb782..e3c44c78898 100644 --- a/cpp/src/io/parquet/parquet_gpu.cuh +++ b/cpp/src/io/parquet/parquet_gpu.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,7 +28,7 @@ namespace cudf::io::parquet::detail { auto constexpr KEY_SENTINEL = size_type{-1}; auto constexpr VALUE_SENTINEL = size_type{-1}; -using map_type = cuco::static_map; +using map_type = cuco::legacy::static_map; /** * @brief The alias of `map_type::pair_atomic_type` class. diff --git a/cpp/src/join/join_common_utils.hpp b/cpp/src/join/join_common_utils.hpp index b88a4fdef58..4d361b23502 100644 --- a/cpp/src/join/join_common_utils.hpp +++ b/cpp/src/join/join_common_utils.hpp @@ -45,13 +45,14 @@ using multimap_type = cudf::hash_join::impl_type::map_type; // Multimap type used for mixed joins. TODO: This is a temporary alias used // until the mixed joins are converted to using CGs properly. Right now it's // using a cooperative group of size 1. -using mixed_multimap_type = cuco::static_multimap>; +using mixed_multimap_type = + cuco::static_multimap>; -using semi_map_type = cuco:: +using semi_map_type = cuco::legacy:: static_map; using row_hash_legacy = diff --git a/cpp/src/search/contains_table.cu b/cpp/src/search/contains_table.cu index ce069abcb78..e1d0fab6025 100644 --- a/cpp/src/search/contains_table.cu +++ b/cpp/src/search/contains_table.cu @@ -158,9 +158,9 @@ void dispatch_nan_comparator( // Distinguish probing scheme CG sizes between nested and flat types for better performance auto const probing_scheme = [&]() { if constexpr (HasNested) { - return cuco::experimental::linear_probing<4, Hasher>{d_hasher}; + return cuco::linear_probing<4, Hasher>{d_hasher}; } else { - return cuco::experimental::linear_probing<1, Hasher>{d_hasher}; + return cuco::linear_probing<1, Hasher>{d_hasher}; } }(); @@ -228,13 +228,14 @@ rmm::device_uvector contains(table_view const& haystack, [&](auto const& d_self_equal, auto const& d_two_table_equal, auto const& probing_scheme) { auto const d_equal = comparator_adapter{d_self_equal, d_two_table_equal}; - auto set = cuco::experimental::static_set{ - cuco::experimental::extent{compute_hash_table_size(haystack.num_rows())}, - cuco::empty_key{lhs_index_type{-1}}, - d_equal, - probing_scheme, - cudf::detail::cuco_allocator{stream}, - stream.value()}; + auto set = cuco::static_set{cuco::extent{compute_hash_table_size(haystack.num_rows())}, + cuco::empty_key{lhs_index_type{-1}}, + d_equal, + probing_scheme, + {}, + {}, + cudf::detail::cuco_allocator{stream}, + stream.value()}; if (haystack_has_nulls && compare_nulls == null_equality::UNEQUAL) { auto const bitmask_buffer_and_ptr = build_row_bitmask(haystack, stream); diff --git a/cpp/src/stream_compaction/distinct_count.cu b/cpp/src/stream_compaction/distinct_count.cu index 507bad777eb..3ec1be42bfe 100644 --- a/cpp/src/stream_compaction/distinct_count.cu +++ b/cpp/src/stream_compaction/distinct_count.cu @@ -141,13 +141,14 @@ cudf::size_type distinct_count(table_view const& keys, auto const comparator_helper = [&](auto const row_equal) { using hasher_type = decltype(hash_key); - auto key_set = - cuco::experimental::static_set{cuco::experimental::extent{compute_hash_table_size(num_rows)}, - cuco::empty_key{-1}, - row_equal, - cuco::experimental::linear_probing<1, hasher_type>{hash_key}, - cudf::detail::cuco_allocator{stream}, - stream.value()}; + auto key_set = cuco::static_set{cuco::extent{compute_hash_table_size(num_rows)}, + cuco::empty_key{-1}, + row_equal, + cuco::linear_probing<1, hasher_type>{hash_key}, + {}, + {}, + cudf::detail::cuco_allocator{stream}, + stream.value()}; auto const iter = thrust::counting_iterator(0); // when nulls are equal, we skip hashing any row that has a null diff --git a/cpp/src/stream_compaction/stream_compaction_common.hpp b/cpp/src/stream_compaction/stream_compaction_common.hpp index ceb62d1d059..dd7d76168d9 100644 --- a/cpp/src/stream_compaction/stream_compaction_common.hpp +++ b/cpp/src/stream_compaction/stream_compaction_common.hpp @@ -27,8 +27,8 @@ namespace cudf { namespace detail { -using hash_map_type = - cuco::static_map; +using hash_map_type = cuco::legacy:: + static_map; } // namespace detail } // namespace cudf diff --git a/cpp/src/text/bpe/byte_pair_encoding.cuh b/cpp/src/text/bpe/byte_pair_encoding.cuh index 1a3f8eadea0..02a8a6c4d0a 100644 --- a/cpp/src/text/bpe/byte_pair_encoding.cuh +++ b/cpp/src/text/bpe/byte_pair_encoding.cuh @@ -44,6 +44,7 @@ namespace detail { using string_hasher_type = cudf::hashing::detail::MurmurHash3_x86_32; using hash_value_type = string_hasher_type::result_type; using merge_pair_type = thrust::pair; +using cuco_storage = cuco::storage<1>; /** * @brief Hasher function used for building and using the cuco static-map @@ -98,15 +99,16 @@ struct bpe_equal { } }; -using bpe_probe_scheme = cuco::experimental::linear_probing<1, bpe_hasher>; +using bpe_probe_scheme = cuco::linear_probing<1, bpe_hasher>; -using merge_pairs_map_type = cuco::experimental::static_map, - cuda::thread_scope_device, - bpe_equal, - bpe_probe_scheme, - cudf::detail::cuco_allocator>; +using merge_pairs_map_type = cuco::static_map, + cuda::thread_scope_device, + bpe_equal, + bpe_probe_scheme, + cudf::detail::cuco_allocator, + cuco_storage>; /** * @brief Hasher function used for building and using the cuco static-map @@ -155,15 +157,16 @@ struct mp_equal { } }; -using mp_probe_scheme = cuco::experimental::linear_probing<1, mp_hasher>; +using mp_probe_scheme = cuco::linear_probing<1, mp_hasher>; -using mp_table_map_type = cuco::experimental::static_map, - cuda::thread_scope_device, - mp_equal, - mp_probe_scheme, - cudf::detail::cuco_allocator>; +using mp_table_map_type = cuco::static_map, + cuda::thread_scope_device, + mp_equal, + mp_probe_scheme, + cudf::detail::cuco_allocator, + cuco_storage>; } // namespace detail @@ -185,8 +188,8 @@ struct bpe_merge_pairs::bpe_merge_pairs_impl { std::unique_ptr&& mp_table_map); auto const get_merge_pairs() const { return *d_merge_pairs; } - auto get_merge_pairs_ref() const { return merge_pairs_map->ref(cuco::experimental::op::find); } - auto get_mp_table_ref() const { return mp_table_map->ref(cuco::experimental::op::find); } + auto get_merge_pairs_ref() const { return merge_pairs_map->ref(cuco::op::find); } + auto get_mp_table_ref() const { return mp_table_map->ref(cuco::op::find); } }; } // namespace nvtext diff --git a/cpp/src/text/bpe/load_merge_pairs.cu b/cpp/src/text/bpe/load_merge_pairs.cu index 3b630886b3e..8da2d745966 100644 --- a/cpp/src/text/bpe/load_merge_pairs.cu +++ b/cpp/src/text/bpe/load_merge_pairs.cu @@ -48,6 +48,8 @@ std::unique_ptr initialize_merge_pairs_map( cuco::empty_value{-1}, bpe_equal{input}, bpe_probe_scheme{bpe_hasher{input}}, + cuco::thread_scope_device, + cuco_storage{}, cudf::detail::cuco_allocator{stream}, stream.value()); @@ -69,6 +71,8 @@ std::unique_ptr initialize_mp_table_map( cuco::empty_value{-1}, mp_equal{input}, mp_probe_scheme{mp_hasher{input}}, + cuco::thread_scope_device, + cuco_storage{}, cudf::detail::cuco_allocator{stream}, stream.value()); diff --git a/cpp/src/text/vocabulary_tokenize.cu b/cpp/src/text/vocabulary_tokenize.cu index c6e90c6fcaa..b6991e534bf 100644 --- a/cpp/src/text/vocabulary_tokenize.cu +++ b/cpp/src/text/vocabulary_tokenize.cu @@ -93,14 +93,16 @@ struct vocab_equal { } }; -using probe_scheme = cuco::experimental::linear_probing<1, vocab_hasher>; -using vocabulary_map_type = cuco::experimental::static_map, - cuda::thread_scope_device, - vocab_equal, - probe_scheme, - cudf::detail::cuco_allocator>; +using probe_scheme = cuco::linear_probing<1, vocab_hasher>; +using cuco_storage = cuco::storage<1>; +using vocabulary_map_type = cuco::static_map, + cuda::thread_scope_device, + vocab_equal, + probe_scheme, + cudf::detail::cuco_allocator, + cuco_storage>; } // namespace } // namespace detail @@ -115,7 +117,7 @@ struct tokenize_vocabulary::tokenize_vocabulary_impl { col_device_view const d_vocabulary; std::unique_ptr vocabulary_map; - auto get_map_ref() const { return vocabulary_map->ref(cuco::experimental::op::find); } + auto get_map_ref() const { return vocabulary_map->ref(cuco::op::find); } tokenize_vocabulary_impl(std::unique_ptr&& vocab, col_device_view&& d_vocab, @@ -149,6 +151,8 @@ tokenize_vocabulary::tokenize_vocabulary(cudf::strings_column_view const& input, cuco::empty_value{-1}, detail::vocab_equal{*d_vocabulary}, detail::probe_scheme{detail::vocab_hasher{*d_vocabulary}}, + cuco::thread_scope_device, + detail::cuco_storage{}, cudf::detail::cuco_allocator{stream}, stream.value()); From 3dbdb149e6b886c29406bbad2b00bf49f50fa605 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 15 Feb 2024 11:06:24 -1000 Subject: [PATCH 252/384] Avoid chained indexing in test_indexing for pandas 2.2 (#15045) Chained indexing raises a `FutureWarning` in pandas 2.2. Since this test doesn't look to specifically test that, refactoring the test to avoid that Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15045 --- python/cudf/cudf/tests/test_indexing.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/python/cudf/cudf/tests/test_indexing.py b/python/cudf/cudf/tests/test_indexing.py index 1cdaa3c52a7..0e6de3d3b4a 100644 --- a/python/cudf/cudf/tests/test_indexing.py +++ b/python/cudf/cudf/tests/test_indexing.py @@ -1242,13 +1242,18 @@ def test_out_of_bounds_indexing(): lambda: psr.__setitem__([0, 1, -4], 2), lambda: gsr.__setitem__([0, 1, -4], 2), ) + + +def test_out_of_bounds_indexing_empty(): + psr = pd.Series(dtype="int64") + gsr = cudf.from_pandas(psr) assert_exceptions_equal( - lambda: psr[4:6].iloc.__setitem__(-1, 2), - lambda: gsr[4:6].iloc.__setitem__(-1, 2), + lambda: psr.iloc.__setitem__(-1, 2), + lambda: gsr.iloc.__setitem__(-1, 2), ) assert_exceptions_equal( - lambda: psr[4:6].iloc.__setitem__(1, 2), - lambda: gsr[4:6].iloc.__setitem__(1, 2), + lambda: psr.iloc.__setitem__(1, 2), + lambda: gsr.iloc.__setitem__(1, 2), ) From 0f694d32bd57121521e3fa7cd1609bca622b6f99 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 15 Feb 2024 12:17:17 -1000 Subject: [PATCH 253/384] Avoid incompatible value type setting in test_rolling for pandas 2.2 (#15050) Related to https://pandas.pydata.org/pdeps/0006-ban-upcasting.html Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Bradley Dice (https://github.com/bdice) - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15050 --- python/cudf/cudf/tests/test_rolling.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/python/cudf/cudf/tests/test_rolling.py b/python/cudf/cudf/tests/test_rolling.py index 9c3c9d1082c..cbd60b8945a 100644 --- a/python/cudf/cudf/tests/test_rolling.py +++ b/python/cudf/cudf/tests/test_rolling.py @@ -90,16 +90,17 @@ def test_rolling_dataframe_basic(data, agg, nulls, center): pdf = pd.DataFrame(data) if len(pdf) > 0: - for col_idx in range(len(pdf.columns)): - if nulls == "one": - p = rng.integers(0, len(data)) - pdf.iloc[p, col_idx] = np.nan - elif nulls == "some": - p1, p2 = rng.integers(0, len(data), (2,)) - pdf.iloc[p1, col_idx] = np.nan - pdf.iloc[p2, col_idx] = np.nan - elif nulls == "all": - pdf.iloc[:, col_idx] = np.nan + if nulls == "all": + pdf = pd.DataFrame(np.nan, columns=pdf.columns, index=pdf.index) + else: + for col_idx in range(len(pdf.columns)): + if nulls == "one": + p = rng.integers(0, len(data)) + pdf.iloc[p, col_idx] = np.nan + elif nulls == "some": + p1, p2 = rng.integers(0, len(data), (2,)) + pdf.iloc[p1, col_idx] = np.nan + pdf.iloc[p2, col_idx] = np.nan gdf = cudf.from_pandas(pdf) for window_size in range(1, len(data) + 1): From aa9d4846e80fad133e7af19aba99cefa04cb8b7b Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 15 Feb 2024 12:17:46 -1000 Subject: [PATCH 254/384] Align concat Series name behavior in pandas 2.2 (#15032) Fixed in pandas by https://github.com/pandas-dev/pandas/pull/56365 Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15032 --- python/cudf/cudf/core/reshape.py | 16 ++--- python/cudf/cudf/tests/test_concat.py | 97 ++++++++++++++++++--------- 2 files changed, 73 insertions(+), 40 deletions(-) diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index 656db855253..2ef39e9357d 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -102,17 +102,17 @@ def _normalize_series_and_dataframe(objs, axis): """Convert any cudf.Series objects in objs to DataFrames in place.""" # Default to naming series by a numerical id if they are not named. sr_name = 0 - for idx, o in enumerate(objs): - if isinstance(o, cudf.Series): - if axis == 1: - name = o.name - if name is None: + for idx, obj in enumerate(objs): + if isinstance(obj, cudf.Series): + name = obj.name + if name is None: + if axis == 0: + name = 0 + else: name = sr_name sr_name += 1 - else: - name = sr_name - objs[idx] = o.to_frame(name=name) + objs[idx] = obj.to_frame(name=name) def concat(objs, axis=0, join="outer", ignore_index=False, sort=None): diff --git a/python/cudf/cudf/tests/test_concat.py b/python/cudf/cudf/tests/test_concat.py index 01c37005271..6e61675ef92 100644 --- a/python/cudf/cudf/tests/test_concat.py +++ b/python/cudf/cudf/tests/test_concat.py @@ -10,7 +10,7 @@ import cudf from cudf.api.types import _is_categorical_dtype -from cudf.core._compat import PANDAS_GE_200 +from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_220 from cudf.core.dtypes import Decimal32Dtype, Decimal64Dtype, Decimal128Dtype from cudf.testing._utils import ( assert_eq, @@ -459,42 +459,75 @@ def test_concat_mixed_input(): [pd.Series([1, 2, 3]), pd.DataFrame({"a": []})], [pd.Series([], dtype="float64"), pd.DataFrame({"a": []})], [pd.Series([], dtype="float64"), pd.DataFrame({"a": [1, 2]})], - [pd.Series([1, 2, 3.0, 1.2], name="abc"), pd.DataFrame({"a": [1, 2]})], - [ - pd.Series( - [1, 2, 3.0, 1.2], name="abc", index=[100, 110, 120, 130] - ), - pd.DataFrame({"a": [1, 2]}), - ], - [ - pd.Series( - [1, 2, 3.0, 1.2], name="abc", index=["a", "b", "c", "d"] + pytest.param( + [ + pd.Series([1, 2, 3.0, 1.2], name="abc"), + pd.DataFrame({"a": [1, 2]}), + ], + marks=pytest.mark.xfail( + not PANDAS_GE_220, + reason="https://github.com/pandas-dev/pandas/pull/56365", ), - pd.DataFrame({"a": [1, 2]}, index=["a", "b"]), - ], - [ - pd.Series( - [1, 2, 3.0, 1.2, 8, 100], - name="New name", - index=["a", "b", "c", "d", "e", "f"], + ), + pytest.param( + [ + pd.Series( + [1, 2, 3.0, 1.2], name="abc", index=[100, 110, 120, 130] + ), + pd.DataFrame({"a": [1, 2]}), + ], + marks=pytest.mark.xfail( + not PANDAS_GE_220, + reason="https://github.com/pandas-dev/pandas/pull/56365", ), - pd.DataFrame( - {"a": [1, 2, 4, 10, 11, 12]}, - index=["a", "b", "c", "d", "e", "f"], + ), + pytest.param( + [ + pd.Series( + [1, 2, 3.0, 1.2], name="abc", index=["a", "b", "c", "d"] + ), + pd.DataFrame({"a": [1, 2]}, index=["a", "b"]), + ], + marks=pytest.mark.xfail( + not PANDAS_GE_220, + reason="https://github.com/pandas-dev/pandas/pull/56365", ), - ], - [ - pd.Series( - [1, 2, 3.0, 1.2, 8, 100], - name="New name", - index=["a", "b", "c", "d", "e", "f"], + ), + pytest.param( + [ + pd.Series( + [1, 2, 3.0, 1.2, 8, 100], + name="New name", + index=["a", "b", "c", "d", "e", "f"], + ), + pd.DataFrame( + {"a": [1, 2, 4, 10, 11, 12]}, + index=["a", "b", "c", "d", "e", "f"], + ), + ], + marks=pytest.mark.xfail( + not PANDAS_GE_220, + reason="https://github.com/pandas-dev/pandas/pull/56365", ), - pd.DataFrame( - {"a": [1, 2, 4, 10, 11, 12]}, - index=["a", "b", "c", "d", "e", "f"], + ), + pytest.param( + [ + pd.Series( + [1, 2, 3.0, 1.2, 8, 100], + name="New name", + index=["a", "b", "c", "d", "e", "f"], + ), + pd.DataFrame( + {"a": [1, 2, 4, 10, 11, 12]}, + index=["a", "b", "c", "d", "e", "f"], + ), + ] + * 7, + marks=pytest.mark.xfail( + not PANDAS_GE_220, + reason="https://github.com/pandas-dev/pandas/pull/56365", ), - ] - * 7, + ), ], ) def test_concat_series_dataframe_input(objs): From 45614e2e372ea420700a9cbe12cf25f8322ab39d Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Fri, 16 Feb 2024 09:15:32 -0500 Subject: [PATCH 255/384] Remove unneeded calls to create_chars_child_column utility (#14997) Removes unneeded calls to `cudf::strings::detail::create_chars_child_column`. This includes all calls except `make_strings_children` which will be modified in a follow-on PR. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Yunsong Wang (https://github.com/PointKernel) URL: https://github.com/rapidsai/cudf/pull/14997 --- cpp/include/cudf/strings/detail/gather.cuh | 26 +++---- .../detail/strings_column_factories.cuh | 72 +++++++++---------- cpp/src/io/csv/durations.cu | 24 +++---- 3 files changed, 60 insertions(+), 62 deletions(-) diff --git a/cpp/include/cudf/strings/detail/gather.cuh b/cpp/include/cudf/strings/detail/gather.cuh index 442155380a2..7092d114009 100644 --- a/cpp/include/cudf/strings/detail/gather.cuh +++ b/cpp/include/cudf/strings/detail/gather.cuh @@ -222,19 +222,19 @@ CUDF_KERNEL void gather_chars_fn_char_parallel(StringIterator strings_begin, * @return New chars column fit for a strings column. */ template -std::unique_ptr gather_chars(StringIterator strings_begin, - MapIterator map_begin, - MapIterator map_end, - cudf::detail::input_offsetalator const offsets, - size_type chars_bytes, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) +rmm::device_uvector gather_chars(StringIterator strings_begin, + MapIterator map_begin, + MapIterator map_end, + cudf::detail::input_offsetalator const offsets, + size_type chars_bytes, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) { auto const output_count = std::distance(map_begin, map_end); - if (output_count == 0) return make_empty_column(type_id::INT8); + if (output_count == 0) return rmm::device_uvector(0, stream, mr); - auto chars_column = create_chars_child_column(chars_bytes, stream, mr); - auto const d_chars = chars_column->mutable_view().template data(); + auto chars_data = rmm::device_uvector(chars_bytes, stream, mr); + auto d_chars = chars_data.data(); constexpr int warps_per_threadblock = 4; // String parallel strategy will be used if average string length is above this threshold. @@ -260,7 +260,7 @@ std::unique_ptr gather_chars(StringIterator strings_begin, stream.value()>>>(strings_begin, d_chars, offsets, map_begin, output_count); } - return chars_column; + return chars_data; } /** @@ -316,12 +316,12 @@ std::unique_ptr gather(strings_column_view const& strings, // build chars column auto const offsets_view = cudf::detail::offsetalator_factory::make_input_iterator(out_offsets_column->view()); - auto out_chars_column = gather_chars( + auto out_chars_data = gather_chars( d_strings->begin(), begin, end, offsets_view, total_bytes, stream, mr); return make_strings_column(output_count, std::move(out_offsets_column), - std::move(out_chars_column->release().data.release()[0]), + out_chars_data.release(), 0, // caller sets these rmm::device_buffer{}); } diff --git a/cpp/include/cudf/strings/detail/strings_column_factories.cuh b/cpp/include/cudf/strings/detail/strings_column_factories.cuh index fcbdfa619f4..0adf6e362be 100644 --- a/cpp/include/cudf/strings/detail/strings_column_factories.cuh +++ b/cpp/include/cudf/strings/detail/strings_column_factories.cuh @@ -98,46 +98,44 @@ std::unique_ptr make_strings_column(IndexPairIterator begin, (null_count > 0) ? std::move(new_nulls.first) : rmm::device_buffer{0, stream, mr}; // build chars column - std::unique_ptr chars_column = - [offsets_view, bytes = bytes, begin, strings_count, null_count, stream, mr] { - auto const avg_bytes_per_row = bytes / std::max(strings_count - null_count, 1); - // use a character-parallel kernel for long string lengths - if (avg_bytes_per_row > FACTORY_BYTES_PER_ROW_THRESHOLD) { - auto const d_offsets = - cudf::detail::offsetalator_factory::make_input_iterator(offsets_view); - auto const str_begin = thrust::make_transform_iterator( - begin, cuda::proclaim_return_type([] __device__(auto ip) { - return string_view{ip.first, ip.second}; - })); - - return gather_chars(str_begin, - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(strings_count), - d_offsets, - bytes, - stream, - mr); - } else { - // this approach is 2-3x faster for a large number of smaller string lengths - auto chars_column = create_chars_child_column(bytes, stream, mr); - auto d_chars = chars_column->mutable_view().template data(); - auto copy_chars = [d_chars] __device__(auto item) { - string_index_pair const str = thrust::get<0>(item); - size_type const offset = thrust::get<1>(item); - if (str.first != nullptr) memcpy(d_chars + offset, str.first, str.second); - }; - thrust::for_each_n(rmm::exec_policy(stream), - thrust::make_zip_iterator( - thrust::make_tuple(begin, offsets_view.template begin())), - strings_count, - copy_chars); - return chars_column; - } - }(); + auto chars_data = [offsets_view, bytes = bytes, begin, strings_count, null_count, stream, mr] { + auto const avg_bytes_per_row = bytes / std::max(strings_count - null_count, 1); + // use a character-parallel kernel for long string lengths + if (avg_bytes_per_row > FACTORY_BYTES_PER_ROW_THRESHOLD) { + auto const d_offsets = cudf::detail::offsetalator_factory::make_input_iterator(offsets_view); + auto const str_begin = thrust::make_transform_iterator( + begin, cuda::proclaim_return_type([] __device__(auto ip) { + return string_view{ip.first, ip.second}; + })); + + return gather_chars(str_begin, + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(strings_count), + d_offsets, + bytes, + stream, + mr); + } else { + // this approach is 2-3x faster for a large number of smaller string lengths + auto chars_data = rmm::device_uvector(bytes, stream, mr); + auto d_chars = chars_data.data(); + auto copy_chars = [d_chars] __device__(auto item) { + string_index_pair const str = thrust::get<0>(item); + size_type const offset = thrust::get<1>(item); + if (str.first != nullptr) memcpy(d_chars + offset, str.first, str.second); + }; + thrust::for_each_n(rmm::exec_policy(stream), + thrust::make_zip_iterator( + thrust::make_tuple(begin, offsets_view.template begin())), + strings_count, + copy_chars); + return chars_data; + } + }(); return make_strings_column(strings_count, std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars_data.release(), null_count, std::move(null_mask)); } diff --git a/cpp/src/io/csv/durations.cu b/cpp/src/io/csv/durations.cu index f4d32edac89..76b1b46dc61 100644 --- a/cpp/src/io/csv/durations.cu +++ b/cpp/src/io/csv/durations.cu @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -88,12 +89,12 @@ struct duration_to_string_size_fn { template struct duration_to_string_fn : public duration_to_string_size_fn { - int32_t const* d_offsets; + cudf::detail::input_offsetalator d_offsets; char* d_chars; using duration_to_string_size_fn::d_durations; duration_to_string_fn(column_device_view const d_durations, - int32_t const* d_offsets, + cudf::detail::input_offsetalator d_offsets, char* d_chars) : duration_to_string_size_fn{d_durations}, d_offsets(d_offsets), d_chars(d_chars) { @@ -181,28 +182,27 @@ struct dispatch_from_durations_fn { // copy null mask rmm::device_buffer null_mask = cudf::detail::copy_bitmask(durations, stream, mr); + // build offsets column - auto offsets_transformer_itr = thrust::make_transform_iterator( - thrust::make_counting_iterator(0), duration_to_string_size_fn{d_column}); - auto [offsets_column, chars_bytes] = cudf::detail::make_offsets_child_column( + auto offsets_transformer_itr = + cudf::detail::make_counting_transform_iterator(0, duration_to_string_size_fn{d_column}); + auto [offsets_column, chars_bytes] = cudf::strings::detail::make_offsets_child_column( offsets_transformer_itr, offsets_transformer_itr + strings_count, stream, mr); - auto offsets_view = offsets_column->view(); - auto d_new_offsets = offsets_view.template data(); + auto d_new_offsets = + cudf::detail::offsetalator_factory::make_input_iterator(offsets_column->view()); // build chars column - auto chars_column = strings::detail::create_chars_child_column(chars_bytes, stream, mr); - auto chars_view = chars_column->mutable_view(); - auto d_chars = chars_view.template data(); + auto chars_data = rmm::device_uvector(chars_bytes, stream, mr); + auto d_chars = chars_data.data(); thrust::for_each_n(rmm::exec_policy(stream), thrust::make_counting_iterator(0), strings_count, duration_to_string_fn{d_column, d_new_offsets, d_chars}); - // return make_strings_column(strings_count, std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars_data.release(), durations.null_count(), std::move(null_mask)); } From 6a9cefdedd0b17a229cc2227c8604e49e7c65d12 Mon Sep 17 00:00:00 2001 From: Chong Gao Date: Mon, 19 Feb 2024 23:35:53 +0800 Subject: [PATCH 256/384] Enable sanitizer check for a test case testORCReadAndWriteForDecimal128 (#14897) Enable sanitizer check for test case TableTest#testORCReadAndWriteForDecimal128 closes https://github.com/NVIDIA/spark-rapids-jni/issues/1338 Authors: - Chong Gao (https://github.com/res-life) Approvers: - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/14897 --- java/src/test/java/ai/rapids/cudf/TableTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/java/src/test/java/ai/rapids/cudf/TableTest.java b/java/src/test/java/ai/rapids/cudf/TableTest.java index 76f127eae77..e270c4a5183 100644 --- a/java/src/test/java/ai/rapids/cudf/TableTest.java +++ b/java/src/test/java/ai/rapids/cudf/TableTest.java @@ -9192,9 +9192,6 @@ void testORCWriteToFileWithColNames() throws IOException { } } - // https://github.com/NVIDIA/spark-rapids-jni/issues/1338 - // Need to remove this tag if #1338 is fixed. - @Tag("noSanitizer") @Test void testORCReadAndWriteForDecimal128() throws IOException { File tempFile = File.createTempFile("test", ".orc"); From 8c20d2ab1896a6d09ccfd607e32457e5acec0e1f Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 19 Feb 2024 12:58:37 -1000 Subject: [PATCH 257/384] Add condition for test_groupby_nulls_basic in pandas 2.2 (#15072) This case for some reason doesn't raise a FutureWarning in pandas in 2.2 while it does in pandas 2.1. It's likely a won't-fix so adding a condition Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Michael Wang (https://github.com/isVoid) URL: https://github.com/rapidsai/cudf/pull/15072 --- python/cudf/cudf/tests/test_groupby.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index 6514053afa7..06fd8f2ea79 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -1433,7 +1433,7 @@ def test_groupby_nulls_basic(agg): # TODO: fillna() used here since we don't follow # Pandas' null semantics. Should we change it? - with expect_warning_if(agg in {"idxmax", "idxmin"}): + with expect_warning_if(agg in {"idxmax", "idxmin"} and not PANDAS_GE_220): assert_groupby_results_equal( getattr(pdf.groupby("a"), agg)().fillna(0), getattr(gdf.groupby("a"), agg)().fillna(0 if agg != "prod" else 1), From 634b4cbb6a7dccff86cec4b21d7a39e66d210941 Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Mon, 19 Feb 2024 17:01:50 -0800 Subject: [PATCH 258/384] Fix `is_device_write_preferred` in `void_sink` and `user_sink_wrapper` (#15064) Addresses a few issues in `data_sink` classes to avoid D2H copies in writers when using a `void_sink`. Provide an `is_device_write_preferred` implementation to always prefer device writes. Implement `is_device_write_preferred` in `user_sink_wrapper` that forwards the call to the wrapped object. Use the `cudf::io::void_sink` in benchmarks instead of the local version, which is not fully implemented. Authors: - Vukasin Milovanovic (https://github.com/vuule) Approvers: - Nghia Truong (https://github.com/ttnghia) - Mark Harris (https://github.com/harrism) URL: https://github.com/rapidsai/cudf/pull/15064 --- cpp/benchmarks/io/cuio_common.cpp | 9 +++++---- cpp/benchmarks/io/cuio_common.hpp | 13 ++----------- cpp/src/io/utilities/data_sink.cpp | 7 +++++++ 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/cpp/benchmarks/io/cuio_common.cpp b/cpp/benchmarks/io/cuio_common.cpp index 943b329a364..b5318b45eb4 100644 --- a/cpp/benchmarks/io/cuio_common.cpp +++ b/cpp/benchmarks/io/cuio_common.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,7 +41,8 @@ std::string random_file_in_dir(std::string const& dir_path) cuio_source_sink_pair::cuio_source_sink_pair(io_type type) : type{type}, d_buffer{0, cudf::get_default_stream()}, - file_name{random_file_in_dir(tmpdir.path())} + file_name{random_file_in_dir(tmpdir.path())}, + void_sink{cudf::io::data_sink::create()} { } @@ -67,7 +68,7 @@ cudf::io::source_info cuio_source_sink_pair::make_source_info() cudf::io::sink_info cuio_source_sink_pair::make_sink_info() { switch (type) { - case io_type::VOID: return cudf::io::sink_info(&void_sink); + case io_type::VOID: return cudf::io::sink_info(void_sink.get()); case io_type::FILEPATH: return cudf::io::sink_info(file_name); case io_type::HOST_BUFFER: [[fallthrough]]; case io_type::DEVICE_BUFFER: return cudf::io::sink_info(&h_buffer); @@ -78,7 +79,7 @@ cudf::io::sink_info cuio_source_sink_pair::make_sink_info() size_t cuio_source_sink_pair::size() { switch (type) { - case io_type::VOID: return void_sink.bytes_written(); + case io_type::VOID: return void_sink->bytes_written(); case io_type::FILEPATH: return static_cast( std::ifstream(file_name, std::ifstream::ate | std::ifstream::binary).tellg()); diff --git a/cpp/benchmarks/io/cuio_common.hpp b/cpp/benchmarks/io/cuio_common.hpp index fe509f196be..3d5be41e25f 100644 --- a/cpp/benchmarks/io/cuio_common.hpp +++ b/cpp/benchmarks/io/cuio_common.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,15 +32,6 @@ std::string random_file_in_dir(std::string const& dir_path); * @brief Class to create a coupled `source_info` and `sink_info` of given type. */ class cuio_source_sink_pair { - class bytes_written_only_sink : public cudf::io::data_sink { - size_t _bytes_written = 0; - - public: - void host_write(void const* data, size_t size) override { _bytes_written += size; } - void flush() override {} - size_t bytes_written() override { return _bytes_written; } - }; - public: cuio_source_sink_pair(io_type type); ~cuio_source_sink_pair() @@ -79,7 +70,7 @@ class cuio_source_sink_pair { std::vector h_buffer; rmm::device_uvector d_buffer; std::string const file_name; - bytes_written_only_sink void_sink; + std::unique_ptr void_sink; }; /** diff --git a/cpp/src/io/utilities/data_sink.cpp b/cpp/src/io/utilities/data_sink.cpp index 0b14d060b05..5786e9dd6d1 100644 --- a/cpp/src/io/utilities/data_sink.cpp +++ b/cpp/src/io/utilities/data_sink.cpp @@ -139,6 +139,8 @@ class void_sink : public data_sink { [[nodiscard]] bool supports_device_write() const override { return true; } + [[nodiscard]] bool is_device_write_preferred(size_t size) const override { return true; } + void device_write(void const* gpu_data, size_t size, rmm::cuda_stream_view stream) override { _bytes_written += size; @@ -189,6 +191,11 @@ class user_sink_wrapper : public data_sink { return user_sink->device_write_async(gpu_data, size, stream); } + [[nodiscard]] bool is_device_write_preferred(size_t size) const override + { + return user_sink->is_device_write_preferred(size); + } + void flush() override { user_sink->flush(); } size_t bytes_written() override { return user_sink->bytes_written(); } From 077eec4dfd5a01b621e9842a97e80645d620e7dd Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 20 Feb 2024 03:47:59 -1000 Subject: [PATCH 259/384] xfail test_join_ordering_pandas_compat for pandas 2.2 (#15080) Right merge is implement by swapping left and right and performing a left merge, but the result ordering of columns that are named similarly changed in pandas 2.2 and I cannot currently narrow down when pandas orders the resulting columns a certain way. Since the merge is still technically correct besides a column ordering, just going to xfail this case for now and have it as a follow up. Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15080 --- python/cudf/cudf/tests/test_join_order.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/tests/test_join_order.py b/python/cudf/cudf/tests/test_join_order.py index 61a2ed239cb..58263faa7bf 100644 --- a/python/cudf/cudf/tests/test_join_order.py +++ b/python/cudf/cudf/tests/test_join_order.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. import itertools import operator @@ -155,7 +155,13 @@ def expected(left, right, sort, *, how): @pytest.mark.parametrize("how", ["inner", "left", "right", "outer"]) -def test_join_ordering_pandas_compat(left, right, sort, how): +def test_join_ordering_pandas_compat(request, left, right, sort, how): + request.applymarker( + pytest.mark.xfail( + PANDAS_GE_220 and how == "right", + reason="TODO: Result ording of suffix'ed columns is incorrect", + ) + ) with cudf.option_context("mode.pandas_compatible", True): actual = left.merge(right, on="key", how=how, sort=sort) expect = expected(left, right, sort, how=how) From 193ab6e877ca676571b5409960d7cb6bf8a694e4 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 20 Feb 2024 03:53:46 -1000 Subject: [PATCH 260/384] Adjust test_binops for pandas 2.2 (#15078) 2 tests needed to be adjusted due to pandas changes in behaviors in https://github.com/pandas-dev/pandas/issues/57447 and https://github.com/pandas-dev/pandas/issues/57448 Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15078 --- python/cudf/cudf/core/column/datetime.py | 4 +- python/cudf/cudf/tests/test_binops.py | 96 ++++++++++++++++++------ 2 files changed, 72 insertions(+), 28 deletions(-) diff --git a/python/cudf/cudf/core/column/datetime.py b/python/cudf/cudf/core/column/datetime.py index 7df22c7d8ea..b2f14b86ed9 100644 --- a/python/cudf/cudf/core/column/datetime.py +++ b/python/cudf/cudf/core/column/datetime.py @@ -567,9 +567,7 @@ def _binaryop(self, other: ColumnBinaryOperand, op: str) -> ColumnBase: if other is NotImplemented: return NotImplemented if isinstance(other, cudf.DateOffset): - return other._datetime_binop(self, op, reflect=reflect).astype( - self.dtype - ) + return other._datetime_binop(self, op, reflect=reflect) # We check this on `other` before reflection since we already know the # dtype of `self`. diff --git a/python/cudf/cudf/tests/test_binops.py b/python/cudf/cudf/tests/test_binops.py index 3ebefa6e071..6c6dae9e22e 100644 --- a/python/cudf/cudf/tests/test_binops.py +++ b/python/cudf/cudf/tests/test_binops.py @@ -13,6 +13,7 @@ import cudf from cudf import Series +from cudf.core._compat import PANDAS_GE_220 from cudf.core.buffer.spill_manager import get_global_manager from cudf.core.index import as_index from cudf.testing import _utils as utils @@ -824,11 +825,21 @@ def test_operator_func_between_series_logical( @pytest.mark.parametrize("fill_value", [None, 1.0]) @pytest.mark.parametrize("use_cudf_scalar", [False, True]) def test_operator_func_series_and_scalar_logical( - dtype, func, has_nulls, scalar, fill_value, use_cudf_scalar + request, dtype, func, has_nulls, scalar, fill_value, use_cudf_scalar ): - gdf_series = utils.gen_rand_series( - dtype, 1000, has_nulls=has_nulls, stride=10000 + request.applymarker( + pytest.mark.xfail( + PANDAS_GE_220 + and fill_value == 1.0 + and scalar is np.nan + and (has_nulls or (not has_nulls and func not in {"eq", "ne"})), + reason="https://github.com/pandas-dev/pandas/issues/57447", + ) ) + if has_nulls: + gdf_series = cudf.Series([-1.0, 0, cudf.NA, 1.1], dtype=dtype) + else: + gdf_series = cudf.Series([-1.0, 0, 10.5, 1.1], dtype=dtype) pdf_series = gdf_series.to_pandas(nullable=True) gdf_series_result = getattr(gdf_series, func)( cudf.Scalar(scalar) if use_cudf_scalar else scalar, @@ -1684,16 +1695,6 @@ def test_scalar_null_binops(op, dtype_l, dtype_r): assert result.dtype == valid_result.dtype -@pytest.mark.parametrize( - "date_col", - [ - [ - "2000-01-01 00:00:00.012345678", - "2000-01-31 00:00:00.012345678", - "2000-02-29 00:00:00.012345678", - ] - ], -) @pytest.mark.parametrize("n_periods", [0, 1, -1, 12, -12]) @pytest.mark.parametrize( "frequency", @@ -1714,8 +1715,40 @@ def test_scalar_null_binops(op, dtype_l, dtype_r): ) @pytest.mark.parametrize("op", [operator.add, operator.sub]) def test_datetime_dateoffset_binaryop( - date_col, n_periods, frequency, dtype, op + request, n_periods, frequency, dtype, op ): + request.applymarker( + pytest.mark.xfail( + PANDAS_GE_220 + and dtype in {"datetime64[ms]", "datetime64[s]"} + and frequency == "microseconds" + and n_periods == 0, + reason="https://github.com/pandas-dev/pandas/issues/57448", + ) + ) + request.applymarker( + pytest.mark.xfail( + not PANDAS_GE_220 + and dtype in {"datetime64[ms]", "datetime64[s]"} + and frequency in ("microseconds", "nanoseconds") + and n_periods != 0, + reason="https://github.com/pandas-dev/pandas/pull/55595", + ) + ) + request.applymarker( + pytest.mark.xfail( + not PANDAS_GE_220 + and dtype == "datetime64[us]" + and frequency == "nanoseconds" + and n_periods != 0, + reason="https://github.com/pandas-dev/pandas/pull/55595", + ) + ) + date_col = [ + "2000-01-01 00:00:00.012345678", + "2000-01-31 00:00:00.012345678", + "2000-02-29 00:00:00.012345678", + ] gsr = cudf.Series(date_col, dtype=dtype) psr = gsr.to_pandas() @@ -1776,16 +1809,6 @@ def test_datetime_dateoffset_binaryop_multiple(date_col, kwargs, op): utils.assert_eq(expect, got) -@pytest.mark.parametrize( - "date_col", - [ - [ - "2000-01-01 00:00:00.012345678", - "2000-01-31 00:00:00.012345678", - "2000-02-29 00:00:00.012345678", - ] - ], -) @pytest.mark.parametrize("n_periods", [0, 1, -1, 12, -12]) @pytest.mark.parametrize( "frequency", @@ -1805,8 +1828,31 @@ def test_datetime_dateoffset_binaryop_multiple(date_col, kwargs, op): ["datetime64[ns]", "datetime64[us]", "datetime64[ms]", "datetime64[s]"], ) def test_datetime_dateoffset_binaryop_reflected( - date_col, n_periods, frequency, dtype + request, n_periods, frequency, dtype ): + request.applymarker( + pytest.mark.xfail( + not PANDAS_GE_220 + and dtype in {"datetime64[ms]", "datetime64[s]"} + and frequency in ("microseconds", "nanoseconds") + and n_periods != 0, + reason="https://github.com/pandas-dev/pandas/pull/55595", + ) + ) + request.applymarker( + pytest.mark.xfail( + not PANDAS_GE_220 + and dtype == "datetime64[us]" + and frequency == "nanoseconds" + and n_periods != 0, + reason="https://github.com/pandas-dev/pandas/pull/55595", + ) + ) + date_col = [ + "2000-01-01 00:00:00.012345678", + "2000-01-31 00:00:00.012345678", + "2000-02-29 00:00:00.012345678", + ] gsr = cudf.Series(date_col, dtype=dtype) psr = gsr.to_pandas() # converts to nanos From d50c9107da35ef40e8262e4cbac5e48fdd1747a4 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 20 Feb 2024 03:57:05 -1000 Subject: [PATCH 261/384] xfail tests in test_udf_masked_ops due to pandas 2.2 bug (#15071) Due to a change in pandas 2.2 with how NA is handled (incorrectly) in UDFs https://github.com/pandas-dev/pandas/issues/57390 Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15071 --- python/cudf/cudf/tests/test_udf_masked_ops.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/python/cudf/cudf/tests/test_udf_masked_ops.py b/python/cudf/cudf/tests/test_udf_masked_ops.py index 0e29d2bfdcc..ed3461578fd 100644 --- a/python/cudf/cudf/tests/test_udf_masked_ops.py +++ b/python/cudf/cudf/tests/test_udf_masked_ops.py @@ -7,6 +7,7 @@ from numba import cuda import cudf +from cudf.core._compat import PANDAS_GE_220 from cudf.core.missing import NA from cudf.core.udf._ops import ( arith_ops, @@ -482,6 +483,9 @@ def func(x): run_masked_udf_series(func, data, check_dtype=False) +@pytest.mark.xfail( + PANDAS_GE_220, reason="https://github.com/pandas-dev/pandas/issues/57390" +) def test_series_apply_null_conditional(): def func(x): if x is NA: @@ -506,6 +510,9 @@ def func(x): run_masked_udf_series(func, data, check_dtype=False) +@pytest.mark.xfail( + PANDAS_GE_220, reason="https://github.com/pandas-dev/pandas/issues/57390" +) @pytest.mark.parametrize("op", comparison_ops) def test_series_compare_masked_vs_masked(op): """ @@ -562,6 +569,9 @@ def func(x): run_masked_udf_series(func, data, check_dtype=False) +@pytest.mark.xfail( + PANDAS_GE_220, reason="https://github.com/pandas-dev/pandas/issues/57390" +) def test_series_masked_is_null_conditional(): def func(x): if x is NA: @@ -742,8 +752,14 @@ def func(x, c): ], ) @pytest.mark.parametrize("op", arith_ops + comparison_ops) -def test_masked_udf_scalar_args_binops_multiple_series(data, op): +def test_masked_udf_scalar_args_binops_multiple_series(request, data, op): data = cudf.Series(data) + request.applymarker( + pytest.mark.xfail( + op in comparison_ops and PANDAS_GE_220 and data.dtype.kind != "b", + reason="https://github.com/pandas-dev/pandas/issues/57390", + ) + ) def func(data, c, k): x = op(data, c) From 44913fc1486d1264bc8db7f3134e4674c8bc783d Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 20 Feb 2024 04:02:30 -1000 Subject: [PATCH 262/384] Adjust test_joining for pandas 2.2 (#15060) As described in https://pandas.pydata.org/docs/dev/whatsnew/v2.2.0.html#merge-and-dataframe-join-now-consistently-follow-documented-sort-behavior Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15060 --- python/cudf/cudf/tests/test_joining.py | 33 ++++++++++++-------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/python/cudf/cudf/tests/test_joining.py b/python/cudf/cudf/tests/test_joining.py index 8b912fe28bc..5fbd1ba602f 100644 --- a/python/cudf/cudf/tests/test_joining.py +++ b/python/cudf/cudf/tests/test_joining.py @@ -7,7 +7,7 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_200 +from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_220 from cudf.core.dtypes import CategoricalDtype, Decimal64Dtype, Decimal128Dtype from cudf.testing._utils import ( INTEGER_TYPES, @@ -160,33 +160,30 @@ def _check_series(expect, got): def test_dataframe_join_suffix(): np.random.seed(0) - df = cudf.DataFrame() - for k in "abc": - df[k] = np.random.randint(0, 5, 5) + df = cudf.DataFrame(np.random.randint(0, 5, (5, 3)), columns=list("abc")) left = df.set_index("a") right = df.set_index("c") - with pytest.raises(ValueError) as raises: - left.join(right) - raises.match( - "there are overlapping columns but lsuffix" - " and rsuffix are not defined" + msg = ( + "there are overlapping columns but lsuffix and rsuffix are not defined" ) + with pytest.raises(ValueError, match=msg): + left.join(right) got = left.join(right, lsuffix="_left", rsuffix="_right", sort=True) - # Get expected value - pddf = df.to_pandas() - expect = pddf.set_index("a").join( - pddf.set_index("c"), lsuffix="_left", rsuffix="_right" + expect = left.to_pandas().join( + right.to_pandas(), + lsuffix="_left", + rsuffix="_right", + sort=PANDAS_GE_220, ) - # Check - assert list(expect.columns) == list(got.columns) - assert_eq(expect.index.values, got.index.values) + # TODO: Retain result index name + expect.index.name = None + assert_eq(got, expect) got_sorted = got.sort_values(by=["b_left", "c", "b_right"], axis=0) expect_sorted = expect.sort_values(by=["b_left", "c", "b_right"], axis=0) - for k in expect_sorted.columns: - _check_series(expect_sorted[k].fillna(-1), got_sorted[k].fillna(-1)) + assert_eq(got_sorted, expect_sorted) def test_dataframe_join_cats(): From 093fe6ad220173446aca8d03d1535f4a09e00dec Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 20 Feb 2024 04:03:27 -1000 Subject: [PATCH 263/384] Fix test_resample index dtype checking for pandas 2.2 (#15058) I think this got unintentionally fixed in pandas 2.2, but `pandas.testing.assert_series_equal` will be strict about checking a Series's Index's dtype for date-likes. Since pandas always returns `ns` in resample and cudf tries to match the resolution frequency (IMO the better behavior), need to specify `check_index=False` in pandas 2.2 Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15058 --- python/cudf/cudf/tests/test_resampling.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/cudf/cudf/tests/test_resampling.py b/python/cudf/cudf/tests/test_resampling.py index ce0fbbfada8..43f7324affe 100644 --- a/python/cudf/cudf/tests/test_resampling.py +++ b/python/cudf/cudf/tests/test_resampling.py @@ -5,7 +5,7 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_200 +from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_220 from cudf.testing._utils import assert_eq @@ -31,6 +31,7 @@ def test_series_downsample_simple(ts_resolution): assert_resample_results_equal( psr.resample("3min").sum(), gsr.resample("3min").sum(), + check_index=not PANDAS_GE_220, ) @@ -43,6 +44,7 @@ def test_series_upsample_simple(): assert_resample_results_equal( psr.resample("3min").sum(), gsr.resample("3min").sum(), + check_index=not PANDAS_GE_220, ) From 0dc9db83f49a2ab789208c72728a522614582e0c Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 20 Feb 2024 04:08:30 -1000 Subject: [PATCH 264/384] Avoid pandas 2.2 `DeprecationWarning` in test_hdf (#15044) The `DeprecationWarning` was from integer data potentially being downcast (e.g. large ints to int8) Additionally did some cleanup in this file: * Used `pytest.importorskip` * Removed testing unsigned ints as they were raising a `NotImplementedError` in tables * Only tested 1 `datetime64` type as the column naming format would conflict with how resolutions were dropped * Made testing data deterministic Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15044 --- python/cudf/cudf/tests/test_hdf.py | 39 ++++++++++++------------------ 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/python/cudf/cudf/tests/test_hdf.py b/python/cudf/cudf/tests/test_hdf.py index 1ddd7f93c3e..d420c95cfb4 100644 --- a/python/cudf/cudf/tests/test_hdf.py +++ b/python/cudf/cudf/tests/test_hdf.py @@ -8,43 +8,35 @@ import pytest import cudf -from cudf.testing._utils import DATETIME_TYPES, NUMERIC_TYPES, assert_eq - -try: - import tables # noqa F401 -except ImportError: - pytest.skip( - "PyTables is not installed and is required for HDF reading/writing", - allow_module_level=True, - ) +from cudf.testing._utils import NUMERIC_TYPES, UNSIGNED_TYPES, assert_eq + +pytest.importorskip("tables") @pytest.fixture(params=[0, 1, 10, 100]) def pdf(request): - types = NUMERIC_TYPES + DATETIME_TYPES + ["bool"] + types = set(NUMERIC_TYPES + ["datetime64[ns]"] + ["bool"]) - set( + UNSIGNED_TYPES + ) typer = {"col_" + val: val for val in types} ncols = len(types) nrows = request.param + rng = np.random.default_rng(1) # Create a pandas dataframe with random data of mixed types test_pdf = pd.DataFrame( - [list(range(ncols * i, ncols * (i + 1))) for i in range(nrows)], - columns=pd.Index([f"col_{typ}" for typ in types], name="foo"), + rng.integers(0, 50, size=(nrows, ncols)), + columns=pd.Index([f"col_{typ}" for typ in types]), + index=pd.RangeIndex(nrows, name="test_index"), ) - # Delete the name of the column index, and rename the row index - test_pdf.columns.name = None - test_pdf.index.name = "test_index" - # Cast all the column dtypes to objects, rename them, and then cast to # appropriate types - test_pdf = ( - test_pdf.astype("object") - .astype(typer) - .rename({"col_datetime64[ms]": "col_datetime64"}, axis=1) + test_pdf = test_pdf.astype(typer).rename( + {"col_datetime64[ns]": "col_datetime64"}, axis=1 ) # Create non-numeric categorical data otherwise may be typecasted - data = [ascii_letters[np.random.randint(0, 52)] for i in range(nrows)] + data = rng.choice(list(ascii_letters), size=nrows) test_pdf["col_category"] = pd.Series(data, dtype="category") return (test_pdf, nrows) @@ -107,6 +99,8 @@ def test_hdf_reader(hdf_files, columns): @pytest.mark.filterwarnings("ignore:Using CPU") def test_hdf_writer(tmpdir, pdf, gdf, complib, format): pdf, nrows = pdf + if format == "table" and nrows == 0: + pytest.skip("Can't read 0 row table with format 'table'") gdf, _ = gdf if format == "fixed": @@ -122,9 +116,6 @@ def test_hdf_writer(tmpdir, pdf, gdf, complib, format): assert os.path.exists(pdf_df_fname) assert os.path.exists(gdf_df_fname) - if format == "table" and nrows == 0: - pytest.skip("Can't read 0 row table with format 'table'") - expect = pd.read_hdf(pdf_df_fname) got = pd.read_hdf(gdf_df_fname) From c9dd3256ee5582fc5e8d742a3d95c7f44b000341 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 20 Feb 2024 04:08:56 -1000 Subject: [PATCH 265/384] Add xfailures for test_applymap for pandas 2.2 (#15034) There were regressions in the `map` methods on the pandas side that is causing some of these applymap tests to fail on pandas 2.2 Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15034 --- python/cudf/cudf/tests/test_applymap.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/tests/test_applymap.py b/python/cudf/cudf/tests/test_applymap.py index 38a34c206d7..adbbbbb1ae4 100644 --- a/python/cudf/cudf/tests/test_applymap.py +++ b/python/cudf/cudf/tests/test_applymap.py @@ -3,7 +3,7 @@ import pytest from cudf import NA, DataFrame -from cudf.core._compat import PANDAS_GE_210 +from cudf.core._compat import PANDAS_GE_210, PANDAS_GE_220 from cudf.testing import _utils as utils @@ -26,7 +26,21 @@ ], ) @pytest.mark.parametrize("na_action", [None, "ignore"]) -def test_applymap_dataframe(data, func, na_action): +def test_applymap_dataframe(data, func, na_action, request): + request.applymarker( + pytest.mark.xfail( + PANDAS_GE_220 + and request.node.callspec.id == "None-2-data3", + reason="https://github.com/pandas-dev/pandas/issues/57390", + ) + ) + request.applymarker( + pytest.mark.xfail( + PANDAS_GE_220 + and request.node.callspec.id == "ignore-3-data3", + reason="https://github.com/pandas-dev/pandas/pull/57388", + ) + ) gdf = DataFrame(data) pdf = gdf.to_pandas(nullable=True) From 2d6be387385faad0e0f2a73e8ca7d62a02f0dd4f Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 20 Feb 2024 04:14:03 -1000 Subject: [PATCH 266/384] Adjust test_datetime_infer_format for pandas 2.2 (#15021) pandas 2.2 is stricter about converting from date string to datetime type if the resolution would lead to loss of precision. This affects `test_datetime_infer_format` where an `astype` is done, so adjusting the test such that the `astypes` don't lead to loss of precision. Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15021 --- python/cudf/cudf/tests/test_datetime.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/python/cudf/cudf/tests/test_datetime.py b/python/cudf/cudf/tests/test_datetime.py index 513123a65d3..6f8e4ec0a1a 100644 --- a/python/cudf/cudf/tests/test_datetime.py +++ b/python/cudf/cudf/tests/test_datetime.py @@ -1275,23 +1275,23 @@ def test_datetime_reductions(data, op, dtype): assert_eq(expected, actual) -@pytest.mark.parametrize("timezone", ["naive", "UTC"]) +@pytest.mark.parametrize("timezone", ["", "Z"]) @pytest.mark.parametrize( "data", [ - np.arange("2002-10-27T04:30", 4 * 60, 60, dtype="M8[m]"), - np.arange("2002-10-27T04:30", 10 * 60, 1, dtype="M8[m]"), - np.arange("2002-10-27T04:30", 10 * 60, 1, dtype="M8[ns]"), - np.arange("2002-10-27T04:30", 10 * 60, 1, dtype="M8[us]"), - np.arange("2002-10-27T04:30", 4 * 60, 60, dtype="M8[s]"), + "2002-10-27T04:30", + "2002-10-27T04:30:00", + "2002-10-27T04:30:00.000", + "2002-10-27T04:30:00.000000", + "2002-10-27T04:30:00.000000000", ], ) @pytest.mark.parametrize("dtype", DATETIME_TYPES) def test_datetime_infer_format(data, timezone, dtype): - ts_data = np.datetime_as_string(data, timezone=timezone) + ts_data = [data + timezone] sr = cudf.Series(ts_data) psr = pd.Series(ts_data) - if timezone == "naive": + if not timezone: expected = psr.astype(dtype) actual = sr.astype(dtype) From c0e370b271849ba5fe79ea324dfb9e4eadeb746c Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 20 Feb 2024 04:20:46 -1000 Subject: [PATCH 267/384] Add groupby.apply(include_groups=) to match pandas 2.2 deprecation (#15006) Matching https://github.com/pandas-dev/pandas/pull/54950 Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15006 --- python/cudf/cudf/core/groupby/groupby.py | 55 +++++-- python/cudf/cudf/tests/test_groupby.py | 135 ++++++++++++------ .../cudf_pandas_tests/test_cudf_pandas.py | 12 +- 3 files changed, 140 insertions(+), 62 deletions(-) diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 12bba3838f3..a236a9b6abf 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -1178,20 +1178,25 @@ def deserialize(cls, header, frames): ) return cls(obj, grouping, **kwargs) - def _grouped(self): + def _grouped(self, *, include_groups: bool = True): offsets, grouped_key_cols, grouped_value_cols = self._groupby.groups( [*self.obj._index._columns, *self.obj._columns] ) grouped_keys = cudf.core.index._index_from_columns(grouped_key_cols) if isinstance(self.grouping.keys, cudf.MultiIndex): grouped_keys.names = self.grouping.keys.names + to_drop = self.grouping.keys.names else: grouped_keys.name = self.grouping.keys.name + to_drop = (self.grouping.keys.name,) grouped_values = self.obj._from_columns_like_self( grouped_value_cols, column_names=self.obj._column_names, index_names=self.obj._index_names, ) + if not include_groups: + for col_name in to_drop: + del grouped_values[col_name] group_names = grouped_keys.unique().sort_values() return (group_names, offsets, grouped_keys, grouped_values) @@ -1348,13 +1353,25 @@ def _post_process_chunk_results( result.index.names = self.grouping.names # When the UDF is like df.x + df.y, the result for each # group is the same length as the original group - elif len(self.obj) == sum(len(chk) for chk in chunk_results): + elif (total_rows := sum(len(chk) for chk in chunk_results)) in { + len(self.obj), + len(group_names), + }: with warnings.catch_warnings(): warnings.simplefilter("ignore", FutureWarning) result = cudf.concat(chunk_results) - index_data = group_keys._data.copy(deep=True) - index_data[None] = grouped_values.index._column - result.index = cudf.MultiIndex._from_data(index_data) + if total_rows == len(group_names): + result.index = group_names + # TODO: Is there a better way to determine what + # the column name should be, especially if we applied + # a nameless UDF. + result = result.to_frame( + name=grouped_values._data.names[0] + ) + else: + index_data = group_keys._data.copy(deep=True) + index_data[None] = grouped_values.index._column + result.index = cudf.MultiIndex._from_data(index_data) else: raise TypeError( "Error handling Groupby apply output with input of " @@ -1372,7 +1389,9 @@ def _post_process_chunk_results( return result @_cudf_nvtx_annotate - def apply(self, function, *args, engine="auto"): + def apply( + self, function, *args, engine="auto", include_groups: bool = True + ): """Apply a python transformation function over the grouped chunk. Parameters @@ -1396,6 +1415,10 @@ def apply(self, function, *args, engine="auto"): The default value `auto` will attempt to use the numba JIT pipeline where possible and will fall back to the iterative algorithm if necessary. + include_groups : bool, default True + When True, will attempt to apply ``func`` to the groupings in + the case that they are columns of the DataFrame. In the future, + this will default to ``False``. Examples -------- @@ -1444,15 +1467,15 @@ def mult(df): ... 'c': [1, 2, 3, 4], ... }) >>> gdf = cudf.from_pandas(df) - >>> df.groupby('a').apply(lambda x: x.iloc[[0]]) - a b c + >>> df.groupby('a')[["b", "c"]].apply(lambda x: x.iloc[[0]]) + b c a - 1 0 1 1 1 - 2 2 2 1 3 - >>> gdf.groupby('a').apply(lambda x: x.iloc[[0]]) - a b c - 0 1 1 1 - 2 2 1 3 + 1 0 1 1 + 2 2 1 3 + >>> gdf.groupby('a')[["b", "c"]].apply(lambda x: x.iloc[[0]]) + b c + 0 1 1 + 2 1 3 ``engine='jit'`` may be used to accelerate certain functions, initially those that contain reductions and arithmetic operations @@ -1487,7 +1510,9 @@ def mult(df): if not callable(function): raise TypeError(f"type {type(function)} is not callable") - group_names, offsets, group_keys, grouped_values = self._grouped() + group_names, offsets, group_keys, grouped_values = self._grouped( + include_groups=include_groups + ) if engine == "auto": if _can_be_jitted(grouped_values, function, args): diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index 06fd8f2ea79..e8dbdd35352 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -188,7 +188,10 @@ def test_groupby_as_index_apply(pdf, gdf, as_index, engine): gdf = gdf.groupby("y", as_index=as_index).apply( lambda df: df["x"].mean(), engine=engine ) - pdf = pdf.groupby("y", as_index=as_index).apply(lambda df: df["x"].mean()) + kwargs = {"func": lambda df: df["x"].mean()} + if PANDAS_GE_220: + kwargs["include_groups"] = False + pdf = pdf.groupby("y", as_index=as_index).apply(**kwargs) assert_groupby_results_equal(pdf, gdf) @@ -311,8 +314,12 @@ def foo(df): df["out"] = df["val1"] + df["val2"] return df - expect = expect_grpby.apply(foo) - got = got_grpby.apply(foo) + if PANDAS_GE_220: + kwargs = {"include_groups": False} + else: + kwargs = {} + expect = expect_grpby.apply(foo, **kwargs) + got = got_grpby.apply(foo, **kwargs) assert_groupby_results_equal(expect, got) @@ -346,9 +353,12 @@ def test_groupby_apply_args(func, args): ["key1", "key2"], as_index=False, group_keys=False ) got_grpby = df.groupby(["key1", "key2"]) - - expect = expect_grpby.apply(func, *args) - got = got_grpby.apply(func, *args) + if PANDAS_GE_220: + kwargs = {"include_groups": False} + else: + kwargs = {} + expect = expect_grpby.apply(func, *args, **kwargs) + got = got_grpby.apply(func, *args, **kwargs) assert_groupby_results_equal(expect, got) @@ -356,14 +366,11 @@ def test_groupby_apply_grouped(): np.random.seed(0) df = DataFrame() nelem = 20 - df["key1"] = np.random.randint(0, 3, nelem) - df["key2"] = np.random.randint(0, 2, nelem) - df["val1"] = np.random.random(nelem) - df["val2"] = np.random.random(nelem) + df["key1"] = range(nelem) + df["key2"] = range(nelem) + df["val1"] = range(nelem) + df["val2"] = range(nelem) - expect_grpby = df.to_pandas().groupby( - ["key1", "key2"], as_index=False, group_keys=False - ) got_grpby = df.groupby(["key1", "key2"]) def foo(key1, val1, com1, com2): @@ -380,14 +387,11 @@ def foo(key1, val1, com1, com2): got = got.to_pandas() - # Get expected result by emulating the operation in pandas - def emulate(df): - df["com1"] = df.key1 * 10000 + df.val1 - df["com2"] = np.arange(len(df), dtype=np.int32) - return df - - expect = expect_grpby.apply(emulate) - expect = expect.sort_values(["key1", "key2"]) + expect = df.copy() + expect["com1"] = (expect["key1"] * 10000 + expect["key1"]).astype( + np.float64 + ) + expect["com2"] = np.zeros(nelem, dtype=np.int32) assert_groupby_results_equal(expect, got) @@ -462,8 +466,14 @@ def run_groupby_apply_jit_test(data, func, keys, *args): got_groupby_obj = data.groupby(keys) # compare cuDF jit to pandas - cudf_jit_result = got_groupby_obj.apply(func, *args, engine="jit") - pandas_result = expect_groupby_obj.apply(func, *args) + if PANDAS_GE_220: + kwargs = {"include_groups": False} + else: + kwargs = {} + cudf_jit_result = got_groupby_obj.apply( + func, *args, engine="jit", **kwargs + ) + pandas_result = expect_groupby_obj.apply(func, *args, **kwargs) assert_groupby_results_equal(cudf_jit_result, pandas_result) @@ -776,7 +786,7 @@ def test_groupby_apply_jit_block_divergence(): ) def diverging_block(grp_df): - if grp_df["a"].mean() > 0: + if grp_df["b"].mean() > 1: return grp_df["b"].mean() return 0 @@ -831,27 +841,41 @@ def f(group): return group.sum() part = partial(f) - - expect = pdf.groupby("a").apply(part) - got = gdf.groupby("a").apply(part, engine="auto") - + if PANDAS_GE_220: + kwargs = {"include_groups": False} + else: + kwargs = {} + expect = pdf.groupby("a").apply(part, **kwargs) + got = gdf.groupby("a").apply(part, engine="auto", **kwargs) assert_groupby_results_equal(expect, got) -@pytest.mark.parametrize("func", [lambda group: group.x + group.y]) -def test_groupby_apply_return_col_from_df(func): +def test_groupby_apply_return_col_from_df(): # tests a UDF that consists of purely colwise # ops, such as `lambda group: group.x + group.y` # which returns a column - df = cudf.datasets.randomdata() + func = lambda group: group.x + group.y # noqa:E731 + df = cudf.DataFrame( + { + "id": range(10), + "x": range(10), + "y": range(10), + } + ) pdf = df.to_pandas() def func(df): return df.x + df.y - expect = pdf.groupby("id").apply(func) - got = df.groupby("id").apply(func) - + if PANDAS_GE_220: + kwargs = {"include_groups": False} + else: + kwargs = {} + got = df.groupby("id").apply(func, **kwargs) + expect = pdf.groupby("id").apply(func, **kwargs) + # pandas seems to erroneously add an extra MI level of ids + # TODO: Figure out how pandas groupby.apply determines the columns + expect = pd.DataFrame(expect.droplevel(1), columns=got.columns) assert_groupby_results_equal(expect, got) @@ -863,8 +887,12 @@ def test_groupby_apply_return_df(func): df = cudf.DataFrame({"a": [1, 1, 2, 2], "b": [1, 2, 3, 4]}) pdf = df.to_pandas() - expect = pdf.groupby("a").apply(func) - got = df.groupby("a").apply(func) + if PANDAS_GE_220: + kwargs = {"include_groups": False} + else: + kwargs = {} + expect = pdf.groupby("a").apply(func, **kwargs) + got = df.groupby("a").apply(func, **kwargs) assert_groupby_results_equal(expect, got) @@ -1910,14 +1938,21 @@ def test_groupby_apply_noempty_group(): {"a": [1, 1, 2, 2], "b": [1, 2, 1, 2], "c": [1, 2, 3, 4]} ) gdf = cudf.from_pandas(pdf) - assert_groupby_results_equal( + if PANDAS_GE_220: + kwargs = {"include_groups": False} + else: + kwargs = {} + expect = ( pdf.groupby("a", group_keys=False) - .apply(lambda x: x.iloc[[0, 1]]) - .reset_index(drop=True), + .apply(lambda x: x.iloc[[0, 1]], **kwargs) + .reset_index(drop=True) + ) + got = ( gdf.groupby("a") - .apply(lambda x: x.iloc[[0, 1]]) - .reset_index(drop=True), + .apply(lambda x: x.iloc[[0, 1]], **kwargs) + .reset_index(drop=True) ) + assert_groupby_results_equal(expect, got) def test_reset_index_after_empty_groupby(): @@ -2198,8 +2233,12 @@ def test_groupby_apply_return_scalars(func, args): ) gdf = cudf.from_pandas(pdf) - expected = pdf.groupby("A").apply(func, *args) - actual = gdf.groupby("A").apply(func, *args) + if PANDAS_GE_220: + kwargs = {"include_groups": False} + else: + kwargs = {} + expected = pdf.groupby("A").apply(func, *args, **kwargs) + actual = gdf.groupby("A").apply(func, *args, **kwargs) assert_groupby_results_equal(expected, actual) @@ -2242,8 +2281,14 @@ def test_groupby_apply_return_series_dataframe(func, args): ) gdf = cudf.from_pandas(pdf) - expected = pdf.groupby(["key"], group_keys=False).apply(func, *args) - actual = gdf.groupby(["key"]).apply(func, *args) + if PANDAS_GE_220: + kwargs = {"include_groups": False} + else: + kwargs = {} + expected = pdf.groupby(["key"], group_keys=False).apply( + func, *args, **kwargs + ) + actual = gdf.groupby(["key"]).apply(func, *args, **kwargs) assert_groupby_results_equal(expected, actual) diff --git a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py index 546f8df95f3..ab4742549f8 100644 --- a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py +++ b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py @@ -17,6 +17,7 @@ import pytest from numba import NumbaDeprecationWarning +from cudf.core._compat import PANDAS_GE_220 from cudf.pandas import LOADED, Profiler from cudf.pandas.fast_slow_proxy import _Unusable @@ -506,10 +507,17 @@ def test_array_ufunc(series): tm.assert_equal(expect, got) +@pytest.mark.xfail(strict=False, reason="Fails in CI, passes locally.") def test_groupby_apply_func_returns_series(dataframe): pdf, df = dataframe - expect = pdf.groupby("a").apply(lambda group: pd.Series({"x": 1})) - got = df.groupby("a").apply(lambda group: xpd.Series({"x": 1})) + if PANDAS_GE_220: + kwargs = {"include_groups": False} + else: + kwargs = {} + expect = pdf.groupby("a").apply( + lambda group: pd.Series({"x": 1}), **kwargs + ) + got = df.groupby("a").apply(lambda group: xpd.Series({"x": 1}), **kwargs) tm.assert_equal(expect, got) From f6c00ff376a7affe561e44f4c1af09f717262016 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 20 Feb 2024 05:11:08 -1000 Subject: [PATCH 268/384] Deprecate datelike isin casting strings to dates to match pandas 2.2 (#15046) Matching https://github.com/pandas-dev/pandas/pull/56427 Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15046 --- python/cudf/cudf/core/tools/datetimes.py | 10 +++++++ python/cudf/cudf/tests/test_index.py | 34 ++++++++++++------------ python/cudf/cudf/tests/test_series.py | 8 ++++-- 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/python/cudf/cudf/core/tools/datetimes.py b/python/cudf/cudf/core/tools/datetimes.py index 529296da6a2..0e0df4ecf6e 100644 --- a/python/cudf/cudf/core/tools/datetimes.py +++ b/python/cudf/cudf/core/tools/datetimes.py @@ -767,10 +767,20 @@ def _isin_datetimelike( rhs = None try: rhs = cudf.core.column.as_column(values) + was_string = len(rhs) and rhs.dtype.kind == "O" if rhs.dtype.kind in {"f", "i", "u"}: return cudf.core.column.full(len(lhs), False, dtype="bool") rhs = rhs.astype(lhs.dtype) + if was_string: + warnings.warn( + f"The behavior of 'isin' with dtype={lhs.dtype} and " + "castable values (e.g. strings) is deprecated. In a " + "future version, these will not be considered matching " + "by isin. Explicitly cast to the appropriate dtype before " + "calling isin instead.", + FutureWarning, + ) res = lhs._isin_earlystop(rhs) if res is not None: return res diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index 3cbfea8063f..defd42b3d00 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -2497,19 +2497,12 @@ def test_index_nan_as_null(data, nan_idx, NA_idx, nan_as_null): @pytest.mark.parametrize( - "data", + "index", [ - [], - pd.Series( - ["this", "is", None, "a", "test"], index=["a", "b", "c", "d", "e"] - ), - pd.Series([0, 15, 10], index=[0, None, 9]), - pd.Series( - range(25), - index=pd.date_range( - start="2019-01-01", end="2019-01-02", freq="h" - ), - ), + pd.Index([]), + pd.Index(["a", "b", "c", "d", "e"]), + pd.Index([0, None, 9]), + pd.date_range("2019-01-01", periods=3), ], ) @pytest.mark.parametrize( @@ -2521,12 +2514,19 @@ def test_index_nan_as_null(data, nan_idx, NA_idx, nan_as_null): ["2019-01-01 04:00:00", "2019-01-01 06:00:00", "2018-03-02 10:00:00"], ], ) -def test_isin_index(data, values): - psr = pd.Series(data) - gsr = cudf.Series.from_pandas(psr) +def test_isin_index(index, values): + pidx = index + gidx = cudf.Index.from_pandas(pidx) - got = gsr.index.isin(values) - expected = psr.index.isin(values) + is_dt_str = ( + next(iter(values), None) == "2019-01-01 04:00:00" + and len(pidx) + and pidx.dtype.kind == "M" + ) + with expect_warning_if(is_dt_str): + got = gidx.isin(values) + with expect_warning_if(PANDAS_GE_220 and is_dt_str): + expected = pidx.isin(values) assert_eq(got, expected) diff --git a/python/cudf/cudf/tests/test_series.py b/python/cudf/cudf/tests/test_series.py index 14006f90b45..252343391be 100644 --- a/python/cudf/cudf/tests/test_series.py +++ b/python/cudf/cudf/tests/test_series.py @@ -15,6 +15,7 @@ import cudf from cudf.api.extensions import no_default +from cudf.core._compat import PANDAS_GE_220 from cudf.errors import MixedTypeError from cudf.testing._utils import ( NUMERIC_TYPES, @@ -1795,8 +1796,11 @@ def test_isin_datetime(data, values): psr = pd.Series(data) gsr = cudf.Series.from_pandas(psr) - got = gsr.isin(values) - expected = psr.isin(values) + is_len_str = isinstance(next(iter(values), None), str) and len(data) + with expect_warning_if(is_len_str): + got = gsr.isin(values) + with expect_warning_if(PANDAS_GE_220 and is_len_str): + expected = psr.isin(values) assert_eq(got, expected) From 4ca9ac83d2b103566d9b053e79b3a787b8ebf7f8 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 20 Feb 2024 05:13:14 -1000 Subject: [PATCH 269/384] Change chained replace inplace test to COW test for pandas 2.2 (#15049) `test_setitem_dataframe_series_inplace` failed with pandas 2.2 because it exhibits a chained indexing behavior that raised a `FutureWarning` in pandas 2.2 and will raise in 3.0. I refactored the test to test cudf copy on write to exhibit the 3.0 behavior, but it still seems to allow this chained indexing behavior, so xfailed it for now. Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15049 --- python/cudf/cudf/tests/test_setitem.py | 33 +++++++++----------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/python/cudf/cudf/tests/test_setitem.py b/python/cudf/cudf/tests/test_setitem.py index de0826d61e9..967c1d27fc1 100644 --- a/python/cudf/cudf/tests/test_setitem.py +++ b/python/cudf/cudf/tests/test_setitem.py @@ -118,34 +118,23 @@ def test_series_setitem_singleton_range(): assert_eq(sr, psr, check_dtype=True) +@pytest.mark.xfail(reason="Copy-on-Write should make a copy") @pytest.mark.parametrize( - "df", + "index", [ - pd.DataFrame( - {"a": [1, 2, 3]}, - index=pd.MultiIndex.from_frame( - pd.DataFrame({"b": [3, 2, 1], "c": ["a", "b", "c"]}) - ), + pd.MultiIndex.from_frame( + pd.DataFrame({"b": [3, 2, 1], "c": ["a", "b", "c"]}) ), - pd.DataFrame({"a": [1, 2, 3]}, index=["a", "b", "c"]), + ["a", "b", "c"], ], ) -def test_setitem_dataframe_series_inplace(df): - pdf = df.copy(deep=True) - gdf = cudf.from_pandas(pdf) - - pdf["a"].replace(1, 500, inplace=True) - gdf["a"].replace(1, 500, inplace=True) - - assert_eq(pdf, gdf) - - psr_a = pdf["a"] - gsr_a = gdf["a"] - - psr_a.replace(500, 501, inplace=True) - gsr_a.replace(500, 501, inplace=True) +def test_setitem_dataframe_series_inplace(index): + gdf = cudf.DataFrame({"a": [1, 2, 3]}, index=index) + expected = gdf.copy() + with cudf.option_context("copy_on_write", True): + gdf["a"].replace(1, 500, inplace=True) - assert_eq(pdf, gdf) + assert_eq(expected, gdf) @pytest.mark.parametrize( From 31506768ff1036d1971a097826229aa49e939c18 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 20 Feb 2024 10:41:06 -0500 Subject: [PATCH 270/384] Rework cudf::strings::detail::copy_range for offsetalator (#15010) This reworks the `cudf::strings::detail::copy_range()` function to use the offsetalator instead of accessing the output offsets directly. Also refactored the code to remove the unnecessary template arguments. And added a benchmark to ensure these changes did not cause a performance impact. Most of the code in `cpp/include/cudf/strings/detail/copy_range.cuh` was rewritten and moved to `cpp/src/strings/copying/copy_range.cu`. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Robert Maynard (https://github.com/robertmaynard) - Vukasin Milovanovic (https://github.com/vuule) URL: https://github.com/rapidsai/cudf/pull/15010 --- cpp/CMakeLists.txt | 1 + cpp/benchmarks/CMakeLists.txt | 1 + cpp/benchmarks/string/copy_range.cpp | 60 +++++ .../cudf/strings/detail/copy_range.cuh | 216 ------------------ .../cudf/strings/detail/copy_range.hpp | 60 +++++ cpp/src/copying/copy_range.cu | 29 +-- cpp/src/strings/copying/copy_range.cu | 143 ++++++++++++ 7 files changed, 269 insertions(+), 241 deletions(-) create mode 100644 cpp/benchmarks/string/copy_range.cpp delete mode 100644 cpp/include/cudf/strings/detail/copy_range.cuh create mode 100644 cpp/include/cudf/strings/detail/copy_range.hpp create mode 100644 cpp/src/strings/copying/copy_range.cu diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d4ed6c113b9..078de27f0ea 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -570,6 +570,7 @@ add_library( src/strings/convert/convert_lists.cu src/strings/copying/concatenate.cu src/strings/copying/copying.cu + src/strings/copying/copy_range.cu src/strings/copying/shift.cu src/strings/count_matches.cu src/strings/extract/extract.cu diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 6ddc5a6b8de..5a014537de0 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -312,6 +312,7 @@ ConfigureNVBench( string/char_types.cpp string/contains.cpp string/copy_if_else.cpp + string/copy_range.cpp string/count.cpp string/extract.cpp string/gather.cpp diff --git a/cpp/benchmarks/string/copy_range.cpp b/cpp/benchmarks/string/copy_range.cpp new file mode 100644 index 00000000000..af217a49195 --- /dev/null +++ b/cpp/benchmarks/string/copy_range.cpp @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include + +#include + +static void bench_copy_range(nvbench::state& state) +{ + auto const num_rows = static_cast(state.get_int64("num_rows")); + auto const row_width = static_cast(state.get_int64("row_width")); + + if (static_cast(num_rows) * static_cast(row_width) >= + static_cast(std::numeric_limits::max())) { + state.skip("Skip benchmarks greater than size_type limit"); + } + + data_profile const table_profile = + data_profile_builder() + .distribution(cudf::type_id::STRING, distribution_id::NORMAL, 0, row_width) + .no_validity(); + auto const source_tables = create_random_table( + {cudf::type_id::STRING, cudf::type_id::STRING}, row_count{num_rows}, table_profile); + + auto const start = num_rows / 4; + auto const end = (num_rows * 3) / 4; + auto const source = source_tables->view().column(0); + auto const target = source_tables->view().column(1); + + state.set_cuda_stream(nvbench::make_cuda_stream_view(cudf::get_default_stream().value())); + auto chars_size = cudf::strings_column_view(target).chars_size(cudf::get_default_stream()); + state.add_global_memory_reads(chars_size); // all bytes are read; + state.add_global_memory_writes(chars_size); // both columns are similar size + + state.exec(nvbench::exec_tag::sync, [&](nvbench::launch& launch) { + [[maybe_unused]] auto result = cudf::copy_range(source, target, start, end, start / 2); + }); +} + +NVBENCH_BENCH(bench_copy_range) + .set_name("copy_range") + .add_int64_axis("row_width", {32, 64, 128, 256, 512, 1024, 2048, 4096}) + .add_int64_axis("num_rows", {4096, 32768, 262144, 2097152, 16777216}); diff --git a/cpp/include/cudf/strings/detail/copy_range.cuh b/cpp/include/cudf/strings/detail/copy_range.cuh deleted file mode 100644 index 567452bac4e..00000000000 --- a/cpp/include/cudf/strings/detail/copy_range.cuh +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Copyright (c) 2019-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include - -namespace { -template -struct compute_element_size { - SourceValueIterator source_value_begin; - SourceValidityIterator source_validity_begin; - cudf::column_device_view d_target; - cudf::size_type target_begin; - cudf::size_type target_end; - - __device__ cudf::size_type operator()(cudf::size_type idx) - { - if (idx >= target_begin && idx < target_end) { - if (source_has_nulls) { - return *(source_validity_begin + (idx - target_begin)) - ? (*(source_value_begin + (idx - target_begin))).size_bytes() - : 0; - } else { - return (*(source_value_begin + (idx - target_begin))).size_bytes(); - } - } else { - if (target_has_nulls) { - return d_target.is_valid_nocheck(idx) - ? d_target.element(idx).size_bytes() - : 0; - } else { - return d_target.element(idx).size_bytes(); - } - } - } -}; - -} // namespace - -namespace cudf { -namespace strings { -namespace detail { -/** - * @brief Internal API to copy a range of string elements out-of-place from - * source iterators to a target column. - * - * Creates a new column as if an in-place copy was performed into @p target. - * The elements indicated by the indices [@p target_begin, @p target_end) were - * replaced with the elements retrieved from source iterators; - * *(@p source_value_begin + idx) if *(@p source_validity_begin + idx) is true, - * invalidate otherwise (where idx = [0, @p target_end - @p target_begin)). - * Elements outside the range are copied from @p target into the new target - * column to return. - * - * @throws cudf::logic_error for invalid range (if @p target_begin < 0, - * target_begin >= @p target.size(), or @p target_end > @p target.size()). - * - * @tparam SourceValueIterator Iterator for retrieving source values - * @tparam SourceValidityIterator Iterator for retrieving source validities - * @param source_value_begin Start of source value iterator - * @param source_validity_begin Start of source validity iterator - * @param target The strings column to copy from outside the range. - * @param target_begin The starting index of the target range (inclusive) - * @param target_end The index of the last element in the target range - * (exclusive) - * @param stream CUDA stream used for device memory operations and kernel launches. - * @param mr Device memory resource used to allocate the returned column's device memory. - * @return std::unique_ptr The result target column - */ -template -std::unique_ptr copy_range(SourceValueIterator source_value_begin, - SourceValidityIterator source_validity_begin, - strings_column_view const& target, - size_type target_begin, - size_type target_end, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) -{ - CUDF_EXPECTS( - (target_begin >= 0) && (target_begin < target.size()) && (target_end <= target.size()), - "Range is out of bounds."); - - if (target_end == target_begin) { - return std::make_unique(target.parent(), stream, mr); - } else { - auto p_target_device_view = column_device_view::create(target.parent(), stream); - auto d_target = *p_target_device_view; - - // create resulting null mask - - std::pair valid_mask{}; - if (target.has_nulls()) { // check validities for both source & target - valid_mask = cudf::detail::valid_if( - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(target.size()), - [source_validity_begin, d_target, target_begin, target_end] __device__(size_type idx) { - return (idx >= target_begin && idx < target_end) - ? *(source_validity_begin + (idx - target_begin)) - : d_target.is_valid_nocheck(idx); - }, - stream, - mr); - } else { // check validities for source only - valid_mask = cudf::detail::valid_if( - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(target.size()), - [source_validity_begin, d_target, target_begin, target_end] __device__(size_type idx) { - return (idx >= target_begin && idx < target_end) - ? *(source_validity_begin + (idx - target_begin)) - : true; - }, - stream, - mr); - } - - auto null_count = valid_mask.second; - rmm::device_buffer null_mask{0, stream, mr}; - if (target.parent().nullable() || null_count > 0) { null_mask = std::move(valid_mask.first); } - - // build offsets column - - std::unique_ptr p_offsets_column{nullptr}; - size_type chars_bytes = 0; - if (target.has_nulls()) { // check validities for both source & target - auto string_size_begin = thrust::make_transform_iterator( - thrust::make_counting_iterator(0), - compute_element_size{ - source_value_begin, source_validity_begin, d_target, target_begin, target_end}); - - std::tie(p_offsets_column, chars_bytes) = cudf::detail::make_offsets_child_column( - string_size_begin, string_size_begin + target.size(), stream, mr); - } else if (null_count > 0) { // check validities for source only - auto string_size_begin = thrust::make_transform_iterator( - thrust::make_counting_iterator(0), - compute_element_size{ - source_value_begin, source_validity_begin, d_target, target_begin, target_end}); - - std::tie(p_offsets_column, chars_bytes) = cudf::detail::make_offsets_child_column( - string_size_begin, string_size_begin + target.size(), stream, mr); - } else { // no need to check validities - auto string_size_begin = thrust::make_transform_iterator( - thrust::make_counting_iterator(0), - compute_element_size{ - source_value_begin, source_validity_begin, d_target, target_begin, target_end}); - - std::tie(p_offsets_column, chars_bytes) = cudf::detail::make_offsets_child_column( - string_size_begin, string_size_begin + target.size(), stream, mr); - } - - // create the chars column - - auto p_offsets = - thrust::device_pointer_cast(p_offsets_column->view().template data()); - auto p_chars_column = strings::detail::create_chars_child_column(chars_bytes, stream, mr); - - // copy to the chars column - - auto p_chars = (p_chars_column->mutable_view()).template data(); - thrust::for_each(rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(target.size()), - [source_value_begin, - source_validity_begin, - d_target, - target_begin, - target_end, - p_offsets, - p_chars] __device__(size_type idx) { - if (p_offsets[idx + 1] - p_offsets[idx] > 0) { - const auto source = (idx >= target_begin && idx < target_end) - ? *(source_value_begin + (idx - target_begin)) - : d_target.element(idx); - memcpy(p_chars + p_offsets[idx], source.data(), source.size_bytes()); - } - }); - - return make_strings_column(target.size(), - std::move(p_offsets_column), - std::move(p_chars_column->release().data.release()[0]), - null_count, - std::move(null_mask)); - } -} - -} // namespace detail -} // namespace strings -} // namespace cudf diff --git a/cpp/include/cudf/strings/detail/copy_range.hpp b/cpp/include/cudf/strings/detail/copy_range.hpp new file mode 100644 index 00000000000..e18f1fdc5ad --- /dev/null +++ b/cpp/include/cudf/strings/detail/copy_range.hpp @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include + +#include + +namespace cudf { +namespace strings { +namespace detail { + +/** + * @brief Internal API to copy a range of string elements out-of-place from + * a source column to a target column + * + * Creates a new column as if an in-place copy was performed into `target`. + * The elements indicated by the indices `source_begin`, `source_end`) + * replace with the elements in the target column starting at `target_begin`. + * Elements outside the range are copied from `target` into the new target + * column to return. + * + * @throws cudf::logic_error for invalid range (if `target_begin < 0`, + * or `target_begin >= target.size()`, + * or `target_begin + (source_end-source_begin)` > target.size()`). + * + * @param source The strings column to copy from inside the `target_begin` range + * @param target The strings column to copy from outside the range + * @param source_end The index of the first element in the source range + * @param source_end The index of the last element in the source range (exclusive) + * @param target_begin The starting index of the target range (inclusive) + * @param stream CUDA stream used for device memory operations and kernel launches. + * @param mr Device memory resource used to allocate the returned column's device memory. + * @return The result target column + */ +std::unique_ptr copy_range(strings_column_view const& source, + strings_column_view const& target, + size_type source_begin, + size_type source_end, + size_type target_begin, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr); + +} // namespace detail +} // namespace strings +} // namespace cudf diff --git a/cpp/src/copying/copy_range.cu b/cpp/src/copying/copy_range.cu index af253858c73..61d51f1d284 100644 --- a/cpp/src/copying/copy_range.cu +++ b/cpp/src/copying/copy_range.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include #include #include @@ -130,29 +130,8 @@ std::unique_ptr out_of_place_copy_range_dispatch::operator()(*p_source_device_view, - cudf::string_view()) + - source_begin, - cudf::detail::make_validity_iterator(*p_source_device_view) + source_begin, - cudf::strings_column_view(target), - target_begin, - target_end, - stream, - mr); - } else { - return cudf::strings::detail::copy_range( - p_source_device_view->begin() + source_begin, - thrust::make_constant_iterator(true), - cudf::strings_column_view(target), - target_begin, - target_end, - stream, - mr); - } + return cudf::strings::detail::copy_range( + source, target, source_begin, source_end, target_begin, stream, mr); } template <> diff --git a/cpp/src/strings/copying/copy_range.cu b/cpp/src/strings/copying/copy_range.cu new file mode 100644 index 00000000000..f4c86389534 --- /dev/null +++ b/cpp/src/strings/copying/copy_range.cu @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +namespace cudf { +namespace strings { +namespace detail { +namespace { +struct compute_element_size { + column_device_view d_source; + column_device_view d_target; + size_type source_begin; + size_type target_begin; + size_type target_end; + bool source_has_nulls; + bool target_has_nulls; + + __device__ cudf::size_type operator()(cudf::size_type idx) + { + if (idx >= target_begin && idx < target_end) { + auto const str_idx = source_begin + (idx - target_begin); + return source_has_nulls && d_source.is_null_nocheck(str_idx) + ? 0 + : d_source.element(str_idx).size_bytes(); + } else { + return target_has_nulls && d_target.is_null_nocheck(idx) + ? 0 + : d_target.element(idx).size_bytes(); + } + } +}; + +} // namespace + +std::unique_ptr copy_range(strings_column_view const& source, + strings_column_view const& target, + size_type source_begin, + size_type source_end, + size_type target_begin, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) +{ + auto target_end = target_begin + (source_end - source_begin); + CUDF_EXPECTS( + (target_begin >= 0) && (target_begin < target.size()) && (target_end <= target.size()), + "Range is out of bounds.", + std::invalid_argument); + + if (target_end == target_begin) { return std::make_unique(target.parent(), stream, mr); } + auto source_device_view = column_device_view::create(source.parent(), stream); + auto d_source = *source_device_view; + auto target_device_view = column_device_view::create(target.parent(), stream); + auto d_target = *target_device_view; + + // create null mask + auto [null_mask, null_count] = [&] { + if (!target.parent().nullable() && !source.parent().nullable()) { + return std::pair(rmm::device_buffer{}, 0); + } + return cudf::detail::valid_if( + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(target.size()), + [d_source, d_target, source_begin, target_begin, target_end] __device__(size_type idx) { + return (idx >= target_begin && idx < target_end) + ? d_source.is_valid(source_begin + (idx - target_begin)) + : d_target.is_valid(idx); + }, + stream, + mr); + }(); + + auto [check_source, check_target] = [target, null_count = null_count] { + // check validities for both source & target + if (target.has_nulls()) { return std::make_pair(true, true); } + // check validities for source only + if (null_count > 0) { return std::make_pair(true, false); } + // no need to check validities + return std::make_pair(false, false); + }(); + + // create offsets + auto sizes_begin = cudf::detail::make_counting_transform_iterator( + 0, + compute_element_size{ + d_source, d_target, source_begin, target_begin, target_end, check_source, check_target}); + auto [offsets_column, chars_bytes] = cudf::strings::detail::make_offsets_child_column( + sizes_begin, sizes_begin + target.size(), stream, mr); + auto d_offsets = cudf::detail::offsetalator_factory::make_input_iterator(offsets_column->view()); + + // create chars + auto chars_data = rmm::device_uvector(chars_bytes, stream, mr); + auto d_chars = chars_data.data(); + thrust::for_each( + rmm::exec_policy_nosync(stream), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(target.size()), + [d_source, d_target, source_begin, target_begin, target_end, d_offsets, d_chars] __device__( + size_type idx) { + if (d_offsets[idx + 1] - d_offsets[idx] > 0) { + const auto source = (idx >= target_begin && idx < target_end) + ? d_source.element(source_begin + (idx - target_begin)) + : d_target.element(idx); + memcpy(d_chars + d_offsets[idx], source.data(), source.size_bytes()); + } + }); + + return make_strings_column(target.size(), + std::move(offsets_column), + chars_data.release(), + null_count, + std::move(null_mask)); +} + +} // namespace detail +} // namespace strings +} // namespace cudf From 66b3a937d18dea141f3807b5cffff3920b4464b9 Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Tue, 20 Feb 2024 16:29:34 +0000 Subject: [PATCH 271/384] Validate types in pylibcudf Column/Table constructors (#15088) Otherwise, someone can pass any random object to the constructor and will receive an unfriendly segfault when interacting with libcudf. Authors: - Lawrence Mitchell (https://github.com/wence-) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15088 --- python/cudf/cudf/_lib/cpp/join.pxd | 32 ++++++++++- python/cudf/cudf/_lib/join.pyx | 2 + python/cudf/cudf/_lib/pylibcudf/column.pxd | 2 +- python/cudf/cudf/_lib/pylibcudf/column.pyx | 2 + python/cudf/cudf/_lib/pylibcudf/join.pxd | 32 +++++++++-- python/cudf/cudf/_lib/pylibcudf/join.pyx | 64 ++++++++++++++++++---- python/cudf/cudf/_lib/pylibcudf/table.pyx | 2 + 7 files changed, 118 insertions(+), 18 deletions(-) diff --git a/python/cudf/cudf/_lib/cpp/join.pxd b/python/cudf/cudf/_lib/cpp/join.pxd index ea05256430a..7508052646a 100644 --- a/python/cudf/cudf/_lib/cpp/join.pxd +++ b/python/cudf/cudf/_lib/cpp/join.pxd @@ -10,7 +10,7 @@ from rmm._lib.device_uvector cimport device_uvector from cudf._lib.cpp.column.column cimport column from cudf._lib.cpp.table.table cimport table from cudf._lib.cpp.table.table_view cimport table_view -from cudf._lib.cpp.types cimport size_type +from cudf._lib.cpp.types cimport null_equality, size_type ctypedef unique_ptr[device_uvector[size_type]] gather_map_type ctypedef pair[gather_map_type, gather_map_type] gather_map_pair_type @@ -40,3 +40,33 @@ cdef extern from "cudf/join.hpp" namespace "cudf" nogil: const table_view left_keys, const table_view right_keys, ) except + + + cdef gather_map_pair_type inner_join( + const table_view left_keys, + const table_view right_keys, + null_equality nulls_equal, + ) except + + + cdef gather_map_pair_type left_join( + const table_view left_keys, + const table_view right_keys, + null_equality nulls_equal, + ) except + + + cdef gather_map_pair_type full_join( + const table_view left_keys, + const table_view right_keys, + null_equality nulls_equal, + ) except + + + cdef gather_map_type left_semi_join( + const table_view left_keys, + const table_view right_keys, + null_equality nulls_equal, + ) except + + + cdef gather_map_type left_anti_join( + const table_view left_keys, + const table_view right_keys, + null_equality nulls_equal, + ) except + diff --git a/python/cudf/cudf/_lib/join.pyx b/python/cudf/cudf/_lib/join.pyx index 65f2f8cdcc8..0a54f0d67a0 100644 --- a/python/cudf/cudf/_lib/join.pyx +++ b/python/cudf/cudf/_lib/join.pyx @@ -20,6 +20,7 @@ def join(list lhs, list rhs, how=None): left_rows, right_rows = join_func( pylibcudf.Table([c.to_pylibcudf(mode="read") for c in lhs]), pylibcudf.Table([c.to_pylibcudf(mode="read") for c in rhs]), + pylibcudf.types.NullEquality.EQUAL ) return Column.from_pylibcudf(left_rows), Column.from_pylibcudf(right_rows) @@ -37,5 +38,6 @@ def semi_join(list lhs, list rhs, how=None): join_func( pylibcudf.Table([c.to_pylibcudf(mode="read") for c in lhs]), pylibcudf.Table([c.to_pylibcudf(mode="read") for c in rhs]), + pylibcudf.types.NullEquality.EQUAL ) ), None diff --git a/python/cudf/cudf/_lib/pylibcudf/column.pxd b/python/cudf/cudf/_lib/pylibcudf/column.pxd index a821c9186a0..fc5cc77c9e7 100644 --- a/python/cudf/cudf/_lib/pylibcudf/column.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/column.pxd @@ -21,7 +21,7 @@ cdef class Column: gpumemoryview _mask size_type _null_count size_type _offset - # children: List[Column] + # _children: List[Column] list _children size_type _num_children diff --git a/python/cudf/cudf/_lib/pylibcudf/column.pyx b/python/cudf/cudf/_lib/pylibcudf/column.pyx index dbe8d4feb37..2a7215099d5 100644 --- a/python/cudf/cudf/_lib/pylibcudf/column.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/column.pyx @@ -45,6 +45,8 @@ cdef class Column: gpumemoryview mask, size_type null_count, size_type offset, list children ): + if not all(isinstance(c, Column) for c in children): + raise ValueError("All children must be pylibcudf Column objects") self._data_type = data_type self._size = size self._data = data diff --git a/python/cudf/cudf/_lib/pylibcudf/join.pxd b/python/cudf/cudf/_lib/pylibcudf/join.pxd index 4014dd4a399..ff7dec97596 100644 --- a/python/cudf/cudf/_lib/pylibcudf/join.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/join.pxd @@ -1,15 +1,37 @@ # Copyright (c) 2024, NVIDIA CORPORATION. +from cudf._lib.cpp.types cimport null_equality + from .column cimport Column from .table cimport Table -cpdef tuple inner_join(Table left_keys, Table right_keys) +cpdef tuple inner_join( + Table left_keys, + Table right_keys, + null_equality nulls_equal +) -cpdef tuple left_join(Table left_keys, Table right_keys) +cpdef tuple left_join( + Table left_keys, + Table right_keys, + null_equality nulls_equal +) -cpdef tuple full_join(Table left_keys, Table right_keys) +cpdef tuple full_join( + Table left_keys, + Table right_keys, + null_equality nulls_equal +) -cpdef Column left_semi_join(Table left_keys, Table right_keys) +cpdef Column left_semi_join( + Table left_keys, + Table right_keys, + null_equality nulls_equal +) -cpdef Column left_anti_join(Table left_keys, Table right_keys) +cpdef Column left_anti_join( + Table left_keys, + Table right_keys, + null_equality nulls_equal +) diff --git a/python/cudf/cudf/_lib/pylibcudf/join.pyx b/python/cudf/cudf/_lib/pylibcudf/join.pyx index e1b61dabe22..3710a84e594 100644 --- a/python/cudf/cudf/_lib/pylibcudf/join.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/join.pyx @@ -9,7 +9,7 @@ from rmm._lib.device_buffer cimport device_buffer from cudf._lib.cpp cimport join as cpp_join from cudf._lib.cpp.column.column cimport column -from cudf._lib.cpp.types cimport data_type, size_type, type_id +from cudf._lib.cpp.types cimport data_type, null_equality, size_type, type_id from .column cimport Column from .table cimport Table @@ -32,7 +32,11 @@ cdef Column _column_from_gather_map(cpp_join.gather_map_type gather_map): ) -cpdef tuple inner_join(Table left_keys, Table right_keys): +cpdef tuple inner_join( + Table left_keys, + Table right_keys, + null_equality nulls_equal +): """Perform an inner join between two tables. For details, see :cpp:func:`inner_join`. @@ -43,6 +47,8 @@ cpdef tuple inner_join(Table left_keys, Table right_keys): The left table to join. right_keys : Table The right table to join. + nulls_equal : NullEquality + Should nulls compare equal? Returns ------- @@ -52,14 +58,18 @@ cpdef tuple inner_join(Table left_keys, Table right_keys): """ cdef cpp_join.gather_map_pair_type c_result with nogil: - c_result = cpp_join.inner_join(left_keys.view(), right_keys.view()) + c_result = cpp_join.inner_join(left_keys.view(), right_keys.view(), nulls_equal) return ( _column_from_gather_map(move(c_result.first)), _column_from_gather_map(move(c_result.second)), ) -cpdef tuple left_join(Table left_keys, Table right_keys): +cpdef tuple left_join( + Table left_keys, + Table right_keys, + null_equality nulls_equal +): """Perform a left join between two tables. For details, see :cpp:func:`left_join`. @@ -70,6 +80,9 @@ cpdef tuple left_join(Table left_keys, Table right_keys): The left table to join. right_keys : Table The right table to join. + nulls_equal : NullEquality + Should nulls compare equal? + Returns ------- @@ -79,14 +92,18 @@ cpdef tuple left_join(Table left_keys, Table right_keys): """ cdef cpp_join.gather_map_pair_type c_result with nogil: - c_result = cpp_join.left_join(left_keys.view(), right_keys.view()) + c_result = cpp_join.left_join(left_keys.view(), right_keys.view(), nulls_equal) return ( _column_from_gather_map(move(c_result.first)), _column_from_gather_map(move(c_result.second)), ) -cpdef tuple full_join(Table left_keys, Table right_keys): +cpdef tuple full_join( + Table left_keys, + Table right_keys, + null_equality nulls_equal +): """Perform a full join between two tables. For details, see :cpp:func:`full_join`. @@ -97,6 +114,9 @@ cpdef tuple full_join(Table left_keys, Table right_keys): The left table to join. right_keys : Table The right table to join. + nulls_equal : NullEquality + Should nulls compare equal? + Returns ------- @@ -106,14 +126,18 @@ cpdef tuple full_join(Table left_keys, Table right_keys): """ cdef cpp_join.gather_map_pair_type c_result with nogil: - c_result = cpp_join.full_join(left_keys.view(), right_keys.view()) + c_result = cpp_join.full_join(left_keys.view(), right_keys.view(), nulls_equal) return ( _column_from_gather_map(move(c_result.first)), _column_from_gather_map(move(c_result.second)), ) -cpdef Column left_semi_join(Table left_keys, Table right_keys): +cpdef Column left_semi_join( + Table left_keys, + Table right_keys, + null_equality nulls_equal +): """Perform a left semi join between two tables. For details, see :cpp:func:`left_semi_join`. @@ -124,6 +148,9 @@ cpdef Column left_semi_join(Table left_keys, Table right_keys): The left table to join. right_keys : Table The right table to join. + nulls_equal : NullEquality + Should nulls compare equal? + Returns ------- @@ -132,11 +159,19 @@ cpdef Column left_semi_join(Table left_keys, Table right_keys): """ cdef cpp_join.gather_map_type c_result with nogil: - c_result = cpp_join.left_semi_join(left_keys.view(), right_keys.view()) + c_result = cpp_join.left_semi_join( + left_keys.view(), + right_keys.view(), + nulls_equal + ) return _column_from_gather_map(move(c_result)) -cpdef Column left_anti_join(Table left_keys, Table right_keys): +cpdef Column left_anti_join( + Table left_keys, + Table right_keys, + null_equality nulls_equal +): """Perform a left anti join between two tables. For details, see :cpp:func:`left_anti_join`. @@ -147,6 +182,9 @@ cpdef Column left_anti_join(Table left_keys, Table right_keys): The left table to join. right_keys : Table The right table to join. + nulls_equal : NullEquality + Should nulls compare equal? + Returns ------- @@ -155,5 +193,9 @@ cpdef Column left_anti_join(Table left_keys, Table right_keys): """ cdef cpp_join.gather_map_type c_result with nogil: - c_result = cpp_join.left_anti_join(left_keys.view(), right_keys.view()) + c_result = cpp_join.left_anti_join( + left_keys.view(), + right_keys.view(), + nulls_equal + ) return _column_from_gather_map(move(c_result)) diff --git a/python/cudf/cudf/_lib/pylibcudf/table.pyx b/python/cudf/cudf/_lib/pylibcudf/table.pyx index 6d25d215f28..0cde346fa9c 100644 --- a/python/cudf/cudf/_lib/pylibcudf/table.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/table.pyx @@ -28,6 +28,8 @@ cdef class Table: The columns in this table. """ def __init__(self, list columns): + if not all(isinstance(c, Column) for c in columns): + raise ValueError("All columns must be pylibcudf Column objects") self._columns = columns cdef table_view view(self) nogil: From ef635967b916abd5416cd864bf60991d60f4b60e Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 20 Feb 2024 12:16:42 -0600 Subject: [PATCH 272/384] target branch-24.04 for GitHub Actions workflows (#15069) Follow-up to #14712 For all GitHub Actions configs, replaces uses of the `test-cuda-12.2` branch on `shared-workflows` with `branch-24.04`, now that https://github.com/rapidsai/shared-workflows/pull/166 has been merged. ### Notes for Reviewers This is part of ongoing work to build and test packages against CUDA 12.2 across all of RAPIDS. For more details see: * https://github.com/rapidsai/build-planning/issues/7 *(created with `rapids-reviser`)* Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/cudf/pull/15069 --- .github/workflows/build.yaml | 16 ++++++++-------- .github/workflows/pr.yaml | 36 ++++++++++++++++++------------------ .github/workflows/test.yaml | 20 ++++++++++---------- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b92e0a53b46..1c68b3504e0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -37,7 +37,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,7 +46,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -57,7 +57,7 @@ jobs: if: github.ref_type == 'branch' needs: python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-cudf: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 with: build_type: ${{ inputs.build_type || 'branch' }} build-2_28-wheels: "true" @@ -80,7 +80,7 @@ jobs: wheel-publish-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -90,7 +90,7 @@ jobs: wheel-build-dask-cudf: needs: wheel-publish-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 with: matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) build_type: ${{ inputs.build_type || 'branch' }} @@ -101,7 +101,7 @@ jobs: wheel-publish-dask-cudf: needs: wheel-build-dask-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 57923dca5d9..4368c3892f5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -32,16 +32,16 @@ jobs: #- pandas-tests-diff #- pandas-tests-diff-comment secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.04 checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.04 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.04 with: build_type: pull-request conda-cpp-checks: @@ -54,19 +54,19 @@ jobs: conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.04 with: build_type: pull-request conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.04 with: build_type: pull-request conda-python-cudf-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 with: build_type: pull-request test_script: "ci/test_python_cudf.sh" @@ -74,14 +74,14 @@ jobs: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 with: build_type: pull-request test_script: "ci/test_python_other.sh" conda-java-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -91,7 +91,7 @@ jobs: conda-notebook-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -101,7 +101,7 @@ jobs: docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -111,7 +111,7 @@ jobs: wheel-build-cudf: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 with: build_type: pull-request build-2_28-wheels: "true" @@ -119,14 +119,14 @@ jobs: wheel-tests-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 with: build_type: pull-request script: ci/test_wheel_cudf.sh wheel-build-dask-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 with: matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) build_type: pull-request @@ -134,14 +134,14 @@ jobs: wheel-tests-dask-cudf: needs: wheel-build-dask-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 with: matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) build_type: pull-request script: ci/test_wheel_dask_cudf.sh devcontainer: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@test-cuda-12.2 + uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.04 with: build_command: | sccache -z; @@ -150,7 +150,7 @@ jobs: unit-tests-cudf-pandas: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 with: matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) build_type: pull-request @@ -159,7 +159,7 @@ jobs: # run the Pandas unit tests using PR branch needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 with: matrix_filter: map(select(.ARCH == "amd64")) | max_by(.CUDA_VER) | [.] build_type: pull-request @@ -171,7 +171,7 @@ jobs: # needs: [pandas-tests-main, pandas-tests-pr] # secrets: inherit # # This branch exports a `job_output` output that the downstream job reads. - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@test-cuda-12.2 + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 # with: # node_type: cpu4 # build_type: pull-request diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e7eef4de1b3..66287d9e515 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -25,7 +25,7 @@ jobs: enable_check_symbols: true conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.04 with: build_type: nightly branch: ${{ inputs.branch }} @@ -33,7 +33,7 @@ jobs: sha: ${{ inputs.sha }} conda-cpp-memcheck-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 with: build_type: nightly branch: ${{ inputs.branch }} @@ -45,7 +45,7 @@ jobs: run_script: "ci/test_cpp_memcheck.sh" conda-python-cudf-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 with: build_type: nightly branch: ${{ inputs.branch }} @@ -55,7 +55,7 @@ jobs: conda-python-other-tests: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 with: build_type: nightly branch: ${{ inputs.branch }} @@ -64,7 +64,7 @@ jobs: test_script: "ci/test_python_other.sh" conda-java-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 with: build_type: nightly branch: ${{ inputs.branch }} @@ -76,7 +76,7 @@ jobs: run_script: "ci/test_java.sh" conda-notebook-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 with: build_type: nightly branch: ${{ inputs.branch }} @@ -88,7 +88,7 @@ jobs: run_script: "ci/test_notebooks.sh" wheel-tests-cudf: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 with: build_type: nightly branch: ${{ inputs.branch }} @@ -97,7 +97,7 @@ jobs: script: ci/test_wheel_cudf.sh wheel-tests-dask-cudf: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 with: matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) build_type: nightly @@ -107,7 +107,7 @@ jobs: script: ci/test_wheel_dask_cudf.sh unit-tests-cudf-pandas: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 with: build_type: nightly branch: ${{ inputs.branch }} @@ -117,7 +117,7 @@ jobs: pandas-tests: # run the Pandas unit tests secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@test-cuda-12.2 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 with: matrix_filter: map(select(.ARCH == "amd64")) | max_by(.CUDA_VER) | [.] build_type: nightly From 12d1500fedacefb34bd62e5f7ac90b001d80f98e Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 20 Feb 2024 13:38:12 -0500 Subject: [PATCH 273/384] Factor out position-offsets logic from strings split_helper utility (#15040) The logic used by `strings::split()` functions is refactored into its own utility for reuse with `strings::replace` and possibly other strings and text functions. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/15040 --- cpp/src/strings/split/split.cu | 40 +++++++++++++++++++ cpp/src/strings/split/split.cuh | 69 +++++++++++++-------------------- 2 files changed, 66 insertions(+), 43 deletions(-) diff --git a/cpp/src/strings/split/split.cu b/cpp/src/strings/split/split.cu index fbab5220383..17293a71b63 100644 --- a/cpp/src/strings/split/split.cu +++ b/cpp/src/strings/split/split.cu @@ -384,6 +384,46 @@ std::unique_ptr
whitespace_split_fn(size_type strings_count, } // namespace +std::unique_ptr create_offsets_from_positions(strings_column_view const& input, + device_span const& positions, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) +{ + auto const d_offsets = + cudf::detail::offsetalator_factory::make_input_iterator(input.offsets(), input.offset()); + + // first, create a vector of string indices for each position + auto indices = rmm::device_uvector(positions.size(), stream); + thrust::upper_bound(rmm::exec_policy_nosync(stream), + d_offsets, + d_offsets + input.size(), + positions.begin(), + positions.end(), + indices.begin()); + + // compute position offsets per string + auto counts = rmm::device_uvector(input.size(), stream); + // memset to zero-out the counts for any null-entries or strings with no positions + thrust::uninitialized_fill(rmm::exec_policy_nosync(stream), counts.begin(), counts.end(), 0); + + // next, count the number of positions per string + auto d_counts = counts.data(); + auto d_indices = indices.data(); + thrust::for_each_n( + rmm::exec_policy_nosync(stream), + thrust::counting_iterator(0), + positions.size(), + [d_indices, d_counts] __device__(int64_t idx) { + auto const str_idx = d_indices[idx] - 1; + cuda::atomic_ref ref{*(d_counts + str_idx)}; + ref.fetch_add(1L, cuda::std::memory_order_relaxed); + }); + + // finally, convert the counts into offsets + return std::get<0>( + cudf::strings::detail::make_offsets_child_column(counts.begin(), counts.end(), stream, mr)); +} + std::unique_ptr
split(strings_column_view const& strings_column, string_scalar const& delimiter, size_type maxsplit, diff --git a/cpp/src/strings/split/split.cuh b/cpp/src/strings/split/split.cuh index 906c522e898..750b18c8b4c 100644 --- a/cpp/src/strings/split/split.cuh +++ b/cpp/src/strings/split/split.cuh @@ -88,7 +88,7 @@ struct base_split_tokenizer { */ __device__ size_type count_tokens(size_type idx, int64_t const* d_positions, - int64_t const* d_delimiter_offsets) const + cudf::detail::input_offsetalator d_delimiter_offsets) const { if (!is_valid(idx)) { return 0; } @@ -132,7 +132,7 @@ struct base_split_tokenizer { __device__ void get_tokens(size_type idx, cudf::detail::input_offsetalator const d_tokens_offsets, int64_t const* d_positions, - int64_t const* d_delimiter_offsets, + cudf::detail::input_offsetalator d_delimiter_offsets, string_index_pair* d_all_tokens) const { auto const d_tokens = // this string's tokens output @@ -280,6 +280,23 @@ struct rsplit_tokenizer_fn : base_split_tokenizer { } }; +/** + * @brief Create offsets for position values within a strings column + * + * The positions usually identify target sub-strings in the input column. + * The offsets identify the set of positions for each string row. + * + * @param input Strings column corresponding to the input positions + * @param positions Indices of target bytes within the input column + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned objects' device memory + * @return Offsets of the position values for each string in input + */ +std::unique_ptr create_offsets_from_positions(strings_column_view const& input, + device_span const& positions, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr); + /** * @brief Helper function used by split/rsplit and split_record/rsplit_record * @@ -316,13 +333,12 @@ std::pair, rmm::device_uvector> split [tokenizer, d_offsets, chars_bytes] __device__(int64_t idx) { return tokenizer.is_delimiter(idx, d_offsets, chars_bytes); }); - // Create a vector of every delimiter position in the chars column. // These may include overlapping or otherwise out-of-bounds delimiters which // will be resolved during token processing. auto delimiter_positions = rmm::device_uvector(delimiter_count, stream); auto d_positions = delimiter_positions.data(); - auto const copy_end = cudf::detail::copy_if_safe( + cudf::detail::copy_if_safe( thrust::counting_iterator(0), thrust::counting_iterator(chars_bytes), delimiter_positions.begin(), @@ -332,48 +348,15 @@ std::pair, rmm::device_uvector> split stream); // create a vector of offsets to each string's delimiter set within delimiter_positions - auto const delimiter_offsets = [&] { - // first, create a vector of string indices for each delimiter - auto string_indices = rmm::device_uvector(delimiter_count, stream); - thrust::upper_bound(rmm::exec_policy(stream), - d_offsets, - d_offsets + strings_count, - delimiter_positions.begin(), - copy_end, - string_indices.begin()); - - // compute delimiter offsets per string - auto delimiter_offsets = rmm::device_uvector(strings_count + 1, stream); - auto d_delimiter_offsets = delimiter_offsets.data(); - - // memset to zero-out the delimiter counts for any null-entries or strings with no delimiters - CUDF_CUDA_TRY(cudaMemsetAsync( - d_delimiter_offsets, 0, delimiter_offsets.size() * sizeof(int64_t), stream.value())); - - // next, count the number of delimiters per string - auto d_string_indices = string_indices.data(); // identifies strings with delimiters only - thrust::for_each_n( - rmm::exec_policy(stream), - thrust::counting_iterator(0), - delimiter_count, - [d_string_indices, d_delimiter_offsets] __device__(int64_t idx) { - auto const str_idx = d_string_indices[idx] - 1; - cuda::atomic_ref ref{*(d_delimiter_offsets + str_idx)}; - ref.fetch_add(1L, cuda::std::memory_order_relaxed); - }); - // finally, convert the delimiter counts into offsets - thrust::exclusive_scan(rmm::exec_policy(stream), - delimiter_offsets.begin(), - delimiter_offsets.end(), - delimiter_offsets.begin()); - return delimiter_offsets; - }(); - auto const d_delimiter_offsets = delimiter_offsets.data(); + auto const delimiter_offsets = + create_offsets_from_positions(input, delimiter_positions, stream, mr); + auto const d_delimiter_offsets = + cudf::detail::offsetalator_factory::make_input_iterator(delimiter_offsets->view()); // compute the number of tokens per string auto token_counts = rmm::device_uvector(strings_count, stream); thrust::transform( - rmm::exec_policy(stream), + rmm::exec_policy_nosync(stream), thrust::make_counting_iterator(0), thrust::make_counting_iterator(strings_count), token_counts.begin(), @@ -391,7 +374,7 @@ std::pair, rmm::device_uvector> split auto tokens = rmm::device_uvector(total_tokens, stream); auto d_tokens = tokens.data(); thrust::for_each_n( - rmm::exec_policy(stream), + rmm::exec_policy_nosync(stream), thrust::make_counting_iterator(0), strings_count, [tokenizer, d_tokens_offsets, d_positions, d_delimiter_offsets, d_tokens] __device__( From 8a673cd6d0bef283861f8b7f38207768e3f57fd2 Mon Sep 17 00:00:00 2001 From: Nghia Truong <7416935+ttnghia@users.noreply.github.com> Date: Tue, 20 Feb 2024 12:09:36 -0700 Subject: [PATCH 274/384] Fix reading offset for data stream in ORC reader (#14911) Fixes a bug in ORC reader, which moves the destination write offset instead of the source read offset when a stream is ignored from reading. Authors: - Nghia Truong (https://github.com/ttnghia) Approvers: - Mike Wilson (https://github.com/hyperbolic2346) - Vukasin Milovanovic (https://github.com/vuule) URL: https://github.com/rapidsai/cudf/pull/14911 --- cpp/src/io/orc/reader_impl_preprocess.cu | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cpp/src/io/orc/reader_impl_preprocess.cu b/cpp/src/io/orc/reader_impl_preprocess.cu index 08f5adb0729..026e2e7d8ed 100644 --- a/cpp/src/io/orc/reader_impl_preprocess.cu +++ b/cpp/src/io/orc/reader_impl_preprocess.cu @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -100,7 +101,9 @@ std::size_t gather_stream_info(std::size_t stripe_index, for (auto const& stream : stripefooter->streams) { if (!stream.column_id || *stream.column_id >= orc2gdf.size()) { - dst_offset += stream.length; + // Ignore reading this stream from source. + cudf::logger().warn("Unexpected stream in the input ORC source. The stream will be ignored."); + src_offset += stream.length; continue; } @@ -125,8 +128,7 @@ std::size_t gather_stream_info(std::size_t stripe_index, } } } - } - if (col != -1) { + } else if (col != -1) { if (src_offset >= stripeinfo->indexLength || use_index) { auto& chunk = chunks[stripe_index][col]; auto const index_type = get_stream_index_type(stream.kind); From 047b112b1ad149407b8fbd1f9e6d6758ad663cad Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 20 Feb 2024 13:15:49 -0600 Subject: [PATCH 275/384] Fix `datetime` binop pytest failures in pandas-2.2 (#15090) This PR handles two datetime binop pytest failures, that are regressions in `pandas-2.2` Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/15090 --- python/cudf/cudf/tests/test_binops.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/tests/test_binops.py b/python/cudf/cudf/tests/test_binops.py index 6c6dae9e22e..92a9fd6636c 100644 --- a/python/cudf/cudf/tests/test_binops.py +++ b/python/cudf/cudf/tests/test_binops.py @@ -1796,7 +1796,13 @@ def test_datetime_dateoffset_binaryop( "ignore:Discarding nonzero nanoseconds:UserWarning" ) @pytest.mark.parametrize("op", [operator.add, operator.sub]) -def test_datetime_dateoffset_binaryop_multiple(date_col, kwargs, op): +def test_datetime_dateoffset_binaryop_multiple(request, date_col, kwargs, op): + request.applymarker( + pytest.mark.xfail( + PANDAS_GE_220 and len(kwargs) == 1 and "milliseconds" in kwargs, + reason="https://github.com/pandas-dev/pandas/issues/57529", + ) + ) gsr = cudf.Series(date_col, dtype="datetime64[ns]") psr = gsr.to_pandas() @@ -1864,7 +1870,9 @@ def test_datetime_dateoffset_binaryop_reflected( expect = poffset + psr got = goffset + gsr - utils.assert_eq(expect, got) + # TODO: Remove check_dtype once we get some clarity on: + # https://github.com/pandas-dev/pandas/issues/57448 + utils.assert_eq(expect, got, check_dtype=not PANDAS_GE_220) with pytest.raises(TypeError): poffset - psr From 44686ca390f766e51cc0c1c3a08a422fc867b061 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 20 Feb 2024 14:57:36 -0500 Subject: [PATCH 276/384] Deprecate cudf::hashing::spark_murmurhash3_x86_32 (#15074) The `cudf::hashing::spark_murmurhash3_x86_32()` function was moved to the Spark plugin since it had common code with the Spark implementation of `xxhash_64` (also implemented in the plugin). This change deprecates the API and the generic `cudf::hashing::hash()` function to be removed in a follow-on release. Reference hash cleanup issue: #13706 Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Karthikeyan (https://github.com/karthikeyann) URL: https://github.com/rapidsai/cudf/pull/15074 --- cpp/include/cudf/hashing.hpp | 6 ++++-- cpp/tests/CMakeLists.txt | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cpp/include/cudf/hashing.hpp b/cpp/include/cudf/hashing.hpp index c3a57af1358..64a78da1803 100644 --- a/cpp/include/cudf/hashing.hpp +++ b/cpp/include/cudf/hashing.hpp @@ -63,7 +63,7 @@ static constexpr uint32_t DEFAULT_HASH_SEED = 0; * * @returns A column where each row is the hash of a column from the input */ -std::unique_ptr hash( +[[deprecated]] std::unique_ptr hash( table_view const& input, hash_id hash_function = hash_id::HASH_MURMUR3, uint32_t seed = DEFAULT_HASH_SEED, @@ -115,6 +115,8 @@ std::unique_ptr
murmurhash3_x64_128( /** * @brief Computes the MurmurHash3 32-bit hash value of each row in the given table * + * @deprecated Since 24.04 + * * This function computes the hash similar to MurmurHash3_x86_32 with special processing * to match Spark's implementation results. * @@ -125,7 +127,7 @@ std::unique_ptr
murmurhash3_x64_128( * * @returns A column where each row is the hash of a row from the input */ -std::unique_ptr spark_murmurhash3_x86_32( +[[deprecated]] std::unique_ptr spark_murmurhash3_x86_32( table_view const& input, uint32_t seed = DEFAULT_HASH_SEED, rmm::cuda_stream_view stream = cudf::get_default_stream(), diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 4c07970714d..94ae349896c 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -175,7 +175,6 @@ ConfigureTest( hashing/sha256_test.cpp hashing/sha384_test.cpp hashing/sha512_test.cpp - hashing/spark_murmurhash3_x86_32_test.cpp hashing/xxhash_64_test.cpp ) From 6903f803041062904a0a3ce37b5f031597cbd0b3 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 20 Feb 2024 15:50:57 -0600 Subject: [PATCH 277/384] Add support for arrow `large_string` in `cudf` (#15093) This PR adds support for `large_string` type of `arrow` arrays in `cudf`. `cudf` strings column lacks 64 bit offset support and it is WIP: https://github.com/rapidsai/cudf/issues/13733 This workaround is essential because `pandas-2.2+` is now defaulting to `large_string` type for arrow-strings instead of `string` type.: https://github.com/pandas-dev/pandas/pull/56220 This PR fixes all 25 `dask-cudf` failures. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Matthew Roeschke (https://github.com/mroeschke) - Ashwin Srinath (https://github.com/shwina) URL: https://github.com/rapidsai/cudf/pull/15093 --- python/cudf/cudf/core/column/column.py | 7 +++++++ python/cudf/cudf/tests/test_series.py | 8 ++++++++ python/cudf/cudf/utils/dtypes.py | 2 ++ 3 files changed, 17 insertions(+) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index f665d83964c..191c55a8a68 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -1920,6 +1920,13 @@ def as_column( return col elif isinstance(arbitrary, (pa.Array, pa.ChunkedArray)): + if pa.types.is_large_string(arbitrary.type): + # Pandas-2.2+: Pandas defaults to `large_string` type + # instead of `string` without data-introspection. + # Temporary workaround until cudf has native + # support for `LARGE_STRING` i.e., 64 bit offsets + arbitrary = arbitrary.cast(pa.string()) + if pa.types.is_float16(arbitrary.type): raise NotImplementedError( "Type casting from `float16` to `float32` is not " diff --git a/python/cudf/cudf/tests/test_series.py b/python/cudf/cudf/tests/test_series.py index 252343391be..caf8947e3b0 100644 --- a/python/cudf/cudf/tests/test_series.py +++ b/python/cudf/cudf/tests/test_series.py @@ -2700,3 +2700,11 @@ def test_series_dtype_astypes(data): result = cudf.Series(data, dtype="float64") expected = cudf.Series([1.0, 2.0, 3.0]) assert_eq(result, expected) + + +def test_series_from_large_string(): + pa_large_string_array = pa.array(["a", "b", "c"]).cast(pa.large_string()) + got = cudf.Series(pa_large_string_array) + expected = pd.Series(pa_large_string_array) + + assert_eq(expected, got) diff --git a/python/cudf/cudf/utils/dtypes.py b/python/cudf/cudf/utils/dtypes.py index 8fa4a230e2c..c8aca94ba19 100644 --- a/python/cudf/cudf/utils/dtypes.py +++ b/python/cudf/cudf/utils/dtypes.py @@ -213,6 +213,8 @@ def cudf_dtype_from_pa_type(typ): return cudf.core.dtypes.StructDtype.from_arrow(typ) elif pa.types.is_decimal(typ): return cudf.core.dtypes.Decimal128Dtype.from_arrow(typ) + elif pa.types.is_large_string(typ): + return cudf.dtype("str") else: return cudf.api.types.pandas_dtype(typ.to_pandas_dtype()) From e7a7e4806af39ff8e220d3ca26c5d402d6be38a3 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 20 Feb 2024 15:54:26 -0600 Subject: [PATCH 278/384] Fix `sort_values` pytest failure with pandas-2.x regression (#15092) pandas-2.x seems to have introduced an ordering regression where the index order is not preserved for cases when there is a tie. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/15092 --- python/cudf/cudf/tests/test_sorting.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/python/cudf/cudf/tests/test_sorting.py b/python/cudf/cudf/tests/test_sorting.py index dd545da4243..b3ecb471bb9 100644 --- a/python/cudf/cudf/tests/test_sorting.py +++ b/python/cudf/cudf/tests/test_sorting.py @@ -48,11 +48,13 @@ def test_dataframe_sort_values(nelem, dtype): @pytest.mark.parametrize("ignore_index", [True, False]) @pytest.mark.parametrize("index", ["a", "b", ["a", "b"]]) -def test_dataframe_sort_values_ignore_index(index, ignore_index): - if not PANDAS_GE_220 and isinstance(index, list) and not ignore_index: - pytest.skip( - reason="TODO: Remove this once pandas-2.2 support is added", +def test_dataframe_sort_values_ignore_index(request, index, ignore_index): + request.applymarker( + pytest.mark.xfail( + PANDAS_GE_220 and isinstance(index, list) and not ignore_index, + reason="https://github.com/pandas-dev/pandas/issues/57531", ) + ) gdf = DataFrame( {"a": [1, 3, 5, 2, 4], "b": [1, 1, 2, 2, 3], "c": [9, 7, 7, 7, 1]} From fab911ac5f6b4454da7677d77c759ab6670f63e1 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 20 Feb 2024 14:25:08 -1000 Subject: [PATCH 279/384] Align MultiIndex.get_indexder with pandas 2.2 change (#15059) Aligns with https://github.com/pandas-dev/pandas/pull/55352 Additionally, refactored a `pandas.PeriodIndex` usage to a non-deprecated version Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15059 --- python/cudf/cudf/core/multiindex.py | 6 ++++ python/cudf/cudf/tests/test_index.py | 47 ++++++++++++++++++---------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index a3f7be7b266..9466d172eb1 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1836,6 +1836,12 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): raise NotImplementedError( f"{method=} is not supported yet for MultiIndex." ) + if method in {"ffill", "bfill", "pad", "backfill"} and not ( + self.is_monotonic_increasing or self.is_monotonic_decreasing + ): + raise ValueError( + "index must be monotonic increasing or decreasing" + ) result = cudf.core.column.full( len(target), diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index defd42b3d00..aff71f1882b 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -2067,14 +2067,6 @@ def test_get_loc_multi_numeric_deviate(idx, key, result): assert_eq(expected, got) -@pytest.mark.parametrize( - "idx", - [ - pd.MultiIndex.from_tuples( - [(2, 1, 1), (1, 2, 3), (1, 2, 1), (1, 1, 10), (1, 1, 1), (2, 2, 1)] - ) - ], -) @pytest.mark.parametrize( "key", [ @@ -2084,21 +2076,42 @@ def test_get_loc_multi_numeric_deviate(idx, key, result): ], ) @pytest.mark.parametrize("method", [None, "ffill", "bfill"]) -def test_get_indexer_multi_numeric_deviate(request, idx, key, method): - pi = idx +def test_get_indexer_multi_numeric_deviate(key, method): + pi = pd.MultiIndex.from_tuples( + [(2, 1, 1), (1, 2, 3), (1, 2, 1), (1, 1, 10), (1, 1, 1), (2, 2, 1)] + ).sort_values() gi = cudf.from_pandas(pi) - request.applymarker( - pytest.mark.xfail( - condition=method is not None and key == ((1, 2, 3),), - reason="https://github.com/pandas-dev/pandas/issues/53452", - ) - ) + expected = pi.get_indexer(key, method=method) got = gi.get_indexer(key, method=method) assert_eq(expected, got) +@pytest.mark.xfail( + not PANDAS_GE_220, reason="Remove after pandas-2.2+ upgrade" +) +@pytest.mark.parametrize("method", ["ffill", "bfill"]) +def test_get_indexer_multi_error(method): + pi = pd.MultiIndex.from_tuples( + [(2, 1, 1), (1, 2, 3), (1, 2, 1), (1, 1, 10), (1, 1, 1), (2, 2, 1)] + ) + gi = cudf.from_pandas(pi) + + assert_exceptions_equal( + pi.get_indexer, + gi.get_indexer, + lfunc_args_and_kwargs=( + [], + {"target": ((1, 2, 3),), "method": method}, + ), + rfunc_args_and_kwargs=( + [], + {"target": ((1, 2, 3),), "method": method}, + ), + ) + + @pytest.mark.parametrize( "idx", [ @@ -3094,7 +3107,7 @@ def test_index_with_index_dtype(data, dtype): def test_period_index_error(): - pidx = pd.PeriodIndex(year=[2000, 2002], quarter=[1, 3]) + pidx = pd.PeriodIndex(data=[pd.Period("2020-01")]) with pytest.raises(NotImplementedError): cudf.from_pandas(pidx) with pytest.raises(NotImplementedError): From 8ea716b6202d7d5093e63808d7518717ec23f7d0 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 20 Feb 2024 14:45:18 -1000 Subject: [PATCH 280/384] Fix ORC and JSON tests failures for pandas 2.2 (#15062) `test_order_nested_json_reader` was refactored to use `assert_eq` instead of comparing via pyarrow. This was failing in pandas 2.2 due to https://github.com/pandas-dev/pandas/issues/57429 `test_orc_reader_trailing_nulls` I believe was failing due to a change in how integers are compared with `assert_series_equal`: https://github.com/pandas-dev/pandas/issues/55882. The "casting workaround" doesn't seem necessary in pandas 2.2 so just avoiding it all together Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15062 --- python/cudf/cudf/tests/test_json.py | 8 +++++++- python/cudf/cudf/tests/test_orc.py | 22 ++++++++++++++-------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/python/cudf/cudf/tests/test_json.py b/python/cudf/cudf/tests/test_json.py index ec980adc334..12ea74bd7a7 100644 --- a/python/cudf/cudf/tests/test_json.py +++ b/python/cudf/cudf/tests/test_json.py @@ -13,7 +13,7 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210 +from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210, PANDAS_GE_220 from cudf.testing._utils import ( DATETIME_TYPES, NUMERIC_TYPES, @@ -1179,7 +1179,13 @@ def test_chunked_nested_json_reader(self, tag, data, chunk_size): def test_order_nested_json_reader(self, tag, data): expected = pd.read_json(StringIO(data), lines=True) + if PANDAS_GE_220: + # TODO: Remove after https://github.com/pandas-dev/pandas/issues/57429 + # is fixed + expected = expected.reset_index(drop=True) target = cudf.read_json(StringIO(data), lines=True) + # Using pyarrow instead of assert_eq because pandas + # doesn't handle nested values comparisons correctly if tag == "dtype_mismatch": with pytest.raises(AssertionError): # pandas parses integer values in float representation diff --git a/python/cudf/cudf/tests/test_orc.py b/python/cudf/cudf/tests/test_orc.py index cf2fd29d41e..80fc815dd76 100644 --- a/python/cudf/cudf/tests/test_orc.py +++ b/python/cudf/cudf/tests/test_orc.py @@ -13,6 +13,7 @@ import pytest import cudf +from cudf.core._compat import PANDAS_GE_220 from cudf.io.orc import ORCWriter from cudf.testing import assert_frame_equal from cudf.testing._utils import ( @@ -130,16 +131,21 @@ def test_orc_reader_filepath_or_buffer(path_or_buf, src): def test_orc_reader_trailing_nulls(datadir): path = datadir / "TestOrcFile.nulls-at-end-snappy.orc" + expect = pd.read_orc(path) + got = cudf.read_orc(path) + if PANDAS_GE_220: + check_categorical = True + else: + check_categorical = False + expect = expect.fillna(0) + got = got.fillna(0) - expect = pd.read_orc(path).fillna(0) - got = cudf.read_orc(path).fillna(0) - - # PANDAS uses NaN to represent invalid data, which forces float dtype - # For comparison, we can replace NaN with 0 and cast to the cuDF dtype - for col in expect.columns: - expect[col] = expect[col].astype(got[col].dtype) + # PANDAS uses NaN to represent invalid data, which forces float dtype + # For comparison, we can replace NaN with 0 and cast to the cuDF dtype + for col in expect.columns: + expect[col] = expect[col].astype(got[col].dtype) - assert_eq(expect, got, check_categorical=False) + assert_eq(expect, got, check_categorical=check_categorical) @pytest.mark.parametrize("use_index", [False, True]) From 8e68b37684ee8780f39b43609b3192a982aa9a5f Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Wed, 21 Feb 2024 09:20:09 -0500 Subject: [PATCH 281/384] Fix deprecation warnings for deprecated hash() calls (#15095) Merged #15074 too soon and missed fixing these now deprecated call warnings. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Nghia Truong (https://github.com/ttnghia) - Mark Harris (https://github.com/harrism) URL: https://github.com/rapidsai/cudf/pull/15095 --- cpp/benchmarks/hashing/hash.cpp | 15 +-------------- cpp/tests/partitioning/hash_partition_test.cpp | 4 ++-- cpp/tests/streams/hash_test.cpp | 6 +++--- 3 files changed, 6 insertions(+), 19 deletions(-) diff --git a/cpp/benchmarks/hashing/hash.cpp b/cpp/benchmarks/hashing/hash.cpp index 1da7457eb82..61e79a47a50 100644 --- a/cpp/benchmarks/hashing/hash.cpp +++ b/cpp/benchmarks/hashing/hash.cpp @@ -97,12 +97,6 @@ static void bench_hash(nvbench::state& state) state.exec(nvbench::exec_tag::sync, [&](nvbench::launch& launch) { auto result = cudf::hashing::sha512(data->view()); }); - } else if (hash_name == "spark_murmurhash3_x86_32") { - state.add_global_memory_writes(num_rows); - - state.exec(nvbench::exec_tag::sync, [&](nvbench::launch& launch) { - auto result = cudf::hashing::spark_murmurhash3_x86_32(data->view()); - }); } else { state.skip(hash_name + ": unknown hash name"); } @@ -113,11 +107,4 @@ NVBENCH_BENCH(bench_hash) .add_int64_axis("num_rows", {65536, 16777216}) .add_float64_axis("nulls", {0.0, 0.1}) .add_string_axis("hash_name", - {"murmurhash3_x86_32", - "md5", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "spark_murmurhash3_x86_32"}); + {"murmurhash3_x86_32", "md5", "sha1", "sha224", "sha256", "sha384", "sha512"}); diff --git a/cpp/tests/partitioning/hash_partition_test.cpp b/cpp/tests/partitioning/hash_partition_test.cpp index f1486a49bf9..d7b12417251 100644 --- a/cpp/tests/partitioning/hash_partition_test.cpp +++ b/cpp/tests/partitioning/hash_partition_test.cpp @@ -309,11 +309,11 @@ void run_fixed_width_test(size_t cols, cudf::table_view partitions_table({partitions_col}); // Sort partition numbers by the corresponding row hashes of each output - auto hash1 = cudf::hash(output1->view()); + auto hash1 = cudf::hashing::murmurhash3_x86_32(output1->view()); cudf::table_view hash1_table({hash1->view()}); auto sorted_partitions1 = cudf::sort_by_key(partitions_table, hash1_table); - auto hash2 = cudf::hash(output2->view()); + auto hash2 = cudf::hashing::murmurhash3_x86_32(output2->view()); cudf::table_view hash2_table({hash2->view()}); auto sorted_partitions2 = cudf::sort_by_key(partitions_table, hash2_table); diff --git a/cpp/tests/streams/hash_test.cpp b/cpp/tests/streams/hash_test.cpp index 0f60c506abe..8c6609fdc22 100644 --- a/cpp/tests/streams/hash_test.cpp +++ b/cpp/tests/streams/hash_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -49,6 +49,6 @@ TEST_F(HashTest, MultiValue) auto const input1 = cudf::table_view({strings_col, ints_col, bools_col1, secs_col}); - auto const output1 = cudf::hash( - input1, cudf::hash_id::HASH_MURMUR3, cudf::DEFAULT_HASH_SEED, cudf::test::get_default_stream()); + auto const output1 = cudf::hashing::murmurhash3_x86_32( + input1, cudf::DEFAULT_HASH_SEED, cudf::test::get_default_stream()); } From 3b888a65e5aff9f1ea8adbcb77b26f1d0d103511 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Wed, 21 Feb 2024 09:20:40 -0500 Subject: [PATCH 282/384] Use offsetalator in cudf::detail::has_nonempty_null_rows (#15076) Updates `cudf::detail::has_nonempty_null_rows` to use the offsetalator instead of hardcoded integer type. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Vukasin Milovanovic (https://github.com/vuule) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/15076 --- cpp/src/copying/purge_nonempty_nulls.cu | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cpp/src/copying/purge_nonempty_nulls.cu b/cpp/src/copying/purge_nonempty_nulls.cu index b578f319a89..620a03d8be5 100644 --- a/cpp/src/copying/purge_nonempty_nulls.cu +++ b/cpp/src/copying/purge_nonempty_nulls.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include #include +#include #include #include @@ -41,9 +42,11 @@ bool has_nonempty_null_rows(cudf::column_view const& input, rmm::cuda_stream_vie if ((input.size() == input.null_count()) && (input.num_children() == 0)) { return false; } // Cross-reference nullmask and offsets. - auto const type = input.type().id(); - auto const offsets = (type == type_id::STRING) ? (strings_column_view{input}).offsets_begin() - : (lists_column_view{input}).offsets_begin(); + auto const type = input.type().id(); + auto const offsets = offsetalator_factory::make_input_iterator( + (type == type_id::STRING) ? strings_column_view{input}.offsets() + : lists_column_view{input}.offsets(), + input.offset()); auto const d_input = cudf::column_device_view::create(input, stream); auto const is_dirty_row = [d_input = *d_input, offsets] __device__(size_type const& row_idx) { return d_input.is_null_nocheck(row_idx) && (offsets[row_idx] != offsets[row_idx + 1]); From 14b149ac0f1fcc085cb492a2cbcfebc26ca6f516 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Wed, 21 Feb 2024 09:22:20 -0500 Subject: [PATCH 283/384] Use offsetalator in cudf::row_bit_count() (#15003) Updates `cudf::row_bit_count()` to use the offsetalator to compute chars size for a strings column. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/15003 --- cpp/src/transform/row_bit_count.cu | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cpp/src/transform/row_bit_count.cu b/cpp/src/transform/row_bit_count.cu index a91dc8fbbc6..e4698fb1262 100644 --- a/cpp/src/transform/row_bit_count.cu +++ b/cpp/src/transform/row_bit_count.cu @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -352,11 +353,12 @@ __device__ size_type row_size_functor::operator()(column_device_vie return 0; } - auto const offsets_size = sizeof(size_type) * CHAR_BIT; + auto const offsets_size = + (offsets.type().id() == type_id::INT32 ? sizeof(int32_t) : sizeof(int64_t)) * CHAR_BIT; auto const validity_size = col.nullable() ? 1 : 0; - auto const chars_size = - (offsets.data()[row_end] - offsets.data()[row_start]) * CHAR_BIT; - return ((offsets_size + validity_size) * num_rows) + chars_size; + auto const d_offsets = cudf::detail::input_offsetalator(offsets.head(), offsets.type()); + auto const chars_size = (d_offsets[row_end] - d_offsets[row_start]) * CHAR_BIT; + return static_cast(((offsets_size + validity_size) * num_rows) + chars_size); } /** From 8a226ebbeb9af9f4effa93180cebba89d7b64f90 Mon Sep 17 00:00:00 2001 From: Jake Awe <50372925+AyodeAwe@users.noreply.github.com> Date: Wed, 21 Feb 2024 09:00:31 -0600 Subject: [PATCH 284/384] updating ops-bot.yaml (#14974) --- .github/ops-bot.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/ops-bot.yaml b/.github/ops-bot.yaml index d2ca78924e1..1e59002c616 100644 --- a/.github/ops-bot.yaml +++ b/.github/ops-bot.yaml @@ -5,3 +5,4 @@ auto_merger: true branch_checker: true label_checker: true release_drafter: true +forward_merger: true From 63e9040d0e80a8ccdb52892bfe10a99309d8b2d5 Mon Sep 17 00:00:00 2001 From: Yunsong Wang Date: Wed, 21 Feb 2024 08:19:50 -0800 Subject: [PATCH 285/384] Clean up nvtx macros (#15038) This PR includes several cleanups for the cudf nvtx wrappers: - Removed the unused `NVTX3_FUNC_RANGE` macro - Fixed a typo in the doc - Added an example in the `cudf::thread_range` doc - Updated the `NVTX` section in the developer guide doc Authors: - Yunsong Wang (https://github.com/PointKernel) Approvers: - Karthikeyan (https://github.com/karthikeyann) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/15038 --- .../developer_guide/DEVELOPER_GUIDE.md | 12 +++++--- cpp/include/cudf/detail/nvtx/nvtx3.hpp | 28 ++----------------- cpp/include/cudf/detail/nvtx/ranges.hpp | 12 +++++++- 3 files changed, 21 insertions(+), 31 deletions(-) diff --git a/cpp/doxygen/developer_guide/DEVELOPER_GUIDE.md b/cpp/doxygen/developer_guide/DEVELOPER_GUIDE.md index 2606b487c07..5c137433dc5 100644 --- a/cpp/doxygen/developer_guide/DEVELOPER_GUIDE.md +++ b/cpp/doxygen/developer_guide/DEVELOPER_GUIDE.md @@ -659,10 +659,14 @@ defaults. ## NVTX Ranges In order to aid in performance optimization and debugging, all compute intensive libcudf functions -should have a corresponding NVTX range. libcudf has a convenience macro `CUDF_FUNC_RANGE()` that -automatically annotates the lifetime of the enclosing function and uses the function's name as -the name of the NVTX range. For more information about NVTX, see -[here](https://github.com/NVIDIA/NVTX/tree/dev/c). +should have a corresponding NVTX range. Choose between `CUDF_FUNC_RANGE` or `cudf::thread_range` +for declaring NVTX ranges in the current scope: +- Use the `CUDF_FUNC_RANGE()` macro if you want to use the name of the function as the name of the +NVTX range +- Use `cudf::thread_range rng{"custom_name"};` to provide a custom name for the current scope's +NVTX range + +For more information about NVTX, see [here](https://github.com/NVIDIA/NVTX/tree/dev/c). ## Input/Output Style diff --git a/cpp/include/cudf/detail/nvtx/nvtx3.hpp b/cpp/include/cudf/detail/nvtx/nvtx3.hpp index 4b840724034..5d44c565077 100644 --- a/cpp/include/cudf/detail/nvtx/nvtx3.hpp +++ b/cpp/include/cudf/detail/nvtx/nvtx3.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -1901,33 +1901,9 @@ inline void mark(event_attributes const& attr) noexcept * * @param[in] D Type containing `name` member used to identify the * `domain` to which the `registered_message` belongs. Else, - * `domain::global` to indicate that the global NVTX domain should be used. + * `domain::global` to indicate that the global NVTX domain should be used. */ #define NVTX3_FUNC_RANGE_IN(D) \ static ::nvtx3::registered_message const nvtx3_func_name__{__func__}; \ static ::nvtx3::event_attributes const nvtx3_func_attr__{nvtx3_func_name__}; \ [[maybe_unused]] ::nvtx3::domain_thread_range const nvtx3_range__{nvtx3_func_attr__}; - -/** - * @brief Convenience macro for generating a range in the global domain from the - * lifetime of a function. - * - * This macro is useful for generating an NVTX range in the global domain from - * the entry point of a function to its exit. It is intended to be the first - * line of the function. - * - * Constructs a static `registered_message` using the name of the immediately - * enclosing function returned by `__func__` and constructs a - * `nvtx3::thread_range` using the registered function name as the range's - * message. - * - * Example: - * ``` - * void foo(...){ - * NVTX3_FUNC_RANGE(); // Range begins on entry to foo() - * // do stuff - * ... - * } // Range ends on return from foo() - * ``` - */ -#define NVTX3_FUNC_RANGE() NVTX3_FUNC_RANGE_IN(::nvtx3::domain::global) diff --git a/cpp/include/cudf/detail/nvtx/ranges.hpp b/cpp/include/cudf/detail/nvtx/ranges.hpp index de5f9901506..6ed30e871fa 100644 --- a/cpp/include/cudf/detail/nvtx/ranges.hpp +++ b/cpp/include/cudf/detail/nvtx/ranges.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,6 +28,16 @@ struct libcudf_domain { /** * @brief Alias for an NVTX range in the libcudf domain. + * + * Customizes an NVTX range with the given input. + * + * Example: + * ``` + * void some_function(){ + * cudf::thread_range rng{"custom_name"}; // Customizes range name + * ... + * } + * ``` */ using thread_range = ::nvtx3::domain_thread_range; From 4ce99af7438d38e91ee2540336a278ade2fffd79 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 21 Feb 2024 06:53:38 -1000 Subject: [PATCH 286/384] Fix reductions when DataFrame has MulitIndex columns (#15097) closes #15085 Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/15097 --- python/cudf/cudf/core/dataframe.py | 11 +++++++---- python/cudf/cudf/tests/test_reductions.py | 10 ++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 1a6376d1c00..89abd7be0ba 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -3955,7 +3955,6 @@ def transpose(self): Not supporting *copy* because default and only behavior is copy=True """ - index = self._data.to_pandas_index() columns = self.index.copy(deep=False) if self._num_columns == 0 or self._num_rows == 0: @@ -6202,9 +6201,13 @@ def _reduce( "Columns must all have the same dtype to " f"perform {op=} with {axis=}" ) - return Series._from_data( - {None: as_column(result)}, as_index(source._data.names) - ) + if source._data.multiindex: + idx = MultiIndex.from_tuples( + source._data.names, names=source._data.level_names + ) + else: + idx = as_index(source._data.names) + return Series._from_data({None: as_column(result)}, idx) elif axis == 1: return source._apply_cupy_method_axis_1(op, **kwargs) else: diff --git a/python/cudf/cudf/tests/test_reductions.py b/python/cudf/cudf/tests/test_reductions.py index 1a38cb3dd22..c6ffa1d2bc7 100644 --- a/python/cudf/cudf/tests/test_reductions.py +++ b/python/cudf/cudf/tests/test_reductions.py @@ -366,3 +366,13 @@ def test_reductions_axis_none_warning(op): ): expected = getattr(pdf, op)(axis=None) assert_eq(expected, actual, check_dtype=False) + + +def test_reduction_column_multiindex(): + idx = cudf.MultiIndex.from_tuples( + [("a", 1), ("a", 2)], names=["foo", "bar"] + ) + df = cudf.DataFrame(np.array([[1, 3], [2, 4]]), columns=idx) + result = df.mean() + expected = df.to_pandas().mean() + assert_eq(result, expected) From d05332308bac4a7aecc12b6ace38fc6cdec5a6a1 Mon Sep 17 00:00:00 2001 From: Mark Harris <783069+harrism@users.noreply.github.com> Date: Thu, 22 Feb 2024 06:42:03 +1100 Subject: [PATCH 287/384] Automate include grouping order in .clang-format (#15063) This uses the `IncludeCategories` settings in .clang-format to attempt to enforce our documented `#include` order in libcudf. See https://docs.rapids.ai/api/libcudf/stable/developer_guide I realize that there was a [previous attempt at this](https://github.com/rapidsai/cudf/pull/12760) by @bdice that met with some resistance. Reading it, I wouldn't say it was vetoed; rather, reviewers requested something much simpler. I have a few reasons to attempt this again. 1. To make a separate task much easier. We are undertaking a refactoring of RMM that will replace `rmm::mr::device_memory_resource*` with `rmm::device_async_resource-ref` everywhere in RAPIDS (not just cuDF). This requires adding an include to MANY files. Getting the location of the include right everywhere is very difficult without automatic grouping of headers. I started out writing a bash script to do this before realizing clang-format has the necessary feature. And I realized that my script would never properly handle [files like this](https://github.com/rapidsai/raft/blob/branch-24.04/cpp/bench/ann/src/raft/raft_cagra_wrapper.h). 2. To increase velocity. Everywhere in RAPIDS that we have automated code standard/style/formatting/other, the benefits to velocity have outweighed the costs. To paraphrase @bdice, $auto \nearrow \rightarrow \mu \searrow \rightarrow v \nearrow$ 3. The previous PR #12760 had nearly 50 categories of headers. There was no way this could be applied universally across RAPIDS repos. My proposal has 10 categories. I tried to reduce it further but realized that it wouldn't be much less configuration to maintain, so I stopped at 10. Note that one of the ways that having few categories can work while still maintaining clear groups is that this PR updates many files to use quotes ("") instead of angle brackets (<>) for local cuDF headers that do not live in `cudf/cpp/include`. With our "near to far" include ordering policy, these are arguably the nearest files, and using quotes allows us to have our first category simply check for quotes. These files will be grouped and sorted without blank lines, but in practice this does not lose clarity because typically headers from more than two directories are not included from the same file. The downside of this change is I don't yet know how to automatically enforce it. I hope that when developers accidentally use <> for internal includes that don't start with (e.g.) "cudf", they will be grouped one of the lowest priority categories, and perhaps this will induce them to switch to "" to get the headers listed at the top. The rule is simple: if it's in libcudf but not in `cpp/include/cudf`, then use quotes. For **everything** else, use angle brackets. Other than headers from RAPIDS repos, we have a group for all CCCL/CUDA headers, a group for all other headers that have a file extension, and a final group for all files that have no file extension (e.g. STL). Below I'm listing the (fairly simple, in my opinion) .clang-format settings for this PR. Note that categories 2-5 will require tweaking for different RAPIDS repos. Some may ask why I ordered `cudf_test` headers before `cudf` headers. I tried both orders, and putting `cudf_test` first generated significantly fewer changes in the PR, meaning that it's already the more common ordering (I suppose `cudf_test` is closer to the files that include it, since they are libcudf tests). I've opened a similar PR for RMM with only 5 groups. https://github.com/rapidsai/rmm/pull/1463 CC @davidwendt @vyasr @wence- @GregoryKimball for feedback @isVoid contributed to this PR via pair programming. ``` IncludeBlocks: Regroup IncludeCategories: - Regex: '^"' # quoted includes Priority: 1 - Regex: '^<(benchmarks|tests)/' # benchmark includes Priority: 2 - Regex: '^ #include +#include #include #include #include @@ -53,8 +54,6 @@ #include #include -#include - #include #include #include diff --git a/cpp/benchmarks/fixture/benchmark_fixture.hpp b/cpp/benchmarks/fixture/benchmark_fixture.hpp index 36370560727..adde0ae1720 100644 --- a/cpp/benchmarks/fixture/benchmark_fixture.hpp +++ b/cpp/benchmarks/fixture/benchmark_fixture.hpp @@ -16,7 +16,6 @@ #pragma once -#include #include #include #include @@ -24,6 +23,8 @@ #include #include +#include + namespace cudf { namespace { diff --git a/cpp/benchmarks/io/cuio_common.cpp b/cpp/benchmarks/io/cuio_common.cpp index b5318b45eb4..3a61e5f1e7b 100644 --- a/cpp/benchmarks/io/cuio_common.cpp +++ b/cpp/benchmarks/io/cuio_common.cpp @@ -15,16 +15,17 @@ */ #include + #include #include +#include + #include #include #include #include -#include - temp_directory const cuio_source_sink_pair::tmpdir{"cudf_gbench"}; std::string random_file_in_dir(std::string const& dir_path) diff --git a/cpp/benchmarks/io/fst.cu b/cpp/benchmarks/io/fst.cu index c0c88517d41..ad19bdfdfcb 100644 --- a/cpp/benchmarks/io/fst.cu +++ b/cpp/benchmarks/io/fst.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,11 +14,10 @@ * limitations under the License. */ -#include - -#include -#include //TODO find better replacement +#include "io/fst/lookup_tables.cuh" +#include "io/utilities/hostdevice_vector.hpp" //TODO find better replacement +#include #include #include diff --git a/cpp/benchmarks/io/json/nested_json.cpp b/cpp/benchmarks/io/json/nested_json.cpp index 03ccd4e245d..9fd8de172a3 100644 --- a/cpp/benchmarks/io/json/nested_json.cpp +++ b/cpp/benchmarks/io/json/nested_json.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,11 +14,10 @@ * limitations under the License. */ +#include "io/json/nested_json.hpp" + #include #include - -#include - #include #include diff --git a/cpp/benchmarks/iterator/iterator.cu b/cpp/benchmarks/iterator/iterator.cu index dcd13cf62c4..ada7a9bd73d 100644 --- a/cpp/benchmarks/iterator/iterator.cu +++ b/cpp/benchmarks/iterator/iterator.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,7 +27,6 @@ #include #include - #include #include #include diff --git a/cpp/benchmarks/join/join_common.hpp b/cpp/benchmarks/join/join_common.hpp index 7d1b1c74465..9f869ddb1ac 100644 --- a/cpp/benchmarks/join/join_common.hpp +++ b/cpp/benchmarks/join/join_common.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,14 +31,14 @@ #include #include -#include - #include #include #include #include #include +#include + #include struct null75_generator { diff --git a/cpp/benchmarks/merge/merge.cpp b/cpp/benchmarks/merge/merge.cpp index 2d2f4fd0de5..9bb8ae666ec 100644 --- a/cpp/benchmarks/merge/merge.cpp +++ b/cpp/benchmarks/merge/merge.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ #include #include + #include #include diff --git a/cpp/benchmarks/sort/rank_lists.cpp b/cpp/benchmarks/sort/rank_lists.cpp index c23f3c891f0..fbdb40b3537 100644 --- a/cpp/benchmarks/sort/rank_lists.cpp +++ b/cpp/benchmarks/sort/rank_lists.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,10 +18,10 @@ #include -#include - #include +#include + #include template diff --git a/cpp/benchmarks/sort/rank_structs.cpp b/cpp/benchmarks/sort/rank_structs.cpp index 271b883e62a..4b0da29df9d 100644 --- a/cpp/benchmarks/sort/rank_structs.cpp +++ b/cpp/benchmarks/sort/rank_structs.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include "rank_types_common.hpp" + #include #include diff --git a/cpp/benchmarks/stream_compaction/apply_boolean_mask.cpp b/cpp/benchmarks/stream_compaction/apply_boolean_mask.cpp index f78aa9fa654..492237474ff 100644 --- a/cpp/benchmarks/stream_compaction/apply_boolean_mask.cpp +++ b/cpp/benchmarks/stream_compaction/apply_boolean_mask.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,11 +15,12 @@ */ #include -#include -#include #include +#include +#include + namespace { constexpr cudf::size_type hundredM = 1e8; diff --git a/cpp/benchmarks/string/string_bench_args.hpp b/cpp/benchmarks/string/string_bench_args.hpp index 92a46374438..a34026281e8 100644 --- a/cpp/benchmarks/string/string_bench_args.hpp +++ b/cpp/benchmarks/string/string_bench_args.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,10 @@ */ #pragma once -#include - #include +#include + #include /** diff --git a/cpp/benchmarks/synchronization/synchronization.hpp b/cpp/benchmarks/synchronization/synchronization.hpp index e56d881d459..cc3bf828d60 100644 --- a/cpp/benchmarks/synchronization/synchronization.hpp +++ b/cpp/benchmarks/synchronization/synchronization.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,9 +27,10 @@ * It is built on top of the idea of Resource acquisition is initialization * (RAII). In the following we show a minimal example of how to use this class. - #include #include + #include + static void sample_cuda_benchmark(benchmark::State& state) { for (auto _ : state){ @@ -60,14 +61,12 @@ #pragma once -// Google Benchmark library -#include - #include #include #include +#include #include class cuda_event_timer { diff --git a/cpp/benchmarks/text/edit_distance.cpp b/cpp/benchmarks/text/edit_distance.cpp index 0a1ea52c415..6ffa90edb8f 100644 --- a/cpp/benchmarks/text/edit_distance.cpp +++ b/cpp/benchmarks/text/edit_distance.cpp @@ -20,10 +20,10 @@ #include -#include - #include +#include + static void bench_edit_distance(nvbench::state& state) { auto const num_rows = static_cast(state.get_int64("num_rows")); diff --git a/cpp/benchmarks/text/hash_ngrams.cpp b/cpp/benchmarks/text/hash_ngrams.cpp index 3df0c61fc31..4e5daf83a3c 100644 --- a/cpp/benchmarks/text/hash_ngrams.cpp +++ b/cpp/benchmarks/text/hash_ngrams.cpp @@ -20,10 +20,10 @@ #include -#include - #include +#include + static void bench_hash_ngrams(nvbench::state& state) { auto const num_rows = static_cast(state.get_int64("num_rows")); diff --git a/cpp/benchmarks/text/jaccard.cpp b/cpp/benchmarks/text/jaccard.cpp index 60251c96096..d05c195d077 100644 --- a/cpp/benchmarks/text/jaccard.cpp +++ b/cpp/benchmarks/text/jaccard.cpp @@ -21,10 +21,10 @@ #include -#include - #include +#include + static void bench_jaccard(nvbench::state& state) { auto const num_rows = static_cast(state.get_int64("num_rows")); diff --git a/cpp/benchmarks/text/minhash.cpp b/cpp/benchmarks/text/minhash.cpp index d10d0d307d7..31ce60d8f9a 100644 --- a/cpp/benchmarks/text/minhash.cpp +++ b/cpp/benchmarks/text/minhash.cpp @@ -20,10 +20,10 @@ #include -#include - #include +#include + static void bench_minhash(nvbench::state& state) { auto const num_rows = static_cast(state.get_int64("num_rows")); diff --git a/cpp/benchmarks/text/vocab.cpp b/cpp/benchmarks/text/vocab.cpp index 770519294ad..523d277df18 100644 --- a/cpp/benchmarks/text/vocab.cpp +++ b/cpp/benchmarks/text/vocab.cpp @@ -20,13 +20,13 @@ #include #include -#include - #include #include #include #include +#include + #include static void bench_vocab_tokenize(nvbench::state& state) diff --git a/cpp/examples/strings/custom_optimized.cu b/cpp/examples/strings/custom_optimized.cu index aa1468ea790..cefa3346150 100644 --- a/cpp/examples/strings/custom_optimized.cu +++ b/cpp/examples/strings/custom_optimized.cu @@ -23,10 +23,9 @@ #include #include -#include - #include #include +#include /** * @brief Computes the size of each output row diff --git a/cpp/include/cudf/ast/detail/operators.hpp b/cpp/include/cudf/ast/detail/operators.hpp index ed7f2d97cef..b618f33a6e5 100644 --- a/cpp/include/cudf/ast/detail/operators.hpp +++ b/cpp/include/cudf/ast/detail/operators.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,9 +20,8 @@ #include #include -#include - #include +#include #include #include diff --git a/cpp/include/cudf/column/column.hpp b/cpp/include/cudf/column/column.hpp index a38186458c4..023e58c5300 100644 --- a/cpp/include/cudf/column/column.hpp +++ b/cpp/include/cudf/column/column.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,6 @@ #pragma once #include - #include #include #include diff --git a/cpp/include/cudf/detail/copy_if.cuh b/cpp/include/cudf/detail/copy_if.cuh index 1d051ea32ff..3af050a5da6 100644 --- a/cpp/include/cudf/detail/copy_if.cuh +++ b/cpp/include/cudf/detail/copy_if.cuh @@ -38,12 +38,10 @@ #include #include -#include -#include - #include - #include +#include +#include #include diff --git a/cpp/include/cudf/detail/copy_range.cuh b/cpp/include/cudf/detail/copy_range.cuh index 4bfdaa94c53..9f8b0f8b619 100644 --- a/cpp/include/cudf/detail/copy_range.cuh +++ b/cpp/include/cudf/detail/copy_range.cuh @@ -29,7 +29,6 @@ #include #include - #include #include diff --git a/cpp/include/cudf/detail/gather.cuh b/cpp/include/cudf/detail/gather.cuh index c9975ef2199..311a100a21b 100644 --- a/cpp/include/cudf/detail/gather.cuh +++ b/cpp/include/cudf/detail/gather.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -39,8 +39,6 @@ #include #include -#include - #include #include #include @@ -48,6 +46,8 @@ #include #include +#include + namespace cudf { namespace detail { diff --git a/cpp/include/cudf/detail/hash_reduce_by_row.cuh b/cpp/include/cudf/detail/hash_reduce_by_row.cuh index a740b5c4e93..1df6848c575 100644 --- a/cpp/include/cudf/detail/hash_reduce_by_row.cuh +++ b/cpp/include/cudf/detail/hash_reduce_by_row.cuh @@ -23,12 +23,11 @@ #include #include +#include #include #include #include -#include - namespace cudf::detail { using hash_map_type = cuco::legacy:: diff --git a/cpp/include/cudf/detail/indexalator.cuh b/cpp/include/cudf/detail/indexalator.cuh index 4d261c54b29..b5d57da6cd5 100644 --- a/cpp/include/cudf/detail/indexalator.cuh +++ b/cpp/include/cudf/detail/indexalator.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,10 +16,9 @@ #pragma once -#include - #include #include +#include #include #include diff --git a/cpp/include/cudf/detail/interop.hpp b/cpp/include/cudf/detail/interop.hpp index 8124471982d..683b49e1813 100644 --- a/cpp/include/cudf/detail/interop.hpp +++ b/cpp/include/cudf/detail/interop.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,10 +32,11 @@ #include #include #include -#include #include +#include + namespace cudf { namespace detail { diff --git a/cpp/include/cudf/detail/label_bins.hpp b/cpp/include/cudf/detail/label_bins.hpp index 7f3cf033e66..50eeba58cdd 100644 --- a/cpp/include/cudf/detail/label_bins.hpp +++ b/cpp/include/cudf/detail/label_bins.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,10 +16,9 @@ #pragma once -#include - #include #include +#include #include #include diff --git a/cpp/include/cudf/detail/null_mask.cuh b/cpp/include/cudf/detail/null_mask.cuh index e57d85f2998..3b55a62cec0 100644 --- a/cpp/include/cudf/detail/null_mask.cuh +++ b/cpp/include/cudf/detail/null_mask.cuh @@ -29,7 +29,7 @@ #include #include - +#include #include #include #include @@ -37,8 +37,6 @@ #include #include -#include - #include #include #include diff --git a/cpp/include/cudf/detail/sizes_to_offsets_iterator.cuh b/cpp/include/cudf/detail/sizes_to_offsets_iterator.cuh index 358dcca02b9..08917bfce24 100644 --- a/cpp/include/cudf/detail/sizes_to_offsets_iterator.cuh +++ b/cpp/include/cudf/detail/sizes_to_offsets_iterator.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,11 +24,10 @@ #include #include +#include #include #include -#include - #include namespace cudf { diff --git a/cpp/include/cudf/detail/utilities/device_atomics.cuh b/cpp/include/cudf/detail/utilities/device_atomics.cuh index c56e88f07a8..1e3fe3d08dc 100644 --- a/cpp/include/cudf/detail/utilities/device_atomics.cuh +++ b/cpp/include/cudf/detail/utilities/device_atomics.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,6 +37,7 @@ #include #include #include + #include namespace cudf { diff --git a/cpp/include/cudf/detail/utilities/pinned_host_vector.hpp b/cpp/include/cudf/detail/utilities/pinned_host_vector.hpp index eee974c8399..c22b6a6ba15 100644 --- a/cpp/include/cudf/detail/utilities/pinned_host_vector.hpp +++ b/cpp/include/cudf/detail/utilities/pinned_host_vector.hpp @@ -1,5 +1,5 @@ /* - * Copyright 2008-2023 NVIDIA Corporation + * Copyright (c) 2008-2024, NVIDIA CORPORATION * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,14 +16,14 @@ #pragma once -#include -#include -#include // for bad_alloc - #include #include +#include +#include +#include // for bad_alloc + namespace cudf::detail { /*! \p pinned_allocator is a CUDA-specific host memory allocator diff --git a/cpp/include/cudf/fixed_point/temporary.hpp b/cpp/include/cudf/fixed_point/temporary.hpp index 1de7f66127b..17dba6c2452 100644 --- a/cpp/include/cudf/fixed_point/temporary.hpp +++ b/cpp/include/cudf/fixed_point/temporary.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,9 +15,6 @@ */ #pragma once -// To avoid https://github.com/NVIDIA/libcudacxx/issues/460 -// in libcudacxx with CTK 12.0/12.1 -#include #include diff --git a/cpp/include/cudf/io/detail/orc.hpp b/cpp/include/cudf/io/detail/orc.hpp index a0bf8b24b80..3c1486b60c2 100644 --- a/cpp/include/cudf/io/detail/orc.hpp +++ b/cpp/include/cudf/io/detail/orc.hpp @@ -22,12 +22,12 @@ #include #include +#include + #include #include #include -#include - namespace cudf::io { // Forward declaration diff --git a/cpp/include/cudf/io/text/detail/bgzip_utils.hpp b/cpp/include/cudf/io/text/detail/bgzip_utils.hpp index 627df5f358a..515bcf16de2 100644 --- a/cpp/include/cudf/io/text/detail/bgzip_utils.hpp +++ b/cpp/include/cudf/io/text/detail/bgzip_utils.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,11 +16,11 @@ #pragma once -#include - #include #include +#include + #include #include #include diff --git a/cpp/include/cudf/io/text/detail/tile_state.hpp b/cpp/include/cudf/io/text/detail/tile_state.hpp index 6ae399fbe75..d42624aa9b7 100644 --- a/cpp/include/cudf/io/text/detail/tile_state.hpp +++ b/cpp/include/cudf/io/text/detail/tile_state.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,6 @@ #pragma once #include - #include namespace cudf { diff --git a/cpp/include/cudf/lists/detail/gather.cuh b/cpp/include/cudf/lists/detail/gather.cuh index 4484a9995c3..03428bc347f 100644 --- a/cpp/include/cudf/lists/detail/gather.cuh +++ b/cpp/include/cudf/lists/detail/gather.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,12 +26,11 @@ #include #include +#include #include #include #include -#include - namespace cudf { namespace lists { namespace detail { diff --git a/cpp/include/cudf/lists/detail/scatter.cuh b/cpp/include/cudf/lists/detail/scatter.cuh index ea2f2bbf544..5fc52ff1c04 100644 --- a/cpp/include/cudf/lists/detail/scatter.cuh +++ b/cpp/include/cudf/lists/detail/scatter.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,6 +31,7 @@ #include #include +#include #include #include #include @@ -39,8 +40,6 @@ #include #include -#include - #include namespace cudf { diff --git a/cpp/include/cudf/lists/detail/set_operations.hpp b/cpp/include/cudf/lists/detail/set_operations.hpp index 1411c65448e..51fc58bee07 100644 --- a/cpp/include/cudf/lists/detail/set_operations.hpp +++ b/cpp/include/cudf/lists/detail/set_operations.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,8 +19,8 @@ #include #include #include -#include +#include #include namespace cudf::lists::detail { diff --git a/cpp/include/cudf/lists/list_device_view.cuh b/cpp/include/cudf/lists/list_device_view.cuh index 07346e78261..170a20bd7f5 100644 --- a/cpp/include/cudf/lists/list_device_view.cuh +++ b/cpp/include/cudf/lists/list_device_view.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,12 @@ */ #pragma once -#include #include #include #include #include +#include #include #include #include diff --git a/cpp/include/cudf/lists/lists_column_device_view.cuh b/cpp/include/cudf/lists/lists_column_device_view.cuh index 943ccbfb2cd..4d12ee1cab4 100644 --- a/cpp/include/cudf/lists/lists_column_device_view.cuh +++ b/cpp/include/cudf/lists/lists_column_device_view.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,11 +15,12 @@ */ #pragma once -#include #include #include #include +#include + namespace cudf { namespace detail { diff --git a/cpp/include/cudf/reduction/detail/reduction.cuh b/cpp/include/cudf/reduction/detail/reduction.cuh index 48b65a3fc54..9807d4cb4ea 100644 --- a/cpp/include/cudf/reduction/detail/reduction.cuh +++ b/cpp/include/cudf/reduction/detail/reduction.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,7 +28,6 @@ #include #include - #include #include diff --git a/cpp/include/cudf/reduction/detail/reduction_operators.cuh b/cpp/include/cudf/reduction/detail/reduction_operators.cuh index a747f7bade7..4cf8564ab3a 100644 --- a/cpp/include/cudf/reduction/detail/reduction_operators.cuh +++ b/cpp/include/cudf/reduction/detail/reduction_operators.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,9 +21,10 @@ #include #include //for CUDF_HOST_DEVICE -#include #include +#include + namespace cudf { namespace reduction { namespace detail { diff --git a/cpp/include/cudf/reduction/detail/segmented_reduction.cuh b/cpp/include/cudf/reduction/detail/segmented_reduction.cuh index e86506681eb..89ca78f1213 100644 --- a/cpp/include/cudf/reduction/detail/segmented_reduction.cuh +++ b/cpp/include/cudf/reduction/detail/segmented_reduction.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,6 @@ #include #include - #include #include diff --git a/cpp/include/cudf/row_conversion.hpp b/cpp/include/cudf/row_conversion.hpp index 89453d49856..e2c0577b885 100644 --- a/cpp/include/cudf/row_conversion.hpp +++ b/cpp/include/cudf/row_conversion.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,13 +16,14 @@ #pragma once -#include - #include #include #include + #include +#include + namespace cudf { //! @cond Doxygen_Suppress diff --git a/cpp/include/cudf/strings/detail/convert/fixed_point.cuh b/cpp/include/cudf/strings/detail/convert/fixed_point.cuh index dd55cae4537..5f51da967d3 100644 --- a/cpp/include/cudf/strings/detail/convert/fixed_point.cuh +++ b/cpp/include/cudf/strings/detail/convert/fixed_point.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,11 +17,10 @@ #include +#include #include #include -#include - namespace cudf { namespace strings { namespace detail { diff --git a/cpp/include/cudf/strings/detail/copy_if_else.cuh b/cpp/include/cudf/strings/detail/copy_if_else.cuh index e1ef97b7803..08ba99e90d8 100644 --- a/cpp/include/cudf/strings/detail/copy_if_else.cuh +++ b/cpp/include/cudf/strings/detail/copy_if_else.cuh @@ -23,12 +23,11 @@ #include #include +#include #include #include #include -#include - namespace cudf { namespace strings { namespace detail { diff --git a/cpp/include/cudf/strings/detail/gather.cuh b/cpp/include/cudf/strings/detail/gather.cuh index 7092d114009..06d959acffb 100644 --- a/cpp/include/cudf/strings/detail/gather.cuh +++ b/cpp/include/cudf/strings/detail/gather.cuh @@ -28,6 +28,7 @@ #include #include +#include #include #include #include @@ -36,8 +37,6 @@ #include #include -#include - namespace cudf { namespace strings { namespace detail { diff --git a/cpp/include/cudf/strings/detail/merge.cuh b/cpp/include/cudf/strings/detail/merge.cuh index 8049895c3c2..f05e957783f 100644 --- a/cpp/include/cudf/strings/detail/merge.cuh +++ b/cpp/include/cudf/strings/detail/merge.cuh @@ -27,13 +27,12 @@ #include #include +#include #include #include #include #include -#include - namespace cudf { namespace strings { namespace detail { diff --git a/cpp/include/cudf/strings/detail/scatter.cuh b/cpp/include/cudf/strings/detail/scatter.cuh index 56eeec01715..8b8c11dcd5c 100644 --- a/cpp/include/cudf/strings/detail/scatter.cuh +++ b/cpp/include/cudf/strings/detail/scatter.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,12 +25,11 @@ #include #include +#include #include #include #include -#include - namespace cudf { namespace strings { namespace detail { diff --git a/cpp/include/cudf/strings/detail/strings_column_factories.cuh b/cpp/include/cudf/strings/detail/strings_column_factories.cuh index 0adf6e362be..8e19f08a5cc 100644 --- a/cpp/include/cudf/strings/detail/strings_column_factories.cuh +++ b/cpp/include/cudf/strings/detail/strings_column_factories.cuh @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -37,8 +38,6 @@ #include #include -#include - namespace cudf { namespace strings { namespace detail { diff --git a/cpp/include/cudf/table/experimental/row_operators.cuh b/cpp/include/cudf/table/experimental/row_operators.cuh index 6946ccdb213..e9b81a525fc 100644 --- a/cpp/include/cudf/table/experimental/row_operators.cuh +++ b/cpp/include/cudf/table/experimental/row_operators.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,6 +33,8 @@ #include #include +#include +#include #include #include #include @@ -46,9 +48,6 @@ #include #include -#include -#include - #include #include #include diff --git a/cpp/include/cudf/utilities/bit.hpp b/cpp/include/cudf/utilities/bit.hpp index 11a797ec466..9bdc372419f 100644 --- a/cpp/include/cudf/utilities/bit.hpp +++ b/cpp/include/cudf/utilities/bit.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,10 +16,12 @@ #pragma once -#include -#include #include +#include + +#include + /** * @file bit.hpp * @brief Utilities for bit and bitmask operations. diff --git a/cpp/include/cudf/utilities/error.hpp b/cpp/include/cudf/utilities/error.hpp index bf8b87e2563..719d44a9ab3 100644 --- a/cpp/include/cudf/utilities/error.hpp +++ b/cpp/include/cudf/utilities/error.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ #include #include + #include #include #include diff --git a/cpp/include/cudf/wrappers/dictionary.hpp b/cpp/include/cudf/wrappers/dictionary.hpp index 329f1fa7754..37264c5a33c 100644 --- a/cpp/include/cudf/wrappers/dictionary.hpp +++ b/cpp/include/cudf/wrappers/dictionary.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,9 +16,10 @@ #pragma once -#include #include +#include + #include /** diff --git a/cpp/include/cudf_test/column_wrapper.hpp b/cpp/include/cudf_test/column_wrapper.hpp index e7ca8400246..151fe50be4f 100644 --- a/cpp/include/cudf_test/column_wrapper.hpp +++ b/cpp/include/cudf_test/column_wrapper.hpp @@ -16,6 +16,10 @@ #pragma once +#include +#include +#include + #include #include #include @@ -33,10 +37,6 @@ #include #include -#include -#include -#include - #include #include diff --git a/cpp/include/cudf_test/file_utilities.hpp b/cpp/include/cudf_test/file_utilities.hpp index 202e7604fa0..defc6f95823 100644 --- a/cpp/include/cudf_test/file_utilities.hpp +++ b/cpp/include/cudf_test/file_utilities.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,10 +18,11 @@ #include +#include + #include #include #include -#include #include /** diff --git a/cpp/include/cudf_test/random.hpp b/cpp/include/cudf_test/random.hpp index 498bacc81c9..f4d539ecffe 100644 --- a/cpp/include/cudf_test/random.hpp +++ b/cpp/include/cudf_test/random.hpp @@ -16,10 +16,10 @@ #pragma once -#include - #include +#include + namespace cudf { namespace test { diff --git a/cpp/include/cudf_test/timestamp_utilities.cuh b/cpp/include/cudf_test/timestamp_utilities.cuh index 6cab8b92283..ebd93862151 100644 --- a/cpp/include/cudf_test/timestamp_utilities.cuh +++ b/cpp/include/cudf_test/timestamp_utilities.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,11 +16,11 @@ #pragma once +#include + #include #include -#include - #include #include diff --git a/cpp/include/cudf_test/type_lists.hpp b/cpp/include/cudf_test/type_lists.hpp index 2404cf0d134..bbff45e2102 100644 --- a/cpp/include/cudf_test/type_lists.hpp +++ b/cpp/include/cudf_test/type_lists.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,8 @@ #pragma once +#include + #include #include #include @@ -23,7 +25,6 @@ #include #include #include -#include #include diff --git a/cpp/include/nvtext/detail/load_hash_file.hpp b/cpp/include/nvtext/detail/load_hash_file.hpp index 9f4640f1daf..f4107adb07e 100644 --- a/cpp/include/nvtext/detail/load_hash_file.hpp +++ b/cpp/include/nvtext/detail/load_hash_file.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,10 @@ */ #pragma once -#include - #include +#include + #include #include diff --git a/cpp/libcudf_kafka/tests/kafka_consumer_tests.cpp b/cpp/libcudf_kafka/tests/kafka_consumer_tests.cpp index 53d87e04ddc..1400bc75b44 100644 --- a/cpp/libcudf_kafka/tests/kafka_consumer_tests.cpp +++ b/cpp/libcudf_kafka/tests/kafka_consumer_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,17 @@ * limitations under the License. */ +#include +#include + #include + #include + #include #include #include -#include -#include - namespace kafka = cudf::io::external::kafka; struct KafkaDatasourceTest : public ::testing::Test {}; diff --git a/cpp/src/binaryop/binaryop.cpp b/cpp/src/binaryop/binaryop.cpp index 53b04c4ca80..be91c3b4d08 100644 --- a/cpp/src/binaryop/binaryop.cpp +++ b/cpp/src/binaryop/binaryop.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Copyright 2018-2019 BlazingDB, Inc. * Copyright 2018 Christian Noboa Mardini @@ -18,12 +18,9 @@ */ #include "compiled/binary_ops.hpp" - -#include - -#include -#include -#include +#include "jit/cache.hpp" +#include "jit/parser.hpp" +#include "jit/util.hpp" #include #include @@ -43,10 +40,12 @@ #include -#include - #include +#include + +#include + namespace cudf { namespace binops { diff --git a/cpp/src/binaryop/compiled/binary_ops.cu b/cpp/src/binaryop/compiled/binary_ops.cu index 73ba15e39f3..1429635b803 100644 --- a/cpp/src/binaryop/compiled/binary_ops.cu +++ b/cpp/src/binaryop/compiled/binary_ops.cu @@ -28,13 +28,12 @@ #include #include +#include #include #include #include #include -#include - namespace cudf { namespace binops { namespace compiled { diff --git a/cpp/src/binaryop/jit/kernel.cu b/cpp/src/binaryop/jit/kernel.cu index 39735a43474..985fc87521c 100644 --- a/cpp/src/binaryop/jit/kernel.cu +++ b/cpp/src/binaryop/jit/kernel.cu @@ -24,9 +24,12 @@ #include #include -#include #include +// clang-format off +#include "binaryop/jit/operation-udf.hpp" +// clang-format on + namespace cudf { namespace binops { namespace jit { diff --git a/cpp/src/bitmask/null_mask.cu b/cpp/src/bitmask/null_mask.cu index bb320e4b81a..806beeb4efe 100644 --- a/cpp/src/bitmask/null_mask.cu +++ b/cpp/src/bitmask/null_mask.cu @@ -34,13 +34,12 @@ #include #include +#include #include #include #include #include -#include - #include #include #include diff --git a/cpp/src/column/column.cu b/cpp/src/column/column.cu index 8276dbe78d2..d4a8fff69e2 100644 --- a/cpp/src/column/column.cu +++ b/cpp/src/column/column.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,11 +33,11 @@ #include #include -#include - #include #include +#include + #include #include #include diff --git a/cpp/src/copying/contiguous_split.cu b/cpp/src/copying/contiguous_split.cu index d711f40605a..c28237587eb 100644 --- a/cpp/src/copying/contiguous_split.cu +++ b/cpp/src/copying/contiguous_split.cu @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -45,8 +46,6 @@ #include #include -#include - #include #include diff --git a/cpp/src/copying/gather.cu b/cpp/src/copying/gather.cu index 2083d3ed618..921f84b6b50 100644 --- a/cpp/src/copying/gather.cu +++ b/cpp/src/copying/gather.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,9 +26,8 @@ #include -#include - #include +#include namespace cudf { namespace detail { diff --git a/cpp/src/copying/reverse.cu b/cpp/src/copying/reverse.cu index 884c93e268c..78d1b54882c 100644 --- a/cpp/src/copying/reverse.cu +++ b/cpp/src/copying/reverse.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,13 +27,12 @@ #include #include +#include #include #include #include #include -#include - namespace cudf { namespace detail { std::unique_ptr
reverse(table_view const& source_table, diff --git a/cpp/src/copying/sample.cu b/cpp/src/copying/sample.cu index e7f5522d3b3..0211f97deb3 100644 --- a/cpp/src/copying/sample.cu +++ b/cpp/src/copying/sample.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,13 +26,12 @@ #include +#include #include #include #include #include -#include - namespace cudf { namespace detail { diff --git a/cpp/src/copying/scatter.cu b/cpp/src/copying/scatter.cu index 517435503ee..baa5d85d4d4 100644 --- a/cpp/src/copying/scatter.cu +++ b/cpp/src/copying/scatter.cu @@ -35,6 +35,7 @@ #include +#include #include #include #include @@ -43,8 +44,6 @@ #include #include -#include - namespace cudf { namespace detail { namespace { diff --git a/cpp/src/datetime/timezone.cpp b/cpp/src/datetime/timezone.cpp index 1b40a994ba9..a75eea7172f 100644 --- a/cpp/src/datetime/timezone.cpp +++ b/cpp/src/datetime/timezone.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include - #include #include +#include #include #include diff --git a/cpp/src/dictionary/detail/concatenate.cu b/cpp/src/dictionary/detail/concatenate.cu index 024acaa872d..17295fb0345 100644 --- a/cpp/src/dictionary/detail/concatenate.cu +++ b/cpp/src/dictionary/detail/concatenate.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -43,8 +44,6 @@ #include #include -#include - #include #include diff --git a/cpp/src/filling/repeat.cu b/cpp/src/filling/repeat.cu index b3ed9743953..bd53eeddbb5 100644 --- a/cpp/src/filling/repeat.cu +++ b/cpp/src/filling/repeat.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,6 +34,7 @@ #include #include +#include #include #include #include @@ -43,8 +44,6 @@ #include #include -#include - #include #include diff --git a/cpp/src/groupby/hash/groupby.cu b/cpp/src/groupby/hash/groupby.cu index 32693487c32..7b85dd02c10 100644 --- a/cpp/src/groupby/hash/groupby.cu +++ b/cpp/src/groupby/hash/groupby.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,8 +14,9 @@ * limitations under the License. */ -#include -#include +#include "groupby/common/utils.hpp" +#include "groupby/hash/groupby_kernels.cuh" +#include "hash/concurrent_unordered_map.cuh" #include #include @@ -45,18 +46,16 @@ #include #include #include -#include #include +#include +#include #include #include #include #include -#include -#include - #include #include #include diff --git a/cpp/src/groupby/hash/groupby_kernels.cuh b/cpp/src/groupby/hash/groupby_kernels.cuh index eedb07200a5..4dfb191480b 100644 --- a/cpp/src/groupby/hash/groupby_kernels.cuh +++ b/cpp/src/groupby/hash/groupby_kernels.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ #pragma once #include "multi_pass_kernels.cuh" + #include #include #include diff --git a/cpp/src/groupby/sort/aggregate.cpp b/cpp/src/groupby/sort/aggregate.cpp index 9233ad1932c..2d6f99de25a 100644 --- a/cpp/src/groupby/sort/aggregate.cpp +++ b/cpp/src/groupby/sort/aggregate.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,9 +14,9 @@ * limitations under the License. */ -#include -#include -#include +#include "groupby/common/utils.hpp" +#include "groupby/sort/functors.hpp" +#include "groupby/sort/group_reductions.hpp" #include #include diff --git a/cpp/src/groupby/sort/group_argmax.cu b/cpp/src/groupby/sort/group_argmax.cu index 466171ec80b..a9c098bcf61 100644 --- a/cpp/src/groupby/sort/group_argmax.cu +++ b/cpp/src/groupby/sort/group_argmax.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "groupby/sort/group_single_pass_reduction_util.cuh" #include #include diff --git a/cpp/src/groupby/sort/group_argmin.cu b/cpp/src/groupby/sort/group_argmin.cu index 4f7b2b713e6..53a514ac8a7 100644 --- a/cpp/src/groupby/sort/group_argmin.cu +++ b/cpp/src/groupby/sort/group_argmin.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "groupby/sort/group_single_pass_reduction_util.cuh" #include #include diff --git a/cpp/src/groupby/sort/group_correlation.cu b/cpp/src/groupby/sort/group_correlation.cu index 887e82e66df..4389b833c33 100644 --- a/cpp/src/groupby/sort/group_correlation.cu +++ b/cpp/src/groupby/sort/group_correlation.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "groupby/sort/group_reductions.hpp" #include #include diff --git a/cpp/src/groupby/sort/group_count.cu b/cpp/src/groupby/sort/group_count.cu index e35b0c2b2fe..2f289c8c8a7 100644 --- a/cpp/src/groupby/sort/group_count.cu +++ b/cpp/src/groupby/sort/group_count.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,14 +23,13 @@ #include #include +#include #include #include #include #include #include -#include - namespace cudf { namespace groupby { namespace detail { diff --git a/cpp/src/groupby/sort/group_count_scan.cu b/cpp/src/groupby/sort/group_count_scan.cu index 0caef47f0e3..2e8fd41d984 100644 --- a/cpp/src/groupby/sort/group_count_scan.cu +++ b/cpp/src/groupby/sort/group_count_scan.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,12 +19,12 @@ #include #include -#include -#include - #include #include +#include +#include + namespace cudf { namespace groupby { namespace detail { diff --git a/cpp/src/groupby/sort/group_histogram.cu b/cpp/src/groupby/sort/group_histogram.cu index bb70037aaef..67c30adcd47 100644 --- a/cpp/src/groupby/sort/group_histogram.cu +++ b/cpp/src/groupby/sort/group_histogram.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "lists/utilities.hpp" #include #include diff --git a/cpp/src/groupby/sort/group_max.cu b/cpp/src/groupby/sort/group_max.cu index 5da15266233..148188f5fdf 100644 --- a/cpp/src/groupby/sort/group_max.cu +++ b/cpp/src/groupby/sort/group_max.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "groupby/sort/group_single_pass_reduction_util.cuh" #include diff --git a/cpp/src/groupby/sort/group_max_scan.cu b/cpp/src/groupby/sort/group_max_scan.cu index 1551dc00a04..8679ab09df6 100644 --- a/cpp/src/groupby/sort/group_max_scan.cu +++ b/cpp/src/groupby/sort/group_max_scan.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "groupby/sort/group_scan_util.cuh" #include diff --git a/cpp/src/groupby/sort/group_min.cu b/cpp/src/groupby/sort/group_min.cu index c42a0b94de0..3939fc41b65 100644 --- a/cpp/src/groupby/sort/group_min.cu +++ b/cpp/src/groupby/sort/group_min.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2020, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "groupby/sort/group_single_pass_reduction_util.cuh" #include diff --git a/cpp/src/groupby/sort/group_min_scan.cu b/cpp/src/groupby/sort/group_min_scan.cu index daaeb6bb6f7..7d2a88fb038 100644 --- a/cpp/src/groupby/sort/group_min_scan.cu +++ b/cpp/src/groupby/sort/group_min_scan.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "groupby/sort/group_scan_util.cuh" #include diff --git a/cpp/src/groupby/sort/group_nth_element.cu b/cpp/src/groupby/sort/group_nth_element.cu index 037fa9a735c..694c052e42d 100644 --- a/cpp/src/groupby/sort/group_nth_element.cu +++ b/cpp/src/groupby/sort/group_nth_element.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,13 +23,14 @@ #include #include #include -#include #include #include +#include #include #include +#include #include #include #include @@ -37,8 +38,6 @@ #include #include -#include - namespace cudf { namespace groupby { namespace detail { diff --git a/cpp/src/groupby/sort/group_product.cu b/cpp/src/groupby/sort/group_product.cu index 74f5cbed041..c53362f2095 100644 --- a/cpp/src/groupby/sort/group_product.cu +++ b/cpp/src/groupby/sort/group_product.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,9 +14,10 @@ * limitations under the License. */ +#include "groupby/sort/group_single_pass_reduction_util.cuh" + #include #include -#include #include diff --git a/cpp/src/groupby/sort/group_quantiles.cu b/cpp/src/groupby/sort/group_quantiles.cu index a456d4b5964..a6bc2d5b38d 100644 --- a/cpp/src/groupby/sort/group_quantiles.cu +++ b/cpp/src/groupby/sort/group_quantiles.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ #include "group_reductions.hpp" -#include +#include "quantiles/quantiles_util.hpp" #include #include diff --git a/cpp/src/groupby/sort/group_scan_util.cuh b/cpp/src/groupby/sort/group_scan_util.cuh index 3ed53944172..1cfbf400062 100644 --- a/cpp/src/groupby/sort/group_scan_util.cuh +++ b/cpp/src/groupby/sort/group_scan_util.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ #pragma once -#include +#include "reductions/nested_type_minmax_util.cuh" #include #include diff --git a/cpp/src/groupby/sort/group_single_pass_reduction_util.cuh b/cpp/src/groupby/sort/group_single_pass_reduction_util.cuh index 34543147b1c..42d4b654346 100644 --- a/cpp/src/groupby/sort/group_single_pass_reduction_util.cuh +++ b/cpp/src/groupby/sort/group_single_pass_reduction_util.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ #pragma once -#include +#include "reductions/nested_type_minmax_util.cuh" #include #include diff --git a/cpp/src/groupby/sort/group_sum.cu b/cpp/src/groupby/sort/group_sum.cu index e3c2ce7c864..0af7cb22159 100644 --- a/cpp/src/groupby/sort/group_sum.cu +++ b/cpp/src/groupby/sort/group_sum.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2020, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,9 +14,10 @@ * limitations under the License. */ +#include "groupby/sort/group_single_pass_reduction_util.cuh" + #include #include -#include #include diff --git a/cpp/src/groupby/sort/group_sum_scan.cu b/cpp/src/groupby/sort/group_sum_scan.cu index 632fde3b9d5..2efa1185899 100644 --- a/cpp/src/groupby/sort/group_sum_scan.cu +++ b/cpp/src/groupby/sort/group_sum_scan.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "groupby/sort/group_scan_util.cuh" #include diff --git a/cpp/src/groupby/sort/scan.cpp b/cpp/src/groupby/sort/scan.cpp index 32120988065..ae183474810 100644 --- a/cpp/src/groupby/sort/scan.cpp +++ b/cpp/src/groupby/sort/scan.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,10 @@ * limitations under the License. */ -#include -#include -#include -#include +#include "groupby/common/utils.hpp" +#include "groupby/sort/functors.hpp" +#include "groupby/sort/group_reductions.hpp" +#include "groupby/sort/group_scan.hpp" #include #include diff --git a/cpp/src/groupby/sort/sort_helper.cu b/cpp/src/groupby/sort/sort_helper.cu index 61cdfe16ab8..1e6c7a9393f 100644 --- a/cpp/src/groupby/sort/sort_helper.cu +++ b/cpp/src/groupby/sort/sort_helper.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,8 +15,7 @@ */ #include "common_utils.cuh" - -#include +#include "stream_compaction/stream_compaction_common.cuh" #include #include @@ -37,13 +36,12 @@ #include #include +#include #include #include #include #include -#include - #include #include #include diff --git a/cpp/src/hash/concurrent_unordered_map.cuh b/cpp/src/hash/concurrent_unordered_map.cuh index adc87c2400e..a010a462de3 100644 --- a/cpp/src/hash/concurrent_unordered_map.cuh +++ b/cpp/src/hash/concurrent_unordered_map.cuh @@ -16,7 +16,7 @@ #pragma once -#include +#include "hash/managed.cuh" #include #include @@ -27,6 +27,7 @@ #include #include +#include #include #include @@ -34,8 +35,6 @@ #include #include -#include - namespace { template struct packed { diff --git a/cpp/src/hash/managed.cuh b/cpp/src/hash/managed.cuh index 10aeb6e52be..aa7bff85ea6 100644 --- a/cpp/src/hash/managed.cuh +++ b/cpp/src/hash/managed.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2023, NVIDIA CORPORATION. + * Copyright (c) 2017-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ #pragma once +#include #include struct managed { diff --git a/cpp/src/interop/detail/arrow_allocator.cpp b/cpp/src/interop/detail/arrow_allocator.cpp index 41fb68a5748..3e6a337457a 100644 --- a/cpp/src/interop/detail/arrow_allocator.cpp +++ b/cpp/src/interop/detail/arrow_allocator.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,10 +16,11 @@ #include -#include #include #include +#include + namespace cudf { namespace detail { diff --git a/cpp/src/interop/dlpack.cpp b/cpp/src/interop/dlpack.cpp index 1759c998c75..9f36280930d 100644 --- a/cpp/src/interop/dlpack.cpp +++ b/cpp/src/interop/dlpack.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -27,7 +28,6 @@ #include #include -#include namespace cudf { namespace { diff --git a/cpp/src/interop/to_arrow.cu b/cpp/src/interop/to_arrow.cu index 04ca1250ed5..e871e656c48 100644 --- a/cpp/src/interop/to_arrow.cu +++ b/cpp/src/interop/to_arrow.cu @@ -14,6 +14,8 @@ * limitations under the License. */ +#include "detail/arrow_allocator.hpp" + #include #include #include @@ -41,8 +43,6 @@ #include #include -#include "detail/arrow_allocator.hpp" - namespace cudf { namespace detail { namespace { diff --git a/cpp/src/io/avro/avro_common.hpp b/cpp/src/io/avro/avro_common.hpp index 0058d236d8c..9bf66369d6a 100644 --- a/cpp/src/io/avro/avro_common.hpp +++ b/cpp/src/io/avro/avro_common.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ #pragma once -#include +#include "io/utilities/column_buffer.hpp" #include #include diff --git a/cpp/src/io/avro/avro_gpu.cu b/cpp/src/io/avro/avro_gpu.cu index 59177a68ee7..612b2d32b7d 100644 --- a/cpp/src/io/avro/avro_gpu.cu +++ b/cpp/src/io/avro/avro_gpu.cu @@ -14,8 +14,7 @@ * limitations under the License. */ #include "avro_gpu.hpp" - -#include +#include "io/utilities/block_utils.cuh" #include diff --git a/cpp/src/io/avro/reader_impl.cu b/cpp/src/io/avro/reader_impl.cu index f73e1db91c3..03fd663040a 100644 --- a/cpp/src/io/avro/reader_impl.cu +++ b/cpp/src/io/avro/reader_impl.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,10 +16,9 @@ #include "avro.hpp" #include "avro_gpu.hpp" - -#include -#include -#include +#include "io/comp/gpuinflate.hpp" +#include "io/utilities/column_buffer.hpp" +#include "io/utilities/hostdevice_vector.hpp" #include #include diff --git a/cpp/src/io/comp/debrotli.cu b/cpp/src/io/comp/debrotli.cu index 9c936fefd6c..861820f47e7 100644 --- a/cpp/src/io/comp/debrotli.cu +++ b/cpp/src/io/comp/debrotli.cu @@ -56,8 +56,7 @@ THE SOFTWARE. #include "brotli_dict.hpp" #include "gpuinflate.hpp" - -#include +#include "io/utilities/block_utils.cuh" #include diff --git a/cpp/src/io/comp/gpuinflate.cu b/cpp/src/io/comp/gpuinflate.cu index cd50545afbd..f29e830eb41 100644 --- a/cpp/src/io/comp/gpuinflate.cu +++ b/cpp/src/io/comp/gpuinflate.cu @@ -44,10 +44,9 @@ Mark Adler madler@alumni.caltech.edu */ #include "gpuinflate.hpp" +#include "io/utilities/block_utils.cuh" #include "io_uncomp.hpp" -#include - #include namespace cudf { diff --git a/cpp/src/io/comp/nvcomp_adapter.cpp b/cpp/src/io/comp/nvcomp_adapter.cpp index 7d98e047c7c..f8920bf82c2 100644 --- a/cpp/src/io/comp/nvcomp_adapter.cpp +++ b/cpp/src/io/comp/nvcomp_adapter.cpp @@ -14,10 +14,11 @@ * limitations under the License. */ #include "nvcomp_adapter.hpp" + +#include "io/utilities/config_utils.hpp" #include "nvcomp_adapter.cuh" #include -#include #include #include diff --git a/cpp/src/io/comp/nvcomp_adapter.cuh b/cpp/src/io/comp/nvcomp_adapter.cuh index dfc803d91bf..4a7b6463fa0 100644 --- a/cpp/src/io/comp/nvcomp_adapter.cuh +++ b/cpp/src/io/comp/nvcomp_adapter.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,11 +20,11 @@ #include -#include - #include #include +#include + #include namespace cudf::io::nvcomp { diff --git a/cpp/src/io/comp/nvcomp_adapter.hpp b/cpp/src/io/comp/nvcomp_adapter.hpp index ebaec617c10..1a680a050fd 100644 --- a/cpp/src/io/comp/nvcomp_adapter.hpp +++ b/cpp/src/io/comp/nvcomp_adapter.hpp @@ -17,8 +17,7 @@ #pragma once #include "gpuinflate.hpp" - -#include +#include "io/utilities/config_utils.hpp" #include #include diff --git a/cpp/src/io/comp/snap.cu b/cpp/src/io/comp/snap.cu index a45e8b2083b..252c96f496a 100644 --- a/cpp/src/io/comp/snap.cu +++ b/cpp/src/io/comp/snap.cu @@ -15,8 +15,7 @@ */ #include "gpuinflate.hpp" - -#include +#include "io/utilities/block_utils.cuh" #include diff --git a/cpp/src/io/comp/statistics.cu b/cpp/src/io/comp/statistics.cu index e0f7e1ec6dd..2a9eb782800 100644 --- a/cpp/src/io/comp/statistics.cu +++ b/cpp/src/io/comp/statistics.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ #include "gpuinflate.hpp" #include + #include namespace cudf::io { diff --git a/cpp/src/io/comp/uncomp.cpp b/cpp/src/io/comp/uncomp.cpp index 0d2d21333bb..3e5d966282d 100644 --- a/cpp/src/io/comp/uncomp.cpp +++ b/cpp/src/io/comp/uncomp.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. + * Copyright (c) 2018-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,22 +14,21 @@ * limitations under the License. */ +#include "io/utilities/hostdevice_vector.hpp" #include "io_uncomp.hpp" #include "nvcomp_adapter.hpp" #include "unbz2.hpp" // bz2 uncompress -#include - #include #include #include #include -#include // memset - #include // uncompress +#include // memset + using cudf::host_span; namespace cudf { diff --git a/cpp/src/io/comp/unsnap.cu b/cpp/src/io/comp/unsnap.cu index 46555a97e9c..b48e49ffd78 100644 --- a/cpp/src/io/comp/unsnap.cu +++ b/cpp/src/io/comp/unsnap.cu @@ -15,8 +15,7 @@ */ #include "gpuinflate.hpp" - -#include +#include "io/utilities/block_utils.cuh" #include diff --git a/cpp/src/io/csv/csv_common.hpp b/cpp/src/io/csv/csv_common.hpp index 7c9c0b00103..9b48e191aca 100644 --- a/cpp/src/io/csv/csv_common.hpp +++ b/cpp/src/io/csv/csv_common.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,8 +16,9 @@ #pragma once +#include "io/utilities/column_type_histogram.hpp" + #include -#include namespace cudf { namespace io { diff --git a/cpp/src/io/csv/csv_gpu.cu b/cpp/src/io/csv/csv_gpu.cu index 8252cccbdb9..9c186f161b3 100644 --- a/cpp/src/io/csv/csv_gpu.cu +++ b/cpp/src/io/csv/csv_gpu.cu @@ -16,9 +16,9 @@ #include "csv_common.hpp" #include "csv_gpu.hpp" - -#include -#include +#include "io/utilities/block_utils.cuh" +#include "io/utilities/parsing_utils.cuh" +#include "io/utilities/trie.cuh" #include #include @@ -31,7 +31,6 @@ #include #include #include -#include #include #include diff --git a/cpp/src/io/csv/csv_gpu.hpp b/cpp/src/io/csv/csv_gpu.hpp index 62bd8f1eff2..06c60319371 100644 --- a/cpp/src/io/csv/csv_gpu.hpp +++ b/cpp/src/io/csv/csv_gpu.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ #pragma once -#include +#include "io/utilities/parsing_utils.cuh" #include #include diff --git a/cpp/src/io/csv/datetime.cuh b/cpp/src/io/csv/datetime.cuh index 50d2106ec42..bfdba238a1e 100644 --- a/cpp/src/io/csv/datetime.cuh +++ b/cpp/src/io/csv/datetime.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,8 +16,8 @@ #pragma once -#include -#include +#include "io/utilities/parsing_utils.cuh" +#include "io/utilities/time_utils.cuh" #include diff --git a/cpp/src/io/csv/reader_impl.cu b/cpp/src/io/csv/reader_impl.cu index 393e44bddf4..02daf4655db 100644 --- a/cpp/src/io/csv/reader_impl.cu +++ b/cpp/src/io/csv/reader_impl.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,11 +21,10 @@ #include "csv_common.hpp" #include "csv_gpu.hpp" - -#include -#include -#include -#include +#include "io/comp/io_uncomp.hpp" +#include "io/utilities/column_buffer.hpp" +#include "io/utilities/hostdevice_vector.hpp" +#include "io/utilities/parsing_utils.cuh" #include #include diff --git a/cpp/src/io/csv/writer_impl.cu b/cpp/src/io/csv/writer_impl.cu index 65473073e31..cedcd97e44e 100644 --- a/cpp/src/io/csv/writer_impl.cu +++ b/cpp/src/io/csv/writer_impl.cu @@ -19,10 +19,9 @@ * @brief cuDF-IO CSV writer class implementation */ -#include "durations.hpp" - #include "csv_common.hpp" #include "csv_gpu.hpp" +#include "durations.hpp" #include #include diff --git a/cpp/src/io/fst/agent_dfa.cuh b/cpp/src/io/fst/agent_dfa.cuh index 9bb087e788d..9ba8696370a 100644 --- a/cpp/src/io/fst/agent_dfa.cuh +++ b/cpp/src/io/fst/agent_dfa.cuh @@ -18,7 +18,6 @@ #include "in_reg_array.cuh" #include - #include #include diff --git a/cpp/src/io/fst/device_dfa.cuh b/cpp/src/io/fst/device_dfa.cuh index 7eeff27eef1..4729c1c1b15 100644 --- a/cpp/src/io/fst/device_dfa.cuh +++ b/cpp/src/io/fst/device_dfa.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,8 +16,7 @@ #pragma once #include "dispatch_dfa.cuh" - -#include +#include "io/utilities/hostdevice_vector.hpp" #include diff --git a/cpp/src/io/fst/logical_stack.cuh b/cpp/src/io/fst/logical_stack.cuh index 22385d33c7b..0f1fc7d572b 100644 --- a/cpp/src/io/fst/logical_stack.cuh +++ b/cpp/src/io/fst/logical_stack.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,24 +15,24 @@ */ #pragma once +#include + #include #include #include #include -#include #include #include #include +#include #include #include #include #include #include -#include - #include #include #include diff --git a/cpp/src/io/fst/lookup_tables.cuh b/cpp/src/io/fst/lookup_tables.cuh index a4e519d180d..5532a7f994b 100644 --- a/cpp/src/io/fst/lookup_tables.cuh +++ b/cpp/src/io/fst/lookup_tables.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,12 @@ #pragma once +#include "io/fst/device_dfa.cuh" +#include "io/utilities/hostdevice_vector.hpp" + #include -#include -#include #include - #include #include diff --git a/cpp/src/io/functions.cpp b/cpp/src/io/functions.cpp index 42f2fd02d52..315562e9183 100644 --- a/cpp/src/io/functions.cpp +++ b/cpp/src/io/functions.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "io/orc/orc.hpp" #include #include diff --git a/cpp/src/io/json/byte_range_info.cu b/cpp/src/io/json/byte_range_info.cu index d359e917dfa..258a40b0dd3 100644 --- a/cpp/src/io/json/byte_range_info.cu +++ b/cpp/src/io/json/byte_range_info.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ #include #include + #include namespace cudf::io::json::detail { diff --git a/cpp/src/io/json/json_column.cu b/cpp/src/io/json/json_column.cu index b1dc2c9dd7f..56da1095b81 100644 --- a/cpp/src/io/json/json_column.cu +++ b/cpp/src/io/json/json_column.cu @@ -14,9 +14,9 @@ * limitations under the License. */ +#include "io/utilities/parsing_utils.cuh" +#include "io/utilities/string_parsing.hpp" #include "nested_json.hpp" -#include -#include #include #include @@ -32,6 +32,8 @@ #include #include +#include +#include #include #include #include @@ -46,9 +48,6 @@ #include #include -#include -#include - #include #include diff --git a/cpp/src/io/json/json_quote_normalization.cu b/cpp/src/io/json/json_quote_normalization.cu index 7c9466748cd..a13b6e0b016 100644 --- a/cpp/src/io/json/json_quote_normalization.cu +++ b/cpp/src/io/json/json_quote_normalization.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "io/fst/lookup_tables.cuh" #include #include diff --git a/cpp/src/io/json/json_tree.cu b/cpp/src/io/json/json_tree.cu index 148aeb5ec7a..1b7976dab89 100644 --- a/cpp/src/io/json/json_tree.cu +++ b/cpp/src/io/json/json_tree.cu @@ -14,8 +14,8 @@ * limitations under the License. */ +#include "io/utilities/hostdevice_vector.hpp" #include "nested_json.hpp" -#include #include #include @@ -33,9 +33,8 @@ #include #include - #include - +#include #include #include #include @@ -54,8 +53,6 @@ #include #include -#include - #include namespace cudf::io::json { diff --git a/cpp/src/io/json/legacy/json_gpu.cu b/cpp/src/io/json/legacy/json_gpu.cu index 4d5293e12fd..9beeecdd6fb 100644 --- a/cpp/src/io/json/legacy/json_gpu.cu +++ b/cpp/src/io/json/legacy/json_gpu.cu @@ -14,11 +14,11 @@ * limitations under the License. */ +#include "io/utilities/column_type_histogram.hpp" +#include "io/utilities/parsing_utils.cuh" +#include "io/utilities/trie.cuh" #include "json_gpu.hpp" -#include -#include - #include #include #include @@ -27,7 +27,6 @@ #include #include #include -#include #include #include diff --git a/cpp/src/io/json/legacy/json_gpu.hpp b/cpp/src/io/json/legacy/json_gpu.hpp index 48fe6c69390..853e30c9427 100644 --- a/cpp/src/io/json/legacy/json_gpu.hpp +++ b/cpp/src/io/json/legacy/json_gpu.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,10 +16,9 @@ #pragma once -#include -#include - -#include +#include "hash/concurrent_unordered_map.cuh" +#include "io/utilities/column_type_histogram.hpp" +#include "io/utilities/parsing_utils.cuh" #include #include diff --git a/cpp/src/io/json/legacy/read_json.hpp b/cpp/src/io/json/legacy/read_json.hpp index e3fa010e08e..32d05c432b4 100644 --- a/cpp/src/io/json/legacy/read_json.hpp +++ b/cpp/src/io/json/legacy/read_json.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,6 +23,10 @@ #include #include +namespace cudf::io { +class json_reader_options; // forward decl +} + namespace cudf::io::json::detail::legacy { table_with_metadata read_json(host_span> sources, diff --git a/cpp/src/io/json/legacy/reader_impl.cu b/cpp/src/io/json/legacy/reader_impl.cu index d461f27c921..f9d0f6895b9 100644 --- a/cpp/src/io/json/legacy/reader_impl.cu +++ b/cpp/src/io/json/legacy/reader_impl.cu @@ -14,14 +14,12 @@ * limitations under the License. */ +#include "hash/concurrent_unordered_map.cuh" +#include "io/comp/io_uncomp.hpp" +#include "io/utilities/column_buffer.hpp" +#include "io/utilities/parsing_utils.cuh" #include "json_gpu.hpp" -#include - -#include -#include -#include - #include #include #include diff --git a/cpp/src/io/json/nested_json_gpu.cu b/cpp/src/io/json/nested_json_gpu.cu index 5eb3883dc64..73af983d108 100644 --- a/cpp/src/io/json/nested_json_gpu.cu +++ b/cpp/src/io/json/nested_json_gpu.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,12 @@ * limitations under the License. */ +#include "io/fst/logical_stack.cuh" +#include "io/fst/lookup_tables.cuh" +#include "io/utilities/parsing_utils.cuh" +#include "io/utilities/string_parsing.hpp" #include "nested_json.hpp" -#include -#include -#include -#include - #include #include #include diff --git a/cpp/src/io/json/read_json.cu b/cpp/src/io/json/read_json.cu index 2cfb5fa03c9..ba8acf2d47a 100644 --- a/cpp/src/io/json/read_json.cu +++ b/cpp/src/io/json/read_json.cu @@ -14,12 +14,11 @@ * limitations under the License. */ +#include "io/comp/io_uncomp.hpp" +#include "io/json/legacy/read_json.hpp" +#include "io/json/nested_json.hpp" #include "read_json.hpp" -#include -#include -#include - #include #include #include diff --git a/cpp/src/io/json/write_json.cu b/cpp/src/io/json/write_json.cu index 84e0ac9e74d..8c5b309244d 100644 --- a/cpp/src/io/json/write_json.cu +++ b/cpp/src/io/json/write_json.cu @@ -19,9 +19,9 @@ * @brief cuDF-IO JSON writer implementation */ -#include -#include -#include +#include "io/csv/durations.hpp" +#include "io/utilities/parsing_utils.cuh" +#include "lists/utilities.hpp" #include #include @@ -48,6 +48,7 @@ #include #include +#include #include #include #include @@ -56,8 +57,6 @@ #include #include -#include - #include #include #include diff --git a/cpp/src/io/orc/aggregate_orc_metadata.cpp b/cpp/src/io/orc/aggregate_orc_metadata.cpp index 8cae1ff5309..ea091099b6e 100644 --- a/cpp/src/io/orc/aggregate_orc_metadata.cpp +++ b/cpp/src/io/orc/aggregate_orc_metadata.cpp @@ -16,7 +16,7 @@ #include "aggregate_orc_metadata.hpp" -#include +#include "io/utilities/row_selection.hpp" #include #include diff --git a/cpp/src/io/orc/orc.cpp b/cpp/src/io/orc/orc.cpp index eab951efe36..de0d7a88614 100644 --- a/cpp/src/io/orc/orc.cpp +++ b/cpp/src/io/orc/orc.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include "orc.hpp" + #include "orc_field_reader.hpp" #include "orc_field_writer.hpp" diff --git a/cpp/src/io/orc/orc.hpp b/cpp/src/io/orc/orc.hpp index 4f3e0a82768..6fbee2824eb 100644 --- a/cpp/src/io/orc/orc.hpp +++ b/cpp/src/io/orc/orc.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,13 @@ #pragma once +#include "io/comp/io_uncomp.hpp" + #include #include #include #include #include -#include #include diff --git a/cpp/src/io/orc/orc_field_reader.hpp b/cpp/src/io/orc/orc_field_reader.hpp index 58f3fff7eb4..3689e4d958b 100644 --- a/cpp/src/io/orc/orc_field_reader.hpp +++ b/cpp/src/io/orc/orc_field_reader.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ #pragma once #include "orc.hpp" + #include /** diff --git a/cpp/src/io/orc/orc_gpu.hpp b/cpp/src/io/orc/orc_gpu.hpp index c2570d71c24..b69722bbded 100644 --- a/cpp/src/io/orc/orc_gpu.hpp +++ b/cpp/src/io/orc/orc_gpu.hpp @@ -16,12 +16,11 @@ #pragma once +#include "io/comp/gpuinflate.hpp" +#include "io/statistics/statistics.cuh" +#include "io/utilities/column_buffer.hpp" #include "orc.hpp" -#include -#include -#include - #include #include #include diff --git a/cpp/src/io/orc/reader_impl.hpp b/cpp/src/io/orc/reader_impl.hpp index 6561c08f2d9..7746bacd188 100644 --- a/cpp/src/io/orc/reader_impl.hpp +++ b/cpp/src/io/orc/reader_impl.hpp @@ -17,8 +17,7 @@ #pragma once #include "aggregate_orc_metadata.hpp" - -#include +#include "io/utilities/column_buffer.hpp" #include #include diff --git a/cpp/src/io/orc/reader_impl_chunking.hpp b/cpp/src/io/orc/reader_impl_chunking.hpp index 44ece671155..0ad0f9af589 100644 --- a/cpp/src/io/orc/reader_impl_chunking.hpp +++ b/cpp/src/io/orc/reader_impl_chunking.hpp @@ -16,10 +16,9 @@ #pragma once +#include "io/utilities/hostdevice_vector.hpp" #include "orc_gpu.hpp" -#include - #include #include diff --git a/cpp/src/io/orc/reader_impl_helpers.hpp b/cpp/src/io/orc/reader_impl_helpers.hpp index f0d91c75fc3..48742b5fc8c 100644 --- a/cpp/src/io/orc/reader_impl_helpers.hpp +++ b/cpp/src/io/orc/reader_impl_helpers.hpp @@ -17,10 +17,9 @@ #pragma once #include "aggregate_orc_metadata.hpp" +#include "io/utilities/column_buffer.hpp" #include "orc.hpp" -#include - #include #include diff --git a/cpp/src/io/orc/reader_impl_preprocess.cu b/cpp/src/io/orc/reader_impl_preprocess.cu index 026e2e7d8ed..ea191f67785 100644 --- a/cpp/src/io/orc/reader_impl_preprocess.cu +++ b/cpp/src/io/orc/reader_impl_preprocess.cu @@ -14,14 +14,13 @@ * limitations under the License. */ +#include "io/comp/gpuinflate.hpp" +#include "io/comp/nvcomp_adapter.hpp" +#include "io/utilities/config_utils.hpp" #include "reader_impl.hpp" #include "reader_impl_chunking.hpp" #include "reader_impl_helpers.hpp" -#include -#include -#include - #include #include #include diff --git a/cpp/src/io/orc/stats_enc.cu b/cpp/src/io/orc/stats_enc.cu index 31159ae0341..2fce981e8a5 100644 --- a/cpp/src/io/orc/stats_enc.cu +++ b/cpp/src/io/orc/stats_enc.cu @@ -14,10 +14,9 @@ * limitations under the License. */ +#include "io/utilities/block_utils.cuh" #include "orc_gpu.hpp" -#include - #include #include diff --git a/cpp/src/io/orc/stripe_data.cu b/cpp/src/io/orc/stripe_data.cu index 14072d79172..5e10d90ae9b 100644 --- a/cpp/src/io/orc/stripe_data.cu +++ b/cpp/src/io/orc/stripe_data.cu @@ -14,14 +14,15 @@ * limitations under the License. */ +#include "io/utilities/block_utils.cuh" #include "orc_gpu.hpp" #include -#include -#include #include +#include + namespace cudf { namespace io { namespace orc { diff --git a/cpp/src/io/orc/stripe_enc.cu b/cpp/src/io/orc/stripe_enc.cu index 516922219d1..748e4d2c27b 100644 --- a/cpp/src/io/orc/stripe_enc.cu +++ b/cpp/src/io/orc/stripe_enc.cu @@ -14,24 +14,23 @@ * limitations under the License. */ +#include "io/comp/nvcomp_adapter.hpp" +#include "io/utilities/block_utils.cuh" +#include "io/utilities/config_utils.hpp" +#include "io/utilities/time_utils.cuh" #include "orc_gpu.hpp" -#include -#include -#include -#include -#include - #include #include #include +#include #include #include -#include #include #include +#include #include #include #include diff --git a/cpp/src/io/orc/stripe_init.cu b/cpp/src/io/orc/stripe_init.cu index 327b9557176..350700a22fd 100644 --- a/cpp/src/io/orc/stripe_init.cu +++ b/cpp/src/io/orc/stripe_init.cu @@ -14,13 +14,14 @@ * limitations under the License. */ +#include "io/utilities/block_utils.cuh" #include "orc_gpu.hpp" #include -#include -#include #include + +#include #include #include diff --git a/cpp/src/io/orc/writer_impl.cu b/cpp/src/io/orc/writer_impl.cu index cc1a18c9173..f0235e13422 100644 --- a/cpp/src/io/orc/writer_impl.cu +++ b/cpp/src/io/orc/writer_impl.cu @@ -19,12 +19,11 @@ * @brief cuDF-IO ORC writer class implementation */ +#include "io/comp/nvcomp_adapter.hpp" +#include "io/statistics/column_statistics.cuh" +#include "io/utilities/column_utils.cuh" #include "writer_impl.hpp" -#include -#include -#include - #include #include #include @@ -39,6 +38,10 @@ #include #include +#include +#include +#include +#include #include #include #include @@ -56,12 +59,6 @@ #include #include -#include -#include - -#include -#include - #include #include #include diff --git a/cpp/src/io/orc/writer_impl.hpp b/cpp/src/io/orc/writer_impl.hpp index f8ac5515f2e..f1dc45087d5 100644 --- a/cpp/src/io/orc/writer_impl.hpp +++ b/cpp/src/io/orc/writer_impl.hpp @@ -16,11 +16,10 @@ #pragma once +#include "io/utilities/hostdevice_vector.hpp" #include "orc.hpp" #include "orc_gpu.hpp" -#include - #include #include #include diff --git a/cpp/src/io/parquet/decode_preprocess.cu b/cpp/src/io/parquet/decode_preprocess.cu index 2d000600028..fea4777af43 100644 --- a/cpp/src/io/parquet/decode_preprocess.cu +++ b/cpp/src/io/parquet/decode_preprocess.cu @@ -14,13 +14,13 @@ * limitations under the License. */ +#include "io/utilities/column_buffer.hpp" #include "page_decode.cuh" -#include - #include #include + #include namespace cudf::io::parquet::detail { diff --git a/cpp/src/io/parquet/page_data.cu b/cpp/src/io/parquet/page_data.cu index 8d220e6fa96..2a9f2d56755 100644 --- a/cpp/src/io/parquet/page_data.cu +++ b/cpp/src/io/parquet/page_data.cu @@ -14,13 +14,13 @@ * limitations under the License. */ +#include "io/utilities/column_buffer.hpp" #include "page_decode.cuh" -#include - #include #include + #include namespace cudf::io::parquet::detail { diff --git a/cpp/src/io/parquet/page_decode.cuh b/cpp/src/io/parquet/page_decode.cuh index 409b1464cd1..4353e079496 100644 --- a/cpp/src/io/parquet/page_decode.cuh +++ b/cpp/src/io/parquet/page_decode.cuh @@ -17,11 +17,10 @@ #pragma once #include "error.hpp" +#include "io/utilities/block_utils.cuh" #include "parquet_gpu.hpp" #include "rle_stream.cuh" -#include - #include #include diff --git a/cpp/src/io/parquet/page_delta_decode.cu b/cpp/src/io/parquet/page_delta_decode.cu index d0557446f14..ebad1434c7f 100644 --- a/cpp/src/io/parquet/page_delta_decode.cu +++ b/cpp/src/io/parquet/page_delta_decode.cu @@ -15,14 +15,14 @@ */ #include "delta_binary.cuh" +#include "io/utilities/block_utils.cuh" #include "page_string_utils.cuh" #include "parquet_gpu.hpp" -#include - #include #include + #include namespace cudf::io::parquet::detail { diff --git a/cpp/src/io/parquet/page_enc.cu b/cpp/src/io/parquet/page_enc.cu index 2f351edd2b9..5aad31bd057 100644 --- a/cpp/src/io/parquet/page_enc.cu +++ b/cpp/src/io/parquet/page_enc.cu @@ -15,10 +15,9 @@ */ #include "delta_enc.cuh" +#include "io/utilities/block_utils.cuh" #include "parquet_gpu.cuh" -#include - #include #include #include @@ -29,9 +28,7 @@ #include #include - #include - #include #include #include diff --git a/cpp/src/io/parquet/page_hdr.cu b/cpp/src/io/parquet/page_hdr.cu index 888d9452612..a15ccf328de 100644 --- a/cpp/src/io/parquet/page_hdr.cu +++ b/cpp/src/io/parquet/page_hdr.cu @@ -15,15 +15,15 @@ */ #include "error.hpp" +#include "io/utilities/block_utils.cuh" #include "parquet_gpu.hpp" -#include #include -#include - #include +#include + namespace cudf::io::parquet::detail { // Minimal thrift implementation for parsing page headers diff --git a/cpp/src/io/parquet/parquet_gpu.hpp b/cpp/src/io/parquet/parquet_gpu.hpp index b215cd7a20b..64e1c199779 100644 --- a/cpp/src/io/parquet/parquet_gpu.hpp +++ b/cpp/src/io/parquet/parquet_gpu.hpp @@ -17,7 +17,6 @@ #pragma once #include "error.hpp" - #include "io/comp/gpuinflate.hpp" #include "io/parquet/parquet.hpp" #include "io/parquet/parquet_common.hpp" @@ -34,7 +33,6 @@ #include #include - #include #include diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 24d46d91dbb..26d810a3337 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -15,6 +15,7 @@ */ #include "reader_impl.hpp" + #include "error.hpp" #include diff --git a/cpp/src/io/parquet/reader_impl_chunking.cu b/cpp/src/io/parquet/reader_impl_chunking.cu index 69141faa7fc..a7af20f5d7c 100644 --- a/cpp/src/io/parquet/reader_impl_chunking.cu +++ b/cpp/src/io/parquet/reader_impl_chunking.cu @@ -14,6 +14,9 @@ * limitations under the License. */ +#include "io/comp/nvcomp_adapter.hpp" +#include "io/utilities/config_utils.hpp" +#include "io/utilities/time_utils.cuh" #include "reader_impl.hpp" #include "reader_impl_chunking.hpp" @@ -21,13 +24,9 @@ #include #include -#include - -#include -#include - #include +#include #include #include #include @@ -35,8 +34,6 @@ #include #include -#include - #include namespace cudf::io::parquet::detail { diff --git a/cpp/src/io/parquet/reader_impl_helpers.cpp b/cpp/src/io/parquet/reader_impl_helpers.cpp index ef51f373b24..6f11debb8df 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.cpp +++ b/cpp/src/io/parquet/reader_impl_helpers.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ #include "reader_impl_helpers.hpp" -#include +#include "io/utilities/row_selection.hpp" #include #include diff --git a/cpp/src/io/parquet/reader_impl_preprocess.cu b/cpp/src/io/parquet/reader_impl_preprocess.cu index ee3b1c466e0..48ff32038b3 100644 --- a/cpp/src/io/parquet/reader_impl_preprocess.cu +++ b/cpp/src/io/parquet/reader_impl_preprocess.cu @@ -23,6 +23,7 @@ #include +#include #include #include #include @@ -38,8 +39,6 @@ #include #include -#include - #include #include diff --git a/cpp/src/io/parquet/rle_stream.cuh b/cpp/src/io/parquet/rle_stream.cuh index 799d6d9fd64..5faadf1369b 100644 --- a/cpp/src/io/parquet/rle_stream.cuh +++ b/cpp/src/io/parquet/rle_stream.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ #pragma once #include "parquet_gpu.hpp" + #include #include diff --git a/cpp/src/io/parquet/writer_impl.cu b/cpp/src/io/parquet/writer_impl.cu index 3dcc9716579..ecdbdd0fd5f 100644 --- a/cpp/src/io/parquet/writer_impl.cu +++ b/cpp/src/io/parquet/writer_impl.cu @@ -21,15 +21,14 @@ #include "compact_protocol_reader.hpp" #include "compact_protocol_writer.hpp" +#include "io/comp/nvcomp_adapter.hpp" +#include "io/statistics/column_statistics.cuh" +#include "io/utilities/column_utils.cuh" +#include "io/utilities/config_utils.hpp" #include "parquet_common.hpp" #include "parquet_gpu.cuh" #include "writer_impl.hpp" -#include -#include -#include -#include - #include #include #include diff --git a/cpp/src/io/parquet/writer_impl.hpp b/cpp/src/io/parquet/writer_impl.hpp index 3415205d179..2f6608b0ae7 100644 --- a/cpp/src/io/parquet/writer_impl.hpp +++ b/cpp/src/io/parquet/writer_impl.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,9 +24,8 @@ #include "parquet.hpp" #include "parquet_gpu.hpp" -#include - #include +#include #include #include #include diff --git a/cpp/src/io/statistics/column_statistics.cuh b/cpp/src/io/statistics/column_statistics.cuh index db0d56ac321..b2cabe24a50 100644 --- a/cpp/src/io/statistics/column_statistics.cuh +++ b/cpp/src/io/statistics/column_statistics.cuh @@ -21,12 +21,10 @@ #pragma once +#include "statistics.cuh" #include "temp_storage_wrapper.cuh" - #include "typed_statistics_chunk.cuh" -#include "statistics.cuh" - namespace cudf { namespace io { diff --git a/cpp/src/io/statistics/statistics_type_identification.cuh b/cpp/src/io/statistics/statistics_type_identification.cuh index ea8c71f0dcb..5e11646be6b 100644 --- a/cpp/src/io/statistics/statistics_type_identification.cuh +++ b/cpp/src/io/statistics/statistics_type_identification.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,18 +22,13 @@ #pragma once #include "byte_array_view.cuh" +#include "conversion_type_select.cuh" #include - -#include - #include - -#include - #include - -#include "conversion_type_select.cuh" +#include +#include #include diff --git a/cpp/src/io/statistics/typed_statistics_chunk.cuh b/cpp/src/io/statistics/typed_statistics_chunk.cuh index e6ec1471cb7..01db781c766 100644 --- a/cpp/src/io/statistics/typed_statistics_chunk.cuh +++ b/cpp/src/io/statistics/typed_statistics_chunk.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,6 @@ #include #include - #include namespace cudf { diff --git a/cpp/src/io/text/bgzip_data_chunk_source.cu b/cpp/src/io/text/bgzip_data_chunk_source.cu index 77647c18b20..faa09e586ab 100644 --- a/cpp/src/io/text/bgzip_data_chunk_source.cu +++ b/cpp/src/io/text/bgzip_data_chunk_source.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,9 @@ * limitations under the License. */ +#include "io/comp/nvcomp_adapter.hpp" #include "io/text/device_data_chunks.hpp" - -#include -#include +#include "io/utilities/config_utils.hpp" #include #include diff --git a/cpp/src/io/text/bgzip_utils.cpp b/cpp/src/io/text/bgzip_utils.cpp index 43e2c26f132..cb412828e2d 100644 --- a/cpp/src/io/text/bgzip_utils.cpp +++ b/cpp/src/io/text/bgzip_utils.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,12 @@ * limitations under the License. */ -#include - #include #include #include +#include + #include #include #include diff --git a/cpp/src/io/text/multibyte_split.cu b/cpp/src/io/text/multibyte_split.cu index 34a476974e4..8e37564fc35 100644 --- a/cpp/src/io/text/multibyte_split.cu +++ b/cpp/src/io/text/multibyte_split.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "io/utilities/output_builder.cuh" #include #include @@ -38,16 +38,14 @@ #include #include +#include +#include +#include #include #include #include #include -#include -#include - -#include - #include #include #include diff --git a/cpp/src/io/utilities/column_buffer.cpp b/cpp/src/io/utilities/column_buffer.cpp index 951217dc442..96503e4907b 100644 --- a/cpp/src/io/utilities/column_buffer.cpp +++ b/cpp/src/io/utilities/column_buffer.cpp @@ -20,6 +20,7 @@ */ #include "column_buffer.hpp" + #include #include #include diff --git a/cpp/src/io/utilities/column_utils.cuh b/cpp/src/io/utilities/column_utils.cuh index 5f4bf646452..a0c20a56233 100644 --- a/cpp/src/io/utilities/column_utils.cuh +++ b/cpp/src/io/utilities/column_utils.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ #pragma once -#include +#include "io/statistics/statistics.cuh" #include #include diff --git a/cpp/src/io/utilities/data_casting.cu b/cpp/src/io/utilities/data_casting.cu index 8fd860d9492..4b5d47e71fb 100644 --- a/cpp/src/io/utilities/data_casting.cu +++ b/cpp/src/io/utilities/data_casting.cu @@ -14,8 +14,8 @@ * limitations under the License. */ -#include -#include +#include "io/utilities/parsing_utils.cuh" +#include "io/utilities/string_parsing.hpp" #include #include @@ -32,12 +32,11 @@ #include #include +#include #include #include #include -#include - #include #include diff --git a/cpp/src/io/utilities/data_sink.cpp b/cpp/src/io/utilities/data_sink.cpp index 5786e9dd6d1..5557648ebbe 100644 --- a/cpp/src/io/utilities/data_sink.cpp +++ b/cpp/src/io/utilities/data_sink.cpp @@ -14,16 +14,18 @@ * limitations under the License. */ -#include - #include "file_io_utilities.hpp" +#include "io/utilities/config_utils.hpp" + #include #include -#include #include + #include +#include + namespace cudf { namespace io { diff --git a/cpp/src/io/utilities/datasource.cpp b/cpp/src/io/utilities/datasource.cpp index 687764be911..cf2ba369023 100644 --- a/cpp/src/io/utilities/datasource.cpp +++ b/cpp/src/io/utilities/datasource.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,19 +15,19 @@ */ #include "file_io_utilities.hpp" +#include "io/utilities/config_utils.hpp" #include #include #include #include #include -#include #include + #include #include - #include #include #include diff --git a/cpp/src/io/utilities/file_io_utilities.cpp b/cpp/src/io/utilities/file_io_utilities.cpp index 63c1114c9ce..01090a43a0e 100644 --- a/cpp/src/io/utilities/file_io_utilities.cpp +++ b/cpp/src/io/utilities/file_io_utilities.cpp @@ -15,8 +15,10 @@ */ #include "file_io_utilities.hpp" + +#include "io/utilities/config_utils.hpp" + #include -#include #include diff --git a/cpp/src/io/utilities/file_io_utilities.hpp b/cpp/src/io/utilities/file_io_utilities.hpp index 90bf591fe0c..0d5a5b218da 100644 --- a/cpp/src/io/utilities/file_io_utilities.hpp +++ b/cpp/src/io/utilities/file_io_utilities.hpp @@ -20,14 +20,15 @@ #include "thread_pool.hpp" #include + #include #endif -#include - #include #include +#include + #include namespace cudf { diff --git a/cpp/src/io/utilities/parsing_utils.cu b/cpp/src/io/utilities/parsing_utils.cu index d02ce99e6e5..c1cbcd0baca 100644 --- a/cpp/src/io/utilities/parsing_utils.cu +++ b/cpp/src/io/utilities/parsing_utils.cu @@ -19,10 +19,10 @@ #include #include -#include - #include +#include + #include namespace cudf { diff --git a/cpp/src/io/utilities/parsing_utils.cuh b/cpp/src/io/utilities/parsing_utils.cuh index 43d62fcd513..06a0a63c0ab 100644 --- a/cpp/src/io/utilities/parsing_utils.cuh +++ b/cpp/src/io/utilities/parsing_utils.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,8 +16,9 @@ #pragma once -#include -#include +#include "column_type_histogram.hpp" +#include "io/csv/datetime.cuh" +#include "io/utilities/trie.cuh" #include #include @@ -27,8 +28,6 @@ #include #include -#include "column_type_histogram.hpp" - #include #include diff --git a/cpp/src/io/utilities/row_selection.cpp b/cpp/src/io/utilities/row_selection.cpp index ae5c7b5fbda..bb5565d8ce7 100644 --- a/cpp/src/io/utilities/row_selection.cpp +++ b/cpp/src/io/utilities/row_selection.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "io/utilities/row_selection.hpp" #include diff --git a/cpp/src/io/utilities/string_parsing.hpp b/cpp/src/io/utilities/string_parsing.hpp index 12fc0a5b2e7..a98660c98a9 100644 --- a/cpp/src/io/utilities/string_parsing.hpp +++ b/cpp/src/io/utilities/string_parsing.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ #pragma once -#include +#include "io/utilities/parsing_utils.cuh" #include #include diff --git a/cpp/src/io/utilities/type_inference.cu b/cpp/src/io/utilities/type_inference.cu index b446ad41946..dff40cc09ed 100644 --- a/cpp/src/io/utilities/type_inference.cu +++ b/cpp/src/io/utilities/type_inference.cu @@ -14,9 +14,9 @@ * limitations under the License. */ -#include -#include -#include +#include "io/utilities/column_type_histogram.hpp" +#include "io/utilities/string_parsing.hpp" +#include "io/utilities/trie.cuh" #include #include diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index 46b347d39b1..bc8e3e8e392 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ #include #include + #include #include diff --git a/cpp/src/jit/cache.hpp b/cpp/src/jit/cache.hpp index df8d4278f0f..8e6c07911f7 100644 --- a/cpp/src/jit/cache.hpp +++ b/cpp/src/jit/cache.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ #pragma once #include + #include namespace cudf { diff --git a/cpp/src/join/conditional_join.cu b/cpp/src/join/conditional_join.cu index c3073524467..cc729ad5e8b 100644 --- a/cpp/src/join/conditional_join.cu +++ b/cpp/src/join/conditional_join.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,11 @@ * limitations under the License. */ +#include "join/conditional_join.hpp" +#include "join/conditional_join_kernels.cuh" +#include "join/join_common_utils.cuh" +#include "join/join_common_utils.hpp" + #include #include #include @@ -23,10 +28,6 @@ #include #include #include -#include -#include -#include -#include #include diff --git a/cpp/src/join/conditional_join_kernels.cuh b/cpp/src/join/conditional_join_kernels.cuh index 02ce27a36ba..cc57fa7b03b 100644 --- a/cpp/src/join/conditional_join_kernels.cuh +++ b/cpp/src/join/conditional_join_kernels.cuh @@ -16,8 +16,8 @@ #pragma once -#include -#include +#include "join/join_common_utils.cuh" +#include "join/join_common_utils.hpp" #include #include diff --git a/cpp/src/join/join_common_utils.cuh b/cpp/src/join/join_common_utils.cuh index f3ce6de4598..9da41e296e6 100644 --- a/cpp/src/join/join_common_utils.cuh +++ b/cpp/src/join/join_common_utils.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,6 @@ #include #include - #include namespace cudf { diff --git a/cpp/src/join/join_common_utils.hpp b/cpp/src/join/join_common_utils.hpp index 4d361b23502..4157100b67e 100644 --- a/cpp/src/join/join_common_utils.hpp +++ b/cpp/src/join/join_common_utils.hpp @@ -24,7 +24,6 @@ #include #include - #include #include diff --git a/cpp/src/join/mixed_join_common_utils.cuh b/cpp/src/join/mixed_join_common_utils.cuh index 38e5b75ade6..19701816867 100644 --- a/cpp/src/join/mixed_join_common_utils.cuh +++ b/cpp/src/join/mixed_join_common_utils.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ #pragma once -#include +#include "join/join_common_utils.hpp" #include #include diff --git a/cpp/src/join/mixed_join_kernel.cuh b/cpp/src/join/mixed_join_kernel.cuh index 22bbbff967a..0fc1c3718b1 100644 --- a/cpp/src/join/mixed_join_kernel.cuh +++ b/cpp/src/join/mixed_join_kernel.cuh @@ -27,7 +27,6 @@ #include #include - #include #include diff --git a/cpp/src/join/mixed_join_kernels.cuh b/cpp/src/join/mixed_join_kernels.cuh index 1d36a246f02..037c02666d4 100644 --- a/cpp/src/join/mixed_join_kernels.cuh +++ b/cpp/src/join/mixed_join_kernels.cuh @@ -16,8 +16,8 @@ #pragma once -#include -#include +#include "join/join_common_utils.hpp" +#include "join/mixed_join_common_utils.cuh" #include #include diff --git a/cpp/src/join/mixed_join_kernels_semi.cu b/cpp/src/join/mixed_join_kernels_semi.cu index bde75395371..5a543997a50 100644 --- a/cpp/src/join/mixed_join_kernels_semi.cu +++ b/cpp/src/join/mixed_join_kernels_semi.cu @@ -14,9 +14,9 @@ * limitations under the License. */ -#include -#include -#include +#include "join/join_common_utils.cuh" +#include "join/join_common_utils.hpp" +#include "join/mixed_join_common_utils.cuh" #include #include diff --git a/cpp/src/join/mixed_join_kernels_semi.cuh b/cpp/src/join/mixed_join_kernels_semi.cuh index 8e4966e3432..f411d36f0a8 100644 --- a/cpp/src/join/mixed_join_kernels_semi.cuh +++ b/cpp/src/join/mixed_join_kernels_semi.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,8 +16,8 @@ #pragma once -#include -#include +#include "join/join_common_utils.hpp" +#include "join/mixed_join_common_utils.cuh" #include #include diff --git a/cpp/src/join/mixed_join_size_kernel.cuh b/cpp/src/join/mixed_join_size_kernel.cuh index 3bd7bfd7c9a..618e7a9082e 100644 --- a/cpp/src/join/mixed_join_size_kernel.cuh +++ b/cpp/src/join/mixed_join_size_kernel.cuh @@ -25,7 +25,6 @@ #include #include - #include #include diff --git a/cpp/src/join/mixed_join_size_kernels_semi.cu b/cpp/src/join/mixed_join_size_kernels_semi.cu index 31da6677aef..7a22ac60710 100644 --- a/cpp/src/join/mixed_join_size_kernels_semi.cu +++ b/cpp/src/join/mixed_join_size_kernels_semi.cu @@ -14,9 +14,9 @@ * limitations under the License. */ -#include -#include -#include +#include "join/join_common_utils.cuh" +#include "join/join_common_utils.hpp" +#include "join/mixed_join_common_utils.cuh" #include #include diff --git a/cpp/src/join/semi_join.cu b/cpp/src/join/semi_join.cu index dcb6835ec09..b0e5282d97f 100644 --- a/cpp/src/join/semi_join.cu +++ b/cpp/src/join/semi_join.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "join/join_common_utils.hpp" #include #include @@ -36,6 +36,8 @@ #include #include +#include + namespace cudf { namespace detail { diff --git a/cpp/src/json/json_path.cu b/cpp/src/json/json_path.cu index 2be5798098d..25f136e2336 100644 --- a/cpp/src/json/json_path.cu +++ b/cpp/src/json/json_path.cu @@ -14,6 +14,8 @@ * limitations under the License. */ +#include "io/utilities/parsing_utils.cuh" + #include #include #include @@ -33,8 +35,6 @@ #include #include -#include - #include #include diff --git a/cpp/src/lists/combine/concatenate_list_elements.cu b/cpp/src/lists/combine/concatenate_list_elements.cu index 26fb81a600f..579ad8e7dff 100644 --- a/cpp/src/lists/combine/concatenate_list_elements.cu +++ b/cpp/src/lists/combine/concatenate_list_elements.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,6 +31,7 @@ #include #include +#include #include #include #include @@ -40,8 +41,6 @@ #include #include -#include - namespace cudf { namespace lists { namespace detail { diff --git a/cpp/src/lists/combine/concatenate_rows.cu b/cpp/src/lists/combine/concatenate_rows.cu index e143fae5742..baecef3b92d 100644 --- a/cpp/src/lists/combine/concatenate_rows.cu +++ b/cpp/src/lists/combine/concatenate_rows.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,14 +27,13 @@ #include #include +#include #include #include #include #include #include -#include - namespace cudf { namespace lists { namespace detail { diff --git a/cpp/src/lists/contains.cu b/cpp/src/lists/contains.cu index 1a88844928e..378cf678f1f 100644 --- a/cpp/src/lists/contains.cu +++ b/cpp/src/lists/contains.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,6 +31,7 @@ #include +#include #include #include #include @@ -41,8 +42,6 @@ #include #include -#include - #include namespace cudf::lists { diff --git a/cpp/src/lists/copying/scatter_helper.cu b/cpp/src/lists/copying/scatter_helper.cu index a341028d805..1ec66b4f98e 100644 --- a/cpp/src/lists/copying/scatter_helper.cu +++ b/cpp/src/lists/copying/scatter_helper.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -30,8 +31,6 @@ #include #include -#include - namespace cudf { namespace lists { namespace detail { diff --git a/cpp/src/lists/copying/segmented_gather.cu b/cpp/src/lists/copying/segmented_gather.cu index 5439a95966b..156f868c5bd 100644 --- a/cpp/src/lists/copying/segmented_gather.cu +++ b/cpp/src/lists/copying/segmented_gather.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,12 +21,11 @@ #include #include -#include -#include +#include #include - -#include +#include +#include namespace cudf { namespace lists { diff --git a/cpp/src/lists/dremel.cu b/cpp/src/lists/dremel.cu index 7a460d3dfab..5625e1bf05c 100644 --- a/cpp/src/lists/dremel.cu +++ b/cpp/src/lists/dremel.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,6 +25,7 @@ #include +#include #include #include #include @@ -34,8 +35,6 @@ #include #include -#include - namespace cudf::detail { namespace { /** diff --git a/cpp/src/lists/explode.cu b/cpp/src/lists/explode.cu index cdb7857b74a..5f1d30321a2 100644 --- a/cpp/src/lists/explode.cu +++ b/cpp/src/lists/explode.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,6 +27,7 @@ #include #include +#include #include #include #include @@ -38,8 +39,6 @@ #include #include -#include - #include #include diff --git a/cpp/src/lists/interleave_columns.cu b/cpp/src/lists/interleave_columns.cu index 8f05b020a2e..fe5e1e677ca 100644 --- a/cpp/src/lists/interleave_columns.cu +++ b/cpp/src/lists/interleave_columns.cu @@ -30,6 +30,7 @@ #include #include +#include #include #include #include @@ -38,8 +39,6 @@ #include #include -#include - namespace cudf { namespace lists { namespace detail { diff --git a/cpp/src/lists/set_operations.cu b/cpp/src/lists/set_operations.cu index 5647b503cf7..5735c84e3d3 100644 --- a/cpp/src/lists/set_operations.cu +++ b/cpp/src/lists/set_operations.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,16 +28,16 @@ #include #include +#include +#include +#include + #include #include #include #include #include -#include -#include -#include - namespace cudf::lists { namespace detail { diff --git a/cpp/src/lists/stream_compaction/distinct.cu b/cpp/src/lists/stream_compaction/distinct.cu index eb21787b3fa..c8d9c15706f 100644 --- a/cpp/src/lists/stream_compaction/distinct.cu +++ b/cpp/src/lists/stream_compaction/distinct.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "lists/utilities.hpp" #include #include diff --git a/cpp/src/merge/merge.cu b/cpp/src/merge/merge.cu index 073a2a6b97e..8be503025bd 100644 --- a/cpp/src/merge/merge.cu +++ b/cpp/src/merge/merge.cu @@ -36,11 +36,10 @@ #include #include -#include -#include #include #include +#include #include #include #include @@ -50,8 +49,8 @@ #include #include -#include - +#include +#include #include #include diff --git a/cpp/src/partitioning/partitioning.cu b/cpp/src/partitioning/partitioning.cu index 8d8f1a71672..0d2daaddb8c 100644 --- a/cpp/src/partitioning/partitioning.cu +++ b/cpp/src/partitioning/partitioning.cu @@ -32,13 +32,12 @@ #include #include +#include +#include #include #include #include -#include -#include - namespace cudf { namespace { // Launch configuration for optimized hash partition diff --git a/cpp/src/partitioning/round_robin.cu b/cpp/src/partitioning/round_robin.cu index c615f08ff12..3283a7c35ee 100644 --- a/cpp/src/partitioning/round_robin.cu +++ b/cpp/src/partitioning/round_robin.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,6 +31,7 @@ #include #include +#include #include #include #include @@ -40,8 +41,6 @@ #include #include -#include - #include #include // for std::ceil() #include diff --git a/cpp/src/quantiles/quantile.cu b/cpp/src/quantiles/quantile.cu index 946ebd479c5..cba7203483b 100644 --- a/cpp/src/quantiles/quantile.cu +++ b/cpp/src/quantiles/quantile.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "quantiles/quantiles_util.hpp" #include #include @@ -34,13 +34,12 @@ #include #include +#include #include #include #include #include -#include - #include #include diff --git a/cpp/src/quantiles/quantiles.cu b/cpp/src/quantiles/quantiles.cu index f55e9c4cb6a..8fee821dfc4 100644 --- a/cpp/src/quantiles/quantiles.cu +++ b/cpp/src/quantiles/quantiles.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "quantiles/quantiles_util.hpp" #include #include @@ -28,11 +28,10 @@ #include +#include #include #include -#include - #include #include diff --git a/cpp/src/quantiles/quantiles_util.hpp b/cpp/src/quantiles/quantiles_util.hpp index 5edb323fb38..5efafdd0be6 100644 --- a/cpp/src/quantiles/quantiles_util.hpp +++ b/cpp/src/quantiles/quantiles_util.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,13 @@ * limitations under the License. */ -#include #include #include #include #include +#include + namespace cudf { namespace detail { template diff --git a/cpp/src/quantiles/tdigest/tdigest.cu b/cpp/src/quantiles/tdigest/tdigest.cu index c8ac19e01cc..96b0355c6e5 100644 --- a/cpp/src/quantiles/tdigest/tdigest.cu +++ b/cpp/src/quantiles/tdigest/tdigest.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "quantiles/tdigest/tdigest_util.cuh" #include #include @@ -28,6 +28,7 @@ #include #include +#include #include #include #include @@ -40,8 +41,6 @@ #include #include -#include - using namespace cudf::tdigest; namespace cudf { diff --git a/cpp/src/quantiles/tdigest/tdigest_aggregation.cu b/cpp/src/quantiles/tdigest/tdigest_aggregation.cu index fc56d17d73b..56e1bfbe003 100644 --- a/cpp/src/quantiles/tdigest/tdigest_aggregation.cu +++ b/cpp/src/quantiles/tdigest/tdigest_aggregation.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "quantiles/tdigest/tdigest_util.cuh" #include #include @@ -33,6 +33,7 @@ #include #include +#include #include #include #include @@ -52,8 +53,6 @@ #include #include -#include - namespace cudf { namespace tdigest { namespace detail { diff --git a/cpp/src/reductions/all.cu b/cpp/src/reductions/all.cu index 4717c0673e3..6cea4e4ada3 100644 --- a/cpp/src/reductions/all.cu +++ b/cpp/src/reductions/all.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,13 +19,12 @@ #include #include +#include #include #include #include #include -#include - namespace cudf { namespace reduction { namespace detail { diff --git a/cpp/src/reductions/any.cu b/cpp/src/reductions/any.cu index f3093df5ac7..c0c044a1e6f 100644 --- a/cpp/src/reductions/any.cu +++ b/cpp/src/reductions/any.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,13 +19,12 @@ #include #include +#include #include #include #include #include -#include - namespace cudf { namespace reduction { namespace detail { diff --git a/cpp/src/reductions/histogram.cu b/cpp/src/reductions/histogram.cu index 42ef266a684..3e46a34cc6a 100644 --- a/cpp/src/reductions/histogram.cu +++ b/cpp/src/reductions/histogram.cu @@ -21,13 +21,12 @@ #include #include +#include +#include #include #include #include -#include -#include - #include namespace cudf::reduction::detail { diff --git a/cpp/src/reductions/nth_element.cu b/cpp/src/reductions/nth_element.cu index 82035fa78ce..88a1778bb7b 100644 --- a/cpp/src/reductions/nth_element.cu +++ b/cpp/src/reductions/nth_element.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,12 +24,11 @@ #include #include +#include #include #include #include -#include - namespace cudf::reduction::detail { std::unique_ptr nth_element(column_view const& col, diff --git a/cpp/src/reductions/scan/scan_exclusive.cu b/cpp/src/reductions/scan/scan_exclusive.cu index 4d7cb605cd4..47301ad91f6 100644 --- a/cpp/src/reductions/scan/scan_exclusive.cu +++ b/cpp/src/reductions/scan/scan_exclusive.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,9 +27,8 @@ #include #include -#include - #include +#include namespace cudf { namespace detail { diff --git a/cpp/src/reductions/scan/scan_inclusive.cu b/cpp/src/reductions/scan/scan_inclusive.cu index 00b608f36b6..7edf89a0c91 100644 --- a/cpp/src/reductions/scan/scan_inclusive.cu +++ b/cpp/src/reductions/scan/scan_inclusive.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "reductions/scan/scan.cuh" #include #include diff --git a/cpp/src/reductions/segmented/simple.cuh b/cpp/src/reductions/segmented/simple.cuh index 31ad24cd1f9..4d4c6661428 100644 --- a/cpp/src/reductions/segmented/simple.cuh +++ b/cpp/src/reductions/segmented/simple.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,13 +34,12 @@ #include +#include #include #include #include #include -#include - #include #include diff --git a/cpp/src/replace/clamp.cu b/cpp/src/replace/clamp.cu index 23c792ddcae..43358a3b165 100644 --- a/cpp/src/replace/clamp.cu +++ b/cpp/src/replace/clamp.cu @@ -37,6 +37,7 @@ #include #include +#include #include #include #include @@ -44,8 +45,6 @@ #include #include -#include - namespace cudf { namespace detail { namespace { diff --git a/cpp/src/reshape/interleave_columns.cu b/cpp/src/reshape/interleave_columns.cu index 6aa322d4d78..72227ab5dda 100644 --- a/cpp/src/reshape/interleave_columns.cu +++ b/cpp/src/reshape/interleave_columns.cu @@ -31,13 +31,12 @@ #include #include +#include #include #include #include #include -#include - namespace cudf { namespace detail { namespace { diff --git a/cpp/src/rolling/detail/lead_lag_nested.cuh b/cpp/src/rolling/detail/lead_lag_nested.cuh index 734f7d1f565..66104fe5c77 100644 --- a/cpp/src/rolling/detail/lead_lag_nested.cuh +++ b/cpp/src/rolling/detail/lead_lag_nested.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,14 +28,13 @@ #include #include +#include #include #include #include #include #include -#include - #include namespace cudf::detail { diff --git a/cpp/src/rolling/detail/rolling.cuh b/cpp/src/rolling/detail/rolling.cuh index 20845a97c7e..af6d6d7f157 100644 --- a/cpp/src/rolling/detail/rolling.cuh +++ b/cpp/src/rolling/detail/rolling.cuh @@ -16,14 +16,16 @@ #pragma once +#include "jit/cache.hpp" +#include "jit/parser.hpp" +#include "jit/util.hpp" #include "lead_lag_nested.cuh" #include "nth_element.cuh" +#include "reductions/nested_type_minmax_util.cuh" #include "rolling.hpp" #include "rolling_collect_list.cuh" #include "rolling_jit.hpp" -#include - #include #include #include @@ -45,24 +47,19 @@ #include #include -#include -#include -#include - -#include - #include #include #include +#include +#include #include #include #include #include #include -#include -#include +#include #include diff --git a/cpp/src/rolling/detail/rolling_collect_list.cuh b/cpp/src/rolling/detail/rolling_collect_list.cuh index 22e55561eca..0ce14792cfa 100644 --- a/cpp/src/rolling/detail/rolling_collect_list.cuh +++ b/cpp/src/rolling/detail/rolling_collect_list.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,13 +25,12 @@ #include #include +#include #include #include #include #include -#include - namespace cudf { namespace detail { diff --git a/cpp/src/rolling/detail/rolling_fixed_window.cu b/cpp/src/rolling/detail/rolling_fixed_window.cu index 07ecf2730a0..f51937f7a0e 100644 --- a/cpp/src/rolling/detail/rolling_fixed_window.cu +++ b/cpp/src/rolling/detail/rolling_fixed_window.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,14 +16,13 @@ #include "rolling.cuh" -#include -#include - #include -#include +#include +#include #include +#include namespace cudf::detail { diff --git a/cpp/src/rolling/detail/rolling_variable_window.cu b/cpp/src/rolling/detail/rolling_variable_window.cu index 85c5e5cb67e..bb73f305c7b 100644 --- a/cpp/src/rolling/detail/rolling_variable_window.cu +++ b/cpp/src/rolling/detail/rolling_variable_window.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,11 +19,10 @@ #include #include +#include #include #include -#include - namespace cudf::detail { // Applies a variable-size rolling window function to the values in a column. diff --git a/cpp/src/rolling/grouped_rolling.cu b/cpp/src/rolling/grouped_rolling.cu index aa009e47c2a..89a51ad1d87 100644 --- a/cpp/src/rolling/grouped_rolling.cu +++ b/cpp/src/rolling/grouped_rolling.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,6 +29,7 @@ #include #include +#include #include #include #include @@ -36,8 +37,6 @@ #include #include -#include - namespace cudf { std::unique_ptr grouped_rolling_window(table_view const& group_keys, column_view const& input, diff --git a/cpp/src/rolling/jit/kernel.cu b/cpp/src/rolling/jit/kernel.cu index 2c753965c1c..466f120022b 100644 --- a/cpp/src/rolling/jit/kernel.cu +++ b/cpp/src/rolling/jit/kernel.cu @@ -14,8 +14,8 @@ * limitations under the License. */ -#include -#include +#include "rolling/detail/rolling_jit.hpp" +#include "rolling/jit/operation.hpp" #include #include diff --git a/cpp/src/rolling/jit/operation.hpp b/cpp/src/rolling/jit/operation.hpp index 22943f0db95..f8a52c03d4e 100644 --- a/cpp/src/rolling/jit/operation.hpp +++ b/cpp/src/rolling/jit/operation.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,9 +14,9 @@ * limitations under the License. */ -#include +#include "rolling/jit/operation-udf.hpp" -#include +#include #pragma once diff --git a/cpp/src/rolling/range_window_bounds.cpp b/cpp/src/rolling/range_window_bounds.cpp index a136f152d25..68e80c6e84e 100644 --- a/cpp/src/rolling/range_window_bounds.cpp +++ b/cpp/src/rolling/range_window_bounds.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include "detail/range_window_bounds.hpp" + #include #include #include diff --git a/cpp/src/round/round.cu b/cpp/src/round/round.cu index 8a6367a1f87..8336e1ef2b0 100644 --- a/cpp/src/round/round.cu +++ b/cpp/src/round/round.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,6 +37,7 @@ #include #include + #include namespace cudf { diff --git a/cpp/src/scalar/scalar_factories.cpp b/cpp/src/scalar/scalar_factories.cpp index 3a2920f8f1a..2336b9075de 100644 --- a/cpp/src/scalar/scalar_factories.cpp +++ b/cpp/src/scalar/scalar_factories.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,14 @@ * limitations under the License. */ +#include +#include #include #include #include #include #include -#include -#include #include namespace cudf { diff --git a/cpp/src/search/contains_table.cu b/cpp/src/search/contains_table.cu index e1d0fab6025..f7b6d8fdb72 100644 --- a/cpp/src/search/contains_table.cu +++ b/cpp/src/search/contains_table.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "join/join_common_utils.cuh" #include #include @@ -26,11 +26,9 @@ #include #include -#include - #include - #include +#include #include diff --git a/cpp/src/sort/rank.cu b/cpp/src/sort/rank.cu index 9cf07f065d2..cbd0207c20e 100644 --- a/cpp/src/sort/rank.cu +++ b/cpp/src/sort/rank.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,6 +31,8 @@ #include #include +#include +#include #include #include #include @@ -44,9 +46,6 @@ #include #include -#include -#include - namespace cudf { namespace detail { namespace { diff --git a/cpp/src/stream_compaction/distinct.cu b/cpp/src/stream_compaction/distinct.cu index e73bab1345e..11e2e77c253 100644 --- a/cpp/src/stream_compaction/distinct.cu +++ b/cpp/src/stream_compaction/distinct.cu @@ -27,13 +27,12 @@ #include +#include #include #include #include #include -#include - #include #include diff --git a/cpp/src/stream_compaction/distinct_count.cu b/cpp/src/stream_compaction/distinct_count.cu index 3ec1be42bfe..b7aadbe14fa 100644 --- a/cpp/src/stream_compaction/distinct_count.cu +++ b/cpp/src/stream_compaction/distinct_count.cu @@ -35,7 +35,6 @@ #include #include - #include #include #include diff --git a/cpp/src/stream_compaction/stream_compaction_common.hpp b/cpp/src/stream_compaction/stream_compaction_common.hpp index dd7d76168d9..13795f49781 100644 --- a/cpp/src/stream_compaction/stream_compaction_common.hpp +++ b/cpp/src/stream_compaction/stream_compaction_common.hpp @@ -21,6 +21,7 @@ #include #include +#include #include diff --git a/cpp/src/strings/attributes.cu b/cpp/src/strings/attributes.cu index 2856c077fb2..073ed74d8c9 100644 --- a/cpp/src/strings/attributes.cu +++ b/cpp/src/strings/attributes.cu @@ -31,6 +31,8 @@ #include #include +#include +#include #include #include #include @@ -40,10 +42,6 @@ #include #include -#include - -#include - namespace cudf { namespace strings { namespace detail { diff --git a/cpp/src/strings/char_types/char_cases.cu b/cpp/src/strings/char_types/char_cases.cu index 1021d5768c1..3b2b6dfaa6c 100644 --- a/cpp/src/strings/char_types/char_cases.cu +++ b/cpp/src/strings/char_types/char_cases.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,13 @@ * limitations under the License. */ +#include + #include #include #include #include -#include - // namespace cudf { namespace strings { diff --git a/cpp/src/strings/contains.cu b/cpp/src/strings/contains.cu index 4383f358a33..3f0ebc5962b 100644 --- a/cpp/src/strings/contains.cu +++ b/cpp/src/strings/contains.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,9 +14,9 @@ * limitations under the License. */ -#include -#include -#include +#include "strings/count_matches.hpp" +#include "strings/regex/regex_program_impl.h" +#include "strings/regex/utilities.cuh" #include #include diff --git a/cpp/src/strings/convert/convert_fixed_point.cu b/cpp/src/strings/convert/convert_fixed_point.cu index 975f03b37d6..c59952834d6 100644 --- a/cpp/src/strings/convert/convert_fixed_point.cu +++ b/cpp/src/strings/convert/convert_fixed_point.cu @@ -32,6 +32,9 @@ #include #include +#include +#include +#include #include #include #include @@ -39,10 +42,6 @@ #include #include -#include -#include -#include - namespace cudf { namespace strings { namespace detail { diff --git a/cpp/src/strings/copying/copying.cu b/cpp/src/strings/copying/copying.cu index 013028d6df3..6f045fa7ea8 100644 --- a/cpp/src/strings/copying/copying.cu +++ b/cpp/src/strings/copying/copying.cu @@ -26,9 +26,8 @@ #include #include -#include - #include +#include namespace cudf { namespace strings { diff --git a/cpp/src/strings/count_matches.cu b/cpp/src/strings/count_matches.cu index 6de5d43dc94..8a32a46cc2b 100644 --- a/cpp/src/strings/count_matches.cu +++ b/cpp/src/strings/count_matches.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,8 +14,8 @@ * limitations under the License. */ -#include -#include +#include "strings/count_matches.hpp" +#include "strings/regex/utilities.cuh" #include #include diff --git a/cpp/src/strings/extract/extract.cu b/cpp/src/strings/extract/extract.cu index 9af1e54fe66..ffd4e03ea87 100644 --- a/cpp/src/strings/extract/extract.cu +++ b/cpp/src/strings/extract/extract.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,8 +14,8 @@ * limitations under the License. */ -#include -#include +#include "strings/regex/regex_program_impl.h" +#include "strings/regex/utilities.cuh" #include #include @@ -30,14 +30,13 @@ #include +#include #include #include #include #include #include -#include - namespace cudf { namespace strings { namespace detail { diff --git a/cpp/src/strings/extract/extract_all.cu b/cpp/src/strings/extract/extract_all.cu index 0c0d4ae4fbf..63ce04df830 100644 --- a/cpp/src/strings/extract/extract_all.cu +++ b/cpp/src/strings/extract/extract_all.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,9 +14,9 @@ * limitations under the License. */ -#include -#include -#include +#include "strings/count_matches.hpp" +#include "strings/regex/regex_program_impl.h" +#include "strings/regex/utilities.cuh" #include #include diff --git a/cpp/src/strings/regex/regcomp.cpp b/cpp/src/strings/regex/regcomp.cpp index b7a7f19369d..170ed59d2fe 100644 --- a/cpp/src/strings/regex/regcomp.cpp +++ b/cpp/src/strings/regex/regcomp.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "strings/regex/regcomp.h" #include #include diff --git a/cpp/src/strings/regex/regex.cuh b/cpp/src/strings/regex/regex.cuh index c1abbd78b43..c8d846624f8 100644 --- a/cpp/src/strings/regex/regex.cuh +++ b/cpp/src/strings/regex/regex.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ #pragma once -#include +#include "strings/regex/regcomp.h" #include #include @@ -23,11 +23,10 @@ #include +#include #include #include -#include - #include namespace cudf { diff --git a/cpp/src/strings/regex/regexec.cpp b/cpp/src/strings/regex/regexec.cpp index a82f035f61b..b5e7e7e8922 100644 --- a/cpp/src/strings/regex/regexec.cpp +++ b/cpp/src/strings/regex/regexec.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,8 +14,8 @@ * limitations under the License. */ -#include -#include +#include "strings/regex/regcomp.h" +#include "strings/regex/regex.cuh" #include #include diff --git a/cpp/src/strings/regex/utilities.cuh b/cpp/src/strings/regex/utilities.cuh index bc8f5d68a4b..d5dd80aba53 100644 --- a/cpp/src/strings/regex/utilities.cuh +++ b/cpp/src/strings/regex/utilities.cuh @@ -16,7 +16,7 @@ #pragma once -#include +#include "strings/regex/regex.cuh" #include #include diff --git a/cpp/src/strings/replace/backref_re.cu b/cpp/src/strings/replace/backref_re.cu index edec525a913..bb99dc0644c 100644 --- a/cpp/src/strings/replace/backref_re.cu +++ b/cpp/src/strings/replace/backref_re.cu @@ -15,9 +15,8 @@ */ #include "backref_re.cuh" - -#include -#include +#include "strings/regex/regex_program_impl.h" +#include "strings/regex/utilities.cuh" #include #include diff --git a/cpp/src/strings/replace/backref_re.cuh b/cpp/src/strings/replace/backref_re.cuh index aeaea40358f..edd85f29e6c 100644 --- a/cpp/src/strings/replace/backref_re.cuh +++ b/cpp/src/strings/replace/backref_re.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "strings/regex/regex.cuh" #include #include diff --git a/cpp/src/strings/replace/multi.cu b/cpp/src/strings/replace/multi.cu index 3d0210d61b0..ab35393651f 100644 --- a/cpp/src/strings/replace/multi.cu +++ b/cpp/src/strings/replace/multi.cu @@ -34,6 +34,7 @@ #include #include +#include #include #include #include @@ -45,8 +46,6 @@ #include #include -#include - namespace cudf { namespace strings { namespace detail { diff --git a/cpp/src/strings/replace/multi_re.cu b/cpp/src/strings/replace/multi_re.cu index c212d9f44ba..ba122d11e0b 100644 --- a/cpp/src/strings/replace/multi_re.cu +++ b/cpp/src/strings/replace/multi_re.cu @@ -14,8 +14,8 @@ * limitations under the License. */ -#include -#include +#include "strings/regex/regex.cuh" +#include "strings/regex/regex_program_impl.h" #include #include diff --git a/cpp/src/strings/replace/replace.cu b/cpp/src/strings/replace/replace.cu index 936127f254b..d68ec84f68c 100644 --- a/cpp/src/strings/replace/replace.cu +++ b/cpp/src/strings/replace/replace.cu @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -45,8 +46,6 @@ #include #include -#include - namespace cudf { namespace strings { namespace detail { diff --git a/cpp/src/strings/replace/replace_re.cu b/cpp/src/strings/replace/replace_re.cu index 10d83932928..500bc0c5bb5 100644 --- a/cpp/src/strings/replace/replace_re.cu +++ b/cpp/src/strings/replace/replace_re.cu @@ -14,8 +14,8 @@ * limitations under the License. */ -#include -#include +#include "strings/regex/regex_program_impl.h" +#include "strings/regex/utilities.cuh" #include #include diff --git a/cpp/src/strings/search/find.cu b/cpp/src/strings/search/find.cu index 78343d58626..598d48157d9 100644 --- a/cpp/src/strings/search/find.cu +++ b/cpp/src/strings/search/find.cu @@ -31,6 +31,7 @@ #include #include +#include #include #include #include @@ -38,8 +39,6 @@ #include #include -#include - namespace cudf { namespace strings { namespace detail { diff --git a/cpp/src/strings/search/findall.cu b/cpp/src/strings/search/findall.cu index 8df1a67d56d..4b4a1191e1b 100644 --- a/cpp/src/strings/search/findall.cu +++ b/cpp/src/strings/search/findall.cu @@ -14,9 +14,9 @@ * limitations under the License. */ -#include -#include -#include +#include "strings/count_matches.hpp" +#include "strings/regex/regex_program_impl.h" +#include "strings/regex/utilities.cuh" #include #include diff --git a/cpp/src/strings/split/split.cu b/cpp/src/strings/split/split.cu index 17293a71b63..1416b293b75 100644 --- a/cpp/src/strings/split/split.cu +++ b/cpp/src/strings/split/split.cu @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -39,8 +40,6 @@ #include #include -#include - namespace cudf { namespace strings { namespace detail { diff --git a/cpp/src/strings/split/split.cuh b/cpp/src/strings/split/split.cuh index 750b18c8b4c..5f3c9372c39 100644 --- a/cpp/src/strings/split/split.cuh +++ b/cpp/src/strings/split/split.cuh @@ -28,6 +28,7 @@ #include +#include #include #include #include @@ -36,8 +37,6 @@ #include #include -#include - namespace cudf::strings::detail { /** diff --git a/cpp/src/strings/split/split_re.cu b/cpp/src/strings/split/split_re.cu index d8385549840..16725fe006a 100644 --- a/cpp/src/strings/split/split_re.cu +++ b/cpp/src/strings/split/split_re.cu @@ -14,9 +14,9 @@ * limitations under the License. */ -#include -#include -#include +#include "strings/count_matches.hpp" +#include "strings/regex/regex_program_impl.h" +#include "strings/regex/utilities.cuh" #include #include diff --git a/cpp/src/strings/split/split_record.cu b/cpp/src/strings/split/split_record.cu index c9ed7b0ed26..0971069592e 100644 --- a/cpp/src/strings/split/split_record.cu +++ b/cpp/src/strings/split/split_record.cu @@ -30,13 +30,12 @@ #include +#include #include #include #include #include -#include - namespace cudf { namespace strings { namespace detail { diff --git a/cpp/src/strings/utilities.cu b/cpp/src/strings/utilities.cu index 782d9767fb5..72c3ccf4ac5 100644 --- a/cpp/src/strings/utilities.cu +++ b/cpp/src/strings/utilities.cu @@ -14,8 +14,8 @@ * limitations under the License. */ -#include -#include +#include "strings/char_types/char_cases.h" +#include "strings/char_types/char_flags.h" #include #include diff --git a/cpp/src/structs/scan/scan_inclusive.cu b/cpp/src/structs/scan/scan_inclusive.cu index 823e4472960..410a7d9348e 100644 --- a/cpp/src/structs/scan/scan_inclusive.cu +++ b/cpp/src/structs/scan/scan_inclusive.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "reductions/nested_type_minmax_util.cuh" #include #include diff --git a/cpp/src/structs/structs_column_factories.cu b/cpp/src/structs/structs_column_factories.cu index 9e511c62d2a..d94a33ce9fb 100644 --- a/cpp/src/structs/structs_column_factories.cu +++ b/cpp/src/structs/structs_column_factories.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ #include #include #include + #include #include diff --git a/cpp/src/table/row_operators.cu b/cpp/src/table/row_operators.cu index 770a7c775b4..71b437cb47d 100644 --- a/cpp/src/table/row_operators.cu +++ b/cpp/src/table/row_operators.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "lists/utilities.hpp" #include #include diff --git a/cpp/src/text/bpe/byte_pair_encoding.cu b/cpp/src/text/bpe/byte_pair_encoding.cu index 62d91054c14..363e15d74c1 100644 --- a/cpp/src/text/bpe/byte_pair_encoding.cu +++ b/cpp/src/text/bpe/byte_pair_encoding.cu @@ -14,9 +14,7 @@ * limitations under the License. */ -#include - -#include +#include "text/bpe/byte_pair_encoding.cuh" #include #include @@ -32,6 +30,8 @@ #include #include +#include + #include #include diff --git a/cpp/src/text/bpe/byte_pair_encoding.cuh b/cpp/src/text/bpe/byte_pair_encoding.cuh index 02a8a6c4d0a..2ad22fd4e46 100644 --- a/cpp/src/text/bpe/byte_pair_encoding.cuh +++ b/cpp/src/text/bpe/byte_pair_encoding.cuh @@ -16,8 +16,6 @@ #pragma once -#include - #include #include #include @@ -25,11 +23,12 @@ #include #include +#include + #include #include #include - #include #include #include diff --git a/cpp/src/text/bpe/load_merge_pairs.cu b/cpp/src/text/bpe/load_merge_pairs.cu index 8da2d745966..1658f20182b 100644 --- a/cpp/src/text/bpe/load_merge_pairs.cu +++ b/cpp/src/text/bpe/load_merge_pairs.cu @@ -14,9 +14,7 @@ * limitations under the License. */ -#include - -#include +#include "text/bpe/byte_pair_encoding.cuh" #include #include @@ -26,15 +24,17 @@ #include #include +#include + #include #include +#include + #include #include #include -#include - namespace nvtext { namespace detail { namespace { diff --git a/cpp/src/text/detokenize.cu b/cpp/src/text/detokenize.cu index 60625d6383a..a317739e4ca 100644 --- a/cpp/src/text/detokenize.cu +++ b/cpp/src/text/detokenize.cu @@ -14,8 +14,6 @@ * limitations under the License. */ -#include - #include #include #include @@ -30,6 +28,8 @@ #include #include +#include + #include #include diff --git a/cpp/src/text/edit_distance.cu b/cpp/src/text/edit_distance.cu index a1d97409987..606bebe2174 100644 --- a/cpp/src/text/edit_distance.cu +++ b/cpp/src/text/edit_distance.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,8 +14,6 @@ * limitations under the License. */ -#include - #include #include #include @@ -25,6 +23,8 @@ #include #include +#include + #include #include #include diff --git a/cpp/src/text/generate_ngrams.cu b/cpp/src/text/generate_ngrams.cu index 882d9a04501..433237bbf81 100644 --- a/cpp/src/text/generate_ngrams.cu +++ b/cpp/src/text/generate_ngrams.cu @@ -14,8 +14,6 @@ * limitations under the License. */ -#include - #include #include #include @@ -32,15 +30,16 @@ #include #include +#include + #include #include +#include #include #include #include -#include - namespace nvtext { namespace detail { namespace { diff --git a/cpp/src/text/jaccard.cu b/cpp/src/text/jaccard.cu index 1f453f60831..612eb52af01 100644 --- a/cpp/src/text/jaccard.cu +++ b/cpp/src/text/jaccard.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,9 +14,6 @@ * limitations under the License. */ -#include -#include - #include #include #include @@ -28,16 +25,18 @@ #include #include +#include +#include + #include #include +#include #include #include #include #include -#include - namespace nvtext { namespace detail { namespace { diff --git a/cpp/src/text/minhash.cu b/cpp/src/text/minhash.cu index dcb59166cec..8d22c784584 100644 --- a/cpp/src/text/minhash.cu +++ b/cpp/src/text/minhash.cu @@ -14,8 +14,6 @@ * limitations under the License. */ -#include - #include #include #include @@ -31,16 +29,17 @@ #include #include +#include + #include #include +#include #include #include #include -#include - namespace nvtext { namespace detail { namespace { diff --git a/cpp/src/text/ngrams_tokenize.cu b/cpp/src/text/ngrams_tokenize.cu index 3444786ff80..75ad542548b 100644 --- a/cpp/src/text/ngrams_tokenize.cu +++ b/cpp/src/text/ngrams_tokenize.cu @@ -14,10 +14,7 @@ * limitations under the License. */ -#include - -#include -#include +#include "text/utilities/tokenize_ops.cuh" #include #include @@ -31,16 +28,18 @@ #include #include +#include +#include + #include #include +#include #include #include #include #include -#include - namespace nvtext { namespace detail { namespace { diff --git a/cpp/src/text/normalize.cu b/cpp/src/text/normalize.cu index 6044689473c..3d98ae59dc0 100644 --- a/cpp/src/text/normalize.cu +++ b/cpp/src/text/normalize.cu @@ -14,11 +14,9 @@ * limitations under the License. */ -#include -#include -#include - -#include +#include "text/subword/detail/data_normalizer.hpp" +#include "text/subword/detail/tokenizer_utils.cuh" +#include "text/utilities/tokenize_ops.cuh" #include #include @@ -35,6 +33,8 @@ #include #include +#include + #include #include diff --git a/cpp/src/text/replace.cu b/cpp/src/text/replace.cu index 50d7bbd077d..1fa0606424c 100644 --- a/cpp/src/text/replace.cu +++ b/cpp/src/text/replace.cu @@ -14,10 +14,7 @@ * limitations under the License. */ -#include - -#include -#include +#include "text/utilities/tokenize_ops.cuh" #include #include @@ -31,6 +28,9 @@ #include #include +#include +#include + #include #include diff --git a/cpp/src/text/stemmer.cu b/cpp/src/text/stemmer.cu index bdcb0b2af32..5c67b2e5f54 100644 --- a/cpp/src/text/stemmer.cu +++ b/cpp/src/text/stemmer.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,8 +14,6 @@ * limitations under the License. */ -#include - #include #include #include @@ -28,6 +26,8 @@ #include #include +#include + #include #include diff --git a/cpp/src/text/subword/data_normalizer.cu b/cpp/src/text/subword/data_normalizer.cu index a56d71cf951..c662581b3f4 100644 --- a/cpp/src/text/subword/data_normalizer.cu +++ b/cpp/src/text/subword/data_normalizer.cu @@ -14,8 +14,8 @@ * limitations under the License. */ -#include -#include +#include "text/subword/detail/data_normalizer.hpp" +#include "text/subword/detail/tokenizer_utils.cuh" #include #include diff --git a/cpp/src/text/subword/detail/data_normalizer.hpp b/cpp/src/text/subword/detail/data_normalizer.hpp index 897a0f31e15..c70e3734691 100644 --- a/cpp/src/text/subword/detail/data_normalizer.hpp +++ b/cpp/src/text/subword/detail/data_normalizer.hpp @@ -16,7 +16,7 @@ #pragma once -#include +#include "text/subword/detail/cp_data.h" #include #include diff --git a/cpp/src/text/subword/detail/tokenizer_utils.cuh b/cpp/src/text/subword/detail/tokenizer_utils.cuh index f2317518663..01df910d420 100644 --- a/cpp/src/text/subword/detail/tokenizer_utils.cuh +++ b/cpp/src/text/subword/detail/tokenizer_utils.cuh @@ -16,7 +16,7 @@ #pragma once -#include +#include "text/subword/detail/cp_data.h" #include diff --git a/cpp/src/text/subword/detail/wordpiece_tokenizer.hpp b/cpp/src/text/subword/detail/wordpiece_tokenizer.hpp index 71e00c2e852..244fe5092e7 100644 --- a/cpp/src/text/subword/detail/wordpiece_tokenizer.hpp +++ b/cpp/src/text/subword/detail/wordpiece_tokenizer.hpp @@ -16,7 +16,7 @@ #pragma once -#include +#include "text/subword/detail/data_normalizer.hpp" #include diff --git a/cpp/src/text/subword/load_hash_file.cu b/cpp/src/text/subword/load_hash_file.cu index cb18d0e0ecf..0b4f9f729c3 100644 --- a/cpp/src/text/subword/load_hash_file.cu +++ b/cpp/src/text/subword/load_hash_file.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,8 @@ * limitations under the License. */ -#include -#include - -#include +#include "text/subword/detail/codepoint_metadata.ah" +#include "text/subword/detail/tokenizer_utils.cuh" #include #include @@ -25,6 +23,8 @@ #include #include +#include + #include #include #include diff --git a/cpp/src/text/subword/subword_tokenize.cu b/cpp/src/text/subword/subword_tokenize.cu index 6d40882659a..a623450ecad 100644 --- a/cpp/src/text/subword/subword_tokenize.cu +++ b/cpp/src/text/subword/subword_tokenize.cu @@ -14,6 +14,8 @@ * limitations under the License. */ +#include "text/subword/detail/wordpiece_tokenizer.hpp" + #include #include #include @@ -26,7 +28,6 @@ #include #include -#include #include #include diff --git a/cpp/src/text/subword/wordpiece_tokenizer.cu b/cpp/src/text/subword/wordpiece_tokenizer.cu index 6e0c324db7d..c094537ebc2 100644 --- a/cpp/src/text/subword/wordpiece_tokenizer.cu +++ b/cpp/src/text/subword/wordpiece_tokenizer.cu @@ -14,13 +14,14 @@ * limitations under the License. */ -#include -#include -#include +#include "text/subword/detail/hash_utils.cuh" +#include "text/subword/detail/tokenizer_utils.cuh" +#include "text/subword/detail/wordpiece_tokenizer.hpp" #include #include #include + #include #include diff --git a/cpp/src/text/tokenize.cu b/cpp/src/text/tokenize.cu index 97896f20f4f..82c51e72b31 100644 --- a/cpp/src/text/tokenize.cu +++ b/cpp/src/text/tokenize.cu @@ -14,10 +14,7 @@ * limitations under the License. */ -#include - -#include -#include +#include "text/utilities/tokenize_ops.cuh" #include #include @@ -30,6 +27,9 @@ #include #include +#include +#include + #include #include #include diff --git a/cpp/src/text/vocabulary_tokenize.cu b/cpp/src/text/vocabulary_tokenize.cu index b6991e534bf..c99adda3fad 100644 --- a/cpp/src/text/vocabulary_tokenize.cu +++ b/cpp/src/text/vocabulary_tokenize.cu @@ -14,9 +14,7 @@ * limitations under the License. */ -#include - -#include +#include "text/utilities/tokenize_ops.cuh" #include #include @@ -35,10 +33,12 @@ #include #include +#include + #include +#include #include - #include #include #include @@ -46,8 +46,6 @@ #include #include -#include - namespace nvtext { namespace detail { namespace { diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 1e913ecb5bb..4fd0369c26b 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -14,21 +14,20 @@ * limitations under the License. */ -// Include Jitify's cstddef header first -#include +#include +#include +#include #include #include #include #include -#include -#include - -#include +#include -#include -#include +// clang-format off +#include "transform/jit/operation-udf.hpp" +// clang-format on namespace cudf { namespace transformation { diff --git a/cpp/src/transform/mask_to_bools.cu b/cpp/src/transform/mask_to_bools.cu index 1b9a58c4724..73c1a83cfe1 100644 --- a/cpp/src/transform/mask_to_bools.cu +++ b/cpp/src/transform/mask_to_bools.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,13 +22,13 @@ #include #include +#include +#include + #include #include #include -#include -#include - namespace cudf { namespace detail { std::unique_ptr mask_to_bools(bitmask_type const* bitmask, diff --git a/cpp/src/transform/one_hot_encode.cu b/cpp/src/transform/one_hot_encode.cu index c7bb40e3bcb..72f864346a4 100644 --- a/cpp/src/transform/one_hot_encode.cu +++ b/cpp/src/transform/one_hot_encode.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,12 +25,12 @@ #include #include -#include -#include - #include #include +#include +#include + #include namespace cudf { diff --git a/cpp/src/transform/row_bit_count.cu b/cpp/src/transform/row_bit_count.cu index e4698fb1262..eda8ec7a463 100644 --- a/cpp/src/transform/row_bit_count.cu +++ b/cpp/src/transform/row_bit_count.cu @@ -27,13 +27,13 @@ #include #include -#include -#include - #include #include #include +#include +#include + namespace cudf { namespace detail { diff --git a/cpp/src/transform/row_conversion.cu b/cpp/src/transform/row_conversion.cu index b294369a90e..361a3610afa 100644 --- a/cpp/src/transform/row_conversion.cu +++ b/cpp/src/transform/row_conversion.cu @@ -14,7 +14,6 @@ * limitations under the License. */ -#include #include #include #include @@ -32,15 +31,19 @@ #include #include #include + #include #include #include #include + +#include #include #include #include #include #include + #include #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700 @@ -51,6 +54,8 @@ #include #endif // #if !defined(__CUDA_ARCH__) || defined(ASYNC_MEMCPY_SUPPORTED) +#include + #include #include #include @@ -60,8 +65,6 @@ #include #include -#include - namespace { constexpr auto JCUDF_ROW_ALIGNMENT = 8; diff --git a/cpp/src/transform/transform.cpp b/cpp/src/transform/transform.cpp index 53750679dbc..6f61ed80dd8 100644 --- a/cpp/src/transform/transform.cpp +++ b/cpp/src/transform/transform.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,10 @@ * limitations under the License. */ +#include "jit/cache.hpp" +#include "jit/parser.hpp" +#include "jit/util.hpp" + #include #include #include @@ -23,14 +27,10 @@ #include #include -#include - -#include -#include -#include - #include +#include + namespace cudf { namespace transformation { namespace jit { diff --git a/cpp/src/utilities/logger.cpp b/cpp/src/utilities/logger.cpp index 91950850e3b..d54f5677c4c 100644 --- a/cpp/src/utilities/logger.cpp +++ b/cpp/src/utilities/logger.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,8 +17,8 @@ #include #include -#include "spdlog/sinks/stdout_sinks.h" #include +#include #include diff --git a/cpp/src/utilities/traits.cpp b/cpp/src/utilities/traits.cpp index b0078ff85a2..a68dc84e340 100644 --- a/cpp/src/utilities/traits.cpp +++ b/cpp/src/utilities/traits.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,13 @@ * limitations under the License. */ -#include - #include #include #include #include +#include + namespace cudf { namespace { diff --git a/cpp/tests/ast/transform_tests.cpp b/cpp/tests/ast/transform_tests.cpp index 01842969268..ef1d09e5652 100644 --- a/cpp/tests/ast/transform_tests.cpp +++ b/cpp/tests/ast/transform_tests.cpp @@ -14,6 +14,13 @@ * limitations under the License. */ +#include +#include +#include +#include +#include +#include + #include #include #include @@ -26,13 +33,6 @@ #include #include -#include -#include -#include -#include -#include -#include - #include #include diff --git a/cpp/tests/binaryop/util/operation.h b/cpp/tests/binaryop/util/operation.h index f14fe85059a..efebc02bc89 100644 --- a/cpp/tests/binaryop/util/operation.h +++ b/cpp/tests/binaryop/util/operation.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Copyright 2018-2019 BlazingDB, Inc. * Copyright 2018 Christian Noboa Mardini @@ -19,9 +19,10 @@ #pragma once +#include + #include #include -#include #include namespace cudf { diff --git a/cpp/tests/bitmask/bitmask_tests.cpp b/cpp/tests/bitmask/bitmask_tests.cpp index 1dd39c1c7ae..72ef88e4ed1 100644 --- a/cpp/tests/bitmask/bitmask_tests.cpp +++ b/cpp/tests/bitmask/bitmask_tests.cpp @@ -13,6 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include +#include +#include +#include +#include +#include + #include #include #include @@ -21,12 +28,6 @@ #include #include #include -#include -#include -#include -#include -#include -#include #include #include diff --git a/cpp/tests/bitmask/set_nullmask_tests.cu b/cpp/tests/bitmask/set_nullmask_tests.cu index 19e3202a6d7..e95c9fb41c6 100644 --- a/cpp/tests/bitmask/set_nullmask_tests.cu +++ b/cpp/tests/bitmask/set_nullmask_tests.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,9 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include - #include #include @@ -31,6 +28,9 @@ #include #include +#include +#include + struct valid_bit_functor { cudf::bitmask_type const* _null_mask; __device__ bool operator()(cudf::size_type element_index) const noexcept diff --git a/cpp/tests/bitmask/valid_if_tests.cu b/cpp/tests/bitmask/valid_if_tests.cu index 164f8d77838..65143ec17f1 100644 --- a/cpp/tests/bitmask/valid_if_tests.cu +++ b/cpp/tests/bitmask/valid_if_tests.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,15 @@ * limitations under the License. */ -#include -#include -#include - #include #include #include #include +#include +#include +#include + #include struct ValidIfTest : public cudf::test::BaseFixture {}; diff --git a/cpp/tests/column/bit_cast_test.cpp b/cpp/tests/column/bit_cast_test.cpp index a6654bb6f29..ab230ab036e 100644 --- a/cpp/tests/column/bit_cast_test.cpp +++ b/cpp/tests/column/bit_cast_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,16 @@ * limitations under the License. */ -#include -#include -#include - #include #include #include #include #include +#include +#include +#include + #include #include diff --git a/cpp/tests/column/column_view_device_span_test.cpp b/cpp/tests/column/column_view_device_span_test.cpp index 7daf6870eac..6de9121158b 100644 --- a/cpp/tests/column/column_view_device_span_test.cpp +++ b/cpp/tests/column/column_view_device_span_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,18 +14,18 @@ * limitations under the License. */ -#include -#include -#include -#include -#include - #include #include #include #include #include +#include +#include +#include +#include +#include + #include #include diff --git a/cpp/tests/column/column_view_shallow_test.cpp b/cpp/tests/column/column_view_shallow_test.cpp index 3e5650652e1..87187dfe57b 100644 --- a/cpp/tests/column/column_view_shallow_test.cpp +++ b/cpp/tests/column/column_view_shallow_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,17 +14,17 @@ * limitations under the License. */ -#include -#include -#include -#include - #include #include #include #include #include +#include +#include +#include +#include + #include #include diff --git a/cpp/tests/column/compound_test.cu b/cpp/tests/column/compound_test.cu index fc348284e09..d7e93fb22a3 100644 --- a/cpp/tests/column/compound_test.cu +++ b/cpp/tests/column/compound_test.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,14 @@ * limitations under the License. */ +#include +#include + #include #include #include #include #include -#include -#include #include #include diff --git a/cpp/tests/copying/purge_nonempty_nulls_tests.cpp b/cpp/tests/copying/purge_nonempty_nulls_tests.cpp index 8e652cb565a..4f28ff12941 100644 --- a/cpp/tests/copying/purge_nonempty_nulls_tests.cpp +++ b/cpp/tests/copying/purge_nonempty_nulls_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,6 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include +#include +#include +#include +#include + #include #include #include @@ -23,12 +29,6 @@ #include #include -#include -#include -#include -#include -#include - using cudf::test::iterators::no_nulls; using cudf::test::iterators::null_at; using cudf::test::iterators::nulls_at; diff --git a/cpp/tests/copying/slice_tests.cpp b/cpp/tests/copying/slice_tests.cpp index b7f00b49fe6..29ff3e1cf9b 100644 --- a/cpp/tests/copying/slice_tests.cpp +++ b/cpp/tests/copying/slice_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,7 @@ * limitations under the License. */ -#include -#include -#include -#include -#include -#include +#include #include #include @@ -27,7 +22,12 @@ #include #include -#include +#include +#include +#include +#include +#include +#include #include #include diff --git a/cpp/tests/copying/utility_tests.cpp b/cpp/tests/copying/utility_tests.cpp index f69bea2834f..0905f9babdc 100644 --- a/cpp/tests/copying/utility_tests.cpp +++ b/cpp/tests/copying/utility_tests.cpp @@ -14,11 +14,6 @@ * limitations under the License. */ -#include -#include -#include -#include -#include #include #include #include @@ -26,6 +21,12 @@ #include #include +#include +#include +#include +#include +#include + #include #include diff --git a/cpp/tests/datetime/datetime_ops_test.cpp b/cpp/tests/datetime/datetime_ops_test.cpp index b58cd0e0cb9..13577c4d0ea 100644 --- a/cpp/tests/datetime/datetime_ops_test.cpp +++ b/cpp/tests/datetime/datetime_ops_test.cpp @@ -14,12 +14,6 @@ * limitations under the License. */ -#include -#include -#include -#include -#include -#include #include #include #include @@ -28,6 +22,13 @@ #include #include +#include +#include +#include +#include +#include +#include + #include #define XXX false // stub for null values diff --git a/cpp/tests/device_atomics/device_atomics_test.cu b/cpp/tests/device_atomics/device_atomics_test.cu index 6e90d4462df..0d846404ea2 100644 --- a/cpp/tests/device_atomics/device_atomics_test.cu +++ b/cpp/tests/device_atomics/device_atomics_test.cu @@ -14,18 +14,18 @@ * limitations under the License. */ -#include -#include -#include -#include -#include - #include #include #include #include #include +#include +#include +#include +#include +#include + #include #include diff --git a/cpp/tests/dictionary/add_keys_test.cpp b/cpp/tests/dictionary/add_keys_test.cpp index 32a6885df09..1314375f383 100644 --- a/cpp/tests/dictionary/add_keys_test.cpp +++ b/cpp/tests/dictionary/add_keys_test.cpp @@ -14,14 +14,15 @@ * limitations under the License. */ -#include -#include -#include #include #include #include #include +#include +#include +#include + #include struct DictionaryAddKeysTest : public cudf::test::BaseFixture {}; diff --git a/cpp/tests/dictionary/decode_test.cpp b/cpp/tests/dictionary/decode_test.cpp index 25ccb331756..33c8cb23110 100644 --- a/cpp/tests/dictionary/decode_test.cpp +++ b/cpp/tests/dictionary/decode_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,13 @@ * limitations under the License. */ -#include -#include #include #include #include +#include +#include + #include struct DictionaryDecodeTest : public cudf::test::BaseFixture {}; diff --git a/cpp/tests/dictionary/encode_test.cpp b/cpp/tests/dictionary/encode_test.cpp index 6b0b33d4e25..93c2ab4c0ef 100644 --- a/cpp/tests/dictionary/encode_test.cpp +++ b/cpp/tests/dictionary/encode_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,13 @@ * limitations under the License. */ -#include -#include #include #include #include +#include +#include + #include struct DictionaryEncodeTest : public cudf::test::BaseFixture {}; diff --git a/cpp/tests/dictionary/factories_test.cpp b/cpp/tests/dictionary/factories_test.cpp index 5db4bf98a24..35aa19c5558 100644 --- a/cpp/tests/dictionary/factories_test.cpp +++ b/cpp/tests/dictionary/factories_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,14 @@ * limitations under the License. */ -#include -#include -#include #include #include #include +#include +#include +#include + #include struct DictionaryFactoriesTest : public cudf::test::BaseFixture {}; diff --git a/cpp/tests/dictionary/fill_test.cpp b/cpp/tests/dictionary/fill_test.cpp index 60e57d96f97..7f2bb5496f3 100644 --- a/cpp/tests/dictionary/fill_test.cpp +++ b/cpp/tests/dictionary/fill_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,14 @@ * limitations under the License. */ +#include +#include +#include + #include #include #include #include -#include -#include -#include #include diff --git a/cpp/tests/dictionary/gather_test.cpp b/cpp/tests/dictionary/gather_test.cpp index 3267da794ee..8fd8751bc76 100644 --- a/cpp/tests/dictionary/gather_test.cpp +++ b/cpp/tests/dictionary/gather_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,14 @@ * limitations under the License. */ +#include +#include +#include + #include #include #include #include -#include -#include -#include #include diff --git a/cpp/tests/dictionary/remove_keys_test.cpp b/cpp/tests/dictionary/remove_keys_test.cpp index eb48c3e783f..13fe3efd0f4 100644 --- a/cpp/tests/dictionary/remove_keys_test.cpp +++ b/cpp/tests/dictionary/remove_keys_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,14 @@ * limitations under the License. */ +#include +#include +#include + #include #include #include #include -#include -#include -#include #include diff --git a/cpp/tests/dictionary/scatter_test.cpp b/cpp/tests/dictionary/scatter_test.cpp index 7030f1e716a..2a2841827d0 100644 --- a/cpp/tests/dictionary/scatter_test.cpp +++ b/cpp/tests/dictionary/scatter_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,15 @@ * limitations under the License. */ -#include -#include -#include #include #include #include #include +#include +#include +#include + #include struct DictionaryScatterTest : public cudf::test::BaseFixture {}; diff --git a/cpp/tests/dictionary/search_test.cpp b/cpp/tests/dictionary/search_test.cpp index 35972bac375..600d00ac186 100644 --- a/cpp/tests/dictionary/search_test.cpp +++ b/cpp/tests/dictionary/search_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,13 @@ * limitations under the License. */ -#include -#include #include #include #include +#include +#include + struct DictionarySearchTest : public cudf::test::BaseFixture {}; TEST_F(DictionarySearchTest, StringsColumn) diff --git a/cpp/tests/dictionary/set_keys_test.cpp b/cpp/tests/dictionary/set_keys_test.cpp index 9eb4b43b786..d0c37493cf8 100644 --- a/cpp/tests/dictionary/set_keys_test.cpp +++ b/cpp/tests/dictionary/set_keys_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,14 @@ * limitations under the License. */ -#include -#include -#include #include #include #include +#include +#include +#include + #include #include diff --git a/cpp/tests/dictionary/slice_test.cpp b/cpp/tests/dictionary/slice_test.cpp index 6446378b779..42bf7d488d2 100644 --- a/cpp/tests/dictionary/slice_test.cpp +++ b/cpp/tests/dictionary/slice_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,15 @@ * limitations under the License. */ +#include +#include +#include + #include #include #include #include #include -#include -#include -#include #include diff --git a/cpp/tests/encode/encode_tests.cpp b/cpp/tests/encode/encode_tests.cpp index 87818e16bb9..4f3463ef00d 100644 --- a/cpp/tests/encode/encode_tests.cpp +++ b/cpp/tests/encode/encode_tests.cpp @@ -13,8 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include #include #include #include @@ -22,6 +20,9 @@ #include #include +#include +#include + template class EncodeNumericTests : public cudf::test::BaseFixture {}; diff --git a/cpp/tests/hash_map/map_test.cu b/cpp/tests/hash_map/map_test.cu index 0232696a123..4b10716706b 100644 --- a/cpp/tests/hash_map/map_test.cu +++ b/cpp/tests/hash_map/map_test.cu @@ -14,12 +14,12 @@ * limitations under the License. */ +#include "hash/concurrent_unordered_map.cuh" + #include #include #include -#include - #include #include diff --git a/cpp/tests/hashing/md5_test.cpp b/cpp/tests/hashing/md5_test.cpp index 9361c4e748c..081ab7978cd 100644 --- a/cpp/tests/hashing/md5_test.cpp +++ b/cpp/tests/hashing/md5_test.cpp @@ -14,14 +14,14 @@ * limitations under the License. */ -#include - #include #include #include #include #include +#include + constexpr cudf::test::debug_output_level verbosity{cudf::test::debug_output_level::ALL_ERRORS}; class MD5HashTest : public cudf::test::BaseFixture {}; diff --git a/cpp/tests/hashing/murmurhash3_x86_32_test.cpp b/cpp/tests/hashing/murmurhash3_x86_32_test.cpp index c3cc20c28b7..24524140e74 100644 --- a/cpp/tests/hashing/murmurhash3_x86_32_test.cpp +++ b/cpp/tests/hashing/murmurhash3_x86_32_test.cpp @@ -14,9 +14,6 @@ * limitations under the License. */ -#include -#include - #include #include #include @@ -24,6 +21,9 @@ #include #include +#include +#include + constexpr cudf::test::debug_output_level verbosity{cudf::test::debug_output_level::ALL_ERRORS}; class MurmurHashTest : public cudf::test::BaseFixture {}; diff --git a/cpp/tests/hashing/sha1_test.cpp b/cpp/tests/hashing/sha1_test.cpp index 31145e4c3c4..c3d0fe7450a 100644 --- a/cpp/tests/hashing/sha1_test.cpp +++ b/cpp/tests/hashing/sha1_test.cpp @@ -14,15 +14,15 @@ * limitations under the License. */ -#include -#include - #include #include #include #include #include +#include +#include + class SHA1HashTest : public cudf::test::BaseFixture {}; TEST_F(SHA1HashTest, EmptyTable) diff --git a/cpp/tests/hashing/sha224_test.cpp b/cpp/tests/hashing/sha224_test.cpp index 9aa1ee0fac2..def5e934177 100644 --- a/cpp/tests/hashing/sha224_test.cpp +++ b/cpp/tests/hashing/sha224_test.cpp @@ -14,15 +14,15 @@ * limitations under the License. */ -#include -#include - #include #include #include #include #include +#include +#include + class SHA224HashTest : public cudf::test::BaseFixture {}; TEST_F(SHA224HashTest, EmptyTable) diff --git a/cpp/tests/hashing/sha256_test.cpp b/cpp/tests/hashing/sha256_test.cpp index 4fed8c55fc2..410a99edd77 100644 --- a/cpp/tests/hashing/sha256_test.cpp +++ b/cpp/tests/hashing/sha256_test.cpp @@ -14,15 +14,15 @@ * limitations under the License. */ -#include -#include - #include #include #include #include #include +#include +#include + constexpr cudf::test::debug_output_level verbosity{cudf::test::debug_output_level::ALL_ERRORS}; class SHA256HashTest : public cudf::test::BaseFixture {}; diff --git a/cpp/tests/hashing/sha384_test.cpp b/cpp/tests/hashing/sha384_test.cpp index 49b9b5ef3a5..810fbc82d8e 100644 --- a/cpp/tests/hashing/sha384_test.cpp +++ b/cpp/tests/hashing/sha384_test.cpp @@ -14,15 +14,15 @@ * limitations under the License. */ -#include -#include - #include #include #include #include #include +#include +#include + class SHA384HashTest : public cudf::test::BaseFixture {}; TEST_F(SHA384HashTest, EmptyTable) diff --git a/cpp/tests/hashing/sha512_test.cpp b/cpp/tests/hashing/sha512_test.cpp index df0315099fb..93caa16c1c4 100644 --- a/cpp/tests/hashing/sha512_test.cpp +++ b/cpp/tests/hashing/sha512_test.cpp @@ -14,15 +14,15 @@ * limitations under the License. */ -#include -#include - #include #include #include #include #include +#include +#include + class SHA512HashTest : public cudf::test::BaseFixture {}; TEST_F(SHA512HashTest, EmptyTable) diff --git a/cpp/tests/hashing/spark_murmurhash3_x86_32_test.cpp b/cpp/tests/hashing/spark_murmurhash3_x86_32_test.cpp index c228c1e6378..e8bbfaa2cba 100644 --- a/cpp/tests/hashing/spark_murmurhash3_x86_32_test.cpp +++ b/cpp/tests/hashing/spark_murmurhash3_x86_32_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,16 @@ * limitations under the License. */ -#include -#include -#include - #include #include #include #include #include +#include +#include +#include + constexpr cudf::test::debug_output_level verbosity{cudf::test::debug_output_level::ALL_ERRORS}; template diff --git a/cpp/tests/hashing/xxhash_64_test.cpp b/cpp/tests/hashing/xxhash_64_test.cpp index 5916c4c2fb9..ab4ed829681 100644 --- a/cpp/tests/hashing/xxhash_64_test.cpp +++ b/cpp/tests/hashing/xxhash_64_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,16 @@ * limitations under the License. */ -#include -#include -#include - #include #include #include #include #include +#include +#include +#include + using NumericTypesNoBools = cudf::test::Concat; diff --git a/cpp/tests/interop/arrow_utils.hpp b/cpp/tests/interop/arrow_utils.hpp index 2c5f7458ce5..1fdf02e02f1 100644 --- a/cpp/tests/interop/arrow_utils.hpp +++ b/cpp/tests/interop/arrow_utils.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,11 @@ * limitations under the License. */ -#include +#include +#include +#include +#include +#include #include #include @@ -25,11 +29,8 @@ #include #include #include -#include -#include -#include -#include -#include + +#include #pragma once diff --git a/cpp/tests/interop/dlpack_test.cpp b/cpp/tests/interop/dlpack_test.cpp index ed44727b712..895887ee348 100644 --- a/cpp/tests/interop/dlpack_test.cpp +++ b/cpp/tests/interop/dlpack_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,17 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include #include #include #include #include #include -#include +#include #include +#include + struct dlpack_deleter { void operator()(DLManagedTensor* tensor) { tensor->deleter(tensor); } }; diff --git a/cpp/tests/interop/from_arrow_test.cpp b/cpp/tests/interop/from_arrow_test.cpp index a898106a5b2..94b0c75f184 100644 --- a/cpp/tests/interop/from_arrow_test.cpp +++ b/cpp/tests/interop/from_arrow_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,14 @@ * limitations under the License. */ +#include + +#include +#include +#include +#include +#include + #include #include #include @@ -26,16 +34,8 @@ #include #include -#include -#include -#include -#include -#include - #include -#include - std::unique_ptr get_cudf_table() { std::vector> columns; diff --git a/cpp/tests/interop/to_arrow_test.cpp b/cpp/tests/interop/to_arrow_test.cpp index 82c4ad7d2f1..a1ece0ce0f1 100644 --- a/cpp/tests/interop/to_arrow_test.cpp +++ b/cpp/tests/interop/to_arrow_test.cpp @@ -14,6 +14,15 @@ * limitations under the License. */ +#include + +#include +#include +#include +#include +#include +#include + #include #include #include @@ -27,15 +36,6 @@ #include #include -#include -#include -#include -#include -#include -#include - -#include - #include using vector_of_columns = std::vector>; diff --git a/cpp/tests/io/comp/decomp_test.cpp b/cpp/tests/io/comp/decomp_test.cpp index cf5a4f1fda5..38c1a57eca9 100644 --- a/cpp/tests/io/comp/decomp_test.cpp +++ b/cpp/tests/io/comp/decomp_test.cpp @@ -14,18 +14,19 @@ * limitations under the License. */ -#include -#include -#include - -#include +#include "io/comp/gpuinflate.hpp" +#include "io/utilities/hostdevice_vector.hpp" #include #include +#include + #include #include +#include + #include using cudf::device_span; diff --git a/cpp/tests/io/csv_test.cpp b/cpp/tests/io/csv_test.cpp index c6e9114605b..8e3ecd817e4 100644 --- a/cpp/tests/io/csv_test.cpp +++ b/cpp/tests/io/csv_test.cpp @@ -34,13 +34,13 @@ #include #include -#include - #include #include #include #include +#include + #include #include #include diff --git a/cpp/tests/io/fst/fst_test.cu b/cpp/tests/io/fst/fst_test.cu index 4064204c56d..4df0d3ae04d 100644 --- a/cpp/tests/io/fst/fst_test.cu +++ b/cpp/tests/io/fst/fst_test.cu @@ -14,8 +14,9 @@ * limitations under the License. */ -#include -#include +#include "io/fst/lookup_tables.cuh" +#include "io/utilities/hostdevice_vector.hpp" + #include #include diff --git a/cpp/tests/io/fst/logical_stack_test.cu b/cpp/tests/io/fst/logical_stack_test.cu index f434736d7f5..012f24c4e9f 100644 --- a/cpp/tests/io/fst/logical_stack_test.cu +++ b/cpp/tests/io/fst/logical_stack_test.cu @@ -14,17 +14,19 @@ * limitations under the License. */ +#include "io/utilities/hostdevice_vector.hpp" + #include #include #include -#include -#include #include #include #include +#include + #include #include #include diff --git a/cpp/tests/io/json_chunked_reader.cpp b/cpp/tests/io/json_chunked_reader.cpp index e2d5959c19f..8d8fdd2a0e1 100644 --- a/cpp/tests/io/json_chunked_reader.cpp +++ b/cpp/tests/io/json_chunked_reader.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,14 @@ * limitations under the License. */ +#include "io/json/read_json.hpp" + #include #include #include #include #include -#include - /** * @brief Base test fixture for JSON reader tests */ diff --git a/cpp/tests/io/json_quote_normalization_test.cpp b/cpp/tests/io/json_quote_normalization_test.cpp index 50faea5e4d8..b13e5bd4177 100644 --- a/cpp/tests/io/json_quote_normalization_test.cpp +++ b/cpp/tests/io/json_quote_normalization_test.cpp @@ -14,16 +14,16 @@ * limitations under the License. */ -#include -#include -#include - #include #include #include #include #include +#include +#include +#include + #include #include #include diff --git a/cpp/tests/io/json_test.cpp b/cpp/tests/io/json_test.cpp index 22c2f0de924..e4ed09d3962 100644 --- a/cpp/tests/io/json_test.cpp +++ b/cpp/tests/io/json_test.cpp @@ -35,12 +35,12 @@ #include #include -#include #include #include #include +#include #include #define wrapper cudf::test::fixed_width_column_wrapper diff --git a/cpp/tests/io/json_tree.cpp b/cpp/tests/io/json_tree.cpp index 40996e4fffa..3577b47a7e2 100644 --- a/cpp/tests/io/json_tree.cpp +++ b/cpp/tests/io/json_tree.cpp @@ -14,8 +14,13 @@ * limitations under the License. */ -#include -#include +#include "io/json/nested_json.hpp" +#include "io/utilities/hostdevice_vector.hpp" + +#include +#include +#include +#include #include #include @@ -23,11 +28,6 @@ #include #include -#include -#include -#include -#include - #include #include diff --git a/cpp/tests/io/json_type_cast_test.cu b/cpp/tests/io/json_type_cast_test.cu index 6923b7be42d..8a541022ab0 100644 --- a/cpp/tests/io/json_type_cast_test.cu +++ b/cpp/tests/io/json_type_cast_test.cu @@ -14,6 +14,8 @@ * limitations under the License. */ +#include "io/utilities/string_parsing.hpp" + #include #include #include @@ -22,8 +24,6 @@ #include #include -#include - #include #include #include diff --git a/cpp/tests/io/json_whitespace_normalization_test.cu b/cpp/tests/io/json_whitespace_normalization_test.cu index ef4172b0ff7..545d8d2c4f9 100644 --- a/cpp/tests/io/json_whitespace_normalization_test.cu +++ b/cpp/tests/io/json_whitespace_normalization_test.cu @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include - -#include -#include +#include "io/fst/lookup_tables.cuh" +#include "io/utilities/hostdevice_vector.hpp" #include #include #include #include +#include +#include + #include #include diff --git a/cpp/tests/io/nested_json_test.cpp b/cpp/tests/io/nested_json_test.cpp index 070ac5ce870..97e1a78f909 100644 --- a/cpp/tests/io/nested_json_test.cpp +++ b/cpp/tests/io/nested_json_test.cpp @@ -14,16 +14,8 @@ * limitations under the License. */ -#include -#include - -#include -#include -#include -#include -#include -#include -#include +#include "io/json/nested_json.hpp" +#include "io/utilities/hostdevice_vector.hpp" #include #include @@ -34,6 +26,14 @@ #include #include +#include +#include +#include +#include +#include +#include +#include + #include #include diff --git a/cpp/tests/io/orc_test.cpp b/cpp/tests/io/orc_test.cpp index f1a397f1747..0b34b39f739 100644 --- a/cpp/tests/io/orc_test.cpp +++ b/cpp/tests/io/orc_test.cpp @@ -34,6 +34,7 @@ #include #include #include + #include #include diff --git a/cpp/tests/io/parquet_chunked_reader_test.cu b/cpp/tests/io/parquet_chunked_reader_test.cu index dea44f0e7c3..ea6d65a8c14 100644 --- a/cpp/tests/io/parquet_chunked_reader_test.cu +++ b/cpp/tests/io/parquet_chunked_reader_test.cu @@ -40,13 +40,13 @@ #include #include -#include -#include +#include +#include #include -#include -#include +#include +#include #include #include diff --git a/cpp/tests/io/text/multibyte_split_test.cpp b/cpp/tests/io/text/multibyte_split_test.cpp index b207c3f15a6..36338253c9b 100644 --- a/cpp/tests/io/text/multibyte_split_test.cpp +++ b/cpp/tests/io/text/multibyte_split_test.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include "io/utilities/output_builder.cuh" #include #include diff --git a/cpp/tests/io/type_inference_test.cu b/cpp/tests/io/type_inference_test.cu index 3bb15a59aa3..37156292f44 100644 --- a/cpp/tests/io/type_inference_test.cu +++ b/cpp/tests/io/type_inference_test.cu @@ -14,14 +14,15 @@ * limitations under the License. */ -#include -#include +#include "io/utilities/string_parsing.hpp" +#include "io/utilities/trie.cuh" -#include -#include #include #include +#include +#include + #include #include diff --git a/cpp/tests/iterator/iterator_tests.cuh b/cpp/tests/iterator/iterator_tests.cuh index 7a5a9eae91c..c6da6b75930 100644 --- a/cpp/tests/iterator/iterator_tests.cuh +++ b/cpp/tests/iterator/iterator_tests.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -34,8 +35,6 @@ #include #include -#include - #include #include #include diff --git a/cpp/tests/iterator/value_iterator_test_chrono.cu b/cpp/tests/iterator/value_iterator_test_chrono.cu index 73796f589bb..03ca0e503e0 100644 --- a/cpp/tests/iterator/value_iterator_test_chrono.cu +++ b/cpp/tests/iterator/value_iterator_test_chrono.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,11 +12,11 @@ * or implied. See the License for the specific language governing permissions and limitations under * the License. */ +#include + #include #include -#include - using TestingTypes = cudf::test::ChronoTypes; template diff --git a/cpp/tests/iterator/value_iterator_test_numeric.cu b/cpp/tests/iterator/value_iterator_test_numeric.cu index 0d5ab6a857d..39e05ff6832 100644 --- a/cpp/tests/iterator/value_iterator_test_numeric.cu +++ b/cpp/tests/iterator/value_iterator_test_numeric.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,11 +12,11 @@ * or implied. See the License for the specific language governing permissions and limitations under * the License. */ +#include + #include #include -#include - using TestingTypes = cudf::test::NumericTypes; template diff --git a/cpp/tests/jit/parse_ptx_function.cpp b/cpp/tests/jit/parse_ptx_function.cpp index f6df2abc01a..6f9dfd06730 100644 --- a/cpp/tests/jit/parse_ptx_function.cpp +++ b/cpp/tests/jit/parse_ptx_function.cpp @@ -14,12 +14,13 @@ * limitations under the License. */ -#include -#include +#include "jit/parser.hpp" #include #include -#include + +#include +#include struct JitParseTest : public ::testing::Test {}; diff --git a/cpp/tests/join/conditional_join_tests.cu b/cpp/tests/join/conditional_join_tests.cu index ad5a33157fd..79968bcd7f4 100644 --- a/cpp/tests/join/conditional_join_tests.cu +++ b/cpp/tests/join/conditional_join_tests.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,16 @@ * limitations under the License. */ +#include +#include +#include + #include #include #include #include #include -#include -#include -#include - #include #include diff --git a/cpp/tests/join/cross_join_tests.cpp b/cpp/tests/join/cross_join_tests.cpp index 8fe8c449218..d87f5e54153 100644 --- a/cpp/tests/join/cross_join_tests.cpp +++ b/cpp/tests/join/cross_join_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,18 +14,18 @@ * limitations under the License. */ -#include -#include -#include -#include -#include - #include #include #include #include #include +#include +#include +#include +#include +#include + template using column_wrapper = cudf::test::fixed_width_column_wrapper; diff --git a/cpp/tests/join/join_tests.cpp b/cpp/tests/join/join_tests.cpp index 651e44511fb..b42f378d872 100644 --- a/cpp/tests/join/join_tests.cpp +++ b/cpp/tests/join/join_tests.cpp @@ -14,6 +14,14 @@ * limitations under the License. */ +#include +#include +#include +#include +#include +#include +#include + #include #include #include @@ -30,14 +38,6 @@ #include #include -#include -#include -#include -#include -#include -#include -#include - #include template diff --git a/cpp/tests/join/mixed_join_tests.cu b/cpp/tests/join/mixed_join_tests.cu index eb450d44efd..cc37dadffd8 100644 --- a/cpp/tests/join/mixed_join_tests.cu +++ b/cpp/tests/join/mixed_join_tests.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,16 @@ * limitations under the License. */ +#include +#include +#include + #include #include #include #include #include -#include -#include -#include - #include #include #include diff --git a/cpp/tests/join/semi_anti_join_tests.cpp b/cpp/tests/join/semi_anti_join_tests.cpp index 0e0c92bc4a2..5cdf5b2a374 100644 --- a/cpp/tests/join/semi_anti_join_tests.cpp +++ b/cpp/tests/join/semi_anti_join_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,12 @@ * limitations under the License. */ +#include +#include +#include +#include +#include + #include #include #include @@ -23,12 +29,6 @@ #include #include -#include -#include -#include -#include -#include - #include template diff --git a/cpp/tests/json/json_tests.cpp b/cpp/tests/json/json_tests.cpp index 548047f0410..0894472dcc3 100644 --- a/cpp/tests/json/json_tests.cpp +++ b/cpp/tests/json/json_tests.cpp @@ -14,15 +14,15 @@ * limitations under the License. */ +#include +#include +#include + #include #include #include #include -#include -#include -#include - #include // reference: https://jsonpath.herokuapp.com/ diff --git a/cpp/tests/labeling/label_bins_tests.cpp b/cpp/tests/labeling/label_bins_tests.cpp index 2356c7e5ce1..2ac6ad5dd0d 100644 --- a/cpp/tests/labeling/label_bins_tests.cpp +++ b/cpp/tests/labeling/label_bins_tests.cpp @@ -14,17 +14,18 @@ * limitations under the License. */ -#include -#include -#include -#include -#include #include #include #include #include #include +#include +#include +#include +#include +#include + #include #include #include diff --git a/cpp/tests/lists/contains_tests.cpp b/cpp/tests/lists/contains_tests.cpp index 62e6653347b..961437ba81e 100644 --- a/cpp/tests/lists/contains_tests.cpp +++ b/cpp/tests/lists/contains_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,17 +15,17 @@ * */ -#include -#include -#include -#include - #include #include #include #include #include +#include +#include +#include +#include + namespace { template (), void>* = nullptr> auto create_scalar_search_key(T const& value) diff --git a/cpp/tests/lists/count_elements_tests.cpp b/cpp/tests/lists/count_elements_tests.cpp index e099139a2fc..0933740b850 100644 --- a/cpp/tests/lists/count_elements_tests.cpp +++ b/cpp/tests/lists/count_elements_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,14 @@ * limitations under the License. */ -#include -#include - #include #include #include #include +#include +#include + #include #include diff --git a/cpp/tests/lists/sequences_tests.cpp b/cpp/tests/lists/sequences_tests.cpp index 9aed3428d69..e97600a76d3 100644 --- a/cpp/tests/lists/sequences_tests.cpp +++ b/cpp/tests/lists/sequences_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,8 +14,6 @@ * limitations under the License. */ -#include - #include #include #include @@ -23,6 +21,8 @@ #include #include +#include + using namespace cudf::test::iterators; namespace { diff --git a/cpp/tests/lists/stream_compaction/apply_boolean_mask_tests.cpp b/cpp/tests/lists/stream_compaction/apply_boolean_mask_tests.cpp index 4d38dbce569..5625b47e7ea 100644 --- a/cpp/tests/lists/stream_compaction/apply_boolean_mask_tests.cpp +++ b/cpp/tests/lists/stream_compaction/apply_boolean_mask_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include -#include -#include - #include #include #include #include #include +#include +#include +#include +#include + namespace cudf::test { using namespace iterators; diff --git a/cpp/tests/merge/merge_dictionary_test.cpp b/cpp/tests/merge/merge_dictionary_test.cpp index 5a5655e4720..55365cb972a 100644 --- a/cpp/tests/merge/merge_dictionary_test.cpp +++ b/cpp/tests/merge/merge_dictionary_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,17 +14,17 @@ * limitations under the License. */ +#include +#include +#include +#include + #include #include #include #include #include -#include -#include -#include -#include - #include struct MergeDictionaryTest : public cudf::test::BaseFixture {}; diff --git a/cpp/tests/merge/merge_string_test.cpp b/cpp/tests/merge/merge_string_test.cpp index 291167e0f9d..28179a7341c 100644 --- a/cpp/tests/merge/merge_string_test.cpp +++ b/cpp/tests/merge/merge_string_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,12 @@ * limitations under the License. */ +#include +#include +#include +#include +#include + #include #include #include @@ -22,11 +28,7 @@ #include #include -#include -#include -#include -#include -#include +#include #include #include @@ -35,8 +37,6 @@ #include #include -#include - using cudf::test::fixed_width_column_wrapper; using cudf::test::strings_column_wrapper; diff --git a/cpp/tests/merge/merge_test.cpp b/cpp/tests/merge/merge_test.cpp index d73c3192549..2e09f25b51f 100644 --- a/cpp/tests/merge/merge_test.cpp +++ b/cpp/tests/merge/merge_test.cpp @@ -14,15 +14,6 @@ * limitations under the License. */ -#include -#include -#include -#include -#include -#include -#include -#include - #include #include #include @@ -33,6 +24,15 @@ #include #include +#include +#include +#include +#include +#include +#include +#include +#include + #include #include diff --git a/cpp/tests/partitioning/hash_partition_test.cpp b/cpp/tests/partitioning/hash_partition_test.cpp index d7b12417251..4177ee9bc98 100644 --- a/cpp/tests/partitioning/hash_partition_test.cpp +++ b/cpp/tests/partitioning/hash_partition_test.cpp @@ -13,11 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include -#include -#include -#include #include #include #include @@ -26,6 +21,12 @@ #include #include +#include +#include +#include +#include +#include + #include #include diff --git a/cpp/tests/partitioning/round_robin_test.cpp b/cpp/tests/partitioning/round_robin_test.cpp index 7f83b5dafd0..8049c7c3a7a 100644 --- a/cpp/tests/partitioning/round_robin_test.cpp +++ b/cpp/tests/partitioning/round_robin_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,12 @@ * limitations under the License. */ +#include +#include +#include +#include +#include + #include #include #include @@ -22,11 +28,7 @@ #include #include -#include -#include -#include -#include -#include +#include #include #include @@ -36,8 +38,6 @@ #include #include -#include - using cudf::test::fixed_width_column_wrapper; using cudf::test::strings_column_wrapper; diff --git a/cpp/tests/reductions/list_rank_test.cpp b/cpp/tests/reductions/list_rank_test.cpp index 68da95fbb12..f5aeb87a3c0 100644 --- a/cpp/tests/reductions/list_rank_test.cpp +++ b/cpp/tests/reductions/list_rank_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,12 @@ * limitations under the License. */ +#include + #include #include #include -#include - #include #include #include diff --git a/cpp/tests/replace/clamp_test.cpp b/cpp/tests/replace/clamp_test.cpp index 74ee27137ed..bb33de1f1e7 100644 --- a/cpp/tests/replace/clamp_test.cpp +++ b/cpp/tests/replace/clamp_test.cpp @@ -14,11 +14,6 @@ * limitations under the License. */ -#include -#include -#include -#include - #include #include #include @@ -26,10 +21,15 @@ #include #include -#include +#include +#include +#include +#include #include +#include + struct ClampErrorTest : public cudf::test::BaseFixture {}; TEST_F(ClampErrorTest, MisMatchingScalarTypes) diff --git a/cpp/tests/replace/normalize_replace_tests.cpp b/cpp/tests/replace/normalize_replace_tests.cpp index 50736940520..2de17388ee8 100644 --- a/cpp/tests/replace/normalize_replace_tests.cpp +++ b/cpp/tests/replace/normalize_replace_tests.cpp @@ -14,13 +14,14 @@ * limitations under the License. */ -#include -#include #include #include #include #include +#include +#include + // This is the main test fixture struct ReplaceTest : public cudf::test::BaseFixture {}; diff --git a/cpp/tests/replace/replace_tests.cpp b/cpp/tests/replace/replace_tests.cpp index 8b953079d34..8685e7300ba 100644 --- a/cpp/tests/replace/replace_tests.cpp +++ b/cpp/tests/replace/replace_tests.cpp @@ -29,13 +29,14 @@ #include #include #include +#include #include #include -#include -#include #include + +#include #include #include diff --git a/cpp/tests/rolling/grouped_rolling_test.cpp b/cpp/tests/rolling/grouped_rolling_test.cpp index 7dd72ace53c..a4abe5ee608 100644 --- a/cpp/tests/rolling/grouped_rolling_test.cpp +++ b/cpp/tests/rolling/grouped_rolling_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,11 +28,12 @@ #include #include #include -#include #include #include +#include + const std::string cuda_func{ R"***( template diff --git a/cpp/tests/rolling/nth_element_test.cpp b/cpp/tests/rolling/nth_element_test.cpp index 4edbcf0d8a6..9cc8b6dec81 100644 --- a/cpp/tests/rolling/nth_element_test.cpp +++ b/cpp/tests/rolling/nth_element_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,13 +26,13 @@ #include #include -#include - #include #include #include +#include + #include #include diff --git a/cpp/tests/rolling/range_rolling_window_test.cpp b/cpp/tests/rolling/range_rolling_window_test.cpp index eed9db1fe04..fcd0cc18019 100644 --- a/cpp/tests/rolling/range_rolling_window_test.cpp +++ b/cpp/tests/rolling/range_rolling_window_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,13 +26,14 @@ #include #include #include -#include -#include #include #include #include +#include +#include + #include template diff --git a/cpp/tests/rolling/range_window_bounds_test.cpp b/cpp/tests/rolling/range_window_bounds_test.cpp index ceedda70075..b77451bf0bc 100644 --- a/cpp/tests/rolling/range_window_bounds_test.cpp +++ b/cpp/tests/rolling/range_window_bounds_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ #include #include + #include #include diff --git a/cpp/tests/rolling/rolling_test.cpp b/cpp/tests/rolling/rolling_test.cpp index c0307000f5c..c2c22986975 100644 --- a/cpp/tests/rolling/rolling_test.cpp +++ b/cpp/tests/rolling/rolling_test.cpp @@ -33,12 +33,13 @@ #include #include #include -#include #include #include #include +#include + #include #include #include diff --git a/cpp/tests/scalar/scalar_device_view_test.cu b/cpp/tests/scalar/scalar_device_view_test.cu index 8444716bccd..5026954403b 100644 --- a/cpp/tests/scalar/scalar_device_view_test.cu +++ b/cpp/tests/scalar/scalar_device_view_test.cu @@ -14,6 +14,11 @@ * limitations under the License. */ +#include +#include +#include +#include + #include #include #include @@ -21,14 +26,11 @@ #include #include #include -#include -#include -#include -#include -#include #include +#include + template struct TypedScalarDeviceViewTest : public cudf::test::BaseFixture {}; diff --git a/cpp/tests/stream_compaction/apply_boolean_mask_tests.cpp b/cpp/tests/stream_compaction/apply_boolean_mask_tests.cpp index cb7d11dab35..6c0582fb846 100644 --- a/cpp/tests/stream_compaction/apply_boolean_mask_tests.cpp +++ b/cpp/tests/stream_compaction/apply_boolean_mask_tests.cpp @@ -14,6 +14,14 @@ * limitations under the License. */ +#include +#include +#include +#include +#include +#include +#include + #include #include #include @@ -22,13 +30,6 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include #include #include diff --git a/cpp/tests/stream_compaction/distinct_count_tests.cpp b/cpp/tests/stream_compaction/distinct_count_tests.cpp index 864ac8f84c6..ee0ca3f86c1 100644 --- a/cpp/tests/stream_compaction/distinct_count_tests.cpp +++ b/cpp/tests/stream_compaction/distinct_count_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,6 @@ * limitations under the License. */ -#include -#include -#include -#include -#include -#include - #include #include #include @@ -28,6 +21,13 @@ #include #include +#include +#include +#include +#include +#include +#include + #include #include diff --git a/cpp/tests/stream_compaction/drop_nans_tests.cpp b/cpp/tests/stream_compaction/drop_nans_tests.cpp index bce8b19802c..425d9a47ecc 100644 --- a/cpp/tests/stream_compaction/drop_nans_tests.cpp +++ b/cpp/tests/stream_compaction/drop_nans_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,18 +14,20 @@ * limitations under the License. */ -#include -#include -#include -#include -#include -#include #include #include #include #include #include +#include +#include +#include +#include +#include + +#include + struct DropNANsTest : public cudf::test::BaseFixture {}; TEST_F(DropNANsTest, MixedNANsAndNull) diff --git a/cpp/tests/stream_compaction/drop_nulls_tests.cpp b/cpp/tests/stream_compaction/drop_nulls_tests.cpp index bff56eb5b81..47aa2d8ee3e 100644 --- a/cpp/tests/stream_compaction/drop_nulls_tests.cpp +++ b/cpp/tests/stream_compaction/drop_nulls_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,17 +14,18 @@ * limitations under the License. */ -#include -#include -#include -#include -#include #include #include #include #include #include +#include +#include +#include +#include +#include + #include #include diff --git a/cpp/tests/stream_compaction/unique_count_tests.cpp b/cpp/tests/stream_compaction/unique_count_tests.cpp index af0b45b97e3..640d159fc4f 100644 --- a/cpp/tests/stream_compaction/unique_count_tests.cpp +++ b/cpp/tests/stream_compaction/unique_count_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,6 @@ * limitations under the License. */ -#include -#include -#include -#include -#include -#include - #include #include #include @@ -28,6 +21,13 @@ #include #include +#include +#include +#include +#include +#include +#include + #include #include diff --git a/cpp/tests/stream_compaction/unique_tests.cpp b/cpp/tests/stream_compaction/unique_tests.cpp index d64c6f589db..01f5f4d39db 100644 --- a/cpp/tests/stream_compaction/unique_tests.cpp +++ b/cpp/tests/stream_compaction/unique_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,6 @@ * limitations under the License. */ -#include -#include -#include -#include -#include -#include - #include #include #include @@ -28,6 +21,13 @@ #include #include +#include +#include +#include +#include +#include +#include + #include #include diff --git a/cpp/tests/streams/binaryop_test.cpp b/cpp/tests/streams/binaryop_test.cpp index 2520aed0458..2a7b52b1b6b 100644 --- a/cpp/tests/streams/binaryop_test.cpp +++ b/cpp/tests/streams/binaryop_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,14 +16,14 @@ #include -#include -#include -#include - #include #include #include +#include +#include +#include + class BinaryopTest : public cudf::test::BaseFixture {}; TEST_F(BinaryopTest, ColumnColumn) diff --git a/cpp/tests/streams/concatenate_test.cpp b/cpp/tests/streams/concatenate_test.cpp index 6e6ff58686f..648fb01a636 100644 --- a/cpp/tests/streams/concatenate_test.cpp +++ b/cpp/tests/streams/concatenate_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,12 @@ * limitations under the License. */ -#include - #include #include #include +#include + class ConcatenateTest : public cudf::test::BaseFixture {}; TEST_F(ConcatenateTest, Column) diff --git a/cpp/tests/streams/dictionary_test.cpp b/cpp/tests/streams/dictionary_test.cpp index f48e64c078e..9e81c8574b8 100644 --- a/cpp/tests/streams/dictionary_test.cpp +++ b/cpp/tests/streams/dictionary_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,16 @@ * limitations under the License. */ +#include +#include +#include + #include #include #include #include #include -#include -#include -#include - class DictionaryTest : public cudf::test::BaseFixture {}; TEST_F(DictionaryTest, Encode) diff --git a/cpp/tests/streams/filling_test.cpp b/cpp/tests/streams/filling_test.cpp index b822743d4ca..d8d48fe6557 100644 --- a/cpp/tests/streams/filling_test.cpp +++ b/cpp/tests/streams/filling_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,14 @@ * limitations under the License. */ -#include -#include -#include - #include #include #include +#include +#include +#include + class FillingTest : public cudf::test::BaseFixture {}; TEST_F(FillingTest, FillInPlace) diff --git a/cpp/tests/streams/hash_test.cpp b/cpp/tests/streams/hash_test.cpp index 8c6609fdc22..64ae6987a3d 100644 --- a/cpp/tests/streams/hash_test.cpp +++ b/cpp/tests/streams/hash_test.cpp @@ -14,12 +14,12 @@ * limitations under the License. */ -#include - #include #include #include +#include + class HashTest : public cudf::test::BaseFixture {}; TEST_F(HashTest, MultiValue) diff --git a/cpp/tests/streams/interop_test.cpp b/cpp/tests/streams/interop_test.cpp index 7eac9e016eb..cf620749d8f 100644 --- a/cpp/tests/streams/interop_test.cpp +++ b/cpp/tests/streams/interop_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,14 @@ * limitations under the License. */ -#include -#include -#include - #include #include #include +#include +#include +#include + struct ArrowTest : public cudf::test::BaseFixture {}; TEST_F(ArrowTest, ToArrow) diff --git a/cpp/tests/streams/io/csv_test.cpp b/cpp/tests/streams/io/csv_test.cpp index d227446ba94..6e27db02d56 100644 --- a/cpp/tests/streams/io/csv_test.cpp +++ b/cpp/tests/streams/io/csv_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,17 +14,17 @@ * limitations under the License. */ +#include +#include +#include +#include + #include #include #include #include #include -#include -#include -#include -#include - #include #include diff --git a/cpp/tests/streams/io/json_test.cpp b/cpp/tests/streams/io/json_test.cpp index 80619d4d58c..21da19a5a38 100644 --- a/cpp/tests/streams/io/json_test.cpp +++ b/cpp/tests/streams/io/json_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,17 +14,17 @@ * limitations under the License. */ +#include +#include +#include +#include + #include #include #include #include #include -#include -#include -#include -#include - #include #include diff --git a/cpp/tests/streams/io/orc_test.cpp b/cpp/tests/streams/io/orc_test.cpp index 929c3697b3b..57e36d13224 100644 --- a/cpp/tests/streams/io/orc_test.cpp +++ b/cpp/tests/streams/io/orc_test.cpp @@ -14,6 +14,11 @@ * limitations under the License. */ +#include +#include +#include +#include + #include #include #include @@ -22,11 +27,6 @@ #include #include -#include -#include -#include -#include - #include #include #include diff --git a/cpp/tests/streams/io/parquet_test.cpp b/cpp/tests/streams/io/parquet_test.cpp index c6d531bc376..f6bb2cf4336 100644 --- a/cpp/tests/streams/io/parquet_test.cpp +++ b/cpp/tests/streams/io/parquet_test.cpp @@ -14,17 +14,17 @@ * limitations under the License. */ +#include +#include +#include +#include + #include #include #include #include #include -#include -#include -#include -#include - #include #include diff --git a/cpp/tests/streams/labeling_bins_test.cpp b/cpp/tests/streams/labeling_bins_test.cpp index a1d3983aacc..c7dc49436b0 100644 --- a/cpp/tests/streams/labeling_bins_test.cpp +++ b/cpp/tests/streams/labeling_bins_test.cpp @@ -14,12 +14,12 @@ * limitations under the License. */ -#include - #include #include #include +#include + class LabelingBinsStreamTest : public cudf::test::BaseFixture {}; TEST_F(LabelingBinsStreamTest, SimpleStringsTest) diff --git a/cpp/tests/streams/null_mask_test.cpp b/cpp/tests/streams/null_mask_test.cpp index 7e59201c8cf..e96224003f4 100644 --- a/cpp/tests/streams/null_mask_test.cpp +++ b/cpp/tests/streams/null_mask_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,14 +16,14 @@ #include -#include -#include -#include - #include #include #include +#include +#include +#include + class NullMaskTest : public cudf::test::BaseFixture {}; TEST_F(NullMaskTest, CreateNullMask) diff --git a/cpp/tests/streams/pool_test.cu b/cpp/tests/streams/pool_test.cu index 52debe24fe8..92aa43b101a 100644 --- a/cpp/tests/streams/pool_test.cu +++ b/cpp/tests/streams/pool_test.cu @@ -18,6 +18,7 @@ #include #include + #include class StreamPoolTest : public cudf::test::BaseFixture {}; diff --git a/cpp/tests/streams/replace_test.cpp b/cpp/tests/streams/replace_test.cpp index c794f99b6f6..25293db4347 100644 --- a/cpp/tests/streams/replace_test.cpp +++ b/cpp/tests/streams/replace_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,15 @@ * limitations under the License. */ -#include -#include -#include - #include #include #include #include +#include +#include +#include + class ReplaceTest : public cudf::test::BaseFixture {}; TEST_F(ReplaceTest, ReplaceNullsColumn) diff --git a/cpp/tests/streams/search_test.cpp b/cpp/tests/streams/search_test.cpp index fbe17fb0cc4..d0249b0a45e 100644 --- a/cpp/tests/streams/search_test.cpp +++ b/cpp/tests/streams/search_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,14 @@ * limitations under the License. */ -#include -#include -#include - #include #include #include +#include +#include +#include + class SearchTest : public cudf::test::BaseFixture {}; TEST_F(SearchTest, LowerBound) diff --git a/cpp/tests/streams/sorting_test.cpp b/cpp/tests/streams/sorting_test.cpp index e481f95bded..ae0e293c8e6 100644 --- a/cpp/tests/streams/sorting_test.cpp +++ b/cpp/tests/streams/sorting_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,13 @@ * limitations under the License. */ -#include -#include - #include #include #include +#include +#include + class SortingTest : public cudf::test::BaseFixture {}; TEST_F(SortingTest, SortedOrder) diff --git a/cpp/tests/streams/strings/case_test.cpp b/cpp/tests/streams/strings/case_test.cpp index df3eabd773a..4852e8e1c7b 100644 --- a/cpp/tests/streams/strings/case_test.cpp +++ b/cpp/tests/streams/strings/case_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,13 @@ * limitations under the License. */ -#include -#include - #include #include #include +#include +#include + class StringsCaseTest : public cudf::test::BaseFixture {}; TEST_F(StringsCaseTest, LowerUpper) diff --git a/cpp/tests/streams/strings/filter_test.cpp b/cpp/tests/streams/strings/filter_test.cpp index 3c44eb81380..53ebe4e0b0d 100644 --- a/cpp/tests/streams/strings/filter_test.cpp +++ b/cpp/tests/streams/strings/filter_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,13 @@ * limitations under the License. */ -#include -#include - #include #include #include +#include +#include + #include #include diff --git a/cpp/tests/streams/strings/find_test.cpp b/cpp/tests/streams/strings/find_test.cpp index b734a1738cc..52839c6fc9f 100644 --- a/cpp/tests/streams/strings/find_test.cpp +++ b/cpp/tests/streams/strings/find_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,15 @@ * limitations under the License. */ +#include +#include +#include + #include #include #include #include -#include -#include -#include - #include class StringsFindTest : public cudf::test::BaseFixture {}; diff --git a/cpp/tests/streams/strings/reverse_test.cpp b/cpp/tests/streams/strings/reverse_test.cpp index 83dcf24594e..4b4d0a7aff5 100644 --- a/cpp/tests/streams/strings/reverse_test.cpp +++ b/cpp/tests/streams/strings/reverse_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,12 @@ * limitations under the License. */ -#include - #include #include #include +#include + #include #include diff --git a/cpp/tests/streams/strings/strings_tests.cpp b/cpp/tests/streams/strings/strings_tests.cpp index 0db467a6895..482d39e866b 100644 --- a/cpp/tests/streams/strings/strings_tests.cpp +++ b/cpp/tests/streams/strings/strings_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,15 @@ * limitations under the License. */ +#include +#include +#include + #include #include #include #include -#include -#include -#include - #include class StringsTest : public cudf::test::BaseFixture {}; diff --git a/cpp/tests/streams/text/edit_distance_test.cpp b/cpp/tests/streams/text/edit_distance_test.cpp index 59206c39e69..a4545ca577f 100644 --- a/cpp/tests/streams/text/edit_distance_test.cpp +++ b/cpp/tests/streams/text/edit_distance_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,12 @@ * limitations under the License. */ -#include - #include #include #include +#include + class TextEditDistanceTest : public cudf::test::BaseFixture {}; TEST_F(TextEditDistanceTest, EditDistance) diff --git a/cpp/tests/streams/text/ngrams_test.cpp b/cpp/tests/streams/text/ngrams_test.cpp index bce0d2b680b..221c0a62f3e 100644 --- a/cpp/tests/streams/text/ngrams_test.cpp +++ b/cpp/tests/streams/text/ngrams_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,13 @@ * limitations under the License. */ -#include -#include - #include #include #include +#include +#include + class TextNGramsTest : public cudf::test::BaseFixture {}; TEST_F(TextNGramsTest, GenerateNgrams) diff --git a/cpp/tests/streams/text/stemmer_test.cpp b/cpp/tests/streams/text/stemmer_test.cpp index 7aa51befa73..03ed6ec5a72 100644 --- a/cpp/tests/streams/text/stemmer_test.cpp +++ b/cpp/tests/streams/text/stemmer_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,12 @@ * limitations under the License. */ -#include - #include #include #include +#include + class TextStemmerTest : public cudf::test::BaseFixture {}; TEST_F(TextStemmerTest, IsLetter) diff --git a/cpp/tests/streams/text/tokenize_test.cpp b/cpp/tests/streams/text/tokenize_test.cpp index b281fbc2c0c..619aaeeaeab 100644 --- a/cpp/tests/streams/text/tokenize_test.cpp +++ b/cpp/tests/streams/text/tokenize_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,12 @@ * limitations under the License. */ -#include - #include #include #include +#include + class TextTokenizeTest : public cudf::test::BaseFixture {}; TEST_F(TextTokenizeTest, Tokenize) diff --git a/cpp/tests/streams/unary_test.cpp b/cpp/tests/streams/unary_test.cpp index 1734c0c4e9f..15f04df70d3 100644 --- a/cpp/tests/streams/unary_test.cpp +++ b/cpp/tests/streams/unary_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,12 @@ * limitations under the License. */ -#include - #include #include #include +#include + class UnaryTest : public cudf::test::BaseFixture {}; TEST_F(UnaryTest, UnaryOperation) diff --git a/cpp/tests/strings/attrs_tests.cpp b/cpp/tests/strings/attrs_tests.cpp index c5f38697f00..93fe5142f00 100644 --- a/cpp/tests/strings/attrs_tests.cpp +++ b/cpp/tests/strings/attrs_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,14 @@ * limitations under the License. */ -#include -#include -#include - #include #include #include +#include +#include +#include + #include #include diff --git a/cpp/tests/strings/chars_types_tests.cpp b/cpp/tests/strings/chars_types_tests.cpp index c595977c269..fbc059186a8 100644 --- a/cpp/tests/strings/chars_types_tests.cpp +++ b/cpp/tests/strings/chars_types_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,14 @@ * limitations under the License. */ -#include -#include -#include #include #include #include +#include +#include +#include + #include #include diff --git a/cpp/tests/strings/combine/join_list_elements_tests.cpp b/cpp/tests/strings/combine/join_list_elements_tests.cpp index 4637113ca33..00317146088 100644 --- a/cpp/tests/strings/combine/join_list_elements_tests.cpp +++ b/cpp/tests/strings/combine/join_list_elements_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,16 @@ * limitations under the License. */ -#include -#include -#include -#include - #include #include #include #include +#include +#include +#include +#include + using namespace cudf::test::iterators; struct StringsListsConcatenateTest : public cudf::test::BaseFixture {}; diff --git a/cpp/tests/strings/durations_tests.cpp b/cpp/tests/strings/durations_tests.cpp index 1902f907f43..86189b29981 100644 --- a/cpp/tests/strings/durations_tests.cpp +++ b/cpp/tests/strings/durations_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ #include #include + #include struct StringsDurationsTest : public cudf::test::BaseFixture {}; diff --git a/cpp/tests/strings/find_multiple_tests.cpp b/cpp/tests/strings/find_multiple_tests.cpp index 986f86d2b49..57cba495ba0 100644 --- a/cpp/tests/strings/find_multiple_tests.cpp +++ b/cpp/tests/strings/find_multiple_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,15 @@ * limitations under the License. */ +#include +#include +#include + #include #include #include #include -#include -#include -#include - #include #include diff --git a/cpp/tests/strings/find_tests.cpp b/cpp/tests/strings/find_tests.cpp index 3f291e870c0..7f89cc9fb53 100644 --- a/cpp/tests/strings/find_tests.cpp +++ b/cpp/tests/strings/find_tests.cpp @@ -14,6 +14,10 @@ * limitations under the License. */ +#include +#include +#include + #include #include #include @@ -21,10 +25,6 @@ #include #include -#include -#include -#include - #include #include diff --git a/cpp/tests/strings/like_tests.cpp b/cpp/tests/strings/like_tests.cpp index 50d8edfd646..4352a1ed584 100644 --- a/cpp/tests/strings/like_tests.cpp +++ b/cpp/tests/strings/like_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,13 @@ * limitations under the License. */ -#include -#include - #include #include #include +#include +#include + struct StringsLikeTests : public cudf::test::BaseFixture {}; TEST_F(StringsLikeTests, Basic) diff --git a/cpp/tests/structs/structs_column_tests.cpp b/cpp/tests/structs/structs_column_tests.cpp index fb25c67b763..8f492a930a8 100644 --- a/cpp/tests/structs/structs_column_tests.cpp +++ b/cpp/tests/structs/structs_column_tests.cpp @@ -14,6 +14,13 @@ * limitations under the License. */ +#include +#include +#include +#include +#include +#include + #include #include #include @@ -26,20 +33,13 @@ #include #include -#include -#include -#include -#include -#include -#include +#include #include #include #include #include -#include - #include #include #include diff --git a/cpp/tests/structs/utilities_tests.cpp b/cpp/tests/structs/utilities_tests.cpp index 6ecc03b9222..00f7d636530 100644 --- a/cpp/tests/structs/utilities_tests.cpp +++ b/cpp/tests/structs/utilities_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include "cudf_test/default_stream.hpp" + #include #include #include diff --git a/cpp/tests/table/row_operators_tests.cpp b/cpp/tests/table/row_operators_tests.cpp index d37c14fd858..974e7d67658 100644 --- a/cpp/tests/table/row_operators_tests.cpp +++ b/cpp/tests/table/row_operators_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,16 @@ * limitations under the License. */ -#include -#include -#include -#include #include #include #include #include +#include +#include +#include +#include + #include struct RowOperatorTestForNAN : public cudf::test::BaseFixture {}; diff --git a/cpp/tests/table/table_tests.cpp b/cpp/tests/table/table_tests.cpp index 0d6b870c33b..1637ba7d7d3 100644 --- a/cpp/tests/table/table_tests.cpp +++ b/cpp/tests/table/table_tests.cpp @@ -14,18 +14,18 @@ * limitations under the License. */ -#include -#include -#include -#include -#include - #include #include #include #include #include +#include +#include +#include +#include +#include + #include #include diff --git a/cpp/tests/text/bpe_tests.cpp b/cpp/tests/text/bpe_tests.cpp index a13b61e0ba4..b03df12c5ed 100644 --- a/cpp/tests/text/bpe_tests.cpp +++ b/cpp/tests/text/bpe_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,8 +14,6 @@ * limitations under the License. */ -#include - #include #include #include @@ -24,6 +22,8 @@ #include #include +#include + struct TextBytePairEncoding : public cudf::test::BaseFixture {}; TEST_F(TextBytePairEncoding, BytePairEncoding) diff --git a/cpp/tests/text/edit_distance_tests.cpp b/cpp/tests/text/edit_distance_tests.cpp index 837a4eb8de4..04b28460d23 100644 --- a/cpp/tests/text/edit_distance_tests.cpp +++ b/cpp/tests/text/edit_distance_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,15 @@ * limitations under the License. */ -#include -#include -#include - #include #include #include +#include +#include + +#include + #include #include diff --git a/cpp/tests/text/jaccard_tests.cpp b/cpp/tests/text/jaccard_tests.cpp index 987de316e7f..a0aee594609 100644 --- a/cpp/tests/text/jaccard_tests.cpp +++ b/cpp/tests/text/jaccard_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,10 +18,10 @@ #include #include -#include - #include +#include + struct JaccardTest : public cudf::test::BaseFixture {}; TEST_F(JaccardTest, Basic) diff --git a/cpp/tests/text/minhash_tests.cpp b/cpp/tests/text/minhash_tests.cpp index b1c961ec9e1..7575a3ba846 100644 --- a/cpp/tests/text/minhash_tests.cpp +++ b/cpp/tests/text/minhash_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,12 +19,12 @@ #include #include -#include - #include #include #include +#include + #include #include diff --git a/cpp/tests/text/ngrams_tokenize_tests.cpp b/cpp/tests/text/ngrams_tokenize_tests.cpp index c6fb886f7e5..998bddedd18 100644 --- a/cpp/tests/text/ngrams_tokenize_tests.cpp +++ b/cpp/tests/text/ngrams_tokenize_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,15 @@ * limitations under the License. */ +#include +#include +#include + #include #include #include -#include -#include -#include -#include +#include #include diff --git a/cpp/tests/text/normalize_tests.cpp b/cpp/tests/text/normalize_tests.cpp index 5fa3bb24f24..bf619bf49bc 100644 --- a/cpp/tests/text/normalize_tests.cpp +++ b/cpp/tests/text/normalize_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,15 @@ * limitations under the License. */ +#include +#include +#include + #include #include #include #include -#include -#include -#include - #include #include diff --git a/cpp/tests/text/replace_tests.cpp b/cpp/tests/text/replace_tests.cpp index f798d596a3c..8c58c6bcaca 100644 --- a/cpp/tests/text/replace_tests.cpp +++ b/cpp/tests/text/replace_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,14 @@ * limitations under the License. */ -#include -#include -#include - #include #include #include +#include +#include +#include + #include #include diff --git a/cpp/tests/text/stemmer_tests.cpp b/cpp/tests/text/stemmer_tests.cpp index 939d2f1cd2f..bbc145e0fe7 100644 --- a/cpp/tests/text/stemmer_tests.cpp +++ b/cpp/tests/text/stemmer_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,14 @@ * limitations under the License. */ -#include -#include -#include - #include #include #include +#include +#include +#include + #include #include diff --git a/cpp/tests/text/subword_tests.cpp b/cpp/tests/text/subword_tests.cpp index 4db289ac5b8..5a347e5fe68 100644 --- a/cpp/tests/text/subword_tests.cpp +++ b/cpp/tests/text/subword_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,14 @@ * limitations under the License. */ -#include -#include -#include - #include #include #include +#include +#include +#include + #include #include diff --git a/cpp/tests/text/tokenize_tests.cpp b/cpp/tests/text/tokenize_tests.cpp index ea36e13de6f..6a6bcda87cc 100644 --- a/cpp/tests/text/tokenize_tests.cpp +++ b/cpp/tests/text/tokenize_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,12 +19,12 @@ #include #include -#include - #include #include #include +#include + #include #include diff --git a/cpp/tests/transform/bools_to_mask_test.cpp b/cpp/tests/transform/bools_to_mask_test.cpp index ce8ed9285fe..215ca158f37 100644 --- a/cpp/tests/transform/bools_to_mask_test.cpp +++ b/cpp/tests/transform/bools_to_mask_test.cpp @@ -14,17 +14,17 @@ * limitations under the License. */ +#include +#include +#include +#include + #include #include #include #include #include -#include -#include -#include -#include - #include struct MaskToNullTest : public cudf::test::BaseFixture { diff --git a/cpp/tests/transform/nans_to_null_test.cpp b/cpp/tests/transform/nans_to_null_test.cpp index 2de06641c7f..5dcfe18b7a0 100644 --- a/cpp/tests/transform/nans_to_null_test.cpp +++ b/cpp/tests/transform/nans_to_null_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,16 @@ * limitations under the License. */ -#include -#include -#include -#include #include #include #include #include +#include +#include +#include +#include + template struct NaNsToNullTest : public cudf::test::BaseFixture { void run_test(cudf::column_view const& input, cudf::column_view const& expected) diff --git a/cpp/tests/transpose/transpose_test.cpp b/cpp/tests/transpose/transpose_test.cpp index 59094db6cc3..5a88c402b8c 100644 --- a/cpp/tests/transpose/transpose_test.cpp +++ b/cpp/tests/transpose/transpose_test.cpp @@ -13,13 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include #include #include #include #include #include +#include + #include #include #include diff --git a/cpp/tests/types/traits_test.cpp b/cpp/tests/types/traits_test.cpp index 53bf224649e..0d9092c33da 100644 --- a/cpp/tests/types/traits_test.cpp +++ b/cpp/tests/types/traits_test.cpp @@ -14,11 +14,12 @@ * limitations under the License. */ -#include #include #include #include +#include + #include #include diff --git a/cpp/tests/types/type_dispatcher_test.cu b/cpp/tests/types/type_dispatcher_test.cu index 0b26330d323..21e56de4621 100644 --- a/cpp/tests/types/type_dispatcher_test.cu +++ b/cpp/tests/types/type_dispatcher_test.cu @@ -14,16 +14,17 @@ * limitations under the License. */ -#include -#include -#include -#include #include #include #include #include #include +#include +#include +#include +#include + #include struct DispatcherTest : public cudf::test::BaseFixture {}; diff --git a/cpp/tests/unary/cast_tests.cpp b/cpp/tests/unary/cast_tests.cpp index d565359a4ea..a82449ffc10 100644 --- a/cpp/tests/unary/cast_tests.cpp +++ b/cpp/tests/unary/cast_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,11 +27,10 @@ #include #include +#include #include #include -#include - #include #include diff --git a/cpp/tests/unary/math_ops_test.cpp b/cpp/tests/unary/math_ops_test.cpp index b86d798917f..acbf0732522 100644 --- a/cpp/tests/unary/math_ops_test.cpp +++ b/cpp/tests/unary/math_ops_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,18 +14,19 @@ * limitations under the License. */ +#include +#include +#include +#include + #include #include #include #include #include -#include -#include -#include -#include - #include + #include template diff --git a/cpp/tests/unary/unary_ops_test.cpp b/cpp/tests/unary/unary_ops_test.cpp index 19c18a8b0c1..e7477c34642 100644 --- a/cpp/tests/unary/unary_ops_test.cpp +++ b/cpp/tests/unary/unary_ops_test.cpp @@ -23,9 +23,8 @@ #include #include -#include - #include +#include template cudf::test::fixed_width_column_wrapper create_fixed_columns(cudf::size_type start, diff --git a/cpp/tests/utilities/column_utilities.cu b/cpp/tests/utilities/column_utilities.cu index 450e8e935b4..018c6aeec2c 100644 --- a/cpp/tests/utilities/column_utilities.cu +++ b/cpp/tests/utilities/column_utilities.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,6 +35,7 @@ #include +#include #include #include #include @@ -50,8 +51,6 @@ #include #include -#include - #include #include diff --git a/cpp/tests/utilities/default_stream.cpp b/cpp/tests/utilities/default_stream.cpp index 52752f78bb9..747e09115bd 100644 --- a/cpp/tests/utilities/default_stream.cpp +++ b/cpp/tests/utilities/default_stream.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,10 @@ * limitations under the License. */ -#include - #include +#include + namespace cudf { namespace test { diff --git a/cpp/tests/utilities/identify_stream_usage.cpp b/cpp/tests/utilities/identify_stream_usage.cpp index bdc338d2c92..5628f7966c3 100644 --- a/cpp/tests/utilities/identify_stream_usage.cpp +++ b/cpp/tests/utilities/identify_stream_usage.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,11 +22,12 @@ #include -#include -#include #include #include #include + +#include +#include #include #include #include diff --git a/cpp/tests/utilities/tdigest_utilities.cu b/cpp/tests/utilities/tdigest_utilities.cu index 9294aa0f681..ec3ea0d9a83 100644 --- a/cpp/tests/utilities/tdigest_utilities.cu +++ b/cpp/tests/utilities/tdigest_utilities.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,16 @@ * limitations under the License. */ +#include +#include +#include + #include #include #include #include #include -#include -#include -#include - #include #include diff --git a/cpp/tests/utilities_tests/column_debug_tests.cpp b/cpp/tests/utilities_tests/column_debug_tests.cpp index 0dae407ad21..7aa05af4591 100644 --- a/cpp/tests/utilities_tests/column_debug_tests.cpp +++ b/cpp/tests/utilities_tests/column_debug_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,14 @@ * limitations under the License. */ -#include - #include #include #include #include #include +#include + #include #include diff --git a/cpp/tests/utilities_tests/column_utilities_tests.cpp b/cpp/tests/utilities_tests/column_utilities_tests.cpp index f5b7a499243..9d6d5ccb9b5 100644 --- a/cpp/tests/utilities_tests/column_utilities_tests.cpp +++ b/cpp/tests/utilities_tests/column_utilities_tests.cpp @@ -14,10 +14,6 @@ * limitations under the License. */ -#include -#include -#include - #include #include #include @@ -26,6 +22,10 @@ #include #include +#include +#include +#include + #include #include diff --git a/cpp/tests/utilities_tests/default_stream_tests.cpp b/cpp/tests/utilities_tests/default_stream_tests.cpp index f5c55879b9c..469ee1bb78e 100644 --- a/cpp/tests/utilities_tests/default_stream_tests.cpp +++ b/cpp/tests/utilities_tests/default_stream_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,10 @@ * limitations under the License. */ -#include - #include +#include + #ifdef CUDA_API_PER_THREAD_DEFAULT_STREAM TEST(DefaultStreamTest, PtdsIsEnabled) { EXPECT_TRUE(cudf::is_ptds_enabled()); } #else diff --git a/cpp/tests/utilities_tests/lists_column_wrapper_tests.cpp b/cpp/tests/utilities_tests/lists_column_wrapper_tests.cpp index 35e86040e73..5e3fda5e6f7 100644 --- a/cpp/tests/utilities_tests/lists_column_wrapper_tests.cpp +++ b/cpp/tests/utilities_tests/lists_column_wrapper_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,17 +14,17 @@ * limitations under the License. */ -#include -#include -#include -#include - #include #include #include #include #include +#include +#include +#include +#include + #include #include diff --git a/cpp/tests/utilities_tests/span_tests.cu b/cpp/tests/utilities_tests/span_tests.cu index 2075c67a18a..30496728083 100644 --- a/cpp/tests/utilities_tests/span_tests.cu +++ b/cpp/tests/utilities_tests/span_tests.cu @@ -14,16 +14,17 @@ * limitations under the License. */ -#include -#include -#include -#include +#include "io/utilities/hostdevice_vector.hpp" #include #include #include #include +#include +#include +#include + #include #include diff --git a/cpp/tests/utilities_tests/type_check_tests.cpp b/cpp/tests/utilities_tests/type_check_tests.cpp index a8f7eaf5399..9c23798fce6 100644 --- a/cpp/tests/utilities_tests/type_check_tests.cpp +++ b/cpp/tests/utilities_tests/type_check_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,15 @@ * limitations under the License. */ +#include +#include +#include + #include #include #include #include -#include -#include -#include - namespace cudf { namespace test { diff --git a/python/cudf/udf_cpp/shim.cu b/python/cudf/udf_cpp/shim.cu index cabca3154be..5d0aabc3907 100644 --- a/python/cudf/udf_cpp/shim.cu +++ b/python/cudf/udf_cpp/shim.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,9 +23,8 @@ #include #include -#include - #include +#include #include #include diff --git a/python/cudf/udf_cpp/strings/src/strings/udf/udf_apis.cu b/python/cudf/udf_cpp/strings/src/strings/udf/udf_apis.cu index bedaa8e8fff..9cf86b5ea48 100644 --- a/python/cudf/udf_cpp/strings/src/strings/udf/udf_apis.cu +++ b/python/cudf/udf_cpp/strings/src/strings/udf/udf_apis.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,11 @@ * limitations under the License. */ -#include -#include - #include #include #include +#include +#include #include #include From 4948aa25557f07a65ce8d8d4afd8ded66576f3a8 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 21 Feb 2024 11:54:50 -1000 Subject: [PATCH 288/384] Fix Series.groupby.shift with a MultiIndex (#15098) closes #15087 closes #11259 (The typing annotation is incorrect, but I guess there needs to be a check somewhere to make `_copy_type_metadata` stricter) Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/15098 --- python/cudf/cudf/core/multiindex.py | 3 ++- python/cudf/cudf/tests/test_groupby.py | 11 ++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 9466d172eb1..df1b1ea10cd 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -2037,7 +2037,8 @@ def _copy_type_metadata( self: MultiIndex, other: MultiIndex, *, override_dtypes=None ) -> MultiIndex: res = super()._copy_type_metadata(other) - res._names = other._names + if isinstance(other, MultiIndex): + res._names = other._names return res @_cudf_nvtx_annotate diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index e8dbdd35352..c22e47bdf06 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -3308,7 +3308,6 @@ def test_groupby_pct_change(data, gkey, periods, fill_method): assert_eq(expected, actual) -@pytest.mark.xfail(reason="https://github.com/rapidsai/cudf/issues/11259") @pytest.mark.parametrize("periods", [-5, 5]) def test_groupby_pct_change_multiindex_dataframe(periods): gdf = cudf.DataFrame( @@ -3812,3 +3811,13 @@ def test_groupby_internal_groups_empty(gdf): gb = gdf.groupby("y")._groupby _, _, grouped_vals = gb.groups([]) assert grouped_vals == [] + + +def test_groupby_shift_series_multiindex(): + idx = cudf.MultiIndex.from_tuples( + [("a", 1), ("a", 2), ("b", 1), ("b", 2)], names=["f", "s"] + ) + ser = Series(range(4), index=idx) + result = ser.groupby(level=0).shift(1) + expected = ser.to_pandas().groupby(level=0).shift(1) + assert_eq(expected, result) From c8dc33c4470bab91d5ba38a311afde20827de8fc Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 21 Feb 2024 18:19:08 -0600 Subject: [PATCH 289/384] Upgrade to `arrow-14.0.2` (#15108) This PR upgrades `arrow` to `14.0.2` Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Jake Awe (https://github.com/AyodeAwe) - Robert Maynard (https://github.com/robertmaynard) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/15108 --- conda/environments/all_cuda-118_arch-x86_64.yaml | 10 +++++----- conda/environments/all_cuda-122_arch-x86_64.yaml | 10 +++++----- conda/recipes/cudf/meta.yaml | 2 +- conda/recipes/libcudf/conda_build_config.yaml | 2 +- cpp/cmake/thirdparty/get_arrow.cmake | 4 ++-- dependencies.yaml | 10 +++++----- python/cudf/pyproject.toml | 2 +- python/cudf_kafka/pyproject.toml | 2 +- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index fa4ef8ddf68..625e6c6e9db 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -40,15 +40,15 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython -- libarrow-acero==14.0.1.* -- libarrow-dataset==14.0.1.* -- libarrow==14.0.1.* +- libarrow-acero==14.0.2.* +- libarrow-dataset==14.0.2.* +- libarrow==14.0.2.* - libcufile-dev=1.4.0.31 - libcufile=1.4.0.31 - libcurand-dev=10.3.0.86 - libcurand=10.3.0.86 - libkvikio==24.4.* -- libparquet==14.0.1.* +- libparquet==14.0.2.* - librdkafka>=1.9.0,<1.10.0a0 - librmm==24.4.* - make @@ -71,7 +71,7 @@ dependencies: - pre-commit - protobuf>=4.21,<5 - ptxcompiler -- pyarrow==14.0.1.* +- pyarrow==14.0.2.* - pydata-sphinx-theme!=0.14.2 - pytest-benchmark - pytest-cases>=3.8.2 diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index c0950c7da98..871f00a0e8e 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -41,13 +41,13 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython -- libarrow-acero==14.0.1.* -- libarrow-dataset==14.0.1.* -- libarrow==14.0.1.* +- libarrow-acero==14.0.2.* +- libarrow-dataset==14.0.2.* +- libarrow==14.0.2.* - libcufile-dev - libcurand-dev - libkvikio==24.4.* -- libparquet==14.0.1.* +- libparquet==14.0.2.* - librdkafka>=1.9.0,<1.10.0a0 - librmm==24.4.* - make @@ -68,7 +68,7 @@ dependencies: - pip - pre-commit - protobuf>=4.21,<5 -- pyarrow==14.0.1.* +- pyarrow==14.0.2.* - pydata-sphinx-theme!=0.14.2 - pynvjitlink - pytest-benchmark diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 018380bbbd2..d32e6932598 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -65,7 +65,7 @@ requirements: - scikit-build-core >=0.7.0 - setuptools - dlpack >=0.5,<0.6.0a0 - - pyarrow ==14.0.1.* + - pyarrow ==14.0.2.* - libcudf ={{ version }} - rmm ={{ minor_version }} {% if cuda_major == "11" %} diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml index 9ed8c94f2bb..603cbd8fc2a 100644 --- a/conda/recipes/libcudf/conda_build_config.yaml +++ b/conda/recipes/libcudf/conda_build_config.yaml @@ -23,7 +23,7 @@ gtest_version: - ">=1.13.0" libarrow_version: - - "==14.0.1" + - "==14.0.2" dlpack_version: - ">=0.5,<0.6.0a0" diff --git a/cpp/cmake/thirdparty/get_arrow.cmake b/cpp/cmake/thirdparty/get_arrow.cmake index 1bead93c9cc..114a1f98a68 100644 --- a/cpp/cmake/thirdparty/get_arrow.cmake +++ b/cpp/cmake/thirdparty/get_arrow.cmake @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -441,7 +441,7 @@ if(NOT DEFINED CUDF_VERSION_Arrow) set(CUDF_VERSION_Arrow # This version must be kept in sync with the libarrow version pinned for builds in # dependencies.yaml. - 14.0.1 + 14.0.2 CACHE STRING "The version of Arrow to find (or build)" ) endif() diff --git a/dependencies.yaml b/dependencies.yaml index c4c2cd3c764..c5797fbe40a 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -261,7 +261,7 @@ dependencies: - &numpy numpy>=1.21 # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. - - pyarrow==14.0.1.* + - pyarrow==14.0.2.* - output_types: conda packages: - scikit-build-core>=0.7.0 @@ -306,10 +306,10 @@ dependencies: packages: # Hard pin the Arrow patch version used during the build. This must # be kept in sync with the version pinned in get_arrow.cmake. - - libarrow-acero==14.0.1.* - - libarrow-dataset==14.0.1.* - - libarrow==14.0.1.* - - libparquet==14.0.1.* + - libarrow-acero==14.0.2.* + - libarrow-dataset==14.0.2.* + - libarrow==14.0.2.* + - libparquet==14.0.2.* libarrow_run: common: - output_types: conda diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 7f2d8e438d2..82ac84a4022 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -8,7 +8,7 @@ requires = [ "ninja", "numpy>=1.21", "protoc-wheel", - "pyarrow==14.0.1.*", + "pyarrow==14.0.2.*", "rmm==24.4.*", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index af59efa9777..216d83940ce 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -7,7 +7,7 @@ requires = [ "cython>=3.0.3", "ninja", "numpy>=1.21", - "pyarrow==14.0.1.*", + "pyarrow==14.0.2.*", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 90b763cc666c424f919ec8dcb1a0ccb064dde35e Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Thu, 22 Feb 2024 00:41:19 -0800 Subject: [PATCH 290/384] Fix bugs in handling of delta encodings (#15075) Part of #14938 was fixing two bugs discovered during testing. One is in the encoding of DELTA_BINARY_PACKED data where the first non-null value in a page to be encoded is not in the first batch of 129 values. The second is an error in decoding of DELTA_BYTE_ARRAY pages where, again, the first non-null value is not in the first block to be decoded. This PR includes a test for the former, but the latter cannot be easily tested because the python API still lacks `skip_rows`, and we cannot generate DELTA_BYTE_ARRAY encoded data without the changes in #14938. A test for the latter will be added later, but the fix has been validated with data on hand locally. Authors: - Ed Seidl (https://github.com/etseidl) Approvers: - Vukasin Milovanovic (https://github.com/vuule) - David Wendt (https://github.com/davidwendt) URL: https://github.com/rapidsai/cudf/pull/15075 --- cpp/src/io/parquet/delta_enc.cuh | 4 ++-- cpp/src/io/parquet/page_string_decode.cu | 3 +++ cpp/tests/io/parquet_writer_test.cpp | 26 ++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/cpp/src/io/parquet/delta_enc.cuh b/cpp/src/io/parquet/delta_enc.cuh index f90d364f5eb..49f4ccedbf0 100644 --- a/cpp/src/io/parquet/delta_enc.cuh +++ b/cpp/src/io/parquet/delta_enc.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -201,7 +201,7 @@ class delta_binary_packer { if (is_valid) { _buffer[delta::rolling_idx(pos + _current_idx + _values_in_buffer)] = value; } __syncthreads(); - if (threadIdx.x == 0) { + if (num_valid > 0 && threadIdx.x == 0) { _values_in_buffer += num_valid; // if first pass write header if (_current_idx == 0) { diff --git a/cpp/src/io/parquet/page_string_decode.cu b/cpp/src/io/parquet/page_string_decode.cu index d652a43d097..5cd8205b4ba 100644 --- a/cpp/src/io/parquet/page_string_decode.cu +++ b/cpp/src/io/parquet/page_string_decode.cu @@ -535,6 +535,9 @@ __device__ thrust::pair totalDeltaByteArraySize(uint8_t const* d uint32_t const idx = db->current_value_idx + i + lane_id; if (idx >= start_value && idx < end_value && idx < db->value_count) { lane_sum += db->value[rolling_index(idx)]; + } + // need lane_max over all values, not just in bounds + if (idx < db->value_count) { lane_max = max(lane_max, db->value[rolling_index(idx)]); } } diff --git a/cpp/tests/io/parquet_writer_test.cpp b/cpp/tests/io/parquet_writer_test.cpp index 34061cb7bf8..62a24bf0a73 100644 --- a/cpp/tests/io/parquet_writer_test.cpp +++ b/cpp/tests/io/parquet_writer_test.cpp @@ -1426,6 +1426,32 @@ TEST_F(ParquetWriterTest, RowGroupMetadata) static_cast(num_rows * sizeof(column_type))); } +TEST_F(ParquetWriterTest, DeltaBinaryStartsWithNulls) +{ + // test that the DELTA_BINARY_PACKED writer can properly encode a column that begins with + // more than 129 nulls + constexpr int num_rows = 500; + constexpr int num_nulls = 150; + + auto const ones = thrust::make_constant_iterator(1); + auto valids = cudf::detail::make_counting_transform_iterator( + 0, [num_nulls](auto i) { return i >= num_nulls; }); + auto const col = cudf::test::fixed_width_column_wrapper{ones, ones + num_rows, valids}; + auto const expected = table_view({col}); + + auto const filepath = temp_env->get_temp_filepath("DeltaBinaryStartsWithNulls.parquet"); + cudf::io::parquet_writer_options out_opts = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, expected) + .write_v2_headers(true) + .dictionary_policy(cudf::io::dictionary_policy::NEVER); + cudf::io::write_parquet(out_opts); + + cudf::io::parquet_reader_options in_opts = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}); + auto result = cudf::io::read_parquet(in_opts); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result.tbl->view()); +} + ///////////////////////////////////////////////////////////// // custom mem mapped data sink that supports device writes template From 6f6e521257dce5732eea7b6b9d56243f8b0a69cc Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Thu, 22 Feb 2024 08:58:35 -0500 Subject: [PATCH 291/384] Split out strings/replace.cu and rework its gtests (#15054) Splitting out changes in PR #14824 to make it easier to review. The changes here simply move `replace_slice()` and `replace_nulls()` from `replace.cu` into their own source files. The detail functions have been simplified removing the template argument that was only needed for unit tests. The gtests were reworked to force calling either row-parallel or character-parallel based on the data input instead of being executed directly. This simplified the internal logic which had duplicate parameter checking. The `cudf::strings::detail::replace_nulls()` is also fixed to use the appropriate `make_offsets_child_column` utitlity. The PR #14824 changes will add large strings support to `cudf::strings::replace()`. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Robert Maynard (https://github.com/robertmaynard) - Nghia Truong (https://github.com/ttnghia) - https://github.com/nvdbaranec URL: https://github.com/rapidsai/cudf/pull/15054 --- cpp/CMakeLists.txt | 2 + cpp/include/cudf/strings/detail/replace.hpp | 45 ++-- cpp/src/strings/replace/replace.cu | 190 +--------------- cpp/src/strings/replace/replace_nulls.cu | 81 +++++++ cpp/src/strings/replace/replace_slice.cu | 117 ++++++++++ cpp/tests/strings/replace_tests.cpp | 239 +++++++++++--------- 6 files changed, 352 insertions(+), 322 deletions(-) create mode 100644 cpp/src/strings/replace/replace_nulls.cu create mode 100644 cpp/src/strings/replace/replace_slice.cu diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 078de27f0ea..58a43c1def1 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -587,7 +587,9 @@ add_library( src/strings/replace/multi.cu src/strings/replace/multi_re.cu src/strings/replace/replace.cu + src/strings/replace/replace_nulls.cu src/strings/replace/replace_re.cu + src/strings/replace/replace_slice.cu src/strings/reverse.cu src/strings/scan/scan_inclusive.cu src/strings/search/findall.cu diff --git a/cpp/include/cudf/strings/detail/replace.hpp b/cpp/include/cudf/strings/detail/replace.hpp index aa6fb2feb3d..28027291b28 100644 --- a/cpp/include/cudf/strings/detail/replace.hpp +++ b/cpp/include/cudf/strings/detail/replace.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,23 +26,10 @@ namespace cudf { namespace strings { namespace detail { -/** - * @brief The type of algorithm to use for a replace operation. - */ -enum class replace_algorithm { - AUTO, ///< Automatically choose the algorithm based on heuristics - ROW_PARALLEL, ///< Row-level parallelism - CHAR_PARALLEL ///< Character-level parallelism -}; - /** * @copydoc cudf::strings::replace(strings_column_view const&, string_scalar const&, - * string_scalar const&, int32_t, rmm::mr::device_memory_resource*) - * - * @tparam alg Replacement algorithm to use - * @param[in] stream CUDA stream used for device memory operations and kernel launches. + * string_scalar const&, int32_t, rmm::cuda_stream_view, rmm::mr::device_memory_resource*) */ -template std::unique_ptr replace(strings_column_view const& strings, string_scalar const& target, string_scalar const& repl, @@ -50,24 +37,9 @@ std::unique_ptr replace(strings_column_view const& strings, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr); -/** - * @copydoc cudf::strings::replace_slice(strings_column_view const&, string_scalar const&, - * size_type. size_type, rmm::mr::device_memory_resource*) - * - * @param[in] stream CUDA stream used for device memory operations and kernel launches. - */ -std::unique_ptr replace_slice(strings_column_view const& strings, - string_scalar const& repl, - size_type start, - size_type stop, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr); - /** * @copydoc cudf::strings::replace(strings_column_view const&, strings_column_view const&, - * strings_column_view const&, rmm::mr::device_memory_resource*) - * - * @param[in] stream CUDA stream used for device memory operations and kernel launches. + * strings_column_view const&, rmm::cuda_stream_view, rmm::mr::device_memory_resource*) */ std::unique_ptr replace(strings_column_view const& strings, strings_column_view const& targets, @@ -98,6 +70,17 @@ std::unique_ptr replace_nulls(strings_column_view const& strings, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr); +/** + * @copydoc cudf::strings::replace_slice(strings_column_view const&, string_scalar const&, + * size_type, size_type, rmm::cuda_stream_view, rmm::mr::device_memory_resource*) + */ +std::unique_ptr replace_slice(strings_column_view const& strings, + string_scalar const& repl, + size_type start, + size_type stop, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr); + } // namespace detail } // namespace strings } // namespace cudf diff --git a/cpp/src/strings/replace/replace.cu b/cpp/src/strings/replace/replace.cu index d68ec84f68c..2d255e57686 100644 --- a/cpp/src/strings/replace/replace.cu +++ b/cpp/src/strings/replace/replace.cu @@ -542,17 +542,12 @@ std::unique_ptr replace_row_parallel(strings_column_view const& strings, } // namespace -/** - * @copydoc cudf::strings::detail::replace(strings_column_view const&, string_scalar const&, - * string_scalar const&, int32_t, rmm::cuda_stream_view, rmm::mr::device_memory_resource*) - */ -template <> -std::unique_ptr replace(strings_column_view const& strings, - string_scalar const& target, - string_scalar const& repl, - int32_t maxrepl, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) +std::unique_ptr replace(strings_column_view const& strings, + string_scalar const& target, + string_scalar const& repl, + int32_t maxrepl, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) { if (strings.is_empty()) return make_empty_column(type_id::STRING); if (maxrepl == 0) return std::make_unique(strings.parent(), stream, mr); @@ -584,168 +579,6 @@ std::unique_ptr replace(strings_column_view con strings, chars_start, chars_end, d_target, d_repl, maxrepl, stream, mr); } -template <> -std::unique_ptr replace( - strings_column_view const& strings, - string_scalar const& target, - string_scalar const& repl, - int32_t maxrepl, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) -{ - if (strings.is_empty()) return make_empty_column(type_id::STRING); - if (maxrepl == 0) return std::make_unique(strings.parent(), stream, mr); - CUDF_EXPECTS(repl.is_valid(stream), "Parameter repl must be valid."); - CUDF_EXPECTS(target.is_valid(stream), "Parameter target must be valid."); - CUDF_EXPECTS(target.size() > 0, "Parameter target must not be empty string."); - - string_view d_target(target.data(), target.size()); - string_view d_repl(repl.data(), repl.size()); - - // determine range of characters in the base column - auto const strings_count = strings.size(); - auto const offset_count = strings_count + 1; - auto const d_offsets = strings.offsets_begin(); - size_type chars_start = (strings.offset() == 0) ? 0 - : cudf::detail::get_value( - strings.offsets(), strings.offset(), stream); - size_type chars_end = (offset_count == strings.offsets().size()) - ? strings.chars_size(stream) - : cudf::detail::get_value( - strings.offsets(), strings.offset() + strings_count, stream); - return replace_char_parallel( - strings, chars_start, chars_end, d_target, d_repl, maxrepl, stream, mr); -} - -template <> -std::unique_ptr replace( - strings_column_view const& strings, - string_scalar const& target, - string_scalar const& repl, - int32_t maxrepl, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) -{ - if (strings.is_empty()) return make_empty_column(type_id::STRING); - if (maxrepl == 0) return std::make_unique(strings.parent(), stream, mr); - CUDF_EXPECTS(repl.is_valid(stream), "Parameter repl must be valid."); - CUDF_EXPECTS(target.is_valid(stream), "Parameter target must be valid."); - CUDF_EXPECTS(target.size() > 0, "Parameter target must not be empty string."); - - string_view d_target(target.data(), target.size()); - string_view d_repl(repl.data(), repl.size()); - return replace_row_parallel(strings, d_target, d_repl, maxrepl, stream, mr); -} - -namespace { -/** - * @brief Function logic for the replace_slice API. - * - * This will perform a replace_slice operation on each string. - */ -struct replace_slice_fn { - column_device_view const d_strings; - string_view const d_repl; - size_type const start; - size_type const stop; - int32_t* d_offsets{}; - char* d_chars{}; - - __device__ void operator()(size_type idx) - { - if (d_strings.is_null(idx)) { - if (!d_chars) d_offsets[idx] = 0; - return; - } - auto const d_str = d_strings.element(idx); - auto const length = d_str.length(); - char const* in_ptr = d_str.data(); - auto const begin = d_str.byte_offset(((start < 0) || (start > length) ? length : start)); - auto const end = d_str.byte_offset(((stop < 0) || (stop > length) ? length : stop)); - - if (d_chars) { - char* out_ptr = d_chars + d_offsets[idx]; - - out_ptr = copy_and_increment(out_ptr, in_ptr, begin); // copy beginning - out_ptr = copy_string(out_ptr, d_repl); // insert replacement - out_ptr = copy_and_increment(out_ptr, // copy end - in_ptr + end, - d_str.size_bytes() - end); - } else { - d_offsets[idx] = d_str.size_bytes() + d_repl.size_bytes() - (end - begin); - } - } -}; - -} // namespace - -std::unique_ptr replace_slice(strings_column_view const& strings, - string_scalar const& repl, - size_type start, - size_type stop, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) -{ - if (strings.is_empty()) return make_empty_column(type_id::STRING); - CUDF_EXPECTS(repl.is_valid(stream), "Parameter repl must be valid."); - if (stop > 0) CUDF_EXPECTS(start <= stop, "Parameter start must be less than or equal to stop."); - - string_view d_repl(repl.data(), repl.size()); - - auto d_strings = column_device_view::create(strings.parent(), stream); - - // this utility calls the given functor to build the offsets and chars columns - auto [offsets_column, chars_column] = cudf::strings::detail::make_strings_children( - replace_slice_fn{*d_strings, d_repl, start, stop}, strings.size(), stream, mr); - - return make_strings_column(strings.size(), - std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), - strings.null_count(), - cudf::detail::copy_bitmask(strings.parent(), stream, mr)); -} - -std::unique_ptr replace_nulls(strings_column_view const& strings, - string_scalar const& repl, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) -{ - size_type strings_count = strings.size(); - if (strings_count == 0) return make_empty_column(type_id::STRING); - CUDF_EXPECTS(repl.is_valid(stream), "Parameter repl must be valid."); - - string_view d_repl(repl.data(), repl.size()); - - auto strings_column = column_device_view::create(strings.parent(), stream); - auto d_strings = *strings_column; - - // build offsets column - auto offsets_transformer_itr = thrust::make_transform_iterator( - thrust::make_counting_iterator(0), - cuda::proclaim_return_type([d_strings, d_repl] __device__(size_type idx) { - return d_strings.is_null(idx) ? d_repl.size_bytes() - : d_strings.element(idx).size_bytes(); - })); - auto [offsets_column, bytes] = cudf::detail::make_offsets_child_column( - offsets_transformer_itr, offsets_transformer_itr + strings_count, stream, mr); - auto d_offsets = offsets_column->view().data(); - - // build chars column - rmm::device_uvector chars(bytes, stream, mr); - auto d_chars = chars.data(); - thrust::for_each_n(rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - strings_count, - [d_strings, d_repl, d_offsets, d_chars] __device__(size_type idx) { - string_view d_str = d_repl; - if (!d_strings.is_null(idx)) d_str = d_strings.element(idx); - memcpy(d_chars + d_offsets[idx], d_str.data(), d_str.size_bytes()); - }); - - return make_strings_column( - strings_count, std::move(offsets_column), chars.release(), 0, rmm::device_buffer{}); -} - } // namespace detail // external API @@ -761,16 +594,5 @@ std::unique_ptr replace(strings_column_view const& strings, return detail::replace(strings, target, repl, maxrepl, stream, mr); } -std::unique_ptr replace_slice(strings_column_view const& strings, - string_scalar const& repl, - size_type start, - size_type stop, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) -{ - CUDF_FUNC_RANGE(); - return detail::replace_slice(strings, repl, start, stop, stream, mr); -} - } // namespace strings } // namespace cudf diff --git a/cpp/src/strings/replace/replace_nulls.cu b/cpp/src/strings/replace/replace_nulls.cu new file mode 100644 index 00000000000..26fb1c7819f --- /dev/null +++ b/cpp/src/strings/replace/replace_nulls.cu @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +namespace cudf { +namespace strings { +namespace detail { + +std::unique_ptr replace_nulls(strings_column_view const& strings, + string_scalar const& repl, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) +{ + size_type strings_count = strings.size(); + if (strings_count == 0) return make_empty_column(type_id::STRING); + CUDF_EXPECTS(repl.is_valid(stream), "Parameter repl must be valid."); + + string_view d_repl(repl.data(), repl.size()); + + auto strings_column = column_device_view::create(strings.parent(), stream); + auto d_strings = *strings_column; + + // build offsets column + auto offsets_transformer_itr = cudf::detail::make_counting_transform_iterator( + 0, cuda::proclaim_return_type([d_strings, d_repl] __device__(size_type idx) { + return d_strings.is_null(idx) ? d_repl.size_bytes() + : d_strings.element(idx).size_bytes(); + })); + auto [offsets_column, bytes] = cudf::strings::detail::make_offsets_child_column( + offsets_transformer_itr, offsets_transformer_itr + strings_count, stream, mr); + auto d_offsets = offsets_column->view().data(); + + // build chars column + rmm::device_uvector chars(bytes, stream, mr); + auto d_chars = chars.data(); + thrust::for_each_n(rmm::exec_policy(stream), + thrust::make_counting_iterator(0), + strings_count, + [d_strings, d_repl, d_offsets, d_chars] __device__(size_type idx) { + string_view d_str = d_repl; + if (!d_strings.is_null(idx)) d_str = d_strings.element(idx); + memcpy(d_chars + d_offsets[idx], d_str.data(), d_str.size_bytes()); + }); + + return make_strings_column( + strings_count, std::move(offsets_column), chars.release(), 0, rmm::device_buffer{}); +} + +} // namespace detail +} // namespace strings +} // namespace cudf diff --git a/cpp/src/strings/replace/replace_slice.cu b/cpp/src/strings/replace/replace_slice.cu new file mode 100644 index 00000000000..4321f78d2d5 --- /dev/null +++ b/cpp/src/strings/replace/replace_slice.cu @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +namespace cudf { +namespace strings { +namespace detail { +namespace { +/** + * @brief Function logic for the replace_slice API. + * + * This will perform a replace_slice operation on each string. + */ +struct replace_slice_fn { + column_device_view const d_strings; + string_view const d_repl; + size_type const start; + size_type const stop; + size_type* d_offsets{}; + char* d_chars{}; + + __device__ void operator()(size_type idx) + { + if (d_strings.is_null(idx)) { + if (!d_chars) d_offsets[idx] = 0; + return; + } + auto const d_str = d_strings.element(idx); + auto const length = d_str.length(); + char const* in_ptr = d_str.data(); + auto const begin = d_str.byte_offset(((start < 0) || (start > length) ? length : start)); + auto const end = d_str.byte_offset(((stop < 0) || (stop > length) ? length : stop)); + + if (d_chars) { + char* out_ptr = d_chars + d_offsets[idx]; + + out_ptr = copy_and_increment(out_ptr, in_ptr, begin); // copy beginning + out_ptr = copy_string(out_ptr, d_repl); // insert replacement + out_ptr = copy_and_increment(out_ptr, // copy end + in_ptr + end, + d_str.size_bytes() - end); + } else { + d_offsets[idx] = d_str.size_bytes() + d_repl.size_bytes() - (end - begin); + } + } +}; + +} // namespace + +std::unique_ptr replace_slice(strings_column_view const& strings, + string_scalar const& repl, + size_type start, + size_type stop, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) +{ + if (strings.is_empty()) return make_empty_column(type_id::STRING); + CUDF_EXPECTS(repl.is_valid(stream), "Parameter repl must be valid."); + if (stop > 0) CUDF_EXPECTS(start <= stop, "Parameter start must be less than or equal to stop."); + + string_view d_repl(repl.data(), repl.size()); + + auto d_strings = column_device_view::create(strings.parent(), stream); + + // this utility calls the given functor to build the offsets and chars columns + auto [offsets_column, chars_column] = cudf::strings::detail::make_strings_children( + replace_slice_fn{*d_strings, d_repl, start, stop}, strings.size(), stream, mr); + + return make_strings_column(strings.size(), + std::move(offsets_column), + std::move(chars_column->release().data.release()[0]), + strings.null_count(), + cudf::detail::copy_bitmask(strings.parent(), stream, mr)); +} +} // namespace detail + +std::unique_ptr replace_slice(strings_column_view const& strings, + string_scalar const& repl, + size_type start, + size_type stop, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) +{ + CUDF_FUNC_RANGE(); + return detail::replace_slice(strings, repl, start, stop, stream, mr); +} + +} // namespace strings +} // namespace cudf diff --git a/cpp/tests/strings/replace_tests.cpp b/cpp/tests/strings/replace_tests.cpp index f04bb832f09..726d9f95c7d 100644 --- a/cpp/tests/strings/replace_tests.cpp +++ b/cpp/tests/strings/replace_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,17 +20,12 @@ #include #include -#include +#include #include #include -#include -#include - #include -using algorithm = cudf::strings::detail::replace_algorithm; - struct StringsReplaceTest : public cudf::test::BaseFixture { cudf::test::strings_column_wrapper build_corpus() { @@ -47,6 +42,13 @@ struct StringsReplaceTest : public cudf::test::BaseFixture { h_strings.end(), thrust::make_transform_iterator(h_strings.begin(), [](auto str) { return str != nullptr; })); } + + std::unique_ptr build_large(cudf::column_view const& first, + cudf::column_view const& remaining) + { + return cudf::strings::concatenate(cudf::table_view( + {first, remaining, remaining, remaining, remaining, remaining, remaining, remaining})); + } }; TEST_F(StringsReplaceTest, Replace) @@ -64,26 +66,23 @@ TEST_F(StringsReplaceTest, Replace) cudf::test::strings_column_wrapper expected( h_expected.begin(), h_expected.end(), cudf::test::iterators::nulls_from_nullptrs(h_expected)); - auto stream = cudf::get_default_stream(); - auto mr = rmm::mr::get_current_device_resource(); + auto target = cudf::string_scalar("the "); + auto replacement = cudf::string_scalar("++++ "); - auto results = - cudf::strings::replace(strings_view, cudf::string_scalar("the "), cudf::string_scalar("++++ ")); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); - results = cudf::strings::detail::replace( - strings_view, cudf::string_scalar("the "), cudf::string_scalar("++++ "), -1, stream, mr); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); - results = cudf::strings::detail::replace( - strings_view, cudf::string_scalar("the "), cudf::string_scalar("++++ "), -1, stream, mr); + auto results = cudf::strings::replace(strings_view, target, replacement); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + + auto input_large = build_large(input, input); + strings_view = cudf::strings_column_view(input_large->view()); + auto expected_large = build_large(expected, expected); + results = cudf::strings::replace(strings_view, target, replacement); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, *expected_large); } TEST_F(StringsReplaceTest, ReplaceReplLimit) { auto input = build_corpus(); auto strings_view = cudf::strings_column_view(input); - auto stream = cudf::get_default_stream(); - auto mr = rmm::mr::get_current_device_resource(); // only remove the first occurrence of 'the ' std::vector h_expected{"quick brown fox jumps over the lazy dog", @@ -95,15 +94,16 @@ TEST_F(StringsReplaceTest, ReplaceReplLimit) nullptr}; cudf::test::strings_column_wrapper expected( h_expected.begin(), h_expected.end(), cudf::test::iterators::nulls_from_nullptrs(h_expected)); - auto results = - cudf::strings::replace(strings_view, cudf::string_scalar("the "), cudf::string_scalar(""), 1); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); - results = cudf::strings::detail::replace( - strings_view, cudf::string_scalar("the "), cudf::string_scalar(""), 1, stream, mr); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); - results = cudf::strings::detail::replace( - strings_view, cudf::string_scalar("the "), cudf::string_scalar(""), 1, stream, mr); + auto target = cudf::string_scalar("the "); + auto replacement = cudf::string_scalar(""); + auto results = cudf::strings::replace(strings_view, target, replacement, 1); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + + auto input_large = build_large(input, input); + strings_view = cudf::strings_column_view(input_large->view()); + auto expected_large = build_large(expected, input); + results = cudf::strings::replace(strings_view, target, replacement, 1); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, *expected_large); } TEST_F(StringsReplaceTest, ReplaceReplLimitInputSliced) @@ -119,22 +119,28 @@ TEST_F(StringsReplaceTest, ReplaceReplLimitInputSliced) nullptr}; cudf::test::strings_column_wrapper expected( h_expected.begin(), h_expected.end(), cudf::test::iterators::nulls_from_nullptrs(h_expected)); - auto stream = cudf::get_default_stream(); - auto mr = rmm::mr::get_current_device_resource(); std::vector slice_indices{0, 2, 2, 3, 3, 7}; auto sliced_strings = cudf::slice(input, slice_indices); auto sliced_expected = cudf::slice(expected, slice_indices); + + auto input_large = build_large(input, input); + auto expected_large = build_large(expected, input); + + auto sliced_large = cudf::slice(input_large->view(), slice_indices); + auto sliced_expected_large = cudf::slice(expected_large->view(), slice_indices); + + auto target = cudf::string_scalar(" "); + auto replacement = cudf::string_scalar("--"); + for (size_t i = 0; i < sliced_strings.size(); ++i) { auto strings_view = cudf::strings_column_view(sliced_strings[i]); - auto results = - cudf::strings::replace(strings_view, cudf::string_scalar(" "), cudf::string_scalar("--"), 2); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, sliced_expected[i]); - results = cudf::strings::detail::replace( - strings_view, cudf::string_scalar(" "), cudf::string_scalar("--"), 2, stream, mr); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, sliced_expected[i]); - results = cudf::strings::detail::replace( - strings_view, cudf::string_scalar(" "), cudf::string_scalar("--"), 2, stream, mr); + auto results = cudf::strings::replace(strings_view, target, replacement, 2); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, sliced_expected[i]); + + strings_view = cudf::strings_column_view(sliced_large[i]); + results = + cudf::strings::replace(strings_view, cudf::string_scalar(" "), cudf::string_scalar("--"), 2); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, sliced_expected_large[i]); } } @@ -158,68 +164,56 @@ TEST_F(StringsReplaceTest, ReplaceTargetOverlap) cudf::test::strings_column_wrapper expected( h_expected.begin(), h_expected.end(), cudf::test::iterators::nulls_from_nullptrs(h_expected)); - auto stream = cudf::get_default_stream(); - auto mr = rmm::mr::get_current_device_resource(); + auto target = cudf::string_scalar("+++"); + auto replacement = cudf::string_scalar("plus "); - auto results = - cudf::strings::replace(strings_view, cudf::string_scalar("+++"), cudf::string_scalar("plus ")); + auto results = cudf::strings::replace(strings_view, target, replacement); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); - results = cudf::strings::detail::replace( - strings_view, cudf::string_scalar("+++"), cudf::string_scalar("plus "), -1, stream, mr); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); - results = cudf::strings::detail::replace( - strings_view, cudf::string_scalar("+++"), cudf::string_scalar("plus "), -1, stream, mr); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + auto input_large = build_large(input->view(), input->view()); + strings_view = cudf::strings_column_view(input_large->view()); + auto expected_large = build_large(expected, expected); + + results = cudf::strings::replace(strings_view, target, replacement); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, *expected_large); } TEST_F(StringsReplaceTest, ReplaceTargetOverlapsStrings) { auto input = build_corpus(); auto strings_view = cudf::strings_column_view(input); - auto stream = cudf::get_default_stream(); - auto mr = rmm::mr::get_current_device_resource(); // replace all occurrences of 'dogthe' with '+' + auto target = cudf::string_scalar("dogthe"); + auto replacement = cudf::string_scalar("+"); + // should not replace anything unless it incorrectly matches across a string boundary - auto results = - cudf::strings::replace(strings_view, cudf::string_scalar("dogthe"), cudf::string_scalar("+")); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, input); - results = cudf::strings::detail::replace( - strings_view, cudf::string_scalar("dogthe"), cudf::string_scalar("+"), -1, stream, mr); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, input); - results = cudf::strings::detail::replace( - strings_view, cudf::string_scalar("dogthe"), cudf::string_scalar("+"), -1, stream, mr); + auto results = cudf::strings::replace(strings_view, target, replacement); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, input); + + auto input_large = cudf::strings::concatenate( + cudf::table_view({input, input, input, input, input, input, input, input}), + cudf::string_scalar(" ")); + strings_view = cudf::strings_column_view(input_large->view()); + results = cudf::strings::replace(strings_view, target, replacement); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, *input_large); } -TEST_F(StringsReplaceTest, ReplaceNullInput) +TEST_F(StringsReplaceTest, ReplaceAllNullInput) { std::vector h_null_strings(128); auto input = cudf::test::strings_column_wrapper( h_null_strings.begin(), h_null_strings.end(), thrust::make_constant_iterator(false)); auto strings_view = cudf::strings_column_view(input); - auto stream = cudf::get_default_stream(); - auto mr = rmm::mr::get_current_device_resource(); - // replace all occurrences of '+' with '' - // should not replace anything as input is all null auto results = cudf::strings::replace(strings_view, cudf::string_scalar("+"), cudf::string_scalar("")); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, input); - results = cudf::strings::detail::replace( - strings_view, cudf::string_scalar("+"), cudf::string_scalar(""), -1, stream, mr); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, input); - results = cudf::strings::detail::replace( - strings_view, cudf::string_scalar("+"), cudf::string_scalar(""), -1, stream, mr); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, input); } TEST_F(StringsReplaceTest, ReplaceEndOfString) { auto input = build_corpus(); auto strings_view = cudf::strings_column_view(input); - auto stream = cudf::get_default_stream(); - auto mr = rmm::mr::get_current_device_resource(); // replace all occurrences of 'in' with ' ' std::vector h_expected{"the quick brown fox jumps over the lazy dog", @@ -233,39 +227,56 @@ TEST_F(StringsReplaceTest, ReplaceEndOfString) cudf::test::strings_column_wrapper expected( h_expected.begin(), h_expected.end(), cudf::test::iterators::nulls_from_nullptrs(h_expected)); - auto results = - cudf::strings::replace(strings_view, cudf::string_scalar("in"), cudf::string_scalar(" ")); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + auto target = cudf::string_scalar("in"); + auto replacement = cudf::string_scalar(" "); - results = cudf::strings::detail::replace( - strings_view, cudf::string_scalar("in"), cudf::string_scalar(" "), -1, stream, mr); + auto results = cudf::strings::replace(strings_view, target, replacement); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); - results = cudf::strings::detail::replace( - strings_view, cudf::string_scalar("in"), cudf::string_scalar(" "), -1, stream, mr); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + auto input_large = build_large(input, input); + strings_view = cudf::strings_column_view(input_large->view()); + auto expected_large = build_large(expected, expected); + results = cudf::strings::replace(strings_view, target, replacement); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, *expected_large); } TEST_F(StringsReplaceTest, ReplaceAdjacentMultiByteTarget) { - auto input = cudf::test::strings_column_wrapper({"ééééééé", "eéeéeée", "eeeeeee"}); + auto input = cudf::test::strings_column_wrapper({"ééééééééééééééééééééé", + "eéeéeéeeéeéeéeeéeéeée", + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"}); auto strings_view = cudf::strings_column_view(input); // replace all occurrences of 'é' with 'e' - cudf::test::strings_column_wrapper expected({"eeeeeee", "eeeeeee", "eeeeeee"}); + cudf::test::strings_column_wrapper expected({"eeeeeeeeeeeeeeeeeeeee", + "eeeeeeeeeeeeeeeeeeeee", + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"}); - auto stream = cudf::get_default_stream(); - auto mr = rmm::mr::get_current_device_resource(); + auto target = cudf::string_scalar("é"); + auto replacement = cudf::string_scalar("e"); - auto target = cudf::string_scalar("é", true, stream); - auto repl = cudf::string_scalar("e", true, stream); - auto results = cudf::strings::replace(strings_view, target, repl); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); - results = cudf::strings::detail::replace( - strings_view, target, repl, -1, stream, mr); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); - results = cudf::strings::detail::replace( - strings_view, target, repl, -1, stream, mr); + auto results = cudf::strings::replace(strings_view, target, replacement); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + + auto input_large = build_large(input, input); + strings_view = cudf::strings_column_view(input_large->view()); + auto expected_large = build_large(expected, expected); + results = cudf::strings::replace(strings_view, target, replacement); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, *expected_large); +} + +TEST_F(StringsReplaceTest, ReplaceErrors) +{ + auto input = cudf::test::strings_column_wrapper({"this column intentionally left blank"}); + + auto target = cudf::string_scalar(" "); + auto replacement = cudf::string_scalar("_"); + auto null_input = cudf::string_scalar("", false); + auto empty_input = cudf::string_scalar(""); + auto sv = cudf::strings_column_view(input); + + EXPECT_THROW(cudf::strings::replace(sv, target, null_input), cudf::logic_error); + EXPECT_THROW(cudf::strings::replace(sv, null_input, replacement), cudf::logic_error); + EXPECT_THROW(cudf::strings::replace(sv, empty_input, replacement), cudf::logic_error); } TEST_F(StringsReplaceTest, ReplaceSlice) @@ -369,22 +380,30 @@ TEST_F(StringsReplaceTest, ReplaceMulti) TEST_F(StringsReplaceTest, ReplaceMultiLong) { - // The length of the strings are to trigger the code path governed by the AVG_CHAR_BYTES_THRESHOLD - // setting in the multi.cu. + // The length of the strings are to trigger the code path governed by the + // AVG_CHAR_BYTES_THRESHOLD setting in the multi.cu. auto input = cudf::test::strings_column_wrapper( {"This string needs to be very long to trigger the long-replace internal functions. " "This string needs to be very long to trigger the long-replace internal functions. " "This string needs to be very long to trigger the long-replace internal functions. " "This string needs to be very long to trigger the long-replace internal functions.", - "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012" - "345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345" - "678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678" - "901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901" + "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890" + "12" + "3456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123" + "45" + "6789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456" + "78" + "9012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" + "01" "2345678901234567890123456789", - "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012" - "345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345" - "678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678" - "901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901" + "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890" + "12" + "3456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123" + "45" + "6789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456" + "78" + "9012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" + "01" "2345678901234567890123456789", "Test string for overlap check: bananaápple bananá ápplebananá banápple ápple bananá " "Test string for overlap check: bananaápple bananá ápplebananá banápple ápple bananá " @@ -410,11 +429,15 @@ TEST_F(StringsReplaceTest, ReplaceMultiLong) "This string needs to be very long to trigger the long-replace internal functions. " "This string needs to be very long to trigger the long-replace internal functions. " "This string needs to be very long to trigger the long-replace internal functions.", - "0123456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456" - "x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x" + "0123456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x234" + "56" + "x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x2345" + "6x" "23456x23456x23456x23456x23456x23456x23456x23456x23456x23456$$9", - "0123456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456" - "x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x" + "0123456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x234" + "56" + "x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x23456x2345" + "6x" "23456x23456x23456x23456x23456x23456x23456x23456x23456x23456$$9", "Test string for overlap check: bananaavocado PEAR avocadoPEAR banavocado avocado PEAR " "Test string for overlap check: bananaavocado PEAR avocadoPEAR banavocado avocado PEAR " @@ -445,8 +468,10 @@ TEST_F(StringsReplaceTest, ReplaceMultiLong) "23456*23456*23456*23456*23456*23456*23456*23456*23456*23456*23456*23456*23456*23456*" "23456*23456*23456*23456*23456*23456*23456*23456*23456*23456*23456*23456*9", "Test string for overlap check: banana* * ** ban* * * Test string for overlap check: " - "banana* * ** ban* * * Test string for overlap check: banana* * ** ban* * * Test string for " - "overlap check: banana* * ** ban* * * Test string for overlap check: banana* * ** ban* * *", + "banana* * ** ban* * * Test string for overlap check: banana* * ** ban* * * Test string " + "for " + "overlap check: banana* * ** ban* * * Test string for overlap check: banana* * ** ban* * " + "*", "", ""}, {1, 1, 1, 1, 0, 1}); From 296185c09c02d96322be89410a9b45e8cc6d97bc Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Feb 2024 18:05:56 -0500 Subject: [PATCH 292/384] Read version from VERSION file in CMake (#14867) Rather than hard-coding the RAPIDS version throughout CMake code, have a single CMake module that reads it from `VERSION` and provides it as a variable. Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - Bradley Dice (https://github.com/bdice) - Robert Maynard (https://github.com/robertmaynard) - Jason Lowe (https://github.com/jlowe) - Charles Blackmon-Luca (https://github.com/charlesbluca) - Ray Douglass (https://github.com/raydouglass) - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/14867 --- ci/release/update-version.sh | 16 ------------- cpp/CMakeLists.txt | 4 ++-- cpp/libcudf_kafka/CMakeLists.txt | 4 ++-- fetch_rapids.cmake | 19 --------------- java/src/main/native/CMakeLists.txt | 4 ++-- python/cudf/CMakeLists.txt | 8 +++---- python/cudf_kafka/CMakeLists.txt | 8 +++---- rapids_config.cmake | 36 +++++++++++++++++++++++++++++ 8 files changed, 48 insertions(+), 51 deletions(-) delete mode 100644 fetch_rapids.cmake create mode 100644 rapids_config.cmake diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 02dba0d09e4..8f266a1b463 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -38,28 +38,12 @@ function sed_runner() { sed -i.bak ''"$1"'' $2 && rm -f ${2}.bak } -# cpp update -sed_runner 's/'"VERSION ${CURRENT_SHORT_TAG}.*"'/'"VERSION ${NEXT_FULL_TAG}"'/g' cpp/CMakeLists.txt - -# Python CMakeLists updates -sed_runner 's/'"cudf_version .*)"'/'"cudf_version ${NEXT_FULL_TAG})"'/g' python/cudf/CMakeLists.txt -sed_runner 's/'"cudf_kafka_version .*)"'/'"cudf_kafka_version ${NEXT_FULL_TAG})"'/g' python/cudf_kafka/CMakeLists.txt - -# cpp libcudf_kafka update -sed_runner 's/'"VERSION ${CURRENT_SHORT_TAG}.*"'/'"VERSION ${NEXT_FULL_TAG}"'/g' cpp/libcudf_kafka/CMakeLists.txt - -# cpp cudf_jni update -sed_runner 's/'"VERSION ${CURRENT_SHORT_TAG}.*"'/'"VERSION ${NEXT_FULL_TAG}"'/g' java/src/main/native/CMakeLists.txt - # Centralized version file update echo "${NEXT_FULL_TAG}" > VERSION # Wheel testing script sed_runner "s/branch-.*/branch-${NEXT_SHORT_TAG}/g" ci/test_wheel_dask_cudf.sh -# rapids-cmake version -sed_runner 's/'"branch-.*\/RAPIDS.cmake"'/'"branch-${NEXT_SHORT_TAG}\/RAPIDS.cmake"'/g' fetch_rapids.cmake - # cmake-format rapids-cmake definitions sed_runner 's/'"branch-.*\/cmake-format-rapids-cmake.json"'/'"branch-${NEXT_SHORT_TAG}\/cmake-format-rapids-cmake.json"'/g' ci/check_style.sh diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 58a43c1def1..b87582b53c9 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -14,7 +14,7 @@ cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) -include(../fetch_rapids.cmake) +include(../rapids_config.cmake) include(rapids-cmake) include(rapids-cpm) include(rapids-cuda) @@ -26,7 +26,7 @@ rapids_cuda_init_architectures(CUDF) project( CUDF - VERSION 24.04.00 + VERSION "${RAPIDS_VERSION}" LANGUAGES C CXX CUDA ) if(CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA" AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 11.5) diff --git a/cpp/libcudf_kafka/CMakeLists.txt b/cpp/libcudf_kafka/CMakeLists.txt index be2c85d6bd3..9760ecfe067 100644 --- a/cpp/libcudf_kafka/CMakeLists.txt +++ b/cpp/libcudf_kafka/CMakeLists.txt @@ -13,7 +13,7 @@ # ============================================================================= cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) -include(../../fetch_rapids.cmake) +include(../../rapids_config.cmake) include(rapids-cmake) include(rapids-cpm) include(rapids-cuda) @@ -22,7 +22,7 @@ include(rapids-find) project( CUDF_KAFKA - VERSION 24.04.00 + VERSION "${RAPIDS_VERSION}" LANGUAGES CXX ) diff --git a/fetch_rapids.cmake b/fetch_rapids.cmake deleted file mode 100644 index 6942b257c3f..00000000000 --- a/fetch_rapids.cmake +++ /dev/null @@ -1,19 +0,0 @@ -# ============================================================================= -# Copyright (c) 2018-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= -if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/CUDF_RAPIDS.cmake) - file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-24.04/RAPIDS.cmake - ${CMAKE_CURRENT_BINARY_DIR}/CUDF_RAPIDS.cmake - ) -endif() -include(${CMAKE_CURRENT_BINARY_DIR}/CUDF_RAPIDS.cmake) diff --git a/java/src/main/native/CMakeLists.txt b/java/src/main/native/CMakeLists.txt index 1406cc3c3a7..1e7ac1a68ea 100644 --- a/java/src/main/native/CMakeLists.txt +++ b/java/src/main/native/CMakeLists.txt @@ -13,7 +13,7 @@ # ============================================================================= cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) -include(../../../../fetch_rapids.cmake) +include(../../../../rapids_config.cmake) include(rapids-cmake) include(rapids-cuda) include(rapids-find) @@ -28,7 +28,7 @@ rapids_cuda_init_architectures(CUDF_JNI) project( CUDF_JNI - VERSION 24.04.00 + VERSION "${RAPIDS_VERSION}" LANGUAGES C CXX CUDA ) diff --git a/python/cudf/CMakeLists.txt b/python/cudf/CMakeLists.txt index 481d6194a03..23edbbc636c 100644 --- a/python/cudf/CMakeLists.txt +++ b/python/cudf/CMakeLists.txt @@ -14,15 +14,13 @@ cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) -set(cudf_version 24.04.00) - -include(../../fetch_rapids.cmake) +include(../../rapids_config.cmake) include(rapids-cuda) rapids_cuda_init_architectures(cudf-python) project( cudf-python - VERSION ${cudf_version} + VERSION "${RAPIDS_VERSION}" LANGUAGES CXX CUDA ) @@ -55,7 +53,7 @@ if(FIND_CUDF_CPP) include(../../cpp/cmake/thirdparty/get_arrow.cmake) endif() - find_package(cudf ${cudf_version} REQUIRED) + find_package(cudf "${RAPIDS_VERSION}" REQUIRED) # an installed version of libcudf doesn't provide the dlpack headers so we need to download dlpack # for the interop.pyx diff --git a/python/cudf_kafka/CMakeLists.txt b/python/cudf_kafka/CMakeLists.txt index 81be80121dd..fd835010c4e 100644 --- a/python/cudf_kafka/CMakeLists.txt +++ b/python/cudf_kafka/CMakeLists.txt @@ -14,17 +14,15 @@ cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) -set(cudf_kafka_version 24.04.00) - -include(../../fetch_rapids.cmake) +include(../../rapids_config.cmake) project( cudf-kafka-python - VERSION ${cudf_kafka_version} + VERSION "${RAPIDS_VERSION}" LANGUAGES CXX ) -find_package(cudf_kafka ${cudf_kafka_version} REQUIRED) +find_package(cudf_kafka "${RAPIDS_VERSION}" REQUIRED) if(NOT cudf_kafka_FOUND) message( diff --git a/rapids_config.cmake b/rapids_config.cmake new file mode 100644 index 00000000000..3a88769f6e7 --- /dev/null +++ b/rapids_config.cmake @@ -0,0 +1,36 @@ +# ============================================================================= +# Copyright (c) 2018-2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= +file(READ "${CMAKE_CURRENT_LIST_DIR}/VERSION" _rapids_version) +if(_rapids_version MATCHES [[^([0-9][0-9])\.([0-9][0-9])\.([0-9][0-9])]]) + set(RAPIDS_VERSION_MAJOR "${CMAKE_MATCH_1}") + set(RAPIDS_VERSION_MINOR "${CMAKE_MATCH_2}") + set(RAPIDS_VERSION_PATCH "${CMAKE_MATCH_3}") + set(RAPIDS_VERSION_MAJOR_MINOR "${RAPIDS_VERSION_MAJOR}.${RAPIDS_VERSION_MINOR}") + set(RAPIDS_VERSION "${RAPIDS_VERSION_MAJOR}.${RAPIDS_VERSION_MINOR}.${RAPIDS_VERSION_PATCH}") +else() + string(REPLACE "\n" "\n " _rapids_version_formatted " ${_rapids_version}") + message( + FATAL_ERROR + "Could not determine RAPIDS version. Contents of VERSION file:\n${_rapids_version_formatted}" + ) +endif() + +if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/CUDF_RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake") + file( + DOWNLOAD + "https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION_MAJOR_MINOR}/RAPIDS.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/CUDF_RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake" + ) +endif() +include("${CMAKE_CURRENT_BINARY_DIR}/CUDF_RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake") From 4e39e71e24659b477df764cc11c52c0324cdf1fe Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Feb 2024 18:22:11 -0500 Subject: [PATCH 293/384] Read `cudf.__version__` in Sphinx build (#14872) Rather than hard-coding the version number in the Sphinx config, dynamically read `cudf.__version__`. Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/cudf/pull/14872 --- ci/release/update-version.sh | 6 ------ docs/cudf/source/conf.py | 13 +++++++++---- docs/dask_cudf/source/conf.py | 15 ++++++++++++--- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 8f266a1b463..1186b02f244 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -50,12 +50,6 @@ sed_runner 's/'"branch-.*\/cmake-format-rapids-cmake.json"'/'"branch-${NEXT_SHOR # doxyfile update sed_runner 's/PROJECT_NUMBER = .*/PROJECT_NUMBER = '${NEXT_FULL_TAG}'/g' cpp/doxygen/Doxyfile -# sphinx docs update -sed_runner 's/version = .*/version = "'${NEXT_SHORT_TAG}'"/g' docs/cudf/source/conf.py -sed_runner 's/release = .*/release = "'${NEXT_FULL_TAG}'"/g' docs/cudf/source/conf.py -sed_runner 's/version = .*/version = "'${NEXT_SHORT_TAG}'"/g' docs/dask_cudf/source/conf.py -sed_runner 's/release = .*/release = "'${NEXT_FULL_TAG}'"/g' docs/dask_cudf/source/conf.py - DEPENDENCIES=( cudf cudf_kafka diff --git a/docs/cudf/source/conf.py b/docs/cudf/source/conf.py index 035ee586822..1b9e3c179cc 100644 --- a/docs/cudf/source/conf.py +++ b/docs/cudf/source/conf.py @@ -16,6 +16,7 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # +import datetime import filecmp import glob import os @@ -25,12 +26,15 @@ import xml.etree.ElementTree as ET from docutils.nodes import Text +from packaging.version import Version from sphinx.addnodes import pending_xref from sphinx.highlighting import lexers from sphinx.ext import intersphinx from pygments.lexer import RegexLexer from pygments.token import Text as PText +import cudf + class PseudoLexer(RegexLexer): """Trivial lexer for pseudocode.""" @@ -172,17 +176,18 @@ def clean_all_xml_files(path): # General information about the project. project = "cudf" -copyright = "2018-2023, NVIDIA Corporation" +copyright = f"2018-{datetime.datetime.today().year}, NVIDIA Corporation" author = "NVIDIA Corporation" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # +CUDF_VERSION = Version(cudf.__version__) # The short X.Y version. -version = "24.04" -# The full version, including alpha/beta/rc tags. -release = "24.04.00" +version = f"{CUDF_VERSION.major:02}.{CUDF_VERSION.minor:02}" +# The full version. +release = f"{CUDF_VERSION.major:02}.{CUDF_VERSION.minor:02}.{CUDF_VERSION.micro:02}" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/dask_cudf/source/conf.py b/docs/dask_cudf/source/conf.py index 25f0eb41ed5..dc40254312e 100644 --- a/docs/dask_cudf/source/conf.py +++ b/docs/dask_cudf/source/conf.py @@ -8,11 +8,20 @@ # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information +import datetime + +from packaging.version import Version + +import dask_cudf + + +DASK_CUDF_VERSION = Version(dask_cudf.__version__) + project = "dask-cudf" -copyright = "2018-2023, NVIDIA Corporation" +copyright = f"2018-{datetime.datetime.today().year}, NVIDIA Corporation" author = "NVIDIA Corporation" -version = "24.04" -release = "24.04.00" +version = f"{DASK_CUDF_VERSION.major:02}.{DASK_CUDF_VERSION.minor:02}" +release = f"{DASK_CUDF_VERSION.major:02}.{DASK_CUDF_VERSION.minor:02}.{DASK_CUDF_VERSION.micro:02}" language = "en" From 2b57c610ddf75ec0e87e6edabd455e998a0371de Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 22 Feb 2024 13:50:04 -1000 Subject: [PATCH 294/384] Ensure slow private attrs are maybe proxies (#14380) Expected pandas test failures: > tests/indexing/test_indexing.py Due to this PR, it appears an `assert something._values is something_else` fails more after this PR since `._values` wraps objects in an proxy object now (a known failure mode) > tests/series/indexing/test_setitem.py Runs into the issue where a test set up calls `proxy._values[key] = something` using a pandas helper function that isn't proxying correctly Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Bradley Dice (https://github.com/bdice) - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/14380 --- python/cudf/cudf/pandas/fast_slow_proxy.py | 19 +++++++++++- .../cudf_pandas_tests/test_cudf_pandas.py | 7 +++++ .../cudf_pandas_tests/test_fast_slow_proxy.py | 31 ++++++++++++++++++- 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/pandas/fast_slow_proxy.py b/python/cudf/cudf/pandas/fast_slow_proxy.py index d132116af61..a2b14e0c3aa 100644 --- a/python/cudf/cudf/pandas/fast_slow_proxy.py +++ b/python/cudf/cudf/pandas/fast_slow_proxy.py @@ -572,7 +572,24 @@ def __getattr__(self, name: str) -> Any: _raise_attribute_error(self.__class__.__name__, name) if name.startswith("_"): # private attributes always come from `._fsproxy_slow`: - return getattr(self._fsproxy_slow, name) + obj = getattr(self._fsproxy_slow, name) + if name.startswith("__array"): + # TODO: numpy methods raise when given proxy ndarray objects + # https://numpy.org/doc/stable/reference/arrays.classes.html#special-attributes-and-methods # noqa:E501 + return obj + + if not _is_function_or_method(obj): + return _maybe_wrap_result( + obj, getattr, self._fsproxy_slow, name + ) + + @functools.wraps(obj) + def _wrapped_private_slow(*args, **kwargs): + slow_args, slow_kwargs = _slow_arg(args), _slow_arg(kwargs) + result = obj(*slow_args, **slow_kwargs) + return _maybe_wrap_result(result, obj, *args, **kwargs) + + return _wrapped_private_slow attr = _FastSlowAttribute(name) return attr.__get__(self) diff --git a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py index ab4742549f8..0386ec434da 100644 --- a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py +++ b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py @@ -1078,6 +1078,13 @@ def test_dataframe_query(): tm.assert_equal(actual, expected) +def test_private_method_result_wrapped(): + xoffset = xpd.offsets.Day() + dt = datetime.datetime(2020, 1, 1) + result = xoffset._apply(dt) + assert isinstance(result, xpd.Timestamp) + + def test_numpy_var(): np.random.seed(42) data = np.random.rand(1000) diff --git a/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py b/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py index b964dfde4ed..631ad2f37b2 100644 --- a/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py +++ b/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. # All rights reserved. # SPDX-License-Identifier: Apache-2.0 @@ -445,6 +445,35 @@ def __radd__(self, other): assert BarProxy() + Foo() == "sum" +def test_slow_attr_still_proxy(): + class A: + pass + + class B: + @property + def _private(self): + return A() + + pxy_a = make_final_proxy_type( + "A", + _Unusable, + A, + fast_to_slow=_Unusable(), + slow_to_fast=_Unusable(), + ) + + pxy_b = make_final_proxy_type( + "B", + _Unusable, + B, + fast_to_slow=_Unusable(), + slow_to_fast=_Unusable(), + ) + + result = pxy_b()._private + assert isinstance(result, pxy_a) + + def tuple_with_attrs(name, fields: list[str], extra_fields: set[str]): # Build a tuple-like class with some extra attributes and a custom # pickling scheme with __getnewargs_ex__ From c84e1e8b3dde5be9c3c095f5cf89a5c181848b5d Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 23 Feb 2024 05:03:07 -0600 Subject: [PATCH 295/384] Raise an error on import for unsupported GPUs. (#15053) RAPIDS 24.02 dropped support for Pascal GPUs. When using an unsupported GPU, the behavior of cudf is undefined and sometimes produces results that appear valid (and empty) but conceal CUDA kernel launch errors. This PR changes the behavior to error on import if unsupported GPUs are detected. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/15053 --- python/cudf/cudf/utils/gpu_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cudf/cudf/utils/gpu_utils.py b/python/cudf/cudf/utils/gpu_utils.py index 10a2f700cbd..b5387ddeb5f 100644 --- a/python/cudf/cudf/utils/gpu_utils.py +++ b/python/cudf/cudf/utils/gpu_utils.py @@ -86,7 +86,7 @@ def validate_setup(): minor_version = getDeviceAttribute( cudaDeviceAttr.cudaDevAttrComputeCapabilityMinor, 0 ) - warnings.warn( + raise UnsupportedCUDAError( "A GPU with NVIDIA Volta™ (Compute Capability 7.0) " "or newer architecture is required.\n" f"Detected GPU 0: {device_name}\n" From ee3c7699bbae4955e68abd13a522ba87c9ffd28c Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Fri, 23 Feb 2024 06:41:26 -0500 Subject: [PATCH 296/384] Use appropriate make_offsets_child_column for building lists columns (#15043) Fixes `cudf::strings::extract_all()` to use `cudf::detail::make_offsets_child_column` so it properly computes the output-size and checks for overflow when building offsets for a lists column. Also undo some changes from #14745 that incorrectly called `cudf::strings::detail::make_offsets_child_column` to create offsets for a lists column. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/15043 --- cpp/src/strings/extract/extract_all.cu | 27 ++++++++++++-------------- cpp/src/strings/search/findall.cu | 8 ++++---- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/cpp/src/strings/extract/extract_all.cu b/cpp/src/strings/extract/extract_all.cu index 63ce04df830..3a02acb7050 100644 --- a/cpp/src/strings/extract/extract_all.cu +++ b/cpp/src/strings/extract/extract_all.cu @@ -118,12 +118,12 @@ std::unique_ptr extract_all_record(strings_column_view const& input, // Get the match counts for each string. // This column will become the output lists child offsets column. - auto offsets = count_matches(*d_strings, *d_prog, strings_count + 1, stream, mr); - auto d_offsets = offsets->mutable_view().data(); + auto counts = count_matches(*d_strings, *d_prog, strings_count, stream, mr); + auto d_counts = counts->mutable_view().data(); // Compute null output rows auto [null_mask, null_count] = cudf::detail::valid_if( - d_offsets, d_offsets + strings_count, [] __device__(auto v) { return v > 0; }, stream, mr); + d_counts, d_counts + strings_count, [] __device__(auto v) { return v > 0; }, stream, mr); // Return an empty lists column if there are no valid rows if (strings_count == null_count) { @@ -132,18 +132,15 @@ std::unique_ptr extract_all_record(strings_column_view const& input, // Convert counts into offsets. // Multiply each count by the number of groups. - thrust::transform_exclusive_scan( - rmm::exec_policy(stream), - d_offsets, - d_offsets + strings_count + 1, - d_offsets, - [groups] __device__(auto v) { return v * groups; }, - size_type{0}, - thrust::plus{}); - auto const total_groups = - cudf::detail::get_value(offsets->view(), strings_count, stream); - - rmm::device_uvector indices(total_groups, stream); + auto sizes_itr = cudf::detail::make_counting_transform_iterator( + 0, cuda::proclaim_return_type([d_counts, groups] __device__(auto idx) { + return d_counts[idx] * groups; + })); + auto [offsets, total_strings] = + cudf::detail::make_offsets_child_column(sizes_itr, sizes_itr + strings_count, stream, mr); + auto d_offsets = offsets->view().data(); + + rmm::device_uvector indices(total_strings, stream); launch_for_each_kernel( extract_fn{*d_strings, d_offsets, indices.data()}, *d_prog, strings_count, stream); diff --git a/cpp/src/strings/search/findall.cu b/cpp/src/strings/search/findall.cu index 4b4a1191e1b..4e8e3a6a449 100644 --- a/cpp/src/strings/search/findall.cu +++ b/cpp/src/strings/search/findall.cu @@ -48,7 +48,7 @@ namespace { */ struct findall_fn { column_device_view const d_strings; - cudf::detail::input_offsetalator const d_offsets; + size_type const* d_offsets; string_index_pair* d_indices; __device__ void operator()(size_type const idx, reprog_device const prog, int32_t const prog_idx) @@ -76,7 +76,7 @@ struct findall_fn { std::unique_ptr findall_util(column_device_view const& d_strings, reprog_device& d_prog, int64_t total_matches, - cudf::detail::input_offsetalator const d_offsets, + size_type const* d_offsets, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { @@ -104,9 +104,9 @@ std::unique_ptr findall(strings_column_view const& input, // Create lists offsets column auto const sizes = count_matches(*d_strings, *d_prog, strings_count, stream, mr); - auto [offsets, total_matches] = cudf::strings::detail::make_offsets_child_column( + auto [offsets, total_matches] = cudf::detail::make_offsets_child_column( sizes->view().begin(), sizes->view().end(), stream, mr); - auto const d_offsets = cudf::detail::offsetalator_factory::make_input_iterator(offsets->view()); + auto const d_offsets = offsets->view().data(); // Build strings column of the matches auto strings_output = findall_util(*d_strings, *d_prog, total_matches, d_offsets, stream, mr); From 8adf0995f5e16c455e803c18dfd4a9be1ea4c575 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 23 Feb 2024 08:46:18 -1000 Subject: [PATCH 297/384] Remove `build_struct|list_column` (#14786) IMO these do not provide much value compared to constructing with `ListColumn` or `StructColumn` cc https://github.com/rapidsai/cudf/pull/14778#discussion_r1457932822 Authors: - Matthew Roeschke (https://github.com/mroeschke) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Michael Wang (https://github.com/isVoid) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/14786 --- python/cudf/cudf/core/column/__init__.py | 2 - python/cudf/cudf/core/column/column.py | 86 ------------------------ python/cudf/cudf/core/column/lists.py | 11 +-- python/cudf/cudf/core/column/struct.py | 9 +-- python/cudf/cudf/core/dataframe.py | 10 +-- python/cudf/cudf/core/groupby/groupby.py | 10 ++- 6 files changed, 24 insertions(+), 104 deletions(-) diff --git a/python/cudf/cudf/core/column/__init__.py b/python/cudf/cudf/core/column/__init__.py index 3dddcae85dc..a1c86b617b0 100644 --- a/python/cudf/cudf/core/column/__init__.py +++ b/python/cudf/cudf/core/column/__init__.py @@ -11,8 +11,6 @@ as_column, build_categorical_column, build_column, - build_list_column, - build_struct_column, column_empty, column_empty_like, column_empty_like_same_mask, diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 191c55a8a68..cecdaf70750 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -1690,92 +1690,6 @@ def build_categorical_column( return cast("cudf.core.column.CategoricalColumn", result) -def build_list_column( - indices: ColumnBase, - elements: ColumnBase, - mask: Optional[Buffer] = None, - size: Optional[int] = None, - offset: int = 0, - null_count: Optional[int] = None, -) -> "cudf.core.column.ListColumn": - """ - Build a ListColumn - - Parameters - ---------- - indices : ColumnBase - Column of list indices - elements : ColumnBase - Column of list elements - mask: Buffer - Null mask - size: int, optional - offset: int, optional - """ - dtype = ListDtype(element_type=elements.dtype) - if size is None: - if indices.size == 0: - size = 0 - else: - # one less because the last element of offsets is the number of - # bytes in the data buffer - size = indices.size - 1 - size = size - offset - - result = build_column( - data=None, - dtype=dtype, - mask=mask, - size=size, - offset=offset, - null_count=null_count, - children=(indices, elements), - ) - - return cast("cudf.core.column.ListColumn", result) - - -def build_struct_column( - names: Sequence[str], - children: Tuple[ColumnBase, ...], - dtype: Optional[Dtype] = None, - mask: Optional[Buffer] = None, - size: Optional[int] = None, - offset: int = 0, - null_count: Optional[int] = None, -) -> "cudf.core.column.StructColumn": - """ - Build a StructColumn - - Parameters - ---------- - names : sequence of strings - Field names to map to children dtypes, must be strings. - children : tuple - - mask: Buffer - Null mask - size: int, optional - offset: int, optional - """ - if dtype is None: - dtype = StructDtype( - fields={name: col.dtype for name, col in zip(names, children)} - ) - - result = build_column( - data=None, - dtype=dtype, - mask=mask, - size=size, - offset=offset, - null_count=null_count, - children=children, - ) - - return cast("cudf.core.column.StructColumn", result) - - def _make_copy_replacing_NaT_with_null(column): """Return a copy with NaT values replaced with nulls.""" if np.issubdtype(column.dtype, np.timedelta64): diff --git a/python/cudf/cudf/core/column/lists.py b/python/cudf/cudf/core/column/lists.py index c28489a2f98..b2205af34e8 100644 --- a/python/cudf/cudf/core/column/lists.py +++ b/python/cudf/cudf/core/column/lists.py @@ -184,15 +184,16 @@ def _with_type_metadata( self: "cudf.core.column.ListColumn", dtype: Dtype ) -> "cudf.core.column.ListColumn": if isinstance(dtype, ListDtype): - return column.build_list_column( - indices=self.base_children[0], - elements=self.base_children[1]._with_type_metadata( - dtype.element_type - ), + elements = self.base_children[1]._with_type_metadata( + dtype.element_type + ) + return ListColumn( + dtype=dtype, mask=self.base_mask, size=self.size, offset=self.offset, null_count=self.null_count, + children=(self.base_children[0], elements), ) return self diff --git a/python/cudf/cudf/core/column/struct.py b/python/cudf/cudf/core/column/struct.py index 6cfa8db0d96..69e9a50956b 100644 --- a/python/cudf/cudf/core/column/struct.py +++ b/python/cudf/cudf/core/column/struct.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from __future__ import annotations from functools import cached_property @@ -9,7 +9,7 @@ import cudf from cudf._typing import Dtype -from cudf.core.column import ColumnBase, build_struct_column +from cudf.core.column import ColumnBase from cudf.core.column.methods import ColumnMethods from cudf.core.dtypes import StructDtype from cudf.core.missing import NA @@ -134,8 +134,9 @@ def _with_type_metadata(self: StructColumn, dtype: Dtype) -> StructColumn: if isinstance(dtype, IntervalDtype): return IntervalColumn.from_struct_column(self, closed=dtype.closed) elif isinstance(dtype, StructDtype): - return build_struct_column( - names=dtype.fields.keys(), + return StructColumn( + data=None, + dtype=dtype, children=tuple( self.base_children[i]._with_type_metadata(dtype.fields[f]) for i, f in enumerate(dtype.fields.keys()) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 89abd7be0ba..5b300f5e4db 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -61,6 +61,7 @@ from cudf.core.column import ( CategoricalColumn, ColumnBase, + StructColumn, as_column, build_categorical_column, build_column, @@ -7127,12 +7128,13 @@ def to_struct(self, name=None): "requires field name to be string. Non-string column names " "will be casted to string as the field name." ) - field_names = [str(name) for name in self._data.names] - - col = cudf.core.column.build_struct_column( - names=field_names, + fields = {str(name): col.dtype for name, col in self._data.items()} + col = StructColumn( + data=None, + dtype=cudf.StructDtype(fields=fields), children=tuple(col.copy(deep=True) for col in self._data.columns), size=len(self), + offset=0, ) return cudf.Series._from_data( cudf.core.column_accessor.ColumnAccessor({name: col}), diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index a236a9b6abf..9612349a607 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -25,7 +25,7 @@ from cudf.api.types import is_bool_dtype, is_float_dtype, is_list_like from cudf.core._compat import PANDAS_LT_300 from cudf.core.abc import Serializable -from cudf.core.column.column import ColumnBase, as_column +from cudf.core.column.column import ColumnBase, StructDtype, as_column from cudf.core.column_accessor import ColumnAccessor from cudf.core.join._join_helpers import _match_join_keys from cudf.core.mixins import Reducible, Scannable @@ -2036,10 +2036,14 @@ def _cov_or_corr(self, func, method_name): ) x, y = str(x), str(y) - column_pair_structs[(x, y)] = cudf.core.column.build_struct_column( - names=(x, y), + column_pair_structs[(x, y)] = cudf.core.column.StructColumn( + data=None, + dtype=StructDtype( + fields={x: self.obj._data[x].dtype, y: self.obj._data[y]} + ), children=(self.obj._data[x], self.obj._data[y]), size=len(self.obj), + offset=0, ) column_pair_groupby = cudf.DataFrame._from_data( From 8e8733563772f024e7cd525fda1d43c364267ee7 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Fri, 23 Feb 2024 13:08:32 -0600 Subject: [PATCH 298/384] Java: Add leak tracking for Scalar instances (#15121) Adds Scalar as another Closeable instance that can be tracked via the leak tracking framework in the cudf Java bindings. Authors: - Jason Lowe (https://github.com/jlowe) Approvers: - Nghia Truong (https://github.com/ttnghia) - Gera Shegalov (https://github.com/gerashegalov) - Robert (Bobby) Evans (https://github.com/revans2) URL: https://github.com/rapidsai/cudf/pull/15121 --- java/src/main/java/ai/rapids/cudf/MemoryCleaner.java | 7 ++++++- java/src/main/java/ai/rapids/cudf/Scalar.java | 4 +++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/MemoryCleaner.java b/java/src/main/java/ai/rapids/cudf/MemoryCleaner.java index 032b075bab7..4614ce24024 100644 --- a/java/src/main/java/ai/rapids/cudf/MemoryCleaner.java +++ b/java/src/main/java/ai/rapids/cudf/MemoryCleaner.java @@ -1,6 +1,6 @@ /* * - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -261,6 +261,11 @@ static void register(ColumnVector vec, Cleaner cleaner) { all.put(cleaner.id, new CleanerWeakReference(vec, cleaner, collected, true)); } + static void register(Scalar s, Cleaner cleaner) { + // It is now registered... + all.put(cleaner.id, new CleanerWeakReference(s, cleaner, collected, true)); + } + static void register(HostColumnVectorCore vec, Cleaner cleaner) { // It is now registered... all.put(cleaner.id, new CleanerWeakReference(vec, cleaner, collected, false)); diff --git a/java/src/main/java/ai/rapids/cudf/Scalar.java b/java/src/main/java/ai/rapids/cudf/Scalar.java index 70538ab082f..286b5c208c9 100644 --- a/java/src/main/java/ai/rapids/cudf/Scalar.java +++ b/java/src/main/java/ai/rapids/cudf/Scalar.java @@ -1,6 +1,6 @@ /* * - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -524,6 +524,7 @@ private static ColumnVector buildNullColumnVector(HostColumnVector.DataType host Scalar(DType type, long scalarHandle) { this.type = type; this.offHeap = new OffHeapState(scalarHandle); + MemoryCleaner.register(this, offHeap); incRefCount(); } @@ -536,6 +537,7 @@ public synchronized Scalar incRefCount() { offHeap.logRefCountDebug("INC AFTER CLOSE " + this); throw new IllegalStateException("Scalar is already closed"); } + offHeap.addRef(); ++refCount; return this; } From 71c990955ab57dcb1aec0efad9630c91404b2a57 Mon Sep 17 00:00:00 2001 From: Yunsong Wang Date: Fri, 23 Feb 2024 11:16:40 -0800 Subject: [PATCH 299/384] Add distinct key inner join (#14990) Contributes to #14948 This PR adds a public `cudf::distinct_hash_join` class that provides a fast code path for joins with distinct keys. Only distinct inner join is tackled in the current PR. Authors: - Yunsong Wang (https://github.com/PointKernel) Approvers: - Jason Lowe (https://github.com/jlowe) - Bradley Dice (https://github.com/bdice) - Lawrence Mitchell (https://github.com/wence-) - David Wendt (https://github.com/davidwendt) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/14990 --- cpp/CMakeLists.txt | 1 + cpp/benchmarks/CMakeLists.txt | 2 +- cpp/benchmarks/join/distinct_join.cu | 77 ++++ cpp/include/cudf/detail/cuco_helpers.hpp | 3 + .../cudf/detail/distinct_hash_join.cuh | 153 +++++++ cpp/include/cudf/join.hpp | 70 +++- cpp/src/join/distinct_hash_join.cu | 387 ++++++++++++++++++ cpp/tests/CMakeLists.txt | 2 +- cpp/tests/join/distinct_join_tests.cpp | 307 ++++++++++++++ 9 files changed, 999 insertions(+), 3 deletions(-) create mode 100644 cpp/benchmarks/join/distinct_join.cu create mode 100644 cpp/include/cudf/detail/distinct_hash_join.cuh create mode 100644 cpp/src/join/distinct_hash_join.cu create mode 100644 cpp/tests/join/distinct_join_tests.cpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index b87582b53c9..5fd6cd3544a 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -434,6 +434,7 @@ add_library( src/jit/util.cpp src/join/conditional_join.cu src/join/cross_join.cu + src/join/distinct_hash_join.cu src/join/hash_join.cu src/join/join.cu src/join/join_utils.cu diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 5a014537de0..ef25278877e 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -164,7 +164,7 @@ ConfigureNVBench( # ################################################################################################## # * join benchmark -------------------------------------------------------------------------------- ConfigureBench(JOIN_BENCH join/left_join.cu join/conditional_join.cu) -ConfigureNVBench(JOIN_NVBENCH join/join.cu join/mixed_join.cu) +ConfigureNVBench(JOIN_NVBENCH join/join.cu join/mixed_join.cu join/distinct_join.cu) # ################################################################################################## # * iterator benchmark ---------------------------------------------------------------------------- diff --git a/cpp/benchmarks/join/distinct_join.cu b/cpp/benchmarks/join/distinct_join.cu new file mode 100644 index 00000000000..cbdb82275ef --- /dev/null +++ b/cpp/benchmarks/join/distinct_join.cu @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "join_common.hpp" + +template +void distinct_inner_join(nvbench::state& state, + nvbench::type_list>) +{ + skip_helper(state); + + auto join = [](cudf::table_view const& left_input, + cudf::table_view const& right_input, + cudf::null_equality compare_nulls, + rmm::cuda_stream_view stream) { + auto const has_nulls = cudf::has_nested_nulls(left_input) || cudf::has_nested_nulls(right_input) + ? cudf::nullable_join::YES + : cudf::nullable_join::NO; + auto hj_obj = cudf::distinct_hash_join{ + left_input, right_input, has_nulls, compare_nulls, stream}; + return hj_obj.inner_join(stream); + }; + + BM_join(state, join); +} + +// inner join ----------------------------------------------------------------------- +NVBENCH_BENCH_TYPES(distinct_inner_join, + NVBENCH_TYPE_AXES(nvbench::type_list, + nvbench::type_list, + nvbench::enum_type_list)) + .set_name("distinct_inner_join_32bit") + .set_type_axes_names({"Key Type", "Payload Type", "Nullable"}) + .add_int64_axis("Build Table Size", {100'000, 10'000'000, 80'000'000, 100'000'000}) + .add_int64_axis("Probe Table Size", + {100'000, 400'000, 10'000'000, 40'000'000, 100'000'000, 240'000'000}); + +NVBENCH_BENCH_TYPES(distinct_inner_join, + NVBENCH_TYPE_AXES(nvbench::type_list, + nvbench::type_list, + nvbench::enum_type_list)) + .set_name("distinct_inner_join_64bit") + .set_type_axes_names({"Key Type", "Payload Type", "Nullable"}) + .add_int64_axis("Build Table Size", {40'000'000, 50'000'000}) + .add_int64_axis("Probe Table Size", {50'000'000, 120'000'000}); + +NVBENCH_BENCH_TYPES(distinct_inner_join, + NVBENCH_TYPE_AXES(nvbench::type_list, + nvbench::type_list, + nvbench::enum_type_list)) + .set_name("distinct_inner_join_32bit_nulls") + .set_type_axes_names({"Key Type", "Payload Type", "Nullable"}) + .add_int64_axis("Build Table Size", {100'000, 10'000'000, 80'000'000, 100'000'000}) + .add_int64_axis("Probe Table Size", + {100'000, 400'000, 10'000'000, 40'000'000, 100'000'000, 240'000'000}); + +NVBENCH_BENCH_TYPES(distinct_inner_join, + NVBENCH_TYPE_AXES(nvbench::type_list, + nvbench::type_list, + nvbench::enum_type_list)) + .set_name("distinct_inner_join_64bit_nulls") + .set_type_axes_names({"Key Type", "Payload Type", "Nullable"}) + .add_int64_axis("Build Table Size", {40'000'000, 50'000'000}) + .add_int64_axis("Probe Table Size", {50'000'000, 120'000'000}); diff --git a/cpp/include/cudf/detail/cuco_helpers.hpp b/cpp/include/cudf/detail/cuco_helpers.hpp index 5f3c31479de..506f6475637 100644 --- a/cpp/include/cudf/detail/cuco_helpers.hpp +++ b/cpp/include/cudf/detail/cuco_helpers.hpp @@ -21,6 +21,9 @@ namespace cudf::detail { +/// Default load factor for cuco data structures +static double constexpr CUCO_DESIRED_LOAD_FACTOR = 0.5; + /** * @brief Stream-ordered allocator adaptor used for cuco data structures * diff --git a/cpp/include/cudf/detail/distinct_hash_join.cuh b/cpp/include/cudf/detail/distinct_hash_join.cuh new file mode 100644 index 00000000000..7827f861bd8 --- /dev/null +++ b/cpp/include/cudf/detail/distinct_hash_join.cuh @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include + +namespace cudf::detail { + +using cudf::experimental::row::lhs_index_type; +using cudf::experimental::row::rhs_index_type; + +/** + * @brief An comparator adapter wrapping both self comparator and two table comparator + */ +template +struct comparator_adapter { + comparator_adapter(Equal const& d_equal) : _d_equal{d_equal} {} + + __device__ constexpr auto operator()( + cuco::pair const&, + cuco::pair const&) const noexcept + { + // All build table keys are distinct thus `false` no matter what + return false; + } + + __device__ constexpr auto operator()( + cuco::pair const& lhs, + cuco::pair const& rhs) const noexcept + { + if (lhs.first != rhs.first) { return false; } + return _d_equal(lhs.second, rhs.second); + } + + private: + Equal _d_equal; +}; + +template +struct hasher_adapter { + hasher_adapter(Hasher const& d_hasher = {}) : _d_hasher{d_hasher} {} + + template + __device__ constexpr auto operator()(cuco::pair const& key) const noexcept + { + return _d_hasher(key.first); + } + + private: + Hasher _d_hasher; +}; + +/** + * @brief Distinct hash join that builds hash table in creation and probes results in subsequent + * `*_join` member functions. + * + * @tparam HasNested Flag indicating whether there are nested columns in build/probe table + */ +template +struct distinct_hash_join { + private: + /// Row equality type for nested columns + using nested_row_equal = cudf::experimental::row::equality::strong_index_comparator_adapter< + cudf::experimental::row::equality::device_row_comparator>; + /// Row equality type for flat columns + using flat_row_equal = cudf::experimental::row::equality::strong_index_comparator_adapter< + cudf::experimental::row::equality::device_row_comparator>; + + /// Device row equal type + using d_equal_type = + std::conditional_t; + using hasher = hasher_adapter>; + using probing_scheme_type = cuco::linear_probing<1, hasher>; + using cuco_storage_type = cuco::storage<1>; + + /// Hash table type + using hash_table_type = cuco::static_set, + cuco::extent, + cuda::thread_scope_device, + comparator_adapter, + probing_scheme_type, + cudf::detail::cuco_allocator, + cuco_storage_type>; + + bool _has_nulls; ///< true if nulls are present in either build table or probe table + cudf::null_equality _nulls_equal; ///< whether to consider nulls as equal + cudf::table_view _build; ///< input table to build the hash map + cudf::table_view _probe; ///< input table to probe the hash map + std::shared_ptr + _preprocessed_build; ///< input table preprocssed for row operators + std::shared_ptr + _preprocessed_probe; ///< input table preprocssed for row operators + hash_table_type _hash_table; ///< hash table built on `_build` + + public: + distinct_hash_join() = delete; + ~distinct_hash_join() = default; + distinct_hash_join(distinct_hash_join const&) = delete; + distinct_hash_join(distinct_hash_join&&) = delete; + distinct_hash_join& operator=(distinct_hash_join const&) = delete; + distinct_hash_join& operator=(distinct_hash_join&&) = delete; + + /** + * @brief Constructor that internally builds the hash table based on the given `build` table. + * + * @throw cudf::logic_error if the number of columns in `build` table is 0. + * + * @param build The build table, from which the hash table is built + * @param probe The probe table + * @param has_nulls Flag to indicate if any nulls exist in the `build` table or + * any `probe` table that will be used later for join. + * @param compare_nulls Controls whether null join-key values should match or not. + * @param stream CUDA stream used for device memory operations and kernel launches. + */ + distinct_hash_join(cudf::table_view const& build, + cudf::table_view const& probe, + bool has_nulls, + cudf::null_equality compare_nulls, + rmm::cuda_stream_view stream); + + /** + * @copydoc cudf::distinct_hash_join::inner_join + */ + std::pair>, + std::unique_ptr>> + inner_join(rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) const; +}; +} // namespace cudf::detail diff --git a/cpp/include/cudf/join.hpp b/cpp/include/cudf/join.hpp index 6c50e1d5998..d97dc64ac39 100644 --- a/cpp/include/cudf/join.hpp +++ b/cpp/include/cudf/join.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,6 +33,13 @@ namespace cudf { +/** + * @brief Enum to indicate whether the distinct join table has nested columns or not + * + * @ingroup column_join + */ +enum class has_nested : bool { YES, NO }; + // forward declaration namespace hashing::detail { template @@ -41,6 +48,9 @@ class MurmurHash3_x86_32; namespace detail { template class hash_join; + +template +class distinct_hash_join; } // namespace detail /** @@ -438,6 +448,64 @@ class hash_join { const std::unique_ptr _impl; }; +/** + * @brief Distinct hash join that builds hash table in creation and probes results in subsequent + * `*_join` member functions + * + * @note Behavior is undefined if the build table contains duplicates. + * @note All NaNs are considered as equal + * + * @tparam HasNested Flag indicating whether there are nested columns in build/probe table + */ +// TODO: `HasNested` to be removed via dispatching +template +class distinct_hash_join { + public: + distinct_hash_join() = delete; + ~distinct_hash_join(); + distinct_hash_join(distinct_hash_join const&) = delete; + distinct_hash_join(distinct_hash_join&&) = delete; + distinct_hash_join& operator=(distinct_hash_join const&) = delete; + distinct_hash_join& operator=(distinct_hash_join&&) = delete; + + /** + * @brief Constructs a distinct hash join object for subsequent probe calls + * + * @param build The build table that contains distinct elements + * @param probe The probe table, from which the keys are probed + * @param has_nulls Flag to indicate if there exists any nulls in the `build` table or + * any `probe` table that will be used later for join + * @param compare_nulls Controls whether null join-key values should match or not + * @param stream CUDA stream used for device memory operations and kernel launches + */ + distinct_hash_join(cudf::table_view const& build, + cudf::table_view const& probe, + nullable_join has_nulls = nullable_join::YES, + null_equality compare_nulls = null_equality::EQUAL, + rmm::cuda_stream_view stream = cudf::get_default_stream()); + + /** + * Returns the row indices that can be used to construct the result of performing + * an inner join between two tables. @see cudf::inner_join(). + * + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned indices' device memory. + * + * @return A pair of columns [`build_indices`, `probe_indices`] that can be used to construct + * the result of performing an inner join between two tables with `build` and `probe` + * as the join keys. + */ + std::pair>, + std::unique_ptr>> + inner_join(rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()) const; + + private: + using impl_type = typename cudf::detail::distinct_hash_join; ///< Implementation type + + std::unique_ptr _impl; ///< Distinct hash join implementation +}; + /** * @brief Returns a pair of row index vectors corresponding to all pairs * of rows between the specified tables where the predicate evaluates to true. diff --git a/cpp/src/join/distinct_hash_join.cu b/cpp/src/join/distinct_hash_join.cu new file mode 100644 index 00000000000..7c834d1a96b --- /dev/null +++ b/cpp/src/join/distinct_hash_join.cu @@ -0,0 +1,387 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "join_common_utils.cuh" +#include "join_common_utils.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +namespace cudf { +namespace detail { +namespace { + +static auto constexpr DISTINCT_JOIN_BLOCK_SIZE = 256; + +template +auto prepare_device_equal( + std::shared_ptr build, + std::shared_ptr probe, + bool has_nulls, + cudf::null_equality compare_nulls) +{ + auto const two_table_equal = + cudf::experimental::row::equality::two_table_comparator(build, probe); + return comparator_adapter{two_table_equal.equal_to( + nullate::DYNAMIC{has_nulls}, compare_nulls)}; +} + +/** + * @brief Device functor to create a pair of {hash_value, row_index} for a given row. + * + * @tparam Hasher The type of internal hasher to compute row hash. + */ +template +class build_keys_fn { + public: + CUDF_HOST_DEVICE build_keys_fn(Hasher const& hash) : _hash{hash} {} + + __device__ __forceinline__ auto operator()(size_type i) const noexcept + { + return cuco::pair{_hash(i), T{i}}; + } + + private: + Hasher _hash; +}; + +template +__device__ void flush_buffer(Tile const& tile, + cudf::size_type tile_count, + cuco::pair* buffer, + cudf::size_type* counter, + cudf::size_type* build_indices, + cudf::size_type* probe_indices) +{ + cudf::size_type offset; + auto const lane_id = tile.thread_rank(); + if (0 == lane_id) { offset = atomicAdd(counter, tile_count); } + offset = tile.shfl(offset, 0); + + for (cudf::size_type i = lane_id; i < tile_count; i += tile.size()) { + auto const& [build_idx, probe_idx] = buffer[i]; + *(build_indices + offset + i) = build_idx; + *(probe_indices + offset + i) = probe_idx; + } +} + +__device__ void flush_buffer(cooperative_groups::thread_block const& block, + cudf::size_type buffer_size, + cuco::pair* buffer, + cudf::size_type* counter, + cudf::size_type* build_indices, + cudf::size_type* probe_indices) +{ + auto i = block.thread_rank(); + __shared__ cudf::size_type offset; + + if (i == 0) { offset = atomicAdd(counter, buffer_size); } + block.sync(); + + while (i < buffer_size) { + auto const& [build_idx, probe_idx] = buffer[i]; + *(build_indices + offset + i) = build_idx; + *(probe_indices + offset + i) = probe_idx; + + i += block.size(); + } +} + +// TODO: custom kernel to be replaced by cuco::static_set::retrieve +template +CUDF_KERNEL void distinct_join_probe_kernel(Iter iter, + cudf::size_type n, + HashTable hash_table, + cudf::size_type* counter, + cudf::size_type* build_indices, + cudf::size_type* probe_indices) +{ + namespace cg = cooperative_groups; + + auto constexpr tile_size = HashTable::cg_size; + auto constexpr window_size = HashTable::window_size; + + auto idx = cudf::detail::grid_1d::global_thread_id() / tile_size; + auto const stride = cudf::detail::grid_1d::grid_stride() / tile_size; + auto const block = cg::this_thread_block(); + + // CG-based probing algorithm + if constexpr (tile_size != 1) { + auto const tile = cg::tiled_partition(block); + + auto constexpr flushing_tile_size = cudf::detail::warp_size / window_size; + // random choice to tune + auto constexpr flushing_buffer_size = 2 * flushing_tile_size; + auto constexpr num_flushing_tiles = DISTINCT_JOIN_BLOCK_SIZE / flushing_tile_size; + auto constexpr max_matches = flushing_tile_size / tile_size; + + auto const flushing_tile = cg::tiled_partition(block); + auto const flushing_tile_id = block.thread_rank() / flushing_tile_size; + + __shared__ cuco::pair + flushing_tile_buffer[num_flushing_tiles][flushing_tile_size]; + // per flushing-tile counter to track number of filled elements + __shared__ cudf::size_type flushing_counter[num_flushing_tiles]; + + if (flushing_tile.thread_rank() == 0) { flushing_counter[flushing_tile_id] = 0; } + flushing_tile.sync(); // sync still needed since cg.any doesn't imply a memory barrier + + while (flushing_tile.any(idx < n)) { + bool active_flag = idx < n; + auto const active_flushing_tile = + cg::binary_partition(flushing_tile, active_flag); + if (active_flag) { + auto const found = hash_table.find(tile, *(iter + idx)); + if (tile.thread_rank() == 0 and found != hash_table.end()) { + auto const offset = atomicAdd_block(&flushing_counter[flushing_tile_id], 1); + flushing_tile_buffer[flushing_tile_id][offset] = cuco::pair{ + static_cast(found->second), static_cast(idx)}; + } + } + + flushing_tile.sync(); + if (flushing_counter[flushing_tile_id] + max_matches > flushing_buffer_size) { + flush_buffer(flushing_tile, + flushing_counter[flushing_tile_id], + flushing_tile_buffer[flushing_tile_id], + counter, + build_indices, + probe_indices); + flushing_tile.sync(); + if (flushing_tile.thread_rank() == 0) { flushing_counter[flushing_tile_id] = 0; } + flushing_tile.sync(); + } + + idx += stride; + } // while + + if (flushing_counter[flushing_tile_id] > 0) { + flush_buffer(flushing_tile, + flushing_counter[flushing_tile_id], + flushing_tile_buffer[flushing_tile_id], + counter, + build_indices, + probe_indices); + } + } + // Scalar probing for CG size 1 + else { + using block_scan = cub::BlockScan; + __shared__ typename block_scan::TempStorage block_scan_temp_storage; + + auto constexpr buffer_capacity = 2 * DISTINCT_JOIN_BLOCK_SIZE; + __shared__ cuco::pair buffer[buffer_capacity]; + cudf::size_type buffer_size = 0; + + while (idx - block.thread_rank() < n) { // the whole thread block falls into the same iteration + cudf::size_type thread_count{0}; + cudf::size_type build_idx{0}; + if (idx < n) { + auto const found = hash_table.find(*(iter + idx)); + thread_count = found != hash_table.end(); + build_idx = static_cast(found->second); + } + + // Use a whole-block scan to calculate the output location + cudf::size_type offset; + cudf::size_type block_count; + block_scan(block_scan_temp_storage).ExclusiveSum(thread_count, offset, block_count); + + if (buffer_size + block_count > buffer_capacity) { + flush_buffer(block, buffer_size, buffer, counter, build_indices, probe_indices); + block.sync(); + buffer_size = 0; + } + + if (thread_count == 1) { + buffer[buffer_size + offset] = cuco::pair{build_idx, static_cast(idx)}; + } + buffer_size += block_count; + block.sync(); + + idx += stride; + } // while + + if (buffer_size > 0) { + flush_buffer(block, buffer_size, buffer, counter, build_indices, probe_indices); + } + } +} +} // namespace + +template +distinct_hash_join::distinct_hash_join(cudf::table_view const& build, + cudf::table_view const& probe, + bool has_nulls, + cudf::null_equality compare_nulls, + rmm::cuda_stream_view stream) + : _has_nulls{has_nulls}, + _nulls_equal{compare_nulls}, + _build{build}, + _probe{probe}, + _preprocessed_build{ + cudf::experimental::row::equality::preprocessed_table::create(_build, stream)}, + _preprocessed_probe{ + cudf::experimental::row::equality::preprocessed_table::create(_probe, stream)}, + _hash_table{build.num_rows(), + CUCO_DESIRED_LOAD_FACTOR, + cuco::empty_key{cuco::pair{std::numeric_limits::max(), + lhs_index_type{JoinNoneValue}}}, + prepare_device_equal( + _preprocessed_build, _preprocessed_probe, has_nulls, compare_nulls), + {}, + cuco::thread_scope_device, + cuco_storage_type{}, + cudf::detail::cuco_allocator{stream}, + stream.value()} +{ + CUDF_FUNC_RANGE(); + CUDF_EXPECTS(0 != this->_build.num_columns(), "Hash join build table is empty"); + + if (this->_build.num_rows() == 0) { return; } + + auto const row_hasher = experimental::row::hash::row_hasher{this->_preprocessed_build}; + auto const d_hasher = row_hasher.device_hasher(nullate::DYNAMIC{this->_has_nulls}); + + auto const iter = cudf::detail::make_counting_transform_iterator( + 0, build_keys_fn{d_hasher}); + + size_type const build_table_num_rows{build.num_rows()}; + if (this->_nulls_equal == cudf::null_equality::EQUAL or (not cudf::nullable(this->_build))) { + this->_hash_table.insert_async(iter, iter + build_table_num_rows, stream.value()); + } else { + auto stencil = thrust::counting_iterator{0}; + auto const row_bitmask = + cudf::detail::bitmask_and(this->_build, stream, rmm::mr::get_current_device_resource()).first; + auto const pred = + cudf::detail::row_is_valid{reinterpret_cast(row_bitmask.data())}; + + // insert valid rows + this->_hash_table.insert_if_async( + iter, iter + build_table_num_rows, stencil, pred, stream.value()); + } +} + +template +std::pair>, + std::unique_ptr>> +distinct_hash_join::inner_join(rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) const +{ + cudf::thread_range range{"distinct_hash_join::inner_join"}; + + size_type const probe_table_num_rows{this->_probe.num_rows()}; + + // If output size is zero, return immediately + if (probe_table_num_rows == 0) { + return std::pair(std::make_unique>(0, stream, mr), + std::make_unique>(0, stream, mr)); + } + + auto left_indices = + std::make_unique>(probe_table_num_rows, stream, mr); + auto right_indices = + std::make_unique>(probe_table_num_rows, stream, mr); + + auto const probe_row_hasher = + cudf::experimental::row::hash::row_hasher{this->_preprocessed_probe}; + auto const d_probe_hasher = probe_row_hasher.device_hasher(nullate::DYNAMIC{this->_has_nulls}); + auto const iter = cudf::detail::make_counting_transform_iterator( + 0, build_keys_fn{d_probe_hasher}); + auto counter = rmm::device_scalar{stream}; + counter.set_value_to_zero_async(stream); + + cudf::detail::grid_1d grid{probe_table_num_rows, DISTINCT_JOIN_BLOCK_SIZE}; + distinct_join_probe_kernel<<>>( + iter, + probe_table_num_rows, + this->_hash_table.ref(cuco::find), + counter.data(), + left_indices->data(), + right_indices->data()); + + auto const actual_size = counter.value(stream); + left_indices->resize(actual_size, stream); + right_indices->resize(actual_size, stream); + + return {std::move(left_indices), std::move(right_indices)}; +} +} // namespace detail + +template <> +distinct_hash_join::~distinct_hash_join() = default; + +template <> +distinct_hash_join::~distinct_hash_join() = default; + +template <> +distinct_hash_join::distinct_hash_join(cudf::table_view const& build, + cudf::table_view const& probe, + nullable_join has_nulls, + null_equality compare_nulls, + rmm::cuda_stream_view stream) + : _impl{std::make_unique( + build, probe, has_nulls == nullable_join::YES, compare_nulls, stream)} +{ +} + +template <> +distinct_hash_join::distinct_hash_join(cudf::table_view const& build, + cudf::table_view const& probe, + nullable_join has_nulls, + null_equality compare_nulls, + rmm::cuda_stream_view stream) + : _impl{std::make_unique( + build, probe, has_nulls == nullable_join::YES, compare_nulls, stream)} +{ +} + +template <> +std::pair>, + std::unique_ptr>> +distinct_hash_join::inner_join(rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) const +{ + return _impl->inner_join(stream, mr); +} + +template <> +std::pair>, + std::unique_ptr>> +distinct_hash_join::inner_join(rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) const +{ + return _impl->inner_join(stream, mr); +} +} // namespace cudf diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 94ae349896c..3e377b07eee 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -152,7 +152,7 @@ ConfigureTest( # * join tests ------------------------------------------------------------------------------------ ConfigureTest( JOIN_TEST join/join_tests.cpp join/conditional_join_tests.cu join/cross_join_tests.cpp - join/semi_anti_join_tests.cpp join/mixed_join_tests.cu + join/semi_anti_join_tests.cpp join/mixed_join_tests.cu join/distinct_join_tests.cpp ) # ################################################################################################## diff --git a/cpp/tests/join/distinct_join_tests.cpp b/cpp/tests/join/distinct_join_tests.cpp new file mode 100644 index 00000000000..27f4c4fdf61 --- /dev/null +++ b/cpp/tests/join/distinct_join_tests.cpp @@ -0,0 +1,307 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +template +using column_wrapper = cudf::test::fixed_width_column_wrapper; +using strcol_wrapper = cudf::test::strings_column_wrapper; +using CVector = std::vector>; +using Table = cudf::table; + +struct DistinctJoinTest : public cudf::test::BaseFixture { + void compare_to_reference( + cudf::table_view const& build_table, + cudf::table_view const& probe_table, + std::pair>, + std::unique_ptr>> const& result, + cudf::table_view const& expected_table) + { + auto const& [build_join_indices, probe_join_indices] = result; + + auto build_indices_span = cudf::device_span{*build_join_indices}; + auto probe_indices_span = cudf::device_span{*probe_join_indices}; + + auto build_indices_col = cudf::column_view{build_indices_span}; + auto probe_indices_col = cudf::column_view{probe_indices_span}; + + auto constexpr oob_policy = cudf::out_of_bounds_policy::DONT_CHECK; + auto joined_cols = cudf::gather(build_table, build_indices_col, oob_policy)->release(); + auto right_cols = cudf::gather(probe_table, probe_indices_col, oob_policy)->release(); + + joined_cols.insert(joined_cols.end(), + std::make_move_iterator(right_cols.begin()), + std::make_move_iterator(right_cols.end())); + auto joined_table = std::make_unique(std::move(joined_cols)); + auto result_sort_order = cudf::sorted_order(joined_table->view()); + auto sorted_joined_table = cudf::gather(joined_table->view(), *result_sort_order); + + auto expected_sort_order = cudf::sorted_order(expected_table); + auto sorted_expected = cudf::gather(expected_table, *expected_sort_order); + CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*sorted_expected, *sorted_joined_table); + } +}; + +TEST_F(DistinctJoinTest, IntegerInnerJoin) +{ + auto constexpr size = 2024; + + auto const init = cudf::numeric_scalar{0}; + + auto build = cudf::sequence(size, init, cudf::numeric_scalar{1}); + auto probe = cudf::sequence(size, init, cudf::numeric_scalar{2}); + + auto build_table = cudf::table_view{{build->view()}}; + auto probe_table = cudf::table_view{{probe->view()}}; + + auto distinct_join = cudf::distinct_hash_join{ + build_table, probe_table, cudf::nullable_join::NO}; + + auto result = distinct_join.inner_join(); + + auto constexpr gold_size = size / 2; + auto gold = cudf::sequence(gold_size, init, cudf::numeric_scalar{2}); + this->compare_to_reference(build_table, probe_table, result, cudf::table_view{{gold->view()}}); +} + +TEST_F(DistinctJoinTest, InnerJoinNoNulls) +{ + column_wrapper col0_0{{1, 2, 3, 4, 5}}; + strcol_wrapper col0_1({"s0", "s0", "s3", "s4", "s5"}); + column_wrapper col0_2{{9, 9, 9, 9, 9}}; + + column_wrapper col1_0{{1, 2, 3, 4, 9}}; + strcol_wrapper col1_1({"s0", "s0", "s0", "s4", "s4"}); + column_wrapper col1_2{{9, 9, 9, 0, 9}}; + + CVector cols0, cols1; + cols0.push_back(col0_0.release()); + cols0.push_back(col0_1.release()); + cols0.push_back(col0_2.release()); + cols1.push_back(col1_0.release()); + cols1.push_back(col1_1.release()); + cols1.push_back(col1_2.release()); + + Table build(std::move(cols0)); + Table probe(std::move(cols1)); + + auto distinct_join = cudf::distinct_hash_join{build.view(), probe.view()}; + auto result = distinct_join.inner_join(); + + column_wrapper col_gold_0{{1, 2}}; + strcol_wrapper col_gold_1({"s0", "s0"}); + column_wrapper col_gold_2{{9, 9}}; + column_wrapper col_gold_3{{1, 2}}; + strcol_wrapper col_gold_4({"s0", "s0"}); + column_wrapper col_gold_5{{9, 9}}; + CVector cols_gold; + cols_gold.push_back(col_gold_0.release()); + cols_gold.push_back(col_gold_1.release()); + cols_gold.push_back(col_gold_2.release()); + cols_gold.push_back(col_gold_3.release()); + cols_gold.push_back(col_gold_4.release()); + cols_gold.push_back(col_gold_5.release()); + Table gold(std::move(cols_gold)); + + this->compare_to_reference(build.view(), probe.view(), result, gold.view()); +} + +TEST_F(DistinctJoinTest, InnerJoinWithNulls) +{ + column_wrapper col0_0{{3, 1, 2, 0, 2}}; + strcol_wrapper col0_1({"s1", "s1", "s0", "s4", "s0"}, {1, 1, 0, 1, 1}); + column_wrapper col0_2{{1, 1, 2, 4, 1}}; + + column_wrapper col1_0{{1, 2, 0, 2, 3}}; + strcol_wrapper col1_1({"s1", "s0", "s1", "s0", "s1"}); + column_wrapper col1_2{{1, 1, 1, 1, 1}, {0, 1, 1, 0, 1}}; + + CVector cols0, cols1; + cols0.push_back(col0_0.release()); + cols0.push_back(col0_1.release()); + cols0.push_back(col0_2.release()); + cols1.push_back(col1_0.release()); + cols1.push_back(col1_1.release()); + cols1.push_back(col1_2.release()); + + Table build(std::move(cols0)); + Table probe(std::move(cols1)); + + auto distinct_join = cudf::distinct_hash_join{build.view(), probe.view()}; + auto result = distinct_join.inner_join(); + + column_wrapper col_gold_0{{3, 2}}; + strcol_wrapper col_gold_1({"s1", "s0"}, {1, 1}); + column_wrapper col_gold_2{{1, 1}}; + column_wrapper col_gold_3{{3, 2}}; + strcol_wrapper col_gold_4({"s1", "s0"}, {1, 1}); + column_wrapper col_gold_5{{1, 1}}; + CVector cols_gold; + cols_gold.push_back(col_gold_0.release()); + cols_gold.push_back(col_gold_1.release()); + cols_gold.push_back(col_gold_2.release()); + cols_gold.push_back(col_gold_3.release()); + cols_gold.push_back(col_gold_4.release()); + cols_gold.push_back(col_gold_5.release()); + Table gold(std::move(cols_gold)); + + this->compare_to_reference(build.view(), probe.view(), result, gold.view()); +} + +TEST_F(DistinctJoinTest, InnerJoinWithStructsAndNulls) +{ + column_wrapper col0_0{{3, 1, 2, 0, 2}}; + strcol_wrapper col0_1({"s1", "s1", "s0", "s4", "s0"}, {1, 1, 0, 1, 1}); + column_wrapper col0_2{{0, 1, 2, 4, 4}, {1, 1, 1, 1, 0}}; + std::initializer_list col0_names = { + "Samuel Vimes", "Carrot Ironfoundersson", "Detritus", "Samuel Vimes", "Angua von Überwald"}; + auto col0_names_col = strcol_wrapper{col0_names.begin(), col0_names.end()}; + auto col0_ages_col = column_wrapper{{48, 27, 351, 31, 25}}; + + auto col0_is_human_col = column_wrapper{{true, true, false, false, false}, {1, 1, 0, 1, 0}}; + + auto col0_3 = + cudf::test::structs_column_wrapper{{col0_names_col, col0_ages_col, col0_is_human_col}}; + + column_wrapper col1_0{{2, 2, 0, 4, 3}}; + strcol_wrapper col1_1({"s1", "s0", "s1", "s2", "s1"}); + column_wrapper col1_2{{1, 1, 1, 2, 0}, {1, 0, 1, 1, 1}}; + std::initializer_list col1_names = {"Carrot Ironfoundersson", + "Angua von Überwald", + "Detritus", + "Carrot Ironfoundersson", + "Samuel Vimes"}; + auto col1_names_col = strcol_wrapper{col1_names.begin(), col1_names.end()}; + auto col1_ages_col = column_wrapper{{31, 25, 351, 27, 48}}; + + auto col1_is_human_col = column_wrapper{{true, false, false, false, true}, {1, 0, 0, 1, 1}}; + + auto col1_3 = + cudf::test::structs_column_wrapper{{col1_names_col, col1_ages_col, col1_is_human_col}}; + + CVector cols0, cols1; + cols0.push_back(col0_0.release()); + cols0.push_back(col0_1.release()); + cols0.push_back(col0_2.release()); + cols0.push_back(col0_3.release()); + cols1.push_back(col1_0.release()); + cols1.push_back(col1_1.release()); + cols1.push_back(col1_2.release()); + cols1.push_back(col1_3.release()); + + Table probe(std::move(cols0)); + Table build(std::move(cols1)); + + auto distinct_join = cudf::distinct_hash_join{build.view(), probe.view()}; + auto result = distinct_join.inner_join(); + + column_wrapper col_gold_0{{3, 2}}; + strcol_wrapper col_gold_1({"s1", "s0"}, {1, 1}); + column_wrapper col_gold_2{{0, 4}, {1, 0}}; + auto col_gold_3_names_col = strcol_wrapper{"Samuel Vimes", "Angua von Überwald"}; + auto col_gold_3_ages_col = column_wrapper{{48, 25}}; + + auto col_gold_3_is_human_col = column_wrapper{{true, false}, {1, 0}}; + + auto col_gold_3 = cudf::test::structs_column_wrapper{ + {col_gold_3_names_col, col_gold_3_ages_col, col_gold_3_is_human_col}}; + + column_wrapper col_gold_4{{3, 2}}; + strcol_wrapper col_gold_5({"s1", "s0"}, {1, 1}); + column_wrapper col_gold_6{{0, -1}, {1, 0}}; + auto col_gold_7_names_col = strcol_wrapper{"Samuel Vimes", "Angua von Überwald"}; + auto col_gold_7_ages_col = column_wrapper{{48, 25}}; + + auto col_gold_7_is_human_col = column_wrapper{{true, false}, {1, 0}}; + + auto col_gold_7 = cudf::test::structs_column_wrapper{ + {col_gold_7_names_col, col_gold_7_ages_col, col_gold_7_is_human_col}}; + CVector cols_gold; + cols_gold.push_back(col_gold_0.release()); + cols_gold.push_back(col_gold_1.release()); + cols_gold.push_back(col_gold_2.release()); + cols_gold.push_back(col_gold_3.release()); + cols_gold.push_back(col_gold_4.release()); + cols_gold.push_back(col_gold_5.release()); + cols_gold.push_back(col_gold_6.release()); + cols_gold.push_back(col_gold_7.release()); + Table gold(std::move(cols_gold)); + + this->compare_to_reference(build.view(), probe.view(), result, gold.view()); +} + +TEST_F(DistinctJoinTest, EmptyBuildTableInnerJoin) +{ + column_wrapper col0_0; + column_wrapper col0_1; + + column_wrapper col1_0{{2, 2, 0, 4, 3}}; + column_wrapper col1_1{{1, 0, 1, 2, 1}, {1, 0, 1, 1, 1}}; + + CVector cols0, cols1; + cols0.push_back(col0_0.release()); + cols0.push_back(col0_1.release()); + cols1.push_back(col1_0.release()); + cols1.push_back(col1_1.release()); + + Table build(std::move(cols0)); + Table probe(std::move(cols1)); + + auto distinct_join = cudf::distinct_hash_join{build.view(), probe.view()}; + auto result = distinct_join.inner_join(); + + this->compare_to_reference(build.view(), probe.view(), result, build.view()); +} + +TEST_F(DistinctJoinTest, EmptyProbeTableInnerJoin) +{ + column_wrapper col0_0{{2, 2, 0, 4, 3}}; + column_wrapper col0_1{{1, 0, 1, 2, 1}, {1, 0, 1, 1, 1}}; + + column_wrapper col1_0; + column_wrapper col1_1; + + CVector cols0, cols1; + cols0.push_back(col0_0.release()); + cols0.push_back(col0_1.release()); + cols1.push_back(col1_0.release()); + cols1.push_back(col1_1.release()); + + Table build(std::move(cols0)); + Table probe(std::move(cols1)); + + auto distinct_join = cudf::distinct_hash_join{build.view(), probe.view()}; + auto result = distinct_join.inner_join(); + + this->compare_to_reference(build.view(), probe.view(), result, probe.view()); +} From c37367ee22f12cc59c7ec7ed530596b82870334c Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Fri, 23 Feb 2024 19:38:12 -0800 Subject: [PATCH 300/384] Align integral types in ORC to specs (#15008) Use `uint64_t` where specified by the ORC specs: - `PostScript::compressionBlockSize` - `StripeInformation::footerLength` - `StripeInformation::numberOfRows` Using the same type for the derived values. Other changes: - Changed the num_rows in orc_metadata to uint64_t so it works with files that have more than 2B rows. - Modified how the skiprows parameter in Python is converted to a C++ value, so now we can skip more than 2B rows. - Renamed `FileFooter` to `Footer` to match the specs. No measurable impact on performance or on the memory footprint of the ORC reader. Authors: - Vukasin Milovanovic (https://github.com/vuule) - Nghia Truong (https://github.com/ttnghia) Approvers: - Nghia Truong (https://github.com/ttnghia) - Shruti Shivakumar (https://github.com/shrshi) - Yunsong Wang (https://github.com/PointKernel) - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15008 --- cpp/include/cudf/io/orc.hpp | 1 + cpp/include/cudf/io/orc_metadata.hpp | 4 +- cpp/src/io/functions.cpp | 2 +- cpp/src/io/orc/aggregate_orc_metadata.cpp | 8 +-- cpp/src/io/orc/aggregate_orc_metadata.hpp | 8 ++- cpp/src/io/orc/orc.cpp | 7 ++- cpp/src/io/orc/orc.hpp | 32 +++++----- cpp/src/io/orc/orc_gpu.hpp | 54 ++++++++-------- cpp/src/io/orc/reader_impl.cu | 2 +- cpp/src/io/orc/reader_impl.hpp | 4 +- cpp/src/io/orc/reader_impl_helpers.hpp | 10 +-- cpp/src/io/orc/reader_impl_preprocess.cu | 36 +++++------ cpp/src/io/orc/stripe_data.cu | 69 ++++++++++---------- cpp/src/io/orc/stripe_enc.cu | 8 +-- cpp/src/io/orc/stripe_init.cu | 46 +++++++------- cpp/src/io/orc/writer_impl.cu | 76 +++++++++++------------ cpp/src/io/orc/writer_impl.hpp | 2 +- cpp/src/io/utilities/row_selection.cpp | 6 +- cpp/src/io/utilities/row_selection.hpp | 6 +- cpp/tests/io/orc_test.cpp | 58 +++++++++++++++++ python/cudf/cudf/_lib/orc.pyx | 26 ++++---- 21 files changed, 262 insertions(+), 203 deletions(-) diff --git a/cpp/include/cudf/io/orc.hpp b/cpp/include/cudf/io/orc.hpp index a3f76817f8a..5cc9ea81f29 100644 --- a/cpp/include/cudf/io/orc.hpp +++ b/cpp/include/cudf/io/orc.hpp @@ -200,6 +200,7 @@ class orc_reader_options { void set_skip_rows(uint64_t rows) { CUDF_EXPECTS(rows == 0 or _stripes.empty(), "Can't set both skip_rows along with stripes"); + CUDF_EXPECTS(rows <= std::numeric_limits::max(), "skip_rows is too large"); _skip_rows = rows; } diff --git a/cpp/include/cudf/io/orc_metadata.hpp b/cpp/include/cudf/io/orc_metadata.hpp index 19d44263d1b..8f3eb1dff3c 100644 --- a/cpp/include/cudf/io/orc_metadata.hpp +++ b/cpp/include/cudf/io/orc_metadata.hpp @@ -331,7 +331,7 @@ class orc_metadata { * @param num_rows number of rows * @param num_stripes number of stripes */ - orc_metadata(orc_schema schema, size_type num_rows, size_type num_stripes) + orc_metadata(orc_schema schema, uint64_t num_rows, size_type num_stripes) : _schema{std::move(schema)}, _num_rows{num_rows}, _num_stripes{num_stripes} { } @@ -362,7 +362,7 @@ class orc_metadata { private: orc_schema _schema; - size_type _num_rows; + uint64_t _num_rows; size_type _num_stripes; }; diff --git a/cpp/src/io/functions.cpp b/cpp/src/io/functions.cpp index 315562e9183..b8353d312fe 100644 --- a/cpp/src/io/functions.cpp +++ b/cpp/src/io/functions.cpp @@ -404,7 +404,7 @@ orc_metadata read_orc_metadata(source_info const& src_info, rmm::cuda_stream_vie auto const footer = orc::metadata(sources.front().get(), stream).ff; return {{make_orc_column_schema(footer.types, 0, "")}, - static_cast(footer.numberOfRows), + footer.numberOfRows, static_cast(footer.stripes.size())}; } diff --git a/cpp/src/io/orc/aggregate_orc_metadata.cpp b/cpp/src/io/orc/aggregate_orc_metadata.cpp index ea091099b6e..f5f540bc3a4 100644 --- a/cpp/src/io/orc/aggregate_orc_metadata.cpp +++ b/cpp/src/io/orc/aggregate_orc_metadata.cpp @@ -155,7 +155,7 @@ aggregate_orc_metadata::aggregate_orc_metadata( std::tuple> aggregate_orc_metadata::select_stripes( std::vector> const& user_specified_stripes, - uint64_t skip_rows, + int64_t skip_rows, std::optional const& num_rows, rmm::cuda_stream_view stream) { @@ -163,7 +163,7 @@ aggregate_orc_metadata::select_stripes( "Can't use both the row selection and the stripe selection"); auto [rows_to_skip, rows_to_read] = [&]() { - if (not user_specified_stripes.empty()) { return std::pair{0, 0}; } + if (not user_specified_stripes.empty()) { return std::pair{0, 0}; } return cudf::io::detail::skip_rows_num_rows_from_options(skip_rows, num_rows, get_num_rows()); }(); @@ -192,8 +192,8 @@ aggregate_orc_metadata::select_stripes( selected_stripes_mapping.push_back({static_cast(src_file_idx), stripe_infos}); } } else { - uint64_t count = 0; - size_type stripe_skip_rows = 0; + int64_t count = 0; + int64_t stripe_skip_rows = 0; // Iterate all source files, each source file has corelating metadata for (size_t src_file_idx = 0; src_file_idx < per_file_metadata.size() && count < rows_to_skip + rows_to_read; diff --git a/cpp/src/io/orc/aggregate_orc_metadata.hpp b/cpp/src/io/orc/aggregate_orc_metadata.hpp index f05946a4346..d1e053be481 100644 --- a/cpp/src/io/orc/aggregate_orc_metadata.hpp +++ b/cpp/src/io/orc/aggregate_orc_metadata.hpp @@ -79,9 +79,11 @@ class aggregate_orc_metadata { [[nodiscard]] auto const& get_types() const { return per_file_metadata[0].ff.types; } - [[nodiscard]] int get_row_index_stride() const + [[nodiscard]] size_type get_row_index_stride() const { - return static_cast(per_file_metadata[0].ff.rowIndexStride); + CUDF_EXPECTS(per_file_metadata[0].ff.rowIndexStride <= std::numeric_limits::max(), + "Row index stride exceeds size_type max"); + return per_file_metadata[0].ff.rowIndexStride; } [[nodiscard]] auto is_row_grp_idx_present() const { return row_grp_idx_present; } @@ -115,7 +117,7 @@ class aggregate_orc_metadata { */ [[nodiscard]] std::tuple> select_stripes(std::vector> const& user_specified_stripes, - uint64_t skip_rows, + int64_t skip_rows, std::optional const& num_rows, rmm::cuda_stream_view stream); diff --git a/cpp/src/io/orc/orc.cpp b/cpp/src/io/orc/orc.cpp index de0d7a88614..1fe5e5aa41e 100644 --- a/cpp/src/io/orc/orc.cpp +++ b/cpp/src/io/orc/orc.cpp @@ -69,7 +69,7 @@ void ProtobufReader::read(PostScript& s, size_t maxlen) function_builder(s, maxlen, op); } -void ProtobufReader::read(FileFooter& s, size_t maxlen) +void ProtobufReader::read(Footer& s, size_t maxlen) { auto op = std::tuple(field_reader(1, s.headerLength), field_reader(2, s.contentLength), @@ -307,7 +307,7 @@ size_t ProtobufWriter::write(PostScript const& s) return w.value(); } -size_t ProtobufWriter::write(FileFooter const& s) +size_t ProtobufWriter::write(Footer const& s) { ProtobufFieldWriter w(this); w.field_uint(1, s.headerLength); @@ -393,7 +393,8 @@ size_t ProtobufWriter::write(Metadata const& s) return w.value(); } -OrcDecompressor::OrcDecompressor(CompressionKind kind, uint32_t blockSize) : m_blockSize(blockSize) +OrcDecompressor::OrcDecompressor(CompressionKind kind, uint64_t block_size) + : m_blockSize(block_size) { switch (kind) { case NONE: diff --git a/cpp/src/io/orc/orc.hpp b/cpp/src/io/orc/orc.hpp index 6fbee2824eb..88bd260a598 100644 --- a/cpp/src/io/orc/orc.hpp +++ b/cpp/src/io/orc/orc.hpp @@ -73,7 +73,7 @@ static constexpr int32_t DEFAULT_MAX_NANOS = 999'999; struct PostScript { uint64_t footerLength = 0; // the length of the footer section in bytes CompressionKind compression = NONE; // the kind of generic compression used - uint32_t compressionBlockSize{}; // the maximum size of each compression chunk + uint64_t compressionBlockSize{}; // the maximum size of each compression chunk std::vector version; // the version of the file format [major, minor] uint64_t metadataLength = 0; // the length of the metadata section in bytes std::optional writerVersion; // The version of the writer that wrote the file @@ -84,8 +84,8 @@ struct StripeInformation { uint64_t offset = 0; // the start of the stripe within the file uint64_t indexLength = 0; // the length of the indexes in bytes uint64_t dataLength = 0; // the length of the data in bytes - uint32_t footerLength = 0; // the length of the footer in bytes - uint32_t numberOfRows = 0; // the number of rows in the stripe + uint64_t footerLength = 0; // the length of the footer in bytes + uint64_t numberOfRows = 0; // the number of rows in the stripe }; struct SchemaType { @@ -105,7 +105,7 @@ struct UserMetadataItem { using ColStatsBlob = std::vector; // Column statistics blob -struct FileFooter { +struct Footer { uint64_t headerLength = 0; // the length of the file header in bytes (always 3) uint64_t contentLength = 0; // the length of the file header and body in bytes std::vector stripes; // the information about the stripes @@ -237,7 +237,7 @@ class ProtobufReader { read(s, m_end - m_cur); } void read(PostScript&, size_t maxlen); - void read(FileFooter&, size_t maxlen); + void read(Footer&, size_t maxlen); void read(StripeInformation&, size_t maxlen); void read(SchemaType&, size_t maxlen); void read(UserMetadataItem&, size_t maxlen); @@ -519,7 +519,7 @@ class ProtobufWriter { public: size_t write(PostScript const&); - size_t write(FileFooter const&); + size_t write(Footer const&); size_t write(StripeInformation const&); size_t write(SchemaType const&); size_t write(UserMetadataItem const&); @@ -540,7 +540,7 @@ class ProtobufWriter { class OrcDecompressor { public: - OrcDecompressor(CompressionKind kind, uint32_t blockSize); + OrcDecompressor(CompressionKind kind, uint64_t blockSize); /** * @brief ORC block decompression @@ -553,17 +553,17 @@ class OrcDecompressor { host_span decompress_blocks(host_span src, rmm::cuda_stream_view stream); [[nodiscard]] uint32_t GetLog2MaxCompressionRatio() const { return m_log2MaxRatio; } - [[nodiscard]] uint32_t GetMaxUncompressedBlockSize(uint32_t block_len) const + [[nodiscard]] uint64_t GetMaxUncompressedBlockSize(uint32_t block_len) const { - return std::min(block_len << m_log2MaxRatio, m_blockSize); + return std::min(static_cast(block_len) << m_log2MaxRatio, m_blockSize); } [[nodiscard]] compression_type compression() const { return _compression; } - [[nodiscard]] uint32_t GetBlockSize() const { return m_blockSize; } + [[nodiscard]] auto GetBlockSize() const { return m_blockSize; } protected: compression_type _compression; uint32_t m_log2MaxRatio = 24; // log2 of maximum compression ratio - uint32_t m_blockSize; + uint64_t m_blockSize; std::vector m_buf; }; @@ -613,9 +613,9 @@ class metadata { public: explicit metadata(datasource* const src, rmm::cuda_stream_view stream); - [[nodiscard]] size_t get_total_rows() const { return ff.numberOfRows; } - [[nodiscard]] int get_num_stripes() const { return ff.stripes.size(); } - [[nodiscard]] int get_num_columns() const { return ff.types.size(); } + [[nodiscard]] auto get_total_rows() const { return ff.numberOfRows; } + [[nodiscard]] size_type get_num_stripes() const { return ff.stripes.size(); } + [[nodiscard]] size_type get_num_columns() const { return ff.types.size(); } /** * @brief Returns the name of the column with the given ID. * @@ -638,7 +638,7 @@ class metadata { CUDF_EXPECTS(column_id < get_num_columns(), "Out of range column id provided"); return column_paths[column_id]; } - [[nodiscard]] int get_row_index_stride() const { return ff.rowIndexStride; } + [[nodiscard]] auto get_row_index_stride() const { return ff.rowIndexStride; } /** * @brief Returns the ID of the parent column of the given column. @@ -666,7 +666,7 @@ class metadata { public: PostScript ps; - FileFooter ff; + Footer ff; Metadata md; std::vector stripefooters; std::unique_ptr decompressor; diff --git a/cpp/src/io/orc/orc_gpu.hpp b/cpp/src/io/orc/orc_gpu.hpp index b69722bbded..8c7ccf0527f 100644 --- a/cpp/src/io/orc/orc_gpu.hpp +++ b/cpp/src/io/orc/orc_gpu.hpp @@ -101,18 +101,18 @@ struct DictionaryEntry { struct ColumnDesc { uint8_t const* streams[CI_NUM_STREAMS]; // ptr to data stream index uint32_t strm_id[CI_NUM_STREAMS]; // stream ids - uint32_t strm_len[CI_NUM_STREAMS]; // stream length + int64_t strm_len[CI_NUM_STREAMS]; // stream length uint32_t* valid_map_base; // base pointer of valid bit map for this column void* column_data_base; // base pointer of column data - uint32_t start_row; // starting row of the stripe - uint32_t num_rows; // number of rows in stripe - uint32_t column_num_rows; // number of rows in whole column - uint32_t num_child_rows; // store number of child rows if it's list column + int64_t start_row; // starting row of the stripe + int64_t num_rows; // number of rows in stripe + int64_t column_num_rows; // number of rows in whole column + int64_t num_child_rows; // store number of child rows if it's list column uint32_t num_rowgroups; // number of rowgroups in the chunk - uint32_t dictionary_start; // start position in global dictionary + int64_t dictionary_start; // start position in global dictionary uint32_t dict_len; // length of local dictionary - uint32_t null_count; // number of null values in this stripe's column - uint32_t skip_count; // number of non-null values to skip + int64_t null_count; // number of null values in this stripe's column + int64_t skip_count; // number of non-null values to skip uint32_t rowgroup_id; // row group position ColumnEncodingKind encoding_kind; // column encoding kind TypeKind type_kind; // column data type @@ -129,10 +129,10 @@ struct ColumnDesc { */ struct RowGroup { uint32_t chunk_id; // Column chunk this entry belongs to - uint32_t strm_offset[2]; // Index offset for CI_DATA and CI_DATA2 streams + int64_t strm_offset[2]; // Index offset for CI_DATA and CI_DATA2 streams uint16_t run_pos[2]; // Run position for CI_DATA and CI_DATA2 uint32_t num_rows; // number of rows in rowgroup - uint32_t start_row; // starting row of the rowgroup + int64_t start_row; // starting row of the rowgroup uint32_t num_child_rows; // number of rows of children in rowgroup in case of list type }; @@ -140,9 +140,9 @@ struct RowGroup { * @brief Struct to describe an encoder data chunk */ struct EncChunk { - uint32_t start_row; // start row of this chunk + int64_t start_row; // start row of this chunk uint32_t num_rows; // number of rows in this chunk - uint32_t null_mask_start_row; // adjusted to multiple of 8 + int64_t null_mask_start_row; // adjusted to multiple of 8 uint32_t null_mask_num_rows; // adjusted to multiple of 8 ColumnEncodingKind encoding_kind; // column encoding kind TypeKind type_kind; // column data type @@ -253,7 +253,7 @@ constexpr uint32_t encode_block_size = 512; */ void ParseCompressedStripeData(CompressedStreamInfo* strm_info, int32_t num_streams, - uint32_t compression_block_size, + uint64_t compression_block_size, uint32_t log2maxcr, rmm::cuda_stream_view stream); @@ -276,7 +276,6 @@ void PostDecompressionReassemble(CompressedStreamInfo* strm_info, * @param[in] chunks ColumnDesc device array [stripe][column] * @param[in] num_columns Number of columns * @param[in] num_stripes Number of stripes - * @param[in] num_rowgroups Number of row groups * @param[in] rowidx_stride Row index stride * @param[in] use_base_stride Whether to use base stride obtained from meta or use the computed * value @@ -285,10 +284,9 @@ void PostDecompressionReassemble(CompressedStreamInfo* strm_info, void ParseRowGroupIndex(RowGroup* row_groups, CompressedStreamInfo* strm_info, ColumnDesc* chunks, - uint32_t num_columns, - uint32_t num_stripes, - uint32_t num_rowgroups, - uint32_t rowidx_stride, + size_type num_columns, + size_type num_stripes, + size_type rowidx_stride, bool use_base_stride, rmm::cuda_stream_view stream); @@ -304,9 +302,9 @@ void ParseRowGroupIndex(RowGroup* row_groups, */ void DecodeNullsAndStringDictionaries(ColumnDesc* chunks, DictionaryEntry* global_dictionary, - uint32_t num_columns, - uint32_t num_stripes, - size_t first_row, + size_type num_columns, + size_type num_stripes, + int64_t first_row, rmm::cuda_stream_view stream); /** @@ -329,12 +327,12 @@ void DecodeNullsAndStringDictionaries(ColumnDesc* chunks, void DecodeOrcColumnData(ColumnDesc* chunks, DictionaryEntry* global_dictionary, device_2dspan row_groups, - uint32_t num_columns, - uint32_t num_stripes, - size_t first_row, + size_type num_columns, + size_type num_stripes, + int64_t first_row, table_device_view tz_table, - uint32_t num_rowgroups, - uint32_t rowidx_stride, + int64_t num_rowgroups, + size_type rowidx_stride, size_t level, size_type* error_count, rmm::cuda_stream_view stream); @@ -364,8 +362,8 @@ void EncodeOrcColumnData(device_2dspan chunks, void EncodeStripeDictionaries(stripe_dictionary const* stripes, device_span columns, device_2dspan chunks, - uint32_t num_string_columns, - uint32_t num_stripes, + size_type num_string_columns, + size_type num_stripes, device_2dspan enc_streams, rmm::cuda_stream_view stream); diff --git a/cpp/src/io/orc/reader_impl.cu b/cpp/src/io/orc/reader_impl.cu index cf3121fe659..f078e20f7e6 100644 --- a/cpp/src/io/orc/reader_impl.cu +++ b/cpp/src/io/orc/reader_impl.cu @@ -37,7 +37,7 @@ reader::impl::impl(std::vector>&& sources, { } -table_with_metadata reader::impl::read(uint64_t skip_rows, +table_with_metadata reader::impl::read(int64_t skip_rows, std::optional const& num_rows_opt, std::vector> const& stripes) { diff --git a/cpp/src/io/orc/reader_impl.hpp b/cpp/src/io/orc/reader_impl.hpp index 7746bacd188..ab8eaebeb61 100644 --- a/cpp/src/io/orc/reader_impl.hpp +++ b/cpp/src/io/orc/reader_impl.hpp @@ -60,7 +60,7 @@ class reader::impl { * @param stripes Indices of individual stripes to load if non-empty * @return The set of columns along with metadata */ - table_with_metadata read(uint64_t skip_rows, + table_with_metadata read(int64_t skip_rows, std::optional const& num_rows_opt, std::vector> const& stripes); @@ -72,7 +72,7 @@ class reader::impl { * @param num_rows_opt Optional number of rows to read, or `std::nullopt` to read all rows * @param stripes Indices of individual stripes to load if non-empty */ - void prepare_data(uint64_t skip_rows, + void prepare_data(int64_t skip_rows, std::optional const& num_rows_opt, std::vector> const& stripes); diff --git a/cpp/src/io/orc/reader_impl_helpers.hpp b/cpp/src/io/orc/reader_impl_helpers.hpp index 48742b5fc8c..22482bad486 100644 --- a/cpp/src/io/orc/reader_impl_helpers.hpp +++ b/cpp/src/io/orc/reader_impl_helpers.hpp @@ -38,7 +38,7 @@ struct reader_column_meta { std::vector> orc_col_map; // Number of rows in child columns. - std::vector num_child_rows; + std::vector num_child_rows; // Consists of parent column valid_map and null count. std::vector parent_column_data; @@ -46,14 +46,14 @@ struct reader_column_meta { std::vector parent_column_index; // Start row of child columns [stripe][column]. - std::vector child_start_row; + std::vector child_start_row; // Number of rows of child columns [stripe][column]. - std::vector num_child_rows_per_stripe; + std::vector num_child_rows_per_stripe; struct row_group_meta { - uint32_t num_rows; // number of rows in a column in a row group - uint32_t start_row; // start row in a column in a row group + size_type num_rows; // number of rows in a column in a row group + int64_t start_row; // start row in a column in a row group }; // Row group metadata [rowgroup][column]. diff --git a/cpp/src/io/orc/reader_impl_preprocess.cu b/cpp/src/io/orc/reader_impl_preprocess.cu index ea191f67785..6c59f83bc46 100644 --- a/cpp/src/io/orc/reader_impl_preprocess.cu +++ b/cpp/src/io/orc/reader_impl_preprocess.cu @@ -77,7 +77,7 @@ std::size_t gather_stream_info(std::size_t stripe_index, host_span types, bool use_index, bool apply_struct_map, - std::size_t* num_dictionary_entries, + int64_t* num_dictionary_entries, std::vector& stream_info, cudf::detail::hostdevice_2dvector& chunks) { @@ -174,8 +174,8 @@ rmm::device_buffer decompress_stripe_data( host_span stream_info, cudf::detail::hostdevice_2dvector& chunks, cudf::detail::hostdevice_2dvector& row_groups, - std::size_t num_stripes, - std::size_t row_index_stride, + size_type num_stripes, + size_type row_index_stride, bool use_base_stride, rmm::cuda_stream_view stream) { @@ -350,15 +350,15 @@ rmm::device_buffer decompress_stripe_data( // We can check on host after stream synchronize CUDF_EXPECTS(not any_block_failure[0], "Error during decompression"); - auto const num_columns = chunks.size().second; + size_type const num_columns = chunks.size().second; // Update the stream information with the updated uncompressed info // TBD: We could update the value from the information we already // have in stream_info[], but using the gpu results also updates // max_uncompressed_size to the actual uncompressed size, or zero if // decompression failed. - for (std::size_t i = 0; i < num_stripes; ++i) { - for (std::size_t j = 0; j < num_columns; ++j) { + for (size_type i = 0; i < num_stripes; ++i) { + for (size_type j = 0; j < num_columns; ++j) { auto& chunk = chunks[i][j]; for (int k = 0; k < gpu::CI_NUM_STREAMS; ++k) { if (chunk.strm_len[k] > 0 && chunk.strm_id[k] < compinfo.size()) { @@ -377,7 +377,6 @@ rmm::device_buffer decompress_stripe_data( chunks.base_device_ptr(), num_columns, num_stripes, - row_groups.size().first, row_index_stride, use_base_stride, stream); @@ -485,8 +484,8 @@ void update_null_mask(cudf::detail::hostdevice_2dvector& chunks * @param mr Device memory resource to use for device memory allocation */ void decode_stream_data(std::size_t num_dicts, - std::size_t skip_rows, - std::size_t row_index_stride, + int64_t skip_rows, + size_type row_index_stride, std::size_t level, table_view const& tz_table, cudf::detail::hostdevice_2dvector& chunks, @@ -622,9 +621,9 @@ void aggregate_child_meta(std::size_t level, col_meta.num_child_rows_per_stripe.resize(number_of_child_chunks); col_meta.rwgrp_meta.resize(num_of_rowgroups * num_child_cols); - auto child_start_row = cudf::detail::host_2dspan( + auto child_start_row = cudf::detail::host_2dspan( col_meta.child_start_row.data(), num_of_stripes, num_child_cols); - auto num_child_rows_per_stripe = cudf::detail::host_2dspan( + auto num_child_rows_per_stripe = cudf::detail::host_2dspan( col_meta.num_child_rows_per_stripe.data(), num_of_stripes, num_child_cols); auto rwgrp_meta = cudf::detail::host_2dspan( col_meta.rwgrp_meta.data(), num_of_rowgroups, num_child_cols); @@ -634,7 +633,7 @@ void aggregate_child_meta(std::size_t level, // For each parent column, update its child column meta for each stripe. std::for_each(nested_cols.begin(), nested_cols.end(), [&](auto const p_col) { auto const parent_col_idx = col_meta.orc_col_map[level][p_col.id]; - auto start_row = 0; + int64_t start_row = 0; auto processed_row_groups = 0; for (std::size_t stripe_id = 0; stripe_id < num_of_stripes; stripe_id++) { @@ -711,7 +710,7 @@ void generate_offsets_for_list(host_span buff_data, rmm::cuda_ } // namespace -void reader::impl::prepare_data(uint64_t skip_rows, +void reader::impl::prepare_data(int64_t skip_rows, std::optional const& num_rows_opt, std::vector> const& stripes) { @@ -813,7 +812,7 @@ void reader::impl::prepare_data(uint64_t skip_rows, // Only use if we don't have much work with complete columns & stripes // TODO: Consider nrows, gpu, and tune the threshold (rows_to_read > _metadata.get_row_index_stride() && !(_metadata.get_row_index_stride() & 7) && - _metadata.get_row_index_stride() > 0 && num_columns * total_num_stripes < 8 * 128) && + _metadata.get_row_index_stride() != 0 && num_columns * total_num_stripes < 8 * 128) && // Only use if first row is aligned to a stripe boundary // TODO: Fix logic to handle unaligned rows (rows_to_skip == 0); @@ -833,10 +832,10 @@ void reader::impl::prepare_data(uint64_t skip_rows, // Tracker for eventually deallocating compressed and uncompressed data auto& stripe_data = lvl_stripe_data[level]; - std::size_t stripe_start_row = 0; - std::size_t num_dict_entries = 0; - std::size_t num_rowgroups = 0; - int stripe_idx = 0; + int64_t stripe_start_row = 0; + int64_t num_dict_entries = 0; + int64_t num_rowgroups = 0; + size_type stripe_idx = 0; std::vector, std::size_t>> read_tasks; for (auto const& stripe_source_mapping : selected_stripes) { @@ -1003,7 +1002,6 @@ void reader::impl::prepare_data(uint64_t skip_rows, chunks.base_device_ptr(), num_columns, total_num_stripes, - num_rowgroups, _metadata.get_row_index_stride(), level == 0, _stream); diff --git a/cpp/src/io/orc/stripe_data.cu b/cpp/src/io/orc/stripe_data.cu index 5e10d90ae9b..1572b7246c0 100644 --- a/cpp/src/io/orc/stripe_data.cu +++ b/cpp/src/io/orc/stripe_data.cu @@ -94,8 +94,8 @@ struct orc_strdict_state_s { }; struct orc_datadec_state_s { - uint32_t cur_row; // starting row of current batch - uint32_t end_row; // ending row of this chunk (start_row + num_rows) + int64_t cur_row; // starting row of current batch + int64_t end_row; // ending row of this chunk (start_row + num_rows) uint32_t max_vals; // max # of non-zero values to decode in this batch uint32_t nrows; // # of rows in current batch (up to block_size) uint32_t buffered_count; // number of buffered values in the secondary data stream @@ -108,7 +108,7 @@ struct orcdec_state_s { orc_bytestream_s bs; orc_bytestream_s bs2; int is_string; - uint64_t num_child_rows; + int64_t num_child_rows; union { orc_strdict_state_s dict; uint32_t nulls_desc_row; // number of rows processed for nulls. @@ -1086,9 +1086,9 @@ template CUDF_KERNEL void __launch_bounds__(block_size) gpuDecodeNullsAndStringDictionaries(ColumnDesc* chunks, DictionaryEntry* global_dictionary, - uint32_t num_columns, - uint32_t num_stripes, - size_t first_row) + size_type num_columns, + size_type num_stripes, + int64_t first_row) { __shared__ __align__(16) orcdec_state_s state_g; using warp_reduce = cub::WarpReduce; @@ -1132,12 +1132,13 @@ CUDF_KERNEL void __launch_bounds__(block_size) : 0; auto const num_elems = s->chunk.num_rows - parent_null_count; while (s->top.nulls_desc_row < num_elems) { - uint32_t nrows_max = min(num_elems - s->top.nulls_desc_row, blockDim.x * 32); - uint32_t nrows; - size_t row_in; + auto const nrows_max = + static_cast(min(num_elems - s->top.nulls_desc_row, blockDim.x * 32ul)); bytestream_fill(&s->bs, t); __syncthreads(); + + uint32_t nrows; if (s->chunk.strm_len[CI_PRESENT] > 0) { uint32_t nbytes = Byte_RLE(&s->bs, &s->u.rle8, s->vals.u8, (nrows_max + 7) >> 3, t); nrows = min(nrows_max, nbytes * 8u); @@ -1151,7 +1152,7 @@ CUDF_KERNEL void __launch_bounds__(block_size) } __syncthreads(); - row_in = s->chunk.start_row + s->top.nulls_desc_row - prev_parent_null_count; + auto const row_in = s->chunk.start_row + s->top.nulls_desc_row - prev_parent_null_count; if (row_in + nrows > first_row && row_in < first_row + max_num_rows && s->chunk.valid_map_base != nullptr) { int64_t dst_row = row_in - first_row; @@ -1284,7 +1285,10 @@ static __device__ void DecodeRowPositions(orcdec_state_s* s, if (t == 0) { if (s->chunk.skip_count != 0) { - s->u.rowdec.nz_count = min(min(s->chunk.skip_count, s->top.data.max_vals), blockDim.x); + s->u.rowdec.nz_count = + min(static_cast( + min(s->chunk.skip_count, static_cast(s->top.data.max_vals))), + blockDim.x); s->chunk.skip_count -= s->u.rowdec.nz_count; s->top.data.nrows = s->u.rowdec.nz_count; } else { @@ -1297,11 +1301,12 @@ static __device__ void DecodeRowPositions(orcdec_state_s* s, } while (s->u.rowdec.nz_count < s->top.data.max_vals && s->top.data.cur_row + s->top.data.nrows < s->top.data.end_row) { - uint32_t nrows = min(s->top.data.end_row - (s->top.data.cur_row + s->top.data.nrows), - min((row_decoder_buffer_size - s->u.rowdec.nz_count) * 2, blockDim.x)); + uint32_t const remaining_rows = s->top.data.end_row - (s->top.data.cur_row + s->top.data.nrows); + uint32_t nrows = + min(remaining_rows, min((row_decoder_buffer_size - s->u.rowdec.nz_count) * 2, blockDim.x)); if (s->chunk.valid_map_base != nullptr) { // We have a present stream - uint32_t rmax = s->top.data.end_row - min((uint32_t)first_row, s->top.data.end_row); + uint32_t rmax = s->top.data.end_row - min(first_row, s->top.data.end_row); auto r = (uint32_t)(s->top.data.cur_row + s->top.data.nrows + t - first_row); uint32_t valid = (t < nrows && r < rmax) ? (((uint8_t const*)s->chunk.valid_map_base)[r >> 3] >> (r & 7)) & 1 @@ -1364,8 +1369,8 @@ CUDF_KERNEL void __launch_bounds__(block_size) DictionaryEntry* global_dictionary, table_device_view tz_table, device_2dspan row_groups, - size_t first_row, - uint32_t rowidx_stride, + int64_t first_row, + size_type rowidx_stride, size_t level, size_type* error_count) { @@ -1405,8 +1410,8 @@ CUDF_KERNEL void __launch_bounds__(block_size) if (s->top.data.index.strm_offset[1] > s->chunk.strm_len[CI_DATA2]) { atomicAdd(error_count, 1); } - uint32_t ofs0 = min(s->top.data.index.strm_offset[0], s->chunk.strm_len[CI_DATA]); - uint32_t ofs1 = min(s->top.data.index.strm_offset[1], s->chunk.strm_len[CI_DATA2]); + auto const ofs0 = min(s->top.data.index.strm_offset[0], s->chunk.strm_len[CI_DATA]); + auto const ofs1 = min(s->top.data.index.strm_offset[1], s->chunk.strm_len[CI_DATA2]); uint32_t rowgroup_rowofs = (level == 0) ? (blockIdx.y - min(s->chunk.rowgroup_id, blockIdx.y)) * rowidx_stride : s->top.data.index.start_row; @@ -1415,14 +1420,13 @@ CUDF_KERNEL void __launch_bounds__(block_size) s->chunk.strm_len[CI_DATA] -= ofs0; s->chunk.streams[CI_DATA2] += ofs1; s->chunk.strm_len[CI_DATA2] -= ofs1; - rowgroup_rowofs = min(rowgroup_rowofs, s->chunk.num_rows); + rowgroup_rowofs = min(static_cast(rowgroup_rowofs), s->chunk.num_rows); s->chunk.start_row += rowgroup_rowofs; s->chunk.num_rows -= rowgroup_rowofs; } - s->is_string = (s->chunk.type_kind == STRING || s->chunk.type_kind == BINARY || + s->is_string = (s->chunk.type_kind == STRING || s->chunk.type_kind == BINARY || s->chunk.type_kind == VARCHAR || s->chunk.type_kind == CHAR); - s->top.data.cur_row = - max(s->chunk.start_row, max((int32_t)(first_row - s->chunk.skip_count), 0)); + s->top.data.cur_row = max(s->chunk.start_row, max(first_row - s->chunk.skip_count, 0ul)); s->top.data.end_row = s->chunk.start_row + s->chunk.num_rows; s->top.data.buffered_count = 0; if (s->top.data.end_row > first_row + max_num_rows) { @@ -1824,7 +1828,8 @@ CUDF_KERNEL void __launch_bounds__(block_size) if (num_rowgroups > 0) { row_groups[blockIdx.y][blockIdx.x].num_child_rows = s->num_child_rows; } - atomicAdd(&chunks[chunk_id].num_child_rows, s->num_child_rows); + cuda::atomic_ref ref{chunks[chunk_id].num_child_rows}; + ref.fetch_add(s->num_child_rows, cuda::std::memory_order_relaxed); } } @@ -1840,9 +1845,9 @@ CUDF_KERNEL void __launch_bounds__(block_size) */ void __host__ DecodeNullsAndStringDictionaries(ColumnDesc* chunks, DictionaryEntry* global_dictionary, - uint32_t num_columns, - uint32_t num_stripes, - size_t first_row, + size_type num_columns, + size_type num_stripes, + int64_t first_row, rmm::cuda_stream_view stream) { dim3 dim_block(block_size, 1); @@ -1869,17 +1874,17 @@ void __host__ DecodeNullsAndStringDictionaries(ColumnDesc* chunks, void __host__ DecodeOrcColumnData(ColumnDesc* chunks, DictionaryEntry* global_dictionary, device_2dspan row_groups, - uint32_t num_columns, - uint32_t num_stripes, - size_t first_row, + size_type num_columns, + size_type num_stripes, + int64_t first_row, table_device_view tz_table, - uint32_t num_rowgroups, - uint32_t rowidx_stride, + int64_t num_rowgroups, + size_type rowidx_stride, size_t level, size_type* error_count, rmm::cuda_stream_view stream) { - uint32_t num_chunks = num_columns * num_stripes; + auto const num_chunks = num_columns * num_stripes; dim3 dim_block(block_size, 1); // 1024 threads per chunk dim3 dim_grid((num_rowgroups > 0) ? num_columns : num_chunks, (num_rowgroups > 0) ? num_rowgroups : 1); diff --git a/cpp/src/io/orc/stripe_enc.cu b/cpp/src/io/orc/stripe_enc.cu index 748e4d2c27b..b6fc4e3510f 100644 --- a/cpp/src/io/orc/stripe_enc.cu +++ b/cpp/src/io/orc/stripe_enc.cu @@ -647,8 +647,8 @@ static __device__ void encode_null_mask(orcenc_state_s* s, if (t_nrows == 0) return 0; if (mask == nullptr) return 0xff; - auto const begin_offset = row + offset; - auto const end_offset = min(begin_offset + 8, offset + column.size()); + size_type const begin_offset = row + offset; + auto const end_offset = min(begin_offset + 8, offset + column.size()); auto const mask_word = cudf::detail::get_mask_offset_word(mask, 0, begin_offset, end_offset); return mask_word & 0xff; }; @@ -1309,8 +1309,8 @@ void EncodeOrcColumnData(device_2dspan chunks, void EncodeStripeDictionaries(stripe_dictionary const* stripes, device_span columns, device_2dspan chunks, - uint32_t num_string_columns, - uint32_t num_stripes, + size_type num_string_columns, + size_type num_stripes, device_2dspan enc_streams, rmm::cuda_stream_view stream) { diff --git a/cpp/src/io/orc/stripe_init.cu b/cpp/src/io/orc/stripe_init.cu index 350700a22fd..dd44b779402 100644 --- a/cpp/src/io/orc/stripe_init.cu +++ b/cpp/src/io/orc/stripe_init.cu @@ -42,8 +42,11 @@ struct compressed_stream_s { }; // blockDim {128,1,1} -CUDF_KERNEL void __launch_bounds__(128, 8) gpuParseCompressedStripeData( - CompressedStreamInfo* strm_info, int32_t num_streams, uint32_t block_size, uint32_t log2maxcr) +CUDF_KERNEL void __launch_bounds__(128, 8) + gpuParseCompressedStripeData(CompressedStreamInfo* strm_info, + int32_t num_streams, + uint64_t compression_block_size, + uint32_t log2maxcr) { __shared__ compressed_stream_s strm_g[4]; @@ -60,18 +63,18 @@ CUDF_KERNEL void __launch_bounds__(128, 8) gpuParseCompressedStripeData( uint8_t const* end = cur + s->info.compressed_data_size; uint8_t* uncompressed = s->info.uncompressed_data; size_t max_uncompressed_size = 0; - uint32_t max_uncompressed_block_size = 0; + uint64_t max_uncompressed_block_size = 0; uint32_t num_compressed_blocks = 0; uint32_t num_uncompressed_blocks = 0; while (cur + block_header_size < end) { uint32_t block_len = shuffle((lane_id == 0) ? cur[0] | (cur[1] << 8) | (cur[2] << 16) : 0); auto const is_uncompressed = static_cast(block_len & 1); - uint32_t uncompressed_size; + uint64_t uncompressed_size; device_span* init_in_ctl = nullptr; device_span* init_out_ctl = nullptr; block_len >>= 1; cur += block_header_size; - if (block_len > block_size || cur + block_len > end) { + if (block_len > compression_block_size || cur + block_len > end) { // Fatal num_compressed_blocks = 0; max_uncompressed_size = 0; @@ -81,9 +84,10 @@ CUDF_KERNEL void __launch_bounds__(128, 8) gpuParseCompressedStripeData( // TBD: For some codecs like snappy, it wouldn't be too difficult to get the actual // uncompressed size and avoid waste due to block size alignment For now, rely on the max // compression ratio to limit waste for the most extreme cases (small single-block streams) - uncompressed_size = (is_uncompressed) ? block_len - : (block_len < (block_size >> log2maxcr)) ? block_len << log2maxcr - : block_size; + uncompressed_size = (is_uncompressed) ? block_len + : (block_len < (compression_block_size >> log2maxcr)) + ? block_len << log2maxcr + : compression_block_size; if (is_uncompressed) { if (uncompressed_size <= 32) { // For short blocks, copy the uncompressed data to output @@ -446,10 +450,9 @@ static __device__ void gpuMapRowIndexToUncompressed(rowindex_state_s* s, CUDF_KERNEL void __launch_bounds__(128, 8) gpuParseRowGroupIndex(RowGroup* row_groups, CompressedStreamInfo* strm_info, ColumnDesc* chunks, - uint32_t num_columns, - uint32_t num_stripes, - uint32_t num_rowgroups, - uint32_t rowidx_stride, + size_type num_columns, + size_type num_stripes, + size_type rowidx_stride, bool use_base_stride) { __shared__ __align__(16) rowindex_state_s state_g; @@ -554,7 +557,7 @@ CUDF_KERNEL void __launch_bounds__(block_size) void __host__ ParseCompressedStripeData(CompressedStreamInfo* strm_info, int32_t num_streams, - uint32_t compression_block_size, + uint64_t compression_block_size, uint32_t log2maxcr, rmm::cuda_stream_view stream) { @@ -577,23 +580,16 @@ void __host__ PostDecompressionReassemble(CompressedStreamInfo* strm_info, void __host__ ParseRowGroupIndex(RowGroup* row_groups, CompressedStreamInfo* strm_info, ColumnDesc* chunks, - uint32_t num_columns, - uint32_t num_stripes, - uint32_t num_rowgroups, - uint32_t rowidx_stride, + size_type num_columns, + size_type num_stripes, + size_type rowidx_stride, bool use_base_stride, rmm::cuda_stream_view stream) { dim3 dim_block(128, 1); dim3 dim_grid(num_columns, num_stripes); // 1 column chunk per block - gpuParseRowGroupIndex<<>>(row_groups, - strm_info, - chunks, - num_columns, - num_stripes, - num_rowgroups, - rowidx_stride, - use_base_stride); + gpuParseRowGroupIndex<<>>( + row_groups, strm_info, chunks, num_columns, num_stripes, rowidx_stride, use_base_stride); } void __host__ reduce_pushdown_masks(device_span columns, diff --git a/cpp/src/io/orc/writer_impl.cu b/cpp/src/io/orc/writer_impl.cu index f0235e13422..ade0e75de35 100644 --- a/cpp/src/io/orc/writer_impl.cu +++ b/cpp/src/io/orc/writer_impl.cu @@ -1309,15 +1309,15 @@ intermediate_statistics gather_statistic_blobs(statistics_freq const stats_freq, * @param stream CUDA stream used for device memory operations and kernel launches * @return The encoded statistic blobs */ -encoded_footer_statistics finish_statistic_blobs(FileFooter const& file_footer, +encoded_footer_statistics finish_statistic_blobs(Footer const& footer, persisted_statistics& per_chunk_stats, rmm::cuda_stream_view stream) { auto stripe_size_iter = thrust::make_transform_iterator(per_chunk_stats.stripe_stat_merge.begin(), [](auto const& s) { return s.size(); }); - auto const num_columns = file_footer.types.size() - 1; - auto const num_stripes = file_footer.stripes.size(); + auto const num_columns = footer.types.size() - 1; + auto const num_stripes = footer.stripes.size(); auto const num_stripe_blobs = thrust::reduce(stripe_size_iter, stripe_size_iter + per_chunk_stats.stripe_stat_merge.size()); @@ -1333,7 +1333,7 @@ encoded_footer_statistics finish_statistic_blobs(FileFooter const& file_footer, // Fill in stats_merge and stat_chunks on the host for (auto i = 0u; i < num_file_blobs; ++i) { stats_merge[i].col_dtype = per_chunk_stats.col_types[i]; - stats_merge[i].stats_dtype = kind_to_stats_type(file_footer.types[i + 1].kind); + stats_merge[i].stats_dtype = kind_to_stats_type(footer.types[i + 1].kind); // Write the sum for empty columns, equal to zero h_stat_chunks[i].has_sum = true; } @@ -2632,21 +2632,21 @@ void writer::impl::write_orc_data_to_sink(encoded_data const& enc_data, void writer::impl::add_table_to_footer_data(orc_table_view const& orc_table, std::vector& stripes) { - if (_ffooter.headerLength == 0) { + if (_footer.headerLength == 0) { // First call - _ffooter.headerLength = std::strlen(MAGIC); - _ffooter.writer = cudf_writer_code; - _ffooter.rowIndexStride = _row_index_stride; - _ffooter.types.resize(1 + orc_table.num_columns()); - _ffooter.types[0].kind = STRUCT; + _footer.headerLength = std::strlen(MAGIC); + _footer.writer = cudf_writer_code; + _footer.rowIndexStride = _row_index_stride; + _footer.types.resize(1 + orc_table.num_columns()); + _footer.types[0].kind = STRUCT; for (auto const& column : orc_table.columns) { if (!column.is_child()) { - _ffooter.types[0].subtypes.emplace_back(column.id()); - _ffooter.types[0].fieldNames.emplace_back(column.orc_name()); + _footer.types[0].subtypes.emplace_back(column.id()); + _footer.types[0].fieldNames.emplace_back(column.orc_name()); } } for (auto const& column : orc_table.columns) { - auto& schema_type = _ffooter.types[column.id()]; + auto& schema_type = _footer.types[column.id()]; schema_type.kind = column.orc_kind(); if (column.orc_kind() == DECIMAL) { schema_type.scale = static_cast(column.scale()); @@ -2667,18 +2667,18 @@ void writer::impl::add_table_to_footer_data(orc_table_view const& orc_table, } } else { // verify the user isn't passing mismatched tables - CUDF_EXPECTS(_ffooter.types.size() == 1 + orc_table.num_columns(), + CUDF_EXPECTS(_footer.types.size() == 1 + orc_table.num_columns(), "Mismatch in table structure between multiple calls to write"); CUDF_EXPECTS( std::all_of(orc_table.columns.cbegin(), orc_table.columns.cend(), - [&](auto const& col) { return _ffooter.types[col.id()].kind == col.orc_kind(); }), + [&](auto const& col) { return _footer.types[col.id()].kind == col.orc_kind(); }), "Mismatch in column types between multiple calls to write"); } - _ffooter.stripes.insert(_ffooter.stripes.end(), - std::make_move_iterator(stripes.begin()), - std::make_move_iterator(stripes.end())); - _ffooter.numberOfRows += orc_table.num_rows(); + _footer.stripes.insert(_footer.stripes.end(), + std::make_move_iterator(stripes.begin()), + std::make_move_iterator(stripes.end())); + _footer.numberOfRows += orc_table.num_rows(); } void writer::impl::close() @@ -2689,11 +2689,11 @@ void writer::impl::close() if (_stats_freq != statistics_freq::STATISTICS_NONE) { // Write column statistics - auto statistics = finish_statistic_blobs(_ffooter, _persisted_stripe_statistics, _stream); + auto statistics = finish_statistic_blobs(_footer, _persisted_stripe_statistics, _stream); // File-level statistics { - _ffooter.statistics.reserve(_ffooter.types.size()); + _footer.statistics.reserve(_footer.types.size()); ProtobufWriter pbw; // Root column: number of rows @@ -2702,32 +2702,32 @@ void writer::impl::close() // Root column: has nulls pbw.put_uint(encode_field_number(10)); pbw.put_uint(0); - _ffooter.statistics.emplace_back(pbw.release()); + _footer.statistics.emplace_back(pbw.release()); // Add file stats, stored after stripe stats in `column_stats` - _ffooter.statistics.insert(_ffooter.statistics.end(), - std::make_move_iterator(statistics.file_level.begin()), - std::make_move_iterator(statistics.file_level.end())); + _footer.statistics.insert(_footer.statistics.end(), + std::make_move_iterator(statistics.file_level.begin()), + std::make_move_iterator(statistics.file_level.end())); } // Stripe-level statistics if (_stats_freq == statistics_freq::STATISTICS_ROWGROUP or _stats_freq == statistics_freq::STATISTICS_PAGE) { - _orc_meta.stripeStats.resize(_ffooter.stripes.size()); - for (size_t stripe_id = 0; stripe_id < _ffooter.stripes.size(); stripe_id++) { - _orc_meta.stripeStats[stripe_id].colStats.resize(_ffooter.types.size()); + _orc_meta.stripeStats.resize(_footer.stripes.size()); + for (size_t stripe_id = 0; stripe_id < _footer.stripes.size(); stripe_id++) { + _orc_meta.stripeStats[stripe_id].colStats.resize(_footer.types.size()); ProtobufWriter pbw; // Root column: number of rows pbw.put_uint(encode_field_number(1)); - pbw.put_uint(_ffooter.stripes[stripe_id].numberOfRows); + pbw.put_uint(_footer.stripes[stripe_id].numberOfRows); // Root column: has nulls pbw.put_uint(encode_field_number(10)); pbw.put_uint(0); _orc_meta.stripeStats[stripe_id].colStats[0] = pbw.release(); - for (size_t col_idx = 0; col_idx < _ffooter.types.size() - 1; col_idx++) { - size_t idx = _ffooter.stripes.size() * col_idx + stripe_id; + for (size_t col_idx = 0; col_idx < _footer.types.size() - 1; col_idx++) { + size_t idx = _footer.stripes.size() * col_idx + stripe_id; _orc_meta.stripeStats[stripe_id].colStats[1 + col_idx] = std::move(statistics.stripe_level[idx]); } @@ -2737,13 +2737,11 @@ void writer::impl::close() _persisted_stripe_statistics.clear(); - _ffooter.contentLength = _out_sink->bytes_written(); - std::transform(_kv_meta.begin(), - _kv_meta.end(), - std::back_inserter(_ffooter.metadata), - [&](auto const& udata) { - return UserMetadataItem{udata.first, udata.second}; - }); + _footer.contentLength = _out_sink->bytes_written(); + std::transform( + _kv_meta.begin(), _kv_meta.end(), std::back_inserter(_footer.metadata), [&](auto const& udata) { + return UserMetadataItem{udata.first, udata.second}; + }); // Write statistics metadata if (not _orc_meta.stripeStats.empty()) { @@ -2756,7 +2754,7 @@ void writer::impl::close() ps.metadataLength = 0; } ProtobufWriter pbw((_compression_kind != NONE) ? 3 : 0); - pbw.write(_ffooter); + pbw.write(_footer); add_uncompressed_block_headers(_compression_kind, _compression_blocksize, pbw.buffer()); // Write postscript metadata diff --git a/cpp/src/io/orc/writer_impl.hpp b/cpp/src/io/orc/writer_impl.hpp index f1dc45087d5..417d29efb58 100644 --- a/cpp/src/io/orc/writer_impl.hpp +++ b/cpp/src/io/orc/writer_impl.hpp @@ -360,7 +360,7 @@ class writer::impl { // Internal states, filled during `write()` and written to sink during `write` and `close()`. std::unique_ptr _table_meta; - FileFooter _ffooter; + Footer _footer; Metadata _orc_meta; persisted_statistics _persisted_stripe_statistics; // Statistics data saved between calls. bool _closed = false; // To track if the output has been written to sink. diff --git a/cpp/src/io/utilities/row_selection.cpp b/cpp/src/io/utilities/row_selection.cpp index bb5565d8ce7..f136cd11ff7 100644 --- a/cpp/src/io/utilities/row_selection.cpp +++ b/cpp/src/io/utilities/row_selection.cpp @@ -23,8 +23,8 @@ namespace cudf::io::detail { -std::pair skip_rows_num_rows_from_options( - uint64_t skip_rows, std::optional const& num_rows, uint64_t num_source_rows) +std::pair skip_rows_num_rows_from_options( + int64_t skip_rows, std::optional const& num_rows, int64_t num_source_rows) { auto const rows_to_skip = std::min(skip_rows, num_source_rows); if (not num_rows.has_value()) { @@ -36,7 +36,7 @@ std::pair skip_rows_num_rows_from_options( // Limit the number of rows to the end of the input return { rows_to_skip, - static_cast(std::min(num_rows.value(), num_source_rows - rows_to_skip))}; + static_cast(std::min(num_rows.value(), num_source_rows - rows_to_skip))}; } } // namespace cudf::io::detail diff --git a/cpp/src/io/utilities/row_selection.hpp b/cpp/src/io/utilities/row_selection.hpp index 211726816de..0b5d3aef8bd 100644 --- a/cpp/src/io/utilities/row_selection.hpp +++ b/cpp/src/io/utilities/row_selection.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,7 +34,7 @@ namespace cudf::io::detail { * * @throw std::overflow_exception The requested number of rows exceeds the column size limit */ -std::pair skip_rows_num_rows_from_options( - uint64_t skip_rows, std::optional const& num_rows, uint64_t num_source_rows); +std::pair skip_rows_num_rows_from_options( + int64_t skip_rows, std::optional const& num_rows, int64_t num_source_rows); } // namespace cudf::io::detail diff --git a/cpp/tests/io/orc_test.cpp b/cpp/tests/io/orc_test.cpp index 0b34b39f739..24e2e2cfea0 100644 --- a/cpp/tests/io/orc_test.cpp +++ b/cpp/tests/io/orc_test.cpp @@ -2111,4 +2111,62 @@ TEST_F(OrcWriterTest, BounceBufferBug) cudf::io::write_orc(out_opts); } +TEST_F(OrcReaderTest, SizeTypeRowsOverflow) +{ + using cudf::test::iterators::no_nulls; + constexpr auto num_rows = 500'000'000l; + constexpr auto num_reps = 5; + constexpr auto total_rows = num_rows * num_reps; + static_assert(total_rows > std::numeric_limits::max()); + + auto sequence = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i % 127; }); + column_wrapper col(sequence, + sequence + num_rows); + table_view chunk_table({col}); + + std::vector out_buffer; + { + cudf::io::chunked_orc_writer_options write_opts = + cudf::io::chunked_orc_writer_options::builder(cudf::io::sink_info{&out_buffer}); + + auto writer = cudf::io::orc_chunked_writer(write_opts); + for (int i = 0; i < num_reps; i++) { + writer.write(chunk_table); + } + } + + // Test reading the metadata + auto metadata = read_orc_metadata(cudf::io::source_info{out_buffer.data(), out_buffer.size()}); + EXPECT_EQ(metadata.num_rows(), total_rows); + EXPECT_EQ(metadata.num_stripes(), total_rows / 1'000'000); + + constexpr auto num_rows_to_read = 1'000'000; + const auto num_rows_to_skip = metadata.num_rows() - num_rows_to_read; + + // Read the last million rows + cudf::io::orc_reader_options skip_opts = + cudf::io::orc_reader_options::builder( + cudf::io::source_info{out_buffer.data(), out_buffer.size()}) + .use_index(false) + .skip_rows(num_rows_to_skip); + const auto got_with_skip = cudf::io::read_orc(skip_opts).tbl; + + const auto sequence_start = num_rows_to_skip % num_rows; + column_wrapper skipped_col( + sequence + sequence_start, sequence + sequence_start + num_rows_to_read, no_nulls()); + table_view expected({skipped_col}); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, got_with_skip->view()); + + // Read the last stripe (still the last million rows) + cudf::io::orc_reader_options stripe_opts = + cudf::io::orc_reader_options::builder( + cudf::io::source_info{out_buffer.data(), out_buffer.size()}) + .use_index(false) + .stripes({{metadata.num_stripes() - 1}}); + const auto got_with_stripe_selection = cudf::io::read_orc(stripe_opts).tbl; + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, got_with_stripe_selection->view()); +} + CUDF_TEST_PROGRAM_MAIN() diff --git a/python/cudf/cudf/_lib/orc.pyx b/python/cudf/cudf/_lib/orc.pyx index 16feccc12d0..3fc9823b914 100644 --- a/python/cudf/cudf/_lib/orc.pyx +++ b/python/cudf/cudf/_lib/orc.pyx @@ -3,6 +3,7 @@ import cudf from cudf.core.buffer import acquire_spill_lock +from libc.stdint cimport int64_t from libcpp cimport bool, int from libcpp.map cimport map from libcpp.memory cimport unique_ptr @@ -98,8 +99,8 @@ cpdef read_orc(object filepaths_or_buffers, filepaths_or_buffers, columns, stripes or [], - get_size_t_arg(skip_rows, "skip_rows"), - get_size_t_arg(num_rows, "num_rows"), + get_skiprows_arg(skip_rows), + get_num_rows_arg(num_rows), ( type_id.EMPTY if timestamp_type is None else @@ -318,15 +319,16 @@ def write_orc( libcudf_write_orc(c_orc_writer_options) -cdef size_type get_size_t_arg(object arg, str name) except*: - if name == "skip_rows": - arg = 0 if arg is None else arg - if not isinstance(arg, int) or arg < 0: - raise TypeError(f"{name} must be an int >= 0") - else: - arg = -1 if arg is None else arg - if not isinstance(arg, int) or arg < -1: - raise TypeError(f"{name} must be an int >= -1") +cdef int64_t get_skiprows_arg(object arg) except*: + arg = 0 if arg is None else arg + if not isinstance(arg, int) or arg < 0: + raise TypeError("skiprows must be an int >= 0") + return arg + +cdef size_type get_num_rows_arg(object arg) except*: + arg = -1 if arg is None else arg + if not isinstance(arg, int) or arg < -1: + raise TypeError("num_rows must be an int >= -1") return arg @@ -334,7 +336,7 @@ cdef orc_reader_options make_orc_reader_options( object filepaths_or_buffers, object column_names, object stripes, - size_type skip_rows, + int64_t skip_rows, size_type num_rows, type_id timestamp_type, bool use_index From 7d2da0e5bd9bc178ab394506e58207667c59eedb Mon Sep 17 00:00:00 2001 From: MithunR Date: Sun, 25 Feb 2024 20:22:10 -0800 Subject: [PATCH 301/384] Remove `const` from `range_window_bounds::_extent`. (#15138) The `const` on the `_extent` member of `range_window_bounds` is superfluous. It provides no additional protection to `range_window_bounds`'s invariants, and prevents the class from being copy assignable. This change removes the `const`, thus making `range_window_bounds` copy-assignable, and more readily usable from Cython. Authors: - MithunR (https://github.com/mythrocks) Approvers: - David Wendt (https://github.com/davidwendt) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/15138 --- cpp/include/cudf/rolling/range_window_bounds.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/include/cudf/rolling/range_window_bounds.hpp b/cpp/include/cudf/rolling/range_window_bounds.hpp index ebb28d0b5c4..81885ade2f0 100644 --- a/cpp/include/cudf/rolling/range_window_bounds.hpp +++ b/cpp/include/cudf/rolling/range_window_bounds.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -104,7 +104,7 @@ struct range_window_bounds { range_window_bounds() = default; // Required for use as return types from dispatch functors. private: - const extent_type _extent{extent_type::UNBOUNDED}; + extent_type _extent{extent_type::UNBOUNDED}; std::shared_ptr _range_scalar{nullptr}; // To enable copy construction/assignment. range_window_bounds(extent_type extent_, std::unique_ptr range_scalar_); From 4d26596f98b6414d44dbce30e5e1e909ef024169 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 26 Feb 2024 10:27:38 -0600 Subject: [PATCH 302/384] Add support for `pandas-2.2` in `cudf` (#15100) This PR: - [x] Enables `pandas-2.2` in `cudf` by upgrading the upper bound pinnings. - [x] Cleans up a lot of dead-code. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Matthew Roeschke (https://github.com/mroeschke) - Richard (Rick) Zamora (https://github.com/rjzamora) - Ashwin Srinath (https://github.com/shwina) - Bradley Dice (https://github.com/bdice) - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/cudf/pull/15100 --- .github/workflows/pr.yaml | 24 ++-- .github/workflows/test.yaml | 24 ++-- .../all_cuda-118_arch-x86_64.yaml | 2 +- .../all_cuda-122_arch-x86_64.yaml | 2 +- conda/recipes/cudf/meta.yaml | 2 +- dependencies.yaml | 3 +- python/cudf/cudf/core/_compat.py | 1 - python/cudf/cudf/core/column/datetime.py | 13 +- python/cudf/cudf/core/column/timedelta.py | 12 +- python/cudf/cudf/core/dataframe.py | 9 +- python/cudf/cudf/core/index.py | 17 +-- python/cudf/cudf/pandas/fast_slow_proxy.py | 8 +- .../cudf/pandas/scripts/run-pandas-tests.sh | 6 +- .../cudf/cudf/tests/indexes/test_interval.py | 6 +- python/cudf/cudf/tests/test_applymap.py | 7 - python/cudf/cudf/tests/test_array_ufunc.py | 53 +------- python/cudf/cudf/tests/test_binops.py | 49 +------ .../cudf/cudf/tests/test_column_accessor.py | 3 +- python/cudf/cudf/tests/test_concat.py | 116 ++++++---------- python/cudf/cudf/tests/test_csv.py | 12 +- python/cudf/cudf/tests/test_dataframe.py | 116 +++------------- python/cudf/cudf/tests/test_datetime.py | 114 +--------------- python/cudf/cudf/tests/test_groupby.py | 119 +++++----------- python/cudf/cudf/tests/test_index.py | 55 +------- python/cudf/cudf/tests/test_interval.py | 5 - python/cudf/cudf/tests/test_join_order.py | 127 +----------------- python/cudf/cudf/tests/test_joining.py | 20 +-- python/cudf/cudf/tests/test_json.py | 24 ++-- python/cudf/cudf/tests/test_multiindex.py | 13 +- python/cudf/cudf/tests/test_numerical.py | 3 +- python/cudf/cudf/tests/test_parquet.py | 36 ++--- python/cudf/cudf/tests/test_replace.py | 20 ++- python/cudf/cudf/tests/test_resampling.py | 4 +- python/cudf/cudf/tests/test_reshape.py | 7 +- python/cudf/cudf/tests/test_rolling.py | 37 ++--- python/cudf/cudf/tests/test_sorting.py | 10 +- python/cudf/cudf/tests/test_stats.py | 11 +- python/cudf/cudf/tests/test_timedelta.py | 7 +- .../cudf_pandas_tests/test_cudf_pandas.py | 11 +- python/cudf/pyproject.toml | 3 +- .../dask_cudf/io/tests/test_parquet.py | 3 +- python/dask_cudf/pyproject.toml | 2 +- 42 files changed, 246 insertions(+), 870 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4368c3892f5..d7f47f628d6 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -28,7 +28,7 @@ jobs: - wheel-tests-dask-cudf - devcontainer - unit-tests-cudf-pandas - - pandas-tests + # - pandas-tests #- pandas-tests-diff #- pandas-tests-diff-comment secrets: inherit @@ -155,17 +155,17 @@ jobs: matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) build_type: pull-request script: ci/cudf_pandas_scripts/run_tests.sh - pandas-tests: - # run the Pandas unit tests using PR branch - needs: wheel-build-cudf - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 - with: - matrix_filter: map(select(.ARCH == "amd64")) | max_by(.CUDA_VER) | [.] - build_type: pull-request - script: ci/cudf_pandas_scripts/pandas-tests/run.sh pr - # Hide test failures because they exceed the GITHUB_STEP_SUMMARY output limit. - test_summary_show: "none" + # pandas-tests: + # # run the Pandas unit tests using PR branch + # needs: wheel-build-cudf + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + # with: + # matrix_filter: map(select(.ARCH == "amd64")) | max_by(.CUDA_VER) | [.] + # build_type: pull-request + # script: ci/cudf_pandas_scripts/pandas-tests/run.sh pr + # # Hide test failures because they exceed the GITHUB_STEP_SUMMARY output limit. + # test_summary_show: "none" #pandas-tests-diff: # # diff the results of running the Pandas unit tests and publish a job summary # needs: [pandas-tests-main, pandas-tests-pr] diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 66287d9e515..da733f51779 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -114,15 +114,15 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} script: ci/cudf_pandas_scripts/run_tests.sh - pandas-tests: - # run the Pandas unit tests - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 - with: - matrix_filter: map(select(.ARCH == "amd64")) | max_by(.CUDA_VER) | [.] - build_type: nightly - branch: ${{ inputs.branch }} - date: ${{ inputs.date }} - sha: ${{ inputs.sha }} - # pr mode uses the HEAD of the branch, which is also correct for nightlies - script: ci/cudf_pandas_scripts/pandas-tests/run.sh pr + # pandas-tests: + # # run the Pandas unit tests + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + # with: + # matrix_filter: map(select(.ARCH == "amd64")) | max_by(.CUDA_VER) | [.] + # build_type: nightly + # branch: ${{ inputs.branch }} + # date: ${{ inputs.date }} + # sha: ${{ inputs.sha }} + # # pr mode uses the HEAD of the branch, which is also correct for nightlies + # script: ci/cudf_pandas_scripts/pandas-tests/run.sh pr diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 625e6c6e9db..9d1f71594a9 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -65,7 +65,7 @@ dependencies: - nvcomp==3.0.5 - nvtx>=0.2.1 - packaging -- pandas>=2.0,<2.1.5dev0 +- pandas>=2.0,<2.2.2dev0 - pandoc - pip - pre-commit diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index 871f00a0e8e..8585480720e 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -63,7 +63,7 @@ dependencies: - nvcomp==3.0.5 - nvtx>=0.2.1 - packaging -- pandas>=2.0,<2.1.5dev0 +- pandas>=2.0,<2.2.2dev0 - pandoc - pip - pre-commit diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index d32e6932598..80920dc7b5f 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -80,7 +80,7 @@ requirements: - {{ pin_compatible('protobuf', min_pin='x.x', max_pin='x') }} - python - typing_extensions >=4.0.0 - - pandas >=2.0,<2.1.5dev0 + - pandas >=2.0,<2.2.2dev0 - cupy >=12.0.0 - numba >=0.57 - numpy >=1.21 diff --git a/dependencies.yaml b/dependencies.yaml index c5797fbe40a..c43dab2c7bf 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -497,7 +497,7 @@ dependencies: packages: - fsspec>=0.6.0 - *numpy - - pandas>=2.0,<2.1.5dev0 + - pandas>=2.0,<2.2.2dev0 run_cudf: common: - output_types: [conda, requirements, pyproject] @@ -742,6 +742,7 @@ dependencies: - pytest-asyncio - pytest-reportlog - python-snappy + - pytest-timeout - pyxlsb - s3fs - scipy diff --git a/python/cudf/cudf/core/_compat.py b/python/cudf/cudf/core/_compat.py index 3e2890e2ac4..7fcb353a800 100644 --- a/python/cudf/cudf/core/_compat.py +++ b/python/cudf/cudf/core/_compat.py @@ -9,7 +9,6 @@ PANDAS_GE_201 = PANDAS_VERSION >= version.parse("2.0.1") PANDAS_GE_210 = PANDAS_VERSION >= version.parse("2.1.0") PANDAS_GE_214 = PANDAS_VERSION >= version.parse("2.1.4") -PANDAS_GE_220 = PANDAS_VERSION >= version.parse("2.2.0") PANDAS_LT_203 = PANDAS_VERSION < version.parse("2.0.3") PANDAS_GE_220 = PANDAS_VERSION >= version.parse("2.2.0") PANDAS_LT_300 = PANDAS_VERSION < version.parse("3.0.0") diff --git a/python/cudf/cudf/core/column/datetime.py b/python/cudf/cudf/core/column/datetime.py index b2f14b86ed9..b03b21a7aba 100644 --- a/python/cudf/cudf/core/column/datetime.py +++ b/python/cudf/cudf/core/column/datetime.py @@ -23,7 +23,7 @@ ScalarLike, ) from cudf.api.types import is_datetime64_dtype, is_scalar, is_timedelta64_dtype -from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_220 +from cudf.core._compat import PANDAS_GE_220 from cudf.core.buffer import Buffer, cuda_array_interface_wrapper from cudf.core.column import ColumnBase, as_column, column, string from cudf.core.column.timedelta import _unit_to_nanoseconds_conversion @@ -324,17 +324,8 @@ def to_pandas( # `copy=True` workaround until following issue is fixed: # https://issues.apache.org/jira/browse/ARROW-9772 - if PANDAS_GE_200: - host_values = self.to_arrow() - else: - # Pandas<2.0 supports only `datetime64[ns]`, hence the cast. - host_values = self.astype("datetime64[ns]").to_arrow() - - # Pandas only supports `datetime64[ns]` dtype - # and conversion to this type is necessary to make - # arrow to pandas conversion happen for large values. return pd.Series( - host_values, + self.to_arrow(), copy=True, dtype=self.dtype, index=index, diff --git a/python/cudf/cudf/core/column/timedelta.py b/python/cudf/cudf/core/column/timedelta.py index edf05fbb264..b911c86fa01 100644 --- a/python/cudf/cudf/core/column/timedelta.py +++ b/python/cudf/cudf/core/column/timedelta.py @@ -14,7 +14,6 @@ from cudf import _lib as libcudf from cudf._typing import ColumnBinaryOperand, DatetimeLikeScalar, Dtype from cudf.api.types import is_scalar, is_timedelta64_dtype -from cudf.core._compat import PANDAS_GE_200 from cudf.core.buffer import Buffer, acquire_spill_lock from cudf.core.column import ColumnBase, column, string from cudf.utils.dtypes import np_to_pa_dtype @@ -153,20 +152,11 @@ def to_pandas( # `copy=True` workaround until following issue is fixed: # https://issues.apache.org/jira/browse/ARROW-9772 - if PANDAS_GE_200: - host_values = self.to_arrow() - else: - # Pandas<2.0 supports only `timedelta64[ns]`, hence the cast. - host_values = self.astype("timedelta64[ns]").to_arrow() - - # Pandas only supports `timedelta64[ns]` dtype - # and conversion to this type is necessary to make - # arrow to pandas conversion happen for large values. if nullable: raise NotImplementedError(f"{nullable=} is not implemented.") return pd.Series( - host_values, + self.to_arrow(), copy=True, dtype=self.dtype, index=index, diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 5b300f5e4db..9b4a79c6841 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -56,7 +56,7 @@ is_string_dtype, ) from cudf.core import column, df_protocol, indexing_utils, reshape -from cudf.core._compat import PANDAS_GE_200, PANDAS_LT_300 +from cudf.core._compat import PANDAS_LT_300 from cudf.core.abc import Serializable from cudf.core.column import ( CategoricalColumn, @@ -1339,13 +1339,6 @@ def __getitem__(self, arg): mask = arg if is_list_like(mask): dtype = None - if len(mask) == 0 and not PANDAS_GE_200: - # An explicit dtype is needed to avoid pandas - # warnings from empty sets of columns. This - # shouldn't be needed in pandas 2.0, we don't - # need to specify a dtype when we know we're not - # trying to match any columns so the default is fine. - dtype = "float64" mask = pd.Series(mask, dtype=dtype) if mask.dtype == "bool": return self._apply_boolean_mask(BooleanMask(mask, len(self))) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index ea8ba154922..1b9893d1256 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -39,7 +39,7 @@ is_signed_integer_dtype, ) from cudf.core._base_index import BaseIndex -from cudf.core._compat import PANDAS_GE_200, PANDAS_LT_300 +from cudf.core._compat import PANDAS_LT_300 from cudf.core.column import ( CategoricalColumn, ColumnBase, @@ -2098,23 +2098,14 @@ def to_pandas(self, *, nullable: bool = False) -> pd.DatetimeIndex: if nullable: raise NotImplementedError(f"{nullable=} is not implemented.") - if PANDAS_GE_200: - nanos = self._values - else: - # no need to convert to nanos with Pandas 2.x - if isinstance(self.dtype, pd.DatetimeTZDtype): - nanos = self._values.astype( - pd.DatetimeTZDtype("ns", self.dtype.tz) - ) - else: - nanos = self._values.astype("datetime64[ns]") - freq = ( self._freq._maybe_as_fast_pandas_offset() if self._freq is not None else None ) - return pd.DatetimeIndex(nanos.to_pandas(), name=self.name, freq=freq) + return pd.DatetimeIndex( + self._values.to_pandas(), name=self.name, freq=freq + ) @_cudf_nvtx_annotate def _get_dt_field(self, field): diff --git a/python/cudf/cudf/pandas/fast_slow_proxy.py b/python/cudf/cudf/pandas/fast_slow_proxy.py index a2b14e0c3aa..3f5df18eae1 100644 --- a/python/cudf/cudf/pandas/fast_slow_proxy.py +++ b/python/cudf/cudf/pandas/fast_slow_proxy.py @@ -1071,7 +1071,7 @@ def _is_intermediate_type(result: Any) -> bool: def _is_function_or_method(obj: Any) -> bool: - return isinstance( + res = isinstance( obj, ( types.FunctionType, @@ -1083,6 +1083,12 @@ def _is_function_or_method(obj: Any) -> bool: types.BuiltinMethodType, ), ) + if not res: + try: + return "cython_function_or_method" in str(type(obj)) + except Exception: + return False + return res def _replace_closurevars( diff --git a/python/cudf/cudf/pandas/scripts/run-pandas-tests.sh b/python/cudf/cudf/pandas/scripts/run-pandas-tests.sh index 319e5ba80fc..45aee296845 100755 --- a/python/cudf/cudf/pandas/scripts/run-pandas-tests.sh +++ b/python/cudf/cudf/pandas/scripts/run-pandas-tests.sh @@ -22,7 +22,7 @@ set -euo pipefail # of Pandas installed. PANDAS_VERSION=$(python -c "import pandas; print(pandas.__version__)") -PYTEST_IGNORES="--ignore=tests/io/test_user_agent.py --ignore=tests/interchange/test_impl.py" +PYTEST_IGNORES="--ignore=tests/io/test_user_agent.py --ignore=tests/interchange/test_impl.py --ignore=tests/window/test_dtypes.py --ignore=tests/strings/test_api.py --ignore=tests/window/test_numba.py" mkdir -p pandas-testing cd pandas-testing @@ -183,8 +183,8 @@ and not test_numpy_ufuncs_basic[nullable_float-deg2rad] \ and not test_numpy_ufuncs_basic[nullable_float-rad2deg]" PANDAS_CI="1" python -m pytest -p cudf.pandas \ - -m "not single_cpu and not db" \ - -k "not test_overwrite_warns and not test_complex_series_frame_alignment and $TEST_NUMPY_UFUNCS_BASIC_FLAKY" \ + -v -m "not single_cpu and not db" \ + -k "not test_overwrite_warns and not test_complex_series_frame_alignment and not test_to_parquet_gcs_new_file and not test_qcut_nat and not test_add and not test_ismethods and $TEST_NUMPY_UFUNCS_BASIC_FLAKY" \ --durations=50 \ --import-mode=importlib \ -o xfail_strict=True \ diff --git a/python/cudf/cudf/tests/indexes/test_interval.py b/python/cudf/cudf/tests/indexes/test_interval.py index 6b7e397f65c..36be7c5674d 100644 --- a/python/cudf/cudf/tests/indexes/test_interval.py +++ b/python/cudf/cudf/tests/indexes/test_interval.py @@ -5,9 +5,9 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_210, PANDAS_GE_220 +from cudf.core._compat import PANDAS_GE_210 from cudf.core.index import IntervalIndex, interval_range -from cudf.testing._utils import assert_eq, expect_warning_if +from cudf.testing._utils import assert_eq def test_interval_constructor_default_closed(): @@ -142,7 +142,7 @@ def test_interval_range_periods_basic_dtype(start_t, end_t, periods_t): def test_interval_range_periods_warnings(): start_val, end_val, periods_val = 0, 4, 1.0 - with expect_warning_if(PANDAS_GE_220): + with pytest.warns(FutureWarning): pindex = pd.interval_range( start=start_val, end=end_val, periods=periods_val, closed="left" ) diff --git a/python/cudf/cudf/tests/test_applymap.py b/python/cudf/cudf/tests/test_applymap.py index adbbbbb1ae4..cfe4237180e 100644 --- a/python/cudf/cudf/tests/test_applymap.py +++ b/python/cudf/cudf/tests/test_applymap.py @@ -34,13 +34,6 @@ def test_applymap_dataframe(data, func, na_action, request): reason="https://github.com/pandas-dev/pandas/issues/57390", ) ) - request.applymarker( - pytest.mark.xfail( - PANDAS_GE_220 - and request.node.callspec.id == "ignore-3-data3", - reason="https://github.com/pandas-dev/pandas/pull/57388", - ) - ) gdf = DataFrame(data) pdf = gdf.to_pandas(nullable=True) diff --git a/python/cudf/cudf/tests/test_array_ufunc.py b/python/cudf/cudf/tests/test_array_ufunc.py index 3ba0403d67c..0eb1d6de3a4 100644 --- a/python/cudf/cudf/tests/test_array_ufunc.py +++ b/python/cudf/cudf/tests/test_array_ufunc.py @@ -10,7 +10,7 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210, PANDAS_LT_300 +from cudf.core._compat import PANDAS_GE_210, PANDAS_LT_300 from cudf.testing._utils import ( assert_eq, expect_warning_if, @@ -183,10 +183,7 @@ def test_ufunc_series(request, ufunc, has_nulls, indexed): request.applymarker( pytest.mark.xfail( - condition=PANDAS_GE_200 - and fname.startswith("bitwise") - and indexed - and has_nulls, + condition=fname.startswith("bitwise") and indexed and has_nulls, reason="https://github.com/pandas-dev/pandas/issues/52500", ) ) @@ -385,52 +382,6 @@ def test_ufunc_dataframe(request, ufunc, has_nulls, indexed): reason=f"cupy has no support for '{fname}'", ) ) - request.applymarker( - pytest.mark.xfail( - condition=( - not PANDAS_GE_200 - and indexed - in { - "add", - "arctan2", - "bitwise_and", - "bitwise_or", - "bitwise_xor", - "copysign", - "divide", - "divmod", - "float_power", - "floor_divide", - "fmax", - "fmin", - "fmod", - "heaviside", - "gcd", - "hypot", - "lcm", - "ldexp", - "left_shift", - "logaddexp", - "logaddexp2", - "logical_and", - "logical_or", - "logical_xor", - "maximum", - "minimum", - "multiply", - "nextafter", - "power", - "remainder", - "right_shift", - "subtract", - } - ), - reason=( - "pandas<2.0 does not currently support misaligned " - "indexes in DataFrames" - ), - ) - ) N = 100 # Avoid zeros in either array to skip division by 0 errors. Also limit the diff --git a/python/cudf/cudf/tests/test_binops.py b/python/cudf/cudf/tests/test_binops.py index 92a9fd6636c..75b393f513a 100644 --- a/python/cudf/cudf/tests/test_binops.py +++ b/python/cudf/cudf/tests/test_binops.py @@ -1726,24 +1726,7 @@ def test_datetime_dateoffset_binaryop( reason="https://github.com/pandas-dev/pandas/issues/57448", ) ) - request.applymarker( - pytest.mark.xfail( - not PANDAS_GE_220 - and dtype in {"datetime64[ms]", "datetime64[s]"} - and frequency in ("microseconds", "nanoseconds") - and n_periods != 0, - reason="https://github.com/pandas-dev/pandas/pull/55595", - ) - ) - request.applymarker( - pytest.mark.xfail( - not PANDAS_GE_220 - and dtype == "datetime64[us]" - and frequency == "nanoseconds" - and n_periods != 0, - reason="https://github.com/pandas-dev/pandas/pull/55595", - ) - ) + date_col = [ "2000-01-01 00:00:00.012345678", "2000-01-31 00:00:00.012345678", @@ -1796,13 +1779,7 @@ def test_datetime_dateoffset_binaryop( "ignore:Discarding nonzero nanoseconds:UserWarning" ) @pytest.mark.parametrize("op", [operator.add, operator.sub]) -def test_datetime_dateoffset_binaryop_multiple(request, date_col, kwargs, op): - request.applymarker( - pytest.mark.xfail( - PANDAS_GE_220 and len(kwargs) == 1 and "milliseconds" in kwargs, - reason="https://github.com/pandas-dev/pandas/issues/57529", - ) - ) +def test_datetime_dateoffset_binaryop_multiple(date_col, kwargs, op): gsr = cudf.Series(date_col, dtype="datetime64[ns]") psr = gsr.to_pandas() @@ -1833,27 +1810,7 @@ def test_datetime_dateoffset_binaryop_multiple(request, date_col, kwargs, op): "dtype", ["datetime64[ns]", "datetime64[us]", "datetime64[ms]", "datetime64[s]"], ) -def test_datetime_dateoffset_binaryop_reflected( - request, n_periods, frequency, dtype -): - request.applymarker( - pytest.mark.xfail( - not PANDAS_GE_220 - and dtype in {"datetime64[ms]", "datetime64[s]"} - and frequency in ("microseconds", "nanoseconds") - and n_periods != 0, - reason="https://github.com/pandas-dev/pandas/pull/55595", - ) - ) - request.applymarker( - pytest.mark.xfail( - not PANDAS_GE_220 - and dtype == "datetime64[us]" - and frequency == "nanoseconds" - and n_periods != 0, - reason="https://github.com/pandas-dev/pandas/pull/55595", - ) - ) +def test_datetime_dateoffset_binaryop_reflected(n_periods, frequency, dtype): date_col = [ "2000-01-01 00:00:00.012345678", "2000-01-31 00:00:00.012345678", diff --git a/python/cudf/cudf/tests/test_column_accessor.py b/python/cudf/cudf/tests/test_column_accessor.py index bf764b02faa..a8eac2edf2b 100644 --- a/python/cudf/cudf/tests/test_column_accessor.py +++ b/python/cudf/cudf/tests/test_column_accessor.py @@ -5,7 +5,6 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_200 from cudf.core.column_accessor import ColumnAccessor from cudf.testing._utils import assert_eq @@ -60,7 +59,7 @@ def test_to_pandas_simple(simple_data): assert_eq( ca.to_pandas_index(), pd.DataFrame(simple_data).columns, - exact=not PANDAS_GE_200, + exact=False, ) diff --git a/python/cudf/cudf/tests/test_concat.py b/python/cudf/cudf/tests/test_concat.py index 6e61675ef92..cdb47ea79d8 100644 --- a/python/cudf/cudf/tests/test_concat.py +++ b/python/cudf/cudf/tests/test_concat.py @@ -10,7 +10,6 @@ import cudf from cudf.api.types import _is_categorical_dtype -from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_220 from cudf.core.dtypes import Decimal32Dtype, Decimal64Dtype, Decimal128Dtype from cudf.testing._utils import ( assert_eq, @@ -390,13 +389,12 @@ def test_pandas_concat_compatibility_axis1_eq_index(): ps1 = s1.to_pandas() ps2 = s2.to_pandas() - with expect_warning_if(not PANDAS_GE_200): - assert_exceptions_equal( - lfunc=pd.concat, - rfunc=cudf.concat, - lfunc_args_and_kwargs=([], {"objs": [ps1, ps2], "axis": 1}), - rfunc_args_and_kwargs=([], {"objs": [s1, s2], "axis": 1}), - ) + assert_exceptions_equal( + lfunc=pd.concat, + rfunc=cudf.concat, + lfunc_args_and_kwargs=([], {"objs": [ps1, ps2], "axis": 1}), + rfunc_args_and_kwargs=([], {"objs": [s1, s2], "axis": 1}), + ) @pytest.mark.parametrize("name", [None, "a"]) @@ -459,75 +457,45 @@ def test_concat_mixed_input(): [pd.Series([1, 2, 3]), pd.DataFrame({"a": []})], [pd.Series([], dtype="float64"), pd.DataFrame({"a": []})], [pd.Series([], dtype="float64"), pd.DataFrame({"a": [1, 2]})], - pytest.param( - [ - pd.Series([1, 2, 3.0, 1.2], name="abc"), - pd.DataFrame({"a": [1, 2]}), - ], - marks=pytest.mark.xfail( - not PANDAS_GE_220, - reason="https://github.com/pandas-dev/pandas/pull/56365", + [ + pd.Series([1, 2, 3.0, 1.2], name="abc"), + pd.DataFrame({"a": [1, 2]}), + ], + [ + pd.Series( + [1, 2, 3.0, 1.2], name="abc", index=[100, 110, 120, 130] ), - ), - pytest.param( - [ - pd.Series( - [1, 2, 3.0, 1.2], name="abc", index=[100, 110, 120, 130] - ), - pd.DataFrame({"a": [1, 2]}), - ], - marks=pytest.mark.xfail( - not PANDAS_GE_220, - reason="https://github.com/pandas-dev/pandas/pull/56365", + pd.DataFrame({"a": [1, 2]}), + ], + [ + pd.Series( + [1, 2, 3.0, 1.2], name="abc", index=["a", "b", "c", "d"] ), - ), - pytest.param( - [ - pd.Series( - [1, 2, 3.0, 1.2], name="abc", index=["a", "b", "c", "d"] - ), - pd.DataFrame({"a": [1, 2]}, index=["a", "b"]), - ], - marks=pytest.mark.xfail( - not PANDAS_GE_220, - reason="https://github.com/pandas-dev/pandas/pull/56365", + pd.DataFrame({"a": [1, 2]}, index=["a", "b"]), + ], + [ + pd.Series( + [1, 2, 3.0, 1.2, 8, 100], + name="New name", + index=["a", "b", "c", "d", "e", "f"], ), - ), - pytest.param( - [ - pd.Series( - [1, 2, 3.0, 1.2, 8, 100], - name="New name", - index=["a", "b", "c", "d", "e", "f"], - ), - pd.DataFrame( - {"a": [1, 2, 4, 10, 11, 12]}, - index=["a", "b", "c", "d", "e", "f"], - ), - ], - marks=pytest.mark.xfail( - not PANDAS_GE_220, - reason="https://github.com/pandas-dev/pandas/pull/56365", + pd.DataFrame( + {"a": [1, 2, 4, 10, 11, 12]}, + index=["a", "b", "c", "d", "e", "f"], ), - ), - pytest.param( - [ - pd.Series( - [1, 2, 3.0, 1.2, 8, 100], - name="New name", - index=["a", "b", "c", "d", "e", "f"], - ), - pd.DataFrame( - {"a": [1, 2, 4, 10, 11, 12]}, - index=["a", "b", "c", "d", "e", "f"], - ), - ] - * 7, - marks=pytest.mark.xfail( - not PANDAS_GE_220, - reason="https://github.com/pandas-dev/pandas/pull/56365", + ], + [ + pd.Series( + [1, 2, 3.0, 1.2, 8, 100], + name="New name", + index=["a", "b", "c", "d", "e", "f"], ), - ), + pd.DataFrame( + {"a": [1, 2, 4, 10, 11, 12]}, + index=["a", "b", "c", "d", "e", "f"], + ), + ] + * 7, ], ) def test_concat_series_dataframe_input(objs): @@ -663,7 +631,7 @@ def test_concat_empty_dataframes(df, other, ignore_index): expected, actual, check_index_type=not gdf.empty, - check_column_type=not PANDAS_GE_200, + check_column_type=False, ) @@ -1137,7 +1105,7 @@ def test_concat_join_no_overlapping_columns_empty_df_basic( expected, actual, check_index_type=True, - check_column_type=not PANDAS_GE_200, + check_column_type=False, ) diff --git a/python/cudf/cudf/tests/test_csv.py b/python/cudf/cudf/tests/test_csv.py index 9b08ef30545..5942c89b9ef 100644 --- a/python/cudf/cudf/tests/test_csv.py +++ b/python/cudf/cudf/tests/test_csv.py @@ -17,12 +17,8 @@ import cudf from cudf import read_csv -from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_220 -from cudf.testing._utils import ( - assert_eq, - assert_exceptions_equal, - expect_warning_if, -) +from cudf.core._compat import PANDAS_GE_200 +from cudf.testing._utils import assert_eq, assert_exceptions_equal def make_numeric_dataframe(nrows, dtype): @@ -1269,14 +1265,14 @@ def test_csv_reader_delim_whitespace(): # with header row with pytest.warns(FutureWarning): cu_df = read_csv(StringIO(buffer), delim_whitespace=True) - with expect_warning_if(PANDAS_GE_220): + with pytest.warns(FutureWarning): pd_df = pd.read_csv(StringIO(buffer), delim_whitespace=True) assert_eq(pd_df, cu_df) # without header row with pytest.warns(FutureWarning): cu_df = read_csv(StringIO(buffer), delim_whitespace=True, header=None) - with expect_warning_if(PANDAS_GE_220): + with pytest.warns(FutureWarning): pd_df = pd.read_csv( StringIO(buffer), delim_whitespace=True, header=None ) diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 565b9b09001..2084db89909 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -25,12 +25,7 @@ import cudf from cudf.api.extensions import no_default -from cudf.core._compat import ( - PANDAS_GE_200, - PANDAS_GE_210, - PANDAS_GE_220, - PANDAS_LT_203, -) +from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210 from cudf.core.buffer.spill_manager import get_global_manager from cudf.core.column import column from cudf.errors import MixedTypeError @@ -166,12 +161,7 @@ def _dataframe_na_data(): @pytest.mark.parametrize( "rows", [ - pytest.param( - 0, - marks=pytest.mark.xfail( - not PANDAS_GE_200, reason=".column returns Index[object]" - ), - ), + 0, 1, 2, 100, @@ -358,7 +348,7 @@ def test_axes(data): actual = csr.axes for e, a in zip(expected, actual): - assert_eq(e, a, exact=not PANDAS_GE_200) + assert_eq(e, a, exact=False) def test_dataframe_truncate_axis_0(): @@ -1707,24 +1697,7 @@ def test_concat_different_column_dataframe(df1_d, df2_d): pdf1 = pd.DataFrame(df1_d) pdf2 = pd.DataFrame(df2_d) - # pandas(lower than pandas 2.0 only) warns when trying to - # concatenate any empty float columns (or float - # columns with all None values) with any non-empty bool columns. - def is_invalid_concat(left, right): - return ( - pd.api.types.is_bool_dtype(left.dtype) - and pd.api.types.is_float_dtype(right.dtype) - and right.count() == 0 - ) - - cond = (not PANDAS_GE_200) and any( - is_invalid_concat(pdf1[colname], pdf2[colname]) - or is_invalid_concat(pdf2[colname], pdf1[colname]) - for colname in set(pdf1) & set(pdf2) - ) - - with expect_warning_if(cond): - expect = pd.concat([pdf1, pdf2, pdf1], sort=False) + expect = pd.concat([pdf1, pdf2, pdf1], sort=False) # numerical columns are upcasted to float in cudf.DataFrame.to_pandas() # casts nan to 0 in non-float numerical columns @@ -3567,16 +3540,8 @@ def test_dataframe_empty_sort_index(): @pytest.mark.parametrize("inplace", [True, False]) @pytest.mark.parametrize("na_position", ["first", "last"]) def test_dataframe_sort_index( - request, index, axis, ascending, inplace, ignore_index, na_position + index, axis, ascending, inplace, ignore_index, na_position ): - request.applymarker( - pytest.mark.xfail( - condition=not PANDAS_GE_220 - and axis in (1, "columns") - and ignore_index, - reason="Bug fixed in pandas-2.2", - ) - ) pdf = pd.DataFrame( {"b": [1, 3, 2], "a": [1, 4, 3], "c": [4, 1, 5]}, index=index, @@ -3629,15 +3594,6 @@ def test_dataframe_sort_index( def test_dataframe_mulitindex_sort_index( request, axis, level, ascending, inplace, ignore_index, na_position ): - request.applymarker( - pytest.mark.xfail( - condition=not PANDAS_GE_220 - and axis in (1, "columns") - and ignore_index - and not (level is None and not ascending), - reason="https://github.com/pandas-dev/pandas/issues/56478", - ) - ) request.applymarker( pytest.mark.xfail( condition=axis in (1, "columns") @@ -6628,20 +6584,14 @@ def test_df_series_dataframe_astype_dtype_dict(copy): [ ([1, 2, 3, 100, 112, 35464], ["a"]), (range(100), None), - pytest.param( + ( [], None, - marks=pytest.mark.xfail( - not PANDAS_GE_200, reason=".column returns Index[object]" - ), ), ((-10, 21, 32, 32, 1, 2, 3), ["p"]), - pytest.param( + ( (), None, - marks=pytest.mark.xfail( - not PANDAS_GE_200, reason=".column returns Index[object]" - ), ), ([[1, 2, 3], [1, 2, 3]], ["col1", "col2", "col3"]), ([range(100), range(100)], ["range" + str(i) for i in range(100)]), @@ -6660,7 +6610,6 @@ def test_dataframe_init_1d_list(data, columns): expect, actual, check_index_type=len(data) != 0, - check_column_type=not PANDAS_GE_200 and len(data) == 0, ) expect = pd.DataFrame(data, columns=None) @@ -6670,7 +6619,6 @@ def test_dataframe_init_1d_list(data, columns): expect, actual, check_index_type=len(data) != 0, - check_column_type=not PANDAS_GE_200 and len(data) == 0, ) @@ -7536,7 +7484,6 @@ def test_dataframe_keys(df): assert_eq( df.keys(), gdf.keys(), - exact=not (PANDAS_GE_200 and len(gdf.columns) == 0), ) @@ -7915,7 +7862,7 @@ def test_dataframe_concat_dataframe_lists(df, other, sort, ignore_index): def test_dataframe_bfill(df, alias): gdf = cudf.from_pandas(df) - with expect_warning_if(PANDAS_GE_200 and alias == "backfill"): + with expect_warning_if(alias == "backfill"): actual = getattr(df, alias)() with expect_warning_if(alias == "backfill"): expected = getattr(gdf, alias)() @@ -7933,7 +7880,7 @@ def test_dataframe_bfill(df, alias): def test_dataframe_ffill(df, alias): gdf = cudf.from_pandas(df) - with expect_warning_if(PANDAS_GE_200 and alias == "pad"): + with expect_warning_if(alias == "pad"): actual = getattr(df, alias)() with expect_warning_if(alias == "pad"): expected = getattr(gdf, alias)() @@ -8010,7 +7957,7 @@ def test_dataframe_concat_lists(df, other, sort, ignore_index): expected, actual, check_index_type=not gdf.empty, - check_column_type=PANDAS_GE_200 and len(gdf.columns) != 0, + check_column_type=len(gdf.columns) != 0, ) @@ -8287,11 +8234,7 @@ def test_series_empty(ps): "columns", [["a"], ["another column name"], None, pd.Index(["a"], name="index name")], ) -def test_dataframe_init_with_columns(data, columns, request): - if data == [] and columns is None and not PANDAS_GE_200: - request.node.add_marker( - pytest.mark.xfail(reason=".column returns Index[object]") - ) +def test_dataframe_init_with_columns(data, columns): pdf = pd.DataFrame(data, columns=columns) gdf = cudf.DataFrame(data, columns=columns) @@ -8300,7 +8243,7 @@ def test_dataframe_init_with_columns(data, columns, request): gdf, check_index_type=len(pdf.index) != 0, check_dtype=not (pdf.empty and len(pdf.columns)), - check_column_type=not PANDAS_GE_200, + check_column_type=False, ) @@ -8370,11 +8313,7 @@ def test_dataframe_init_with_columns(data, columns, request): pd.Index(["abc"], name="custom_name"), ], ) -def test_dataframe_init_from_series_list(data, ignore_dtype, columns, request): - if columns is None and data[0].empty and not PANDAS_GE_200: - request.applymarker( - pytest.mark.xfail(reason=".column returns Index[object]") - ) +def test_dataframe_init_from_series_list(data, ignore_dtype, columns): gd_data = [cudf.from_pandas(obj) for obj in data] expected = pd.DataFrame(data, columns=columns) @@ -8398,7 +8337,7 @@ def test_dataframe_init_from_series_list(data, ignore_dtype, columns, request): expected, actual, check_index_type=True, - check_column_type=not PANDAS_GE_200, + check_column_type=False, ) @@ -8478,12 +8417,7 @@ def test_dataframe_init_from_series_list_with_index( ignore_dtype, index, columns, - request, ): - if columns is None and data[0].empty and not PANDAS_GE_200: - request.applymarker( - pytest.mark.xfail(reason=".column returns Index[object]") - ) gd_data = [cudf.from_pandas(obj) for obj in data] expected = pd.DataFrame(data, columns=columns, index=index) @@ -8498,7 +8432,7 @@ def test_dataframe_init_from_series_list_with_index( actual = actual.sort_index(axis=1) assert_eq(expected.fillna(-1), actual.fillna(-1), check_dtype=False) else: - assert_eq(expected, actual, check_column_type=not PANDAS_GE_200) + assert_eq(expected, actual, check_column_type=False) @pytest.mark.parametrize( @@ -8754,18 +8688,8 @@ def test_describe_misc_exclude(df, exclude): ) @pytest.mark.parametrize("numeric_only", [True, False]) @pytest.mark.parametrize("dropna", [True, False]) -def test_dataframe_mode(request, df, numeric_only, dropna): +def test_dataframe_mode(df, numeric_only, dropna): pdf = df.to_pandas() - request.applymarker( - pytest.mark.xfail( - condition=PANDAS_GE_200 - and PANDAS_LT_203 - and numeric_only is False - and "b" in df.columns - and df["b"].dtype == np.dtype("timedelta64[s]"), - reason="https://github.com/pandas-dev/pandas/issues/53497", - ) - ) expected = pdf.mode(numeric_only=numeric_only, dropna=dropna) actual = df.mode(numeric_only=numeric_only, dropna=dropna) @@ -9113,15 +9037,9 @@ def assert_local_eq(actual, df, expected, host_columns): expected, actual, check_index_type=check_index_type, - check_column_type=not PANDAS_GE_200, + check_column_type=False, ) - if df.empty and columns is None and not PANDAS_GE_200: - request.node.add_marker( - pytest.mark.xfail( - reason="pandas returns Index[object] instead of RangeIndex" - ) - ) gdf = cudf.from_pandas(df) host_columns = ( columns.to_pandas() if isinstance(columns, cudf.BaseIndex) else columns diff --git a/python/cudf/cudf/tests/test_datetime.py b/python/cudf/cudf/tests/test_datetime.py index 6f8e4ec0a1a..cceb6efaaae 100644 --- a/python/cudf/cudf/tests/test_datetime.py +++ b/python/cudf/cudf/tests/test_datetime.py @@ -13,12 +13,7 @@ import cudf import cudf.testing.dataset_generator as dataset_generator from cudf import DataFrame, Series -from cudf.core._compat import ( - PANDAS_EQ_200, - PANDAS_GE_200, - PANDAS_GE_210, - PANDAS_GE_220, -) +from cudf.core._compat import PANDAS_EQ_200, PANDAS_GE_210 from cudf.core.index import DatetimeIndex from cudf.testing._utils import ( DATETIME_TYPES, @@ -1550,45 +1545,7 @@ def test_date_range_start_end_freq(request, start, end, freq): reason="https://github.com/rapidsai/cudf/issues/12133", ) ) - request.applymarker( - pytest.mark.xfail( - condition=( - not PANDAS_GE_200 - and isinstance(freq, dict) - and freq.get("hours", None) == 10 - and freq.get("days", None) == 57 - and freq.get("nanoseconds", None) == 3 - and ( - ( - start == "1996-11-21 04:05:30" - and end == "2000-02-13 08:41:06" - ) - or ( - start == "1970-01-01 00:00:00" - and end == "2000-02-13 08:41:06" - ) - or ( - start == "1970-01-01 00:00:00" - and end == "1996-11-21 04:05:30" - ) - or ( - start == "1831-05-08 15:23:21" - and end == "2000-02-13 08:41:06" - ) - or ( - start == "1831-05-08 15:23:21" - and end == "1996-11-21 04:05:30" - ) - or ( - start == "1831-05-08 15:23:21" - and end == "1970-01-01 00:00:00" - ) - ) - ), - reason="Nanosecond offsets being dropped by pandas, which is " - "fixed in pandas-2.0+", - ) - ) + if isinstance(freq, str): _gfreq = _pfreq = freq else: @@ -1605,29 +1562,6 @@ def test_date_range_start_end_freq(request, start, end, freq): def test_date_range_start_freq_periods(request, start, freq, periods): - request.applymarker( - pytest.mark.xfail( - condition=( - not PANDAS_GE_200 - and isinstance(freq, dict) - and freq.get("hours", None) == 10 - and freq.get("days", None) == 57 - and freq.get("nanoseconds", None) == 3 - and periods in (10, 100) - and ( - start - in { - "2000-02-13 08:41:06", - "1996-11-21 04:05:30", - "1970-01-01 00:00:00", - "1831-05-08 15:23:21", - } - ) - ), - reason="Nanosecond offsets being dropped by pandas, which is " - "fixed in pandas-2.0+", - ) - ) if isinstance(freq, str): _gfreq = _pfreq = freq else: @@ -1655,29 +1589,7 @@ def test_date_range_end_freq_periods(request, end, freq, periods): reason="https://github.com/pandas-dev/pandas/issues/46877", ) ) - request.applymarker( - pytest.mark.xfail( - condition=( - not PANDAS_GE_220 - and isinstance(freq, dict) - and freq.get("hours", None) == 10 - and freq.get("days", None) == 57 - and freq.get("nanoseconds", None) == 3 - and periods in (10, 100) - and ( - end - in { - "2000-02-13 08:41:06", - "1996-11-21 04:05:30", - "1970-01-01 00:00:00", - "1831-05-08 15:23:21", - } - ) - ), - reason="Nanosecond offsets being dropped by pandas, which is " - "fixed in pandas-2.0+", - ) - ) + if isinstance(freq, str): _gfreq = _pfreq = freq else: @@ -1748,15 +1660,7 @@ def test_date_range_raise_overflow(): "B", ], ) -def test_date_range_raise_unsupported(request, freqstr_unsupported): - request.applymarker( - pytest.mark.xfail( - condition=( - not PANDAS_GE_220 and freqstr_unsupported.endswith("E") - ), - reason="TODO: Remove this once pandas-2.2 support is added", - ) - ) +def test_date_range_raise_unsupported(freqstr_unsupported): s, e = "2001-01-01", "2008-01-31" pd.date_range(start=s, end=e, freq=freqstr_unsupported) with pytest.raises(ValueError, match="does not yet support"): @@ -1768,7 +1672,7 @@ def test_date_range_raise_unsupported(request, freqstr_unsupported): if freqstr_unsupported != "3MS": freqstr_unsupported = freqstr_unsupported.lower() with pytest.raises(ValueError, match="does not yet support"): - with expect_warning_if(PANDAS_GE_220): + with pytest.warns(FutureWarning): cudf.date_range(start=s, end=e, freq=freqstr_unsupported) @@ -2285,13 +2189,7 @@ def test_daterange_pandas_compatibility(): ([101, 201, 301, 401], "datetime64[ms]", "100ms"), ], ) -def test_datetime_index_with_freq(request, data, dtype, freq): - # request.applymarker( - # pytest.mark.xfail( - # condition=(not PANDAS_GE_200 and dtype != "datetime64[ns]"), - # reason="Pandas < 2.0 lacks non-nano-second dtype support.", - # ) - # ) +def test_datetime_index_with_freq(data, dtype, freq): actual = cudf.DatetimeIndex(data, dtype=dtype, freq=freq) expected = pd.DatetimeIndex(data, dtype=dtype, freq=freq) assert_eq(actual, expected) diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index c22e47bdf06..63e0cf98b27 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -20,7 +20,7 @@ import cudf from cudf import DataFrame, Series from cudf.api.extensions import no_default -from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210, PANDAS_GE_220 +from cudf.core._compat import PANDAS_GE_210, PANDAS_GE_220 from cudf.core.udf._ops import arith_ops, comparison_ops, unary_ops from cudf.core.udf.groupby_typing import SUPPORTED_GROUPBY_NUMPY_TYPES from cudf.core.udf.utils import UDFError, precompiled @@ -188,9 +188,7 @@ def test_groupby_as_index_apply(pdf, gdf, as_index, engine): gdf = gdf.groupby("y", as_index=as_index).apply( lambda df: df["x"].mean(), engine=engine ) - kwargs = {"func": lambda df: df["x"].mean()} - if PANDAS_GE_220: - kwargs["include_groups"] = False + kwargs = {"func": lambda df: df["x"].mean(), "include_groups": False} pdf = pdf.groupby("y", as_index=as_index).apply(**kwargs) assert_groupby_results_equal(pdf, gdf) @@ -314,12 +312,8 @@ def foo(df): df["out"] = df["val1"] + df["val2"] return df - if PANDAS_GE_220: - kwargs = {"include_groups": False} - else: - kwargs = {} - expect = expect_grpby.apply(foo, **kwargs) - got = got_grpby.apply(foo, **kwargs) + expect = expect_grpby.apply(foo, include_groups=False) + got = got_grpby.apply(foo, include_groups=False) assert_groupby_results_equal(expect, got) @@ -353,12 +347,8 @@ def test_groupby_apply_args(func, args): ["key1", "key2"], as_index=False, group_keys=False ) got_grpby = df.groupby(["key1", "key2"]) - if PANDAS_GE_220: - kwargs = {"include_groups": False} - else: - kwargs = {} - expect = expect_grpby.apply(func, *args, **kwargs) - got = got_grpby.apply(func, *args, **kwargs) + expect = expect_grpby.apply(func, *args, include_groups=False) + got = got_grpby.apply(func, *args, include_groups=False) assert_groupby_results_equal(expect, got) @@ -466,14 +456,10 @@ def run_groupby_apply_jit_test(data, func, keys, *args): got_groupby_obj = data.groupby(keys) # compare cuDF jit to pandas - if PANDAS_GE_220: - kwargs = {"include_groups": False} - else: - kwargs = {} cudf_jit_result = got_groupby_obj.apply( - func, *args, engine="jit", **kwargs + func, *args, engine="jit", include_groups=False ) - pandas_result = expect_groupby_obj.apply(func, *args, **kwargs) + pandas_result = expect_groupby_obj.apply(func, *args, include_groups=False) assert_groupby_results_equal(cudf_jit_result, pandas_result) @@ -841,12 +827,9 @@ def f(group): return group.sum() part = partial(f) - if PANDAS_GE_220: - kwargs = {"include_groups": False} - else: - kwargs = {} - expect = pdf.groupby("a").apply(part, **kwargs) - got = gdf.groupby("a").apply(part, engine="auto", **kwargs) + + expect = pdf.groupby("a").apply(part, include_groups=False) + got = gdf.groupby("a").apply(part, engine="auto", include_groups=False) assert_groupby_results_equal(expect, got) @@ -867,12 +850,8 @@ def test_groupby_apply_return_col_from_df(): def func(df): return df.x + df.y - if PANDAS_GE_220: - kwargs = {"include_groups": False} - else: - kwargs = {} - got = df.groupby("id").apply(func, **kwargs) - expect = pdf.groupby("id").apply(func, **kwargs) + got = df.groupby("id").apply(func, include_groups=False) + expect = pdf.groupby("id").apply(func, include_groups=False) # pandas seems to erroneously add an extra MI level of ids # TODO: Figure out how pandas groupby.apply determines the columns expect = pd.DataFrame(expect.droplevel(1), columns=got.columns) @@ -887,12 +866,8 @@ def test_groupby_apply_return_df(func): df = cudf.DataFrame({"a": [1, 1, 2, 2], "b": [1, 2, 3, 4]}) pdf = df.to_pandas() - if PANDAS_GE_220: - kwargs = {"include_groups": False} - else: - kwargs = {} - expect = pdf.groupby("a").apply(func, **kwargs) - got = df.groupby("a").apply(func, **kwargs) + expect = pdf.groupby("a").apply(func, include_groups=False) + got = df.groupby("a").apply(func, include_groups=False) assert_groupby_results_equal(expect, got) @@ -1938,18 +1913,15 @@ def test_groupby_apply_noempty_group(): {"a": [1, 1, 2, 2], "b": [1, 2, 1, 2], "c": [1, 2, 3, 4]} ) gdf = cudf.from_pandas(pdf) - if PANDAS_GE_220: - kwargs = {"include_groups": False} - else: - kwargs = {} + expect = ( pdf.groupby("a", group_keys=False) - .apply(lambda x: x.iloc[[0, 1]], **kwargs) + .apply(lambda x: x.iloc[[0, 1]], include_groups=False) .reset_index(drop=True) ) got = ( gdf.groupby("a") - .apply(lambda x: x.iloc[[0, 1]], **kwargs) + .apply(lambda x: x.iloc[[0, 1]], include_groups=False) .reset_index(drop=True) ) assert_groupby_results_equal(expect, got) @@ -2147,19 +2119,8 @@ def test_groupby_list_columns_excluded(): ) gdf = cudf.from_pandas(pdf) - if PANDAS_GE_200: - pandas_result = pdf.groupby("a").mean(numeric_only=True) - pandas_agg_result = pdf.groupby("a").agg("mean", numeric_only=True) - else: - # cudf does not yet support numeric_only, so our default is False, but - # pandas defaults to inferring and throws a warning about it, so - # we need to catch that. pandas future behavior will match ours - # by default (at which point supporting numeric_only=True will - # be the open feature request). - with pytest.warns(FutureWarning): - pandas_result = pdf.groupby("a").mean() - with pytest.warns(FutureWarning): - pandas_agg_result = pdf.groupby("a").agg("mean") + pandas_result = pdf.groupby("a").mean(numeric_only=True) + pandas_agg_result = pdf.groupby("a").agg("mean", numeric_only=True) assert_groupby_results_equal( pandas_result, gdf.groupby("a").mean(), check_dtype=False @@ -2233,12 +2194,8 @@ def test_groupby_apply_return_scalars(func, args): ) gdf = cudf.from_pandas(pdf) - if PANDAS_GE_220: - kwargs = {"include_groups": False} - else: - kwargs = {} - expected = pdf.groupby("A").apply(func, *args, **kwargs) - actual = gdf.groupby("A").apply(func, *args, **kwargs) + expected = pdf.groupby("A").apply(func, *args, include_groups=False) + actual = gdf.groupby("A").apply(func, *args, include_groups=False) assert_groupby_results_equal(expected, actual) @@ -2281,14 +2238,10 @@ def test_groupby_apply_return_series_dataframe(func, args): ) gdf = cudf.from_pandas(pdf) - if PANDAS_GE_220: - kwargs = {"include_groups": False} - else: - kwargs = {} expected = pdf.groupby(["key"], group_keys=False).apply( - func, *args, **kwargs + func, *args, include_groups=False ) - actual = gdf.groupby(["key"]).apply(func, *args, **kwargs) + actual = gdf.groupby(["key"]).apply(func, *args, include_groups=False) assert_groupby_results_equal(expected, actual) @@ -2300,7 +2253,7 @@ def test_groupby_apply_return_series_dataframe(func, args): def test_groupby_no_keys(pdf): gdf = cudf.from_pandas(pdf) if isinstance(pdf, pd.DataFrame): - kwargs = {"check_column_type": not PANDAS_GE_200} + kwargs = {"check_column_type": False} else: kwargs = {} assert_groupby_results_equal( @@ -2319,7 +2272,7 @@ def test_groupby_no_keys(pdf): def test_groupby_apply_no_keys(pdf): gdf = cudf.from_pandas(pdf) if isinstance(pdf, pd.DataFrame): - kwargs = {"check_column_type": not PANDAS_GE_200} + kwargs = {"check_column_type": False} else: kwargs = {} assert_groupby_results_equal( @@ -2790,7 +2743,7 @@ def test_groupby_fillna_multi_value(nelem): } # cudf can't fillna with a pandas.Timedelta type fill_values["4"] = fill_values["4"].to_numpy() - with expect_warning_if(PANDAS_GE_220): + with pytest.warns(FutureWarning): expect = pdf.groupby(key_col).fillna(value=fill_values) with pytest.warns(FutureWarning): got = gdf.groupby(key_col).fillna(value=fill_values) @@ -2836,7 +2789,7 @@ def test_groupby_fillna_multi_value_df(nelem): # cudf can't fillna with a pandas.Timedelta type fill_values["4"] = fill_values["4"].to_numpy() fill_values = pd.DataFrame(fill_values, index=pdf.index) - with expect_warning_if(PANDAS_GE_220): + with pytest.warns(FutureWarning): expect = pdf.groupby(key_col).fillna(value=fill_values) fill_values = cudf.from_pandas(fill_values) @@ -2858,9 +2811,7 @@ def test_groupby_various_by_fillna(by, data, args): ps = pd.Series(data) gs = cudf.from_pandas(ps) - with expect_warning_if( - (PANDAS_GE_210 and "method" in args) or PANDAS_GE_220 - ): + with pytest.warns(FutureWarning): expect = ps.groupby(by).fillna(**args) if isinstance(by, pd.Grouper): by = cudf.Grouper(level=by.level) @@ -3017,7 +2968,7 @@ def test_groupby_freq_week(label, closed): got, check_like=True, check_dtype=False, - check_index_type=not PANDAS_GE_200, + check_index_type=False, ) @@ -3050,7 +3001,7 @@ def test_groupby_freq_day(label, closed): got, check_like=True, check_dtype=False, - check_index_type=not PANDAS_GE_200, + check_index_type=False, ) @@ -3083,7 +3034,7 @@ def test_groupby_freq_min(label, closed): got, check_like=True, check_dtype=False, - check_index_type=not PANDAS_GE_200, + check_index_type=False, ) @@ -3116,7 +3067,7 @@ def test_groupby_freq_s(label, closed): got, check_like=True, check_dtype=False, - check_index_type=not PANDAS_GE_200, + check_index_type=False, ) @@ -3602,12 +3553,12 @@ def test_head_tail_empty(): expected = pdf.groupby(pd.Series(values)).head() got = df.groupby(cudf.Series(values)).head() - assert_eq(expected, got, check_column_type=not PANDAS_GE_200) + assert_eq(expected, got, check_column_type=False) expected = pdf.groupby(pd.Series(values)).tail() got = df.groupby(cudf.Series(values)).tail() - assert_eq(expected, got, check_column_type=not PANDAS_GE_200) + assert_eq(expected, got, check_column_type=False) @pytest.mark.parametrize( diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index aff71f1882b..cced05d2217 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -15,7 +15,6 @@ import cudf from cudf.api.extensions import no_default from cudf.api.types import is_bool_dtype -from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_220 from cudf.core.index import ( CategoricalIndex, DatetimeIndex, @@ -797,26 +796,9 @@ def test_index_to_series(data): "name_data,name_other", [("abc", "c"), (None, "abc"), ("abc", pd.NA), ("abc", "abc")], ) -def test_index_difference(request, data, other, sort, name_data, name_other): +def test_index_difference(data, other, sort, name_data, name_other): pd_data = pd.Index(data, name=name_data) pd_other = pd.Index(other, name=name_other) - request.applymarker( - pytest.mark.xfail( - condition=PANDAS_GE_220 - and isinstance(pd_data.dtype, pd.CategoricalDtype) - and not isinstance(pd_other.dtype, pd.CategoricalDtype) - and pd_other.isnull().any(), - reason="https://github.com/pandas-dev/pandas/issues/57318", - ) - ) - request.applymarker( - pytest.mark.xfail( - condition=not PANDAS_GE_220 - and len(pd_other) == 0 - and len(pd_data) != len(pd_data.unique()), - reason="Bug fixed in pandas-2.2+", - ) - ) gd_data = cudf.from_pandas(pd_data) gd_other = cudf.from_pandas(pd_other) @@ -1534,7 +1516,7 @@ def test_index_from_arrow(data): arrow_array = pa.Array.from_pandas(pdi) expected_index = pd.Index(arrow_array.to_pandas()) gdi = cudf.Index.from_arrow(arrow_array) - if PANDAS_GE_200 and gdi.dtype == cudf.dtype("datetime64[s]"): + if gdi.dtype == cudf.dtype("datetime64[s]"): # Arrow bug: # https://github.com/apache/arrow/issues/33321 # arrow cannot convert non-nanosecond @@ -1748,8 +1730,7 @@ def test_get_indexer_single_unique_numeric(idx, key, method): rfunc_args_and_kwargs=([], {"key": key, "method": method}), ) else: - with expect_warning_if(not PANDAS_GE_200 and method is not None): - expected = pi.get_indexer(key, method=method) + expected = pi.get_indexer(key, method=method) got = gi.get_indexer(key, method=method) assert_eq(expected, got) @@ -2088,9 +2069,6 @@ def test_get_indexer_multi_numeric_deviate(key, method): assert_eq(expected, got) -@pytest.mark.xfail( - not PANDAS_GE_220, reason="Remove after pandas-2.2+ upgrade" -) @pytest.mark.parametrize("method", ["ffill", "bfill"]) def test_get_indexer_multi_error(method): pi = pd.MultiIndex.from_tuples( @@ -2437,10 +2415,7 @@ def test_index_type_methods(data, func): pidx = pd.Index(data) gidx = cudf.from_pandas(pidx) - if PANDAS_GE_200: - with pytest.warns(FutureWarning): - expected = getattr(pidx, func)() - else: + with pytest.warns(FutureWarning): expected = getattr(pidx, func)() with pytest.warns(FutureWarning): actual = getattr(gidx, func)() @@ -2538,7 +2513,7 @@ def test_isin_index(index, values): ) with expect_warning_if(is_dt_str): got = gidx.isin(values) - with expect_warning_if(PANDAS_GE_220 and is_dt_str): + with expect_warning_if(is_dt_str): expected = pidx.isin(values) assert_eq(got, expected) @@ -3048,22 +3023,7 @@ def test_index_getitem_time_duration(dtype): @pytest.mark.parametrize("dtype", ALL_TYPES) -def test_index_empty_from_pandas(request, dtype): - request.node.add_marker( - pytest.mark.xfail( - condition=not PANDAS_GE_200 - and dtype - in { - "datetime64[ms]", - "datetime64[s]", - "datetime64[us]", - "timedelta64[ms]", - "timedelta64[s]", - "timedelta64[us]", - }, - reason="Fixed in pandas-2.0", - ) - ) +def test_index_empty_from_pandas(dtype): pidx = pd.Index([], dtype=dtype) gidx = cudf.from_pandas(pidx) @@ -3087,8 +3047,7 @@ def test_index_to_frame(data, data_name, index, name): pidx = pd.Index(data, name=data_name) gidx = cudf.from_pandas(pidx) - with expect_warning_if(not PANDAS_GE_200 and name is None): - expected = pidx.to_frame(index=index, name=name) + expected = pidx.to_frame(index=index, name=name) actual = gidx.to_frame(index=index, name=name) assert_eq(expected, actual) diff --git a/python/cudf/cudf/tests/test_interval.py b/python/cudf/cudf/tests/test_interval.py index 1c61b378d68..7b923af1f75 100644 --- a/python/cudf/cudf/tests/test_interval.py +++ b/python/cudf/cudf/tests/test_interval.py @@ -6,7 +6,6 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_220 from cudf.testing._utils import assert_eq @@ -167,10 +166,6 @@ def test_interval_index_unique(): assert_eq(expected, actual) -@pytest.mark.xfail( - condition=not PANDAS_GE_220, - reason="TODO: Remove this once pandas-2.2 support is added", -) @pytest.mark.parametrize("box", [pd.Series, pd.IntervalIndex]) @pytest.mark.parametrize("tz", ["US/Eastern", None]) def test_interval_with_datetime(tz, box): diff --git a/python/cudf/cudf/tests/test_join_order.py b/python/cudf/cudf/tests/test_join_order.py index 58263faa7bf..7031a43d7f5 100644 --- a/python/cudf/cudf/tests/test_join_order.py +++ b/python/cudf/cudf/tests/test_join_order.py @@ -1,9 +1,7 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. import itertools -import operator import string -from collections import defaultdict import numpy as np import pytest @@ -34,124 +32,13 @@ def right(): return cudf.DataFrame({"key": right_key, "val": right_val}) -if PANDAS_GE_220: - # Behaviour in sort=False case didn't match documentation in many - # cases prior to https://github.com/pandas-dev/pandas/pull/54611 - # (released as part of pandas 2.2) - def expected(left, right, sort, *, how): - left = left.to_pandas() - right = right.to_pandas() - return left.merge(right, on="key", how=how, sort=sort) - -else: - - def expect_inner(left, right, sort): - left_key = left.key.values_host.tolist() - left_val = left.val.values_host.tolist() - right_key = right.key.values_host.tolist() - right_val = right.val.values_host.tolist() - - right_have = defaultdict(list) - for i, k in enumerate(right_key): - right_have[k].append(i) - keys = [] - val_x = [] - val_y = [] - for k, v in zip(left_key, left_val): - if k not in right_have: - continue - for i in right_have[k]: - keys.append(k) - val_x.append(v) - val_y.append(right_val[i]) - - if sort: - # Python sort is stable, so this will preserve input order for - # equal items. - keys, val_x, val_y = zip( - *sorted(zip(keys, val_x, val_y), key=operator.itemgetter(0)) - ) - return cudf.DataFrame({"key": keys, "val_x": val_x, "val_y": val_y}) - - def expect_left(left, right, sort): - left_key = left.key.values_host.tolist() - left_val = left.val.values_host.tolist() - right_key = right.key.values_host.tolist() - right_val = right.val.values_host.tolist() - - right_have = defaultdict(list) - for i, k in enumerate(right_key): - right_have[k].append(i) - keys = [] - val_x = [] - val_y = [] - for k, v in zip(left_key, left_val): - if k not in right_have: - right_vals = [None] - else: - right_vals = [right_val[i] for i in right_have[k]] - - for rv in right_vals: - keys.append(k) - val_x.append(v) - val_y.append(rv) - - if sort: - # Python sort is stable, so this will preserve input order for - # equal items. - keys, val_x, val_y = zip( - *sorted(zip(keys, val_x, val_y), key=operator.itemgetter(0)) - ) - return cudf.DataFrame({"key": keys, "val_x": val_x, "val_y": val_y}) - - def expect_outer(left, right, sort): - left_key = left.key.values_host.tolist() - left_val = left.val.values_host.tolist() - right_key = right.key.values_host.tolist() - right_val = right.val.values_host.tolist() - right_have = defaultdict(list) - for i, k in enumerate(right_key): - right_have[k].append(i) - keys = [] - val_x = [] - val_y = [] - for k, v in zip(left_key, left_val): - if k not in right_have: - right_vals = [None] - else: - right_vals = [right_val[i] for i in right_have[k]] - for rv in right_vals: - keys.append(k) - val_x.append(v) - val_y.append(rv) - left_have = set(left_key) - for k, v in zip(right_key, right_val): - if k not in left_have: - keys.append(k) - val_x.append(None) - val_y.append(v) - - # Python sort is stable, so this will preserve input order for - # equal items. - # outer joins are always sorted, but we test both sort values - keys, val_x, val_y = zip( - *sorted(zip(keys, val_x, val_y), key=operator.itemgetter(0)) - ) - return cudf.DataFrame({"key": keys, "val_x": val_x, "val_y": val_y}) - - def expected(left, right, sort, *, how): - if how == "inner": - return expect_inner(left, right, sort) - elif how == "outer": - return expect_outer(left, right, sort) - elif how == "left": - return expect_left(left, right, sort) - elif how == "right": - return expect_left(right, left, sort).rename( - {"val_x": "val_y", "val_y": "val_x"}, axis=1 - ) - else: - raise NotImplementedError() +# Behaviour in sort=False case didn't match documentation in many +# cases prior to https://github.com/pandas-dev/pandas/pull/54611 +# (released as part of pandas 2.2) +def expected(left, right, sort, *, how): + left = left.to_pandas() + right = right.to_pandas() + return left.merge(right, on="key", how=how, sort=sort) @pytest.mark.parametrize("how", ["inner", "left", "right", "outer"]) diff --git a/python/cudf/cudf/tests/test_joining.py b/python/cudf/cudf/tests/test_joining.py index 5fbd1ba602f..302051ade05 100644 --- a/python/cudf/cudf/tests/test_joining.py +++ b/python/cudf/cudf/tests/test_joining.py @@ -2156,19 +2156,13 @@ def test_join_multiindex_empty(): rhs = pd.DataFrame(index=["a", "c", "d"]) g_lhs = cudf.from_pandas(lhs) g_rhs = cudf.from_pandas(rhs) - if PANDAS_GE_200: - assert_exceptions_equal( - lfunc=lhs.join, - rfunc=g_lhs.join, - lfunc_args_and_kwargs=([rhs], {"how": "inner"}), - rfunc_args_and_kwargs=([g_rhs], {"how": "inner"}), - check_exception_type=False, - ) - else: - with pytest.warns(FutureWarning): - _ = lhs.join(rhs, how="inner") - with pytest.raises(ValueError): - _ = g_lhs.join(g_rhs, how="inner") + assert_exceptions_equal( + lfunc=lhs.join, + rfunc=g_lhs.join, + lfunc_args_and_kwargs=([rhs], {"how": "inner"}), + rfunc_args_and_kwargs=([g_rhs], {"how": "inner"}), + check_exception_type=False, + ) def test_join_on_index_with_duplicate_names(): diff --git a/python/cudf/cudf/tests/test_json.py b/python/cudf/cudf/tests/test_json.py index 12ea74bd7a7..45f9980ebd6 100644 --- a/python/cudf/cudf/tests/test_json.py +++ b/python/cudf/cudf/tests/test_json.py @@ -13,7 +13,7 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_210, PANDAS_GE_220 +from cudf.core._compat import PANDAS_GE_210, PANDAS_GE_220 from cudf.testing._utils import ( DATETIME_TYPES, NUMERIC_TYPES, @@ -216,18 +216,16 @@ def test_cudf_json_writer_read(gdf_writer_types): if pdf2.empty: pdf2.reset_index(drop=True, inplace=True) pdf2.columns = pdf2.columns.astype("object") - if PANDAS_GE_200: - # Pandas moved to consistent datetimes parsing format: - # https://pandas.pydata.org/docs/dev/whatsnew/v2.0.0.html#datetimes-are-now-parsed-with-a-consistent-format - for unit in ["s", "ms"]: - if f"col_datetime64[{unit}]" in pdf2.columns: - pdf2[f"col_datetime64[{unit}]"] = ( - pd.to_datetime( - pdf2[f"col_datetime64[{unit}]"], format="mixed" - ) - .dt.tz_localize(None) - .astype(f"datetime64[{unit}]") - ) + + # Pandas moved to consistent datetimes parsing format: + # https://pandas.pydata.org/docs/dev/whatsnew/v2.0.0.html#datetimes-are-now-parsed-with-a-consistent-format + for unit in ["s", "ms"]: + if f"col_datetime64[{unit}]" in pdf2.columns: + pdf2[f"col_datetime64[{unit}]"] = ( + pd.to_datetime(pdf2[f"col_datetime64[{unit}]"], format="mixed") + .dt.tz_localize(None) + .astype(f"datetime64[{unit}]") + ) assert_eq(pdf2, gdf2) diff --git a/python/cudf/cudf/tests/test_multiindex.py b/python/cudf/cudf/tests/test_multiindex.py index e15b3f6db40..a13fe333107 100644 --- a/python/cudf/cudf/tests/test_multiindex.py +++ b/python/cudf/cudf/tests/test_multiindex.py @@ -17,7 +17,6 @@ import cudf from cudf.api.extensions import no_default -from cudf.core._compat import PANDAS_GE_200 from cudf.core.column import as_column from cudf.core.index import as_index from cudf.testing._utils import ( @@ -1854,10 +1853,7 @@ def test_pickle_roundtrip_multiindex(names): def test_multiindex_type_methods(pidx, func): gidx = cudf.from_pandas(pidx) - if PANDAS_GE_200: - with pytest.warns(FutureWarning): - expected = getattr(pidx, func)() - else: + with pytest.warns(FutureWarning): expected = getattr(pidx, func)() with pytest.warns(FutureWarning): @@ -1996,10 +1992,9 @@ def test_multiindex_to_frame_allow_duplicates( allow_duplicates=allow_duplicates, ) else: - with expect_warning_if(not PANDAS_GE_200 and name is None): - expected = pidx.to_frame( - index=index, name=name, allow_duplicates=allow_duplicates - ) + expected = pidx.to_frame( + index=index, name=name, allow_duplicates=allow_duplicates + ) actual = gidx.to_frame( index=index, name=name, allow_duplicates=allow_duplicates ) diff --git a/python/cudf/cudf/tests/test_numerical.py b/python/cudf/cudf/tests/test_numerical.py index fb1bc580aa4..2e3be92dbeb 100644 --- a/python/cudf/cudf/tests/test_numerical.py +++ b/python/cudf/cudf/tests/test_numerical.py @@ -5,7 +5,6 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_220 from cudf.testing._utils import NUMERIC_TYPES, assert_eq, expect_warning_if from cudf.utils.dtypes import np_dtypes_to_pandas_dtypes @@ -373,7 +372,7 @@ def test_to_numeric_error(data, errors): ): cudf.to_numeric(data, errors=errors) else: - with expect_warning_if(PANDAS_GE_220 and errors == "ignore"): + with expect_warning_if(errors == "ignore"): expect = pd.to_numeric(data, errors=errors) with expect_warning_if(errors == "ignore"): got = cudf.to_numeric(data, errors=errors) diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index 851f0c30dc8..9bd014ce59f 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -291,7 +291,7 @@ def test_parquet_reader_empty_pandas_dataframe(tmpdir, engine): expect = expect.reset_index(drop=True) got = got.reset_index(drop=True) - assert_eq(expect, got, check_column_type=not PANDAS_GE_200) + assert_eq(expect, got) @pytest.mark.parametrize("has_null", [False, True]) @@ -2412,7 +2412,6 @@ def run_parquet_index(pdf, index): expected, actual, check_index_type=True, - check_column_type=not PANDAS_GE_200, ) @@ -2685,18 +2684,17 @@ def test_parquet_writer_column_validation(): with pytest.warns(UserWarning): df.to_parquet(cudf_parquet) - if PANDAS_GE_200: - with pytest.warns(UserWarning): - pdf.to_parquet(pandas_parquet) + with pytest.warns(UserWarning): + pdf.to_parquet(pandas_parquet) - assert_eq( - pd.read_parquet(cudf_parquet), - cudf.read_parquet(pandas_parquet), - ) - assert_eq( - cudf.read_parquet(cudf_parquet), - pd.read_parquet(pandas_parquet), - ) + assert_eq( + pd.read_parquet(cudf_parquet), + cudf.read_parquet(pandas_parquet), + ) + assert_eq( + cudf.read_parquet(cudf_parquet), + pd.read_parquet(pandas_parquet), + ) with cudf.option_context("mode.pandas_compatible", False): with pytest.raises(ValueError): @@ -2723,16 +2721,6 @@ def test_parquet_writer_nulls_pandas_read(tmpdir, pdf): got = pd.read_parquet(fname) nullable = num_rows > 0 - if not PANDAS_GE_200: - # BUG in pre-2.0.1: - # https://github.com/pandas-dev/pandas/issues/52449 - gdf["col_datetime64[ms]"] = gdf["col_datetime64[ms]"].astype( - "datetime64[ns]" - ) - gdf["col_datetime64[us]"] = gdf["col_datetime64[us]"].astype( - "datetime64[ns]" - ) - if nullable: gdf = gdf.drop(columns="col_datetime64[ms]") gdf = gdf.drop(columns="col_datetime64[us]") @@ -3042,7 +3030,7 @@ def test_parquet_roundtrip_time_delta(): df.to_parquet(buffer) # TODO: Remove `check_dtype` once following issue is fixed in arrow: # https://github.com/apache/arrow/issues/33321 - assert_eq(df, cudf.read_parquet(buffer), check_dtype=not PANDAS_GE_200) + assert_eq(df, cudf.read_parquet(buffer), check_dtype=False) def test_parquet_reader_malformed_file(datadir): diff --git a/python/cudf/cudf/tests/test_replace.py b/python/cudf/cudf/tests/test_replace.py index 0b57f9fe846..c667211b6d8 100644 --- a/python/cudf/cudf/tests/test_replace.py +++ b/python/cudf/cudf/tests/test_replace.py @@ -57,18 +57,14 @@ def test_series_replace_all(gsr, to_replace, value): else: pd_value = value - with expect_warning_if( + expect_warn = ( isinstance(gsr.dtype, cudf.CategoricalDtype) and isinstance(gd_to_replace, str) and gd_to_replace == "one" - ): + ) + with expect_warning_if(expect_warn): actual = gsr.replace(to_replace=gd_to_replace, value=gd_value) - with expect_warning_if( - PANDAS_GE_220 - and isinstance(gsr.dtype, cudf.CategoricalDtype) - and isinstance(gd_to_replace, str) - and gd_to_replace == "one" - ): + with expect_warning_if(expect_warn): if pd_value is None: # TODO: Remove this workaround once cudf # introduces `no_default` values @@ -93,7 +89,7 @@ def test_series_replace(): # Categorical psr3 = pd.Series(["one", "two", "three"], dtype="category") - with expect_warning_if(PANDAS_GE_220): + with pytest.warns(FutureWarning): psr4 = psr3.replace("one", "two") sr3 = cudf.from_pandas(psr3) with pytest.warns(FutureWarning): @@ -102,7 +98,7 @@ def test_series_replace(): psr4.sort_values().reset_index(drop=True), sr4.sort_values().reset_index(drop=True), ) - with expect_warning_if(PANDAS_GE_220): + with pytest.warns(FutureWarning): psr5 = psr3.replace("one", "five") with pytest.warns(FutureWarning): sr5 = sr3.replace("one", "five") @@ -517,7 +513,7 @@ def test_fillna_categorical(psr_data, fill_value, inplace): pd.date_range( "2010-01-01", "2020-01-10", - freq="1YE" if PANDAS_GE_220 else "1y", + freq="1YE", ) ), pd.Series(["2010-01-01", None, "2011-10-10"], dtype="datetime64[ns]"), @@ -564,7 +560,7 @@ def test_fillna_categorical(psr_data, fill_value, inplace): pd.date_range( "2010-01-01", "2020-01-10", - freq="1YE" if PANDAS_GE_220 else "1y", + freq="1YE", ) ) + pd.Timedelta("1d"), diff --git a/python/cudf/cudf/tests/test_resampling.py b/python/cudf/cudf/tests/test_resampling.py index 43f7324affe..a7e04e3fa13 100644 --- a/python/cudf/cudf/tests/test_resampling.py +++ b/python/cudf/cudf/tests/test_resampling.py @@ -5,7 +5,7 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_200, PANDAS_GE_220 +from cudf.core._compat import PANDAS_GE_220 from cudf.testing._utils import assert_eq @@ -15,7 +15,7 @@ def assert_resample_results_equal(lhs, rhs, **kwargs): rhs.sort_index(), check_dtype=False, check_freq=False, - check_index_type=not PANDAS_GE_200, + check_index_type=False, **kwargs, ) diff --git a/python/cudf/cudf/tests/test_reshape.py b/python/cudf/cudf/tests/test_reshape.py index 59c5a0662be..e632078e0d9 100644 --- a/python/cudf/cudf/tests/test_reshape.py +++ b/python/cudf/cudf/tests/test_reshape.py @@ -9,14 +9,13 @@ import cudf from cudf import melt as cudf_melt -from cudf.core._compat import PANDAS_GE_210, PANDAS_GE_220 +from cudf.core._compat import PANDAS_GE_210 from cudf.core.buffer.spill_manager import get_global_manager from cudf.testing._utils import ( ALL_TYPES, DATETIME_TYPES, NUMERIC_TYPES, assert_eq, - expect_warning_if, ) pytest_xfail = pytest.mark.xfail @@ -214,7 +213,7 @@ def test_df_stack_multiindex_column_axis(columns, index, level, dropna): with pytest.warns(FutureWarning): got = gdf.stack(level=level, dropna=dropna, future_stack=False) - with expect_warning_if(PANDAS_GE_220): + with pytest.warns(FutureWarning): expect = pdf.stack(level=level, dropna=dropna, future_stack=False) assert_eq(expect, got, check_dtype=False) @@ -259,7 +258,7 @@ def test_df_stack_multiindex_column_axis_pd_example(level): df = pd.DataFrame(np.random.randn(4, 4), columns=columns) - with expect_warning_if(PANDAS_GE_220): + with pytest.warns(FutureWarning): expect = df.stack(level=level, future_stack=False) gdf = cudf.from_pandas(df) with pytest.warns(FutureWarning): diff --git a/python/cudf/cudf/tests/test_rolling.py b/python/cudf/cudf/tests/test_rolling.py index cbd60b8945a..1d1d7ae8d29 100644 --- a/python/cudf/cudf/tests/test_rolling.py +++ b/python/cudf/cudf/tests/test_rolling.py @@ -1,32 +1,16 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. import math -from contextlib import contextmanager import numpy as np import pandas as pd import pytest import cudf -from cudf.core._compat import PANDAS_GE_200 from cudf.testing._utils import assert_eq from cudf.testing.dataset_generator import rand_dataframe -@contextmanager -def _hide_pandas_rolling_min_periods_warning(agg): - if not PANDAS_GE_200 and agg == "count": - with pytest.warns( - FutureWarning, - match="min_periods=None will default to the size of window " - "consistent with other methods in a future version. Specify " - "min_periods=0 instead.", - ): - yield - else: - yield - - @pytest.mark.parametrize( "data,index", [ @@ -410,10 +394,9 @@ def test_rolling_groupby_simple(agg): gdf = cudf.from_pandas(pdf) for window_size in range(1, len(pdf) + 1): - with _hide_pandas_rolling_min_periods_warning(agg): - expect = getattr( - pdf.groupby("a").rolling(window_size), agg - )().fillna(-1) + expect = getattr(pdf.groupby("a").rolling(window_size), agg)().fillna( + -1 + ) got = getattr(gdf.groupby("a").rolling(window_size), agg)().fillna(-1) assert_eq(expect, got, check_dtype=False) @@ -423,10 +406,9 @@ def test_rolling_groupby_simple(agg): gdf = cudf.from_pandas(pdf) for window_size in range(1, len(pdf) + 1): - with _hide_pandas_rolling_min_periods_warning(agg): - expect = getattr( - pdf.groupby("a").rolling(window_size), agg - )().fillna(-1) + expect = getattr(pdf.groupby("a").rolling(window_size), agg)().fillna( + -1 + ) got = getattr(gdf.groupby("a").rolling(window_size), agg)().fillna(-1) assert_eq(expect, got, check_dtype=False) @@ -445,10 +427,9 @@ def test_rolling_groupby_multi(agg): gdf = cudf.from_pandas(pdf) for window_size in range(1, len(pdf) + 1): - with _hide_pandas_rolling_min_periods_warning(agg): - expect = getattr( - pdf.groupby(["a", "b"], sort=True).rolling(window_size), agg - )().fillna(-1) + expect = getattr( + pdf.groupby(["a", "b"], sort=True).rolling(window_size), agg + )().fillna(-1) got = getattr( gdf.groupby(["a", "b"], sort=True).rolling(window_size), agg )().fillna(-1) diff --git a/python/cudf/cudf/tests/test_sorting.py b/python/cudf/cudf/tests/test_sorting.py index b3ecb471bb9..f9ca0e8ebcb 100644 --- a/python/cudf/cudf/tests/test_sorting.py +++ b/python/cudf/cudf/tests/test_sorting.py @@ -48,13 +48,11 @@ def test_dataframe_sort_values(nelem, dtype): @pytest.mark.parametrize("ignore_index", [True, False]) @pytest.mark.parametrize("index", ["a", "b", ["a", "b"]]) -def test_dataframe_sort_values_ignore_index(request, index, ignore_index): - request.applymarker( - pytest.mark.xfail( - PANDAS_GE_220 and isinstance(index, list) and not ignore_index, - reason="https://github.com/pandas-dev/pandas/issues/57531", +def test_dataframe_sort_values_ignore_index(index, ignore_index): + if PANDAS_GE_220 and isinstance(index, list) and not ignore_index: + pytest.skip( + reason="Unstable sorting by pandas(numpy): https://github.com/pandas-dev/pandas/issues/57531" ) - ) gdf = DataFrame( {"a": [1, 3, 5, 2, 4], "b": [1, 1, 2, 2, 3], "c": [9, 7, 7, 7, 1]} diff --git a/python/cudf/cudf/tests/test_stats.py b/python/cudf/cudf/tests/test_stats.py index b35dd28c4ec..9d5f0cd5eab 100644 --- a/python/cudf/cudf/tests/test_stats.py +++ b/python/cudf/cudf/tests/test_stats.py @@ -356,17 +356,10 @@ def test_series_median(dtype, num_na): @pytest.mark.parametrize( "fill_method", ["ffill", "bfill", "pad", "backfill", no_default, None] ) -def test_series_pct_change(request, data, periods, fill_method): +def test_series_pct_change(data, periods, fill_method): cs = cudf.Series(data) ps = cs.to_pandas() - request.applymarker( - pytest.mark.xfail( - condition=( - len(cs) == 0 and periods == 0 and fill_method is no_default - ), - reason="https://github.com/pandas-dev/pandas/issues/57056", - ) - ) + if np.abs(periods) <= len(cs): with expect_warning_if(fill_method not in (no_default, None)): got = cs.pct_change(periods=periods, fill_method=fill_method) diff --git a/python/cudf/cudf/tests/test_timedelta.py b/python/cudf/cudf/tests/test_timedelta.py index 18fe1700e25..0c591965361 100644 --- a/python/cudf/cudf/tests/test_timedelta.py +++ b/python/cudf/cudf/tests/test_timedelta.py @@ -9,7 +9,6 @@ import pytest import cudf -from cudf.core._compat import PANDAS_GE_200 from cudf.testing import _utils as utils from cudf.testing._utils import assert_eq, assert_exceptions_equal @@ -1324,11 +1323,7 @@ def test_numeric_to_timedelta(data, dtype, timedelta_dtype): psr = sr.to_pandas() actual = sr.astype(timedelta_dtype) - - if PANDAS_GE_200: - expected = psr.astype(timedelta_dtype) - else: - expected = pd.Series(psr.to_numpy().astype(timedelta_dtype)) + expected = psr.astype(timedelta_dtype) assert_eq(expected, actual) diff --git a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py index 0386ec434da..f017b46866f 100644 --- a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py +++ b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py @@ -17,7 +17,6 @@ import pytest from numba import NumbaDeprecationWarning -from cudf.core._compat import PANDAS_GE_220 from cudf.pandas import LOADED, Profiler from cudf.pandas.fast_slow_proxy import _Unusable @@ -510,14 +509,12 @@ def test_array_ufunc(series): @pytest.mark.xfail(strict=False, reason="Fails in CI, passes locally.") def test_groupby_apply_func_returns_series(dataframe): pdf, df = dataframe - if PANDAS_GE_220: - kwargs = {"include_groups": False} - else: - kwargs = {} expect = pdf.groupby("a").apply( - lambda group: pd.Series({"x": 1}), **kwargs + lambda group: pd.Series({"x": 1}), include_groups=False + ) + got = df.groupby("a").apply( + lambda group: xpd.Series({"x": 1}), include_groups=False ) - got = df.groupby("a").apply(lambda group: xpd.Series({"x": 1}), **kwargs) tm.assert_equal(expect, got) diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 82ac84a4022..ef3b439bdf4 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ "numpy>=1.21", "nvtx>=0.2.1", "packaging", - "pandas>=2.0,<2.1.5dev0", + "pandas>=2.0,<2.2.2dev0", "protobuf>=4.21,<5", "ptxcompiler", "pyarrow>=14.0.1,<15.0.0a0", @@ -98,6 +98,7 @@ pandas-tests = [ "pyreadstat", "pytest-asyncio", "pytest-reportlog", + "pytest-timeout", "python-snappy", "pyxlsb", "s3fs", diff --git a/python/dask_cudf/dask_cudf/io/tests/test_parquet.py b/python/dask_cudf/dask_cudf/io/tests/test_parquet.py index 583d4b07f6f..5e4ea578101 100644 --- a/python/dask_cudf/dask_cudf/io/tests/test_parquet.py +++ b/python/dask_cudf/dask_cudf/io/tests/test_parquet.py @@ -13,7 +13,6 @@ from dask.utils import natural_sort_key import cudf -from cudf.core._compat import PANDAS_GE_200 import dask_cudf @@ -168,7 +167,7 @@ def test_dask_timeseries_from_pandas(tmpdir): read_df = dask_cudf.read_parquet(fn) # Workaround until following issue is fixed: # https://github.com/apache/arrow/issues/33321 - dd.assert_eq(ddf2, read_df.compute(), check_index_type=not PANDAS_GE_200) + dd.assert_eq(ddf2, read_df.compute(), check_index_type=False) @pytest.mark.parametrize("index", [False, None]) diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index c23c21f4107..5d4ea429d5f 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "cupy-cuda11x>=12.0.0", "fsspec>=0.6.0", "numpy>=1.21", - "pandas>=2.0,<2.1.5dev0", + "pandas>=2.0,<2.2.2dev0", "rapids-dask-dependency==24.4.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ From e03623ae2ddbc4326201c30f15540ac04d78c0d6 Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Mon, 26 Feb 2024 11:28:33 -0800 Subject: [PATCH 303/384] Add environment-agnostic scripts for running ctests and pytests (#14992) This PR adds environment-agnostic `run_*_{ctests,pytests}.sh` scripts, and updates `test_*_{cpp,python}.sh` to call them. The `test_*_{cpp,python}.sh` scripts assume they're running in our CI environment, and they do more than just run the tests. This PR allows devs and downstream consumers to only run the tests, and skip the unrelated logic in `test_*_{cpp,python}.sh`. Authors: - Paul Taylor (https://github.com/trxcllnt) Approvers: - Jake Awe (https://github.com/AyodeAwe) URL: https://github.com/rapidsai/cudf/pull/14992 --- ci/run_cudf_benchmark_smoketests.sh | 13 +++++++++ ci/run_cudf_ctests.sh | 9 +++++++ ci/run_cudf_kafka_ctests.sh | 9 +++++++ ci/run_cudf_memcheck_ctests.sh | 24 +++++++++++++++++ ci/run_cudf_pandas_pytest_benchmarks.sh | 13 +++++++++ ci/run_cudf_pytest_benchmarks.sh | 12 +++++++++ ci/run_cudf_pytests.sh | 11 ++++++++ ci/run_custreamz_pytests.sh | 11 ++++++++ ci/run_dask_cudf_pytests.sh | 11 ++++++++ ci/test_cpp.sh | 24 +++++------------ ci/test_cpp_memcheck.sh | 25 ++++++------------ ci/test_python_cudf.sh | 35 +++++++++---------------- ci/test_python_other.sh | 23 +++++++--------- 13 files changed, 149 insertions(+), 71 deletions(-) create mode 100755 ci/run_cudf_benchmark_smoketests.sh create mode 100755 ci/run_cudf_ctests.sh create mode 100755 ci/run_cudf_kafka_ctests.sh create mode 100755 ci/run_cudf_memcheck_ctests.sh create mode 100755 ci/run_cudf_pandas_pytest_benchmarks.sh create mode 100755 ci/run_cudf_pytest_benchmarks.sh create mode 100755 ci/run_cudf_pytests.sh create mode 100755 ci/run_custreamz_pytests.sh create mode 100755 ci/run_dask_cudf_pytests.sh diff --git a/ci/run_cudf_benchmark_smoketests.sh b/ci/run_cudf_benchmark_smoketests.sh new file mode 100755 index 00000000000..56e768d68ba --- /dev/null +++ b/ci/run_cudf_benchmark_smoketests.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +# Support customizing the ctests' install location +cd "${INSTALL_PREFIX:-${CONDA_PREFIX:-/usr}}/bin/benchmarks/libcudf/"; + +# Ensure that benchmarks are runnable +# Run a small Google benchmark +./MERGE_BENCH --benchmark_filter=/2/ +# Run a small nvbench benchmark +./STRINGS_NVBENCH --run-once --benchmark 0 --devices 0 diff --git a/ci/run_cudf_ctests.sh b/ci/run_cudf_ctests.sh new file mode 100755 index 00000000000..562201c11b0 --- /dev/null +++ b/ci/run_cudf_ctests.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +# Support customizing the ctests' install location +cd "${INSTALL_PREFIX:-${CONDA_PREFIX:-/usr}}/bin/gtests/libcudf/"; + +ctest --output-on-failure --no-tests=error "$@" diff --git a/ci/run_cudf_kafka_ctests.sh b/ci/run_cudf_kafka_ctests.sh new file mode 100755 index 00000000000..51e5e302a68 --- /dev/null +++ b/ci/run_cudf_kafka_ctests.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +# Support customizing the ctests' install location +cd "${INSTALL_PREFIX:-${CONDA_PREFIX:-/usr}}/bin/gtests/libcudf_kafka/"; + +ctest --output-on-failure --no-tests=error "$@" diff --git a/ci/run_cudf_memcheck_ctests.sh b/ci/run_cudf_memcheck_ctests.sh new file mode 100755 index 00000000000..cfd12cb92b4 --- /dev/null +++ b/ci/run_cudf_memcheck_ctests.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -uo pipefail + +EXITCODE=0 +trap "EXITCODE=1" ERR + +# Support customizing the ctests' install location +cd "${INSTALL_PREFIX:-${CONDA_PREFIX:-/usr}}/bin/gtests/libcudf/"; + +export GTEST_CUDF_RMM_MODE=cuda +for gt in ./*_TEST ; do + test_name=$(basename ${gt}) + # Run gtests with compute-sanitizer + if [[ "$test_name" == "ERROR_TEST" ]] || [[ "$test_name" == "STREAM_IDENTIFICATION_TEST" ]]; then + continue + fi + echo "Running compute-sanitizer on $test_name" + compute-sanitizer --tool memcheck ${gt} "$@" +done +unset GTEST_CUDF_RMM_MODE + +exit ${EXITCODE} diff --git a/ci/run_cudf_pandas_pytest_benchmarks.sh b/ci/run_cudf_pandas_pytest_benchmarks.sh new file mode 100755 index 00000000000..d3ab387a612 --- /dev/null +++ b/ci/run_cudf_pandas_pytest_benchmarks.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +# It is essential to cd into python/cudf as `pytest-xdist` + `coverage` seem to work only at this directory level. + +# Support invoking run_cudf_pandas_pytest_benchmarks.sh outside the script directory +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cudf/ + +CUDF_BENCHMARKS_USE_PANDAS=ON \ +CUDF_BENCHMARKS_DEBUG_ONLY=ON \ +pytest --cache-clear "$@" benchmarks diff --git a/ci/run_cudf_pytest_benchmarks.sh b/ci/run_cudf_pytest_benchmarks.sh new file mode 100755 index 00000000000..5e9b537f2b0 --- /dev/null +++ b/ci/run_cudf_pytest_benchmarks.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +# It is essential to cd into python/cudf as `pytest-xdist` + `coverage` seem to work only at this directory level. + +# Support invoking run_cudf_pytest_benchmarks.sh outside the script directory +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cudf/ + +CUDF_BENCHMARKS_DEBUG_ONLY=ON \ +pytest --cache-clear "$@" benchmarks diff --git a/ci/run_cudf_pytests.sh b/ci/run_cudf_pytests.sh new file mode 100755 index 00000000000..2b7b71b5132 --- /dev/null +++ b/ci/run_cudf_pytests.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +# It is essential to cd into python/cudf/cudf as `pytest-xdist` + `coverage` seem to work only at this directory level. + +# Support invoking run_cudf_pytests.sh outside the script directory +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cudf/cudf/ + +pytest --cache-clear --ignore="benchmarks" "$@" tests diff --git a/ci/run_custreamz_pytests.sh b/ci/run_custreamz_pytests.sh new file mode 100755 index 00000000000..53e27ec64b3 --- /dev/null +++ b/ci/run_custreamz_pytests.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +# It is essential to cd into python/cudf/cudf as `pytest-xdist` + `coverage` seem to work only at this directory level. + +# Support invoking run_custreamz_pytests.sh outside the script directory +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/custreamz/custreamz/ + +pytest --cache-clear "$@" tests diff --git a/ci/run_dask_cudf_pytests.sh b/ci/run_dask_cudf_pytests.sh new file mode 100755 index 00000000000..07658c6d234 --- /dev/null +++ b/ci/run_dask_cudf_pytests.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +# It is essential to cd into python/cudf/cudf as `pytest-xdist` + `coverage` seem to work only at this directory level. + +# Support invoking run_dask_cudf_pytests.sh outside the script directory +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/dask_cudf/dask_cudf/ + +pytest --cache-clear "$@" . diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 7119a79f4de..995c8d7d71f 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -1,7 +1,10 @@ #!/bin/bash # Copyright (c) 2022-2024, NVIDIA CORPORATION. -source "$(dirname "$0")/test_cpp_common.sh" +# Support invoking test_cpp.sh outside the script directory +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ + +source ./ci/test_cpp_common.sh EXITCODE=0 trap "EXITCODE=1" ERR @@ -10,36 +13,23 @@ set +e # Run libcudf and libcudf_kafka gtests from libcudf-tests package export GTEST_OUTPUT=xml:${RAPIDS_TESTS_DIR}/ -pushd $CONDA_PREFIX/bin/gtests/libcudf/ rapids-logger "Run libcudf gtests" -ctest -j20 --output-on-failure --no-tests=error +./ci/run_cudf_ctests.sh -j20 SUITEERROR=$? -popd if (( ${SUITEERROR} == 0 )); then - pushd $CONDA_PREFIX/bin/gtests/libcudf_kafka/ rapids-logger "Run libcudf_kafka gtests" - ctest -j20 --output-on-failure --no-tests=error + ./ci/run_cudf_kafka_ctests.sh -j20 SUITEERROR=$? - popd fi # Ensure that benchmarks are runnable -pushd $CONDA_PREFIX/bin/benchmarks/libcudf/ rapids-logger "Run tests of libcudf benchmarks" if (( ${SUITEERROR} == 0 )); then - # Run a small Google benchmark - ./MERGE_BENCH --benchmark_filter=/2/ - SUITEERROR=$? -fi - -if (( ${SUITEERROR} == 0 )); then - # Run a small nvbench benchmark - ./STRINGS_NVBENCH --run-once --benchmark 0 --devices 0 + ./ci/run_cudf_benchmark_smoketests.sh SUITEERROR=$? fi -popd rapids-logger "Test script exiting with value: $EXITCODE" exit ${EXITCODE} diff --git a/ci/test_cpp_memcheck.sh b/ci/test_cpp_memcheck.sh index 0e85268cb72..0233c2b55f8 100755 --- a/ci/test_cpp_memcheck.sh +++ b/ci/test_cpp_memcheck.sh @@ -1,25 +1,16 @@ #!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. -source "$(dirname "$0")/test_cpp_common.sh" +# Support invoking test_cpp.sh outside the script directory +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ -EXITCODE=0 -trap "EXITCODE=1" ERR -set +e +source ./ci/test_cpp_common.sh -# Run gtests with compute-sanitizer rapids-logger "Memcheck gtests with rmm_mode=cuda" -export GTEST_CUDF_RMM_MODE=cuda -COMPUTE_SANITIZER_CMD="compute-sanitizer --tool memcheck" -for gt in "$CONDA_PREFIX"/bin/gtests/libcudf/*_TEST ; do - test_name=$(basename ${gt}) - if [[ "$test_name" == "ERROR_TEST" ]] || [[ "$test_name" == "STREAM_IDENTIFICATION_TEST" ]]; then - continue - fi - echo "Running compute-sanitizer on $test_name" - ${COMPUTE_SANITIZER_CMD} ${gt} --gtest_output=xml:"${RAPIDS_TESTS_DIR}${test_name}.xml" -done -unset GTEST_CUDF_RMM_MODE + +./ci/run_cudf_memcheck_ctests.sh \ + --gtest_output=xml:"${RAPIDS_TESTS_DIR}${test_name}.xml" \ + && EXITCODE=$? || EXITCODE=$?; rapids-logger "Test script exiting with value: $EXITCODE" exit ${EXITCODE} diff --git a/ci/test_python_cudf.sh b/ci/test_python_cudf.sh index bb33d8473ce..ace71bb0b75 100755 --- a/ci/test_python_cudf.sh +++ b/ci/test_python_cudf.sh @@ -1,8 +1,11 @@ #!/bin/bash -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. + +# Support invoking test_python_cudf.sh outside the script directory +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../; # Common setup steps shared by Python test jobs -source "$(dirname "$0")/test_python_common.sh" +source ./ci/test_python_common.sh rapids-logger "Check GPU usage" nvidia-smi @@ -12,51 +15,37 @@ trap "EXITCODE=1" ERR set +e rapids-logger "pytest cudf" -pushd python/cudf/cudf -# It is essential to cd into python/cudf/cudf as `pytest-xdist` + `coverage` seem to work only at this directory level. -pytest \ - --cache-clear \ - --ignore="benchmarks" \ +./ci/run_cudf_pytests.sh \ --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf.xml" \ --numprocesses=8 \ --dist=loadscope \ --cov-config=../.coveragerc \ --cov=cudf \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-coverage.xml" \ - --cov-report=term \ - tests -popd + --cov-report=term # Run benchmarks with both cudf and pandas to ensure compatibility is maintained. # Benchmarks are run in DEBUG_ONLY mode, meaning that only small data sizes are used. # Therefore, these runs only verify that benchmarks are valid. # They do not generate meaningful performance measurements. -pushd python/cudf + rapids-logger "pytest for cudf benchmarks" -CUDF_BENCHMARKS_DEBUG_ONLY=ON \ -pytest \ - --cache-clear \ +./ci/run_cudf_pytest_benchmarks.sh \ --numprocesses=8 \ --dist=loadscope \ --cov-config=.coveragerc \ --cov=cudf \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-benchmark-coverage.xml" \ - --cov-report=term \ - benchmarks + --cov-report=term rapids-logger "pytest for cudf benchmarks using pandas" -CUDF_BENCHMARKS_USE_PANDAS=ON \ -CUDF_BENCHMARKS_DEBUG_ONLY=ON \ -pytest \ - --cache-clear \ +./ci/run_cudf_pandas_pytest_benchmarks.sh \ --numprocesses=8 \ --dist=loadscope \ --cov-config=.coveragerc \ --cov=cudf \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-benchmark-pandas-coverage.xml" \ - --cov-report=term \ - benchmarks -popd + --cov-report=term rapids-logger "Test script exiting with value: $EXITCODE" exit ${EXITCODE} diff --git a/ci/test_python_other.sh b/ci/test_python_other.sh index 25c1d681029..bc15747b26a 100755 --- a/ci/test_python_other.sh +++ b/ci/test_python_other.sh @@ -1,8 +1,11 @@ #!/bin/bash -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. + +# Support invoking test_python_cudf.sh outside the script directory +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ # Common setup steps shared by Python test jobs -source "$(dirname "$0")/test_python_common.sh" +source ./ci/test_python_common.sh rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ @@ -17,32 +20,24 @@ trap "EXITCODE=1" ERR set +e rapids-logger "pytest dask_cudf" -pushd python/dask_cudf/dask_cudf -pytest \ - --cache-clear \ +./ci/run_dask_cudf_pytests.sh \ --junitxml="${RAPIDS_TESTS_DIR}/junit-dask-cudf.xml" \ --numprocesses=8 \ --dist=loadscope \ --cov-config=../.coveragerc \ --cov=dask_cudf \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/dask-cudf-coverage.xml" \ - --cov-report=term \ - . -popd + --cov-report=term rapids-logger "pytest custreamz" -pushd python/custreamz/custreamz -pytest \ - --cache-clear \ +./ci/run_custreamz_pytests.sh \ --junitxml="${RAPIDS_TESTS_DIR}/junit-custreamz.xml" \ --numprocesses=8 \ --dist=loadscope \ --cov-config=../.coveragerc \ --cov=custreamz \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/custreamz-coverage.xml" \ - --cov-report=term \ - tests -popd + --cov-report=term rapids-logger "Test script exiting with value: $EXITCODE" exit ${EXITCODE} From dc88dcbffcd1183076cff4dcff6bc652c84fe676 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 27 Feb 2024 09:11:26 -0600 Subject: [PATCH 304/384] Bump to nvcomp 3.0.6. (#15128) This PR bumps nvcomp to 3.0.6. This is needed as a hotfix for https://github.com/rapidsai/cudf/issues/15096. Depends on: - https://github.com/conda-forge/nvcomp-feedstock/pull/14 - https://github.com/rapidsai/rapids-cmake/pull/542 - https://github.com/rapidsai/kvikio/pull/346 Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Lawrence Mitchell (https://github.com/wence-) - Ray Douglass (https://github.com/raydouglass) --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-120_arch-x86_64.yaml | 2 +- conda/recipes/libcudf/conda_build_config.yaml | 2 +- dependencies.yaml | 2 +- .../data/parquet/zstd_huff_tables_bug.parquet | Bin 0 -> 2759 bytes python/cudf/cudf/tests/test_parquet.py | 11 +++++++++++ 6 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 python/cudf/cudf/tests/data/parquet/zstd_huff_tables_bug.parquet diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 956c685f7de..f123e7c7bbb 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -62,7 +62,7 @@ dependencies: - numpy>=1.21,<1.25 - numpydoc - nvcc_linux-64=11.8 -- nvcomp==3.0.5 +- nvcomp==3.0.6 - nvtx>=0.2.1 - packaging - pandas>=1.3,<1.6.0dev0 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index cd2c70577f9..9db43a2b938 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -60,7 +60,7 @@ dependencies: - numba>=0.57 - numpy>=1.21,<1.25 - numpydoc -- nvcomp==3.0.5 +- nvcomp==3.0.6 - nvtx>=0.2.1 - packaging - pandas>=1.3,<1.6.0dev0 diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml index 9ed8c94f2bb..084f4651450 100644 --- a/conda/recipes/libcudf/conda_build_config.yaml +++ b/conda/recipes/libcudf/conda_build_config.yaml @@ -38,7 +38,7 @@ spdlog_version: - ">=1.12.0,<1.13" nvcomp_version: - - "=3.0.5" + - "=3.0.6" zlib_version: - ">=1.2.13" diff --git a/dependencies.yaml b/dependencies.yaml index 9a1d11af02d..efd42c838bb 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -251,7 +251,7 @@ dependencies: - libkvikio==24.2.* - librdkafka>=1.9.0,<1.10.0a0 # Align nvcomp version with rapids-cmake - - nvcomp==3.0.5 + - nvcomp==3.0.6 - spdlog>=1.12.0,<1.13 build_wheels: common: diff --git a/python/cudf/cudf/tests/data/parquet/zstd_huff_tables_bug.parquet b/python/cudf/cudf/tests/data/parquet/zstd_huff_tables_bug.parquet new file mode 100644 index 0000000000000000000000000000000000000000..4fb66fd86fc6c689ea522032d5ded66d64a30167 GIT binary patch literal 2759 zcmZuz3p`Y58$WX~#{D*Dnqk^Sa?3L07S_m8W7qw*qA)ZeA`DWWCd3d1iQJW3%AF;` zu3VFAF1h9sYEijgHjKVg?e5op?K$uBJHO|D-t&K+_j#W8Jg@0-b43sUmw3P=4zPt$ z6v;~fYY-5E0?_~eM`nKsa8a5sR0`&l!N5rD~ z&A}=Tam^@xG2Cc>awZ#-Lu^UJ%b-kIZAwgDpCl-cJs$kj`2#5zWths8>X z8(_UyYB{ovls&aKy9MV!sN_9UiZg z8CNE3UJ3R)h^-2=-c35ETaL)5ni3M7vtlSFnf=~gt;9TPv>`^s^@)Y}eo*m1+DX@Z z?)8$%muoyR)GA6y-dO6QntYq4M80A#V48?=kxEYF3>Vlh|B~x zo84&c*;LLBb!*;&vB*~5ypl#mzOBCK*fI|xt%p|0`JJkz0vq<`N<5JaC*JC6ErTcmDKd#jDADtugs(>yms^E+wNL&#doX8Uri;Nhtq!y?_Vow zR{VKauFKLn>-i6$y$&U%$O{Go`nD<(QJ;(KAtyK!eZ3p~b&(}v0RQTH@{T_5P@{Qu z`O|#AmHG2OON|-ZAZhEZv_xKZ_4kz4pU*MrUcn5cHwP*GAYq@E0VG0qLnViA42u${ zI5puQ!Z@?QqOPoNTdZySxyT3Y4^8gpVMZ`hQw;t5r)!L}tyw7(q}o>EnY@?P^-m}F z`uJBE){7Q!ZLTM?!*S}jG^^awU(wVk#rK$Z$FbT(i49C)@9?^#6ecjBUWqN z(bR|QLt1BhvqtYURQ^g*s8}zHZhNC}EjR7fdlqWbjrC%u@+uc3+DuXLq{{QeY7&q7 zzLw<<;@&RT(TfPn!niuF-1jx#G;iS^HsBchT|Lq4t-;ZI$6V_?-$?{o*m$$mkZMVE z6;E`#0}S5V7{i!VHKo(UhJ5A}!jhK9_IU7h@Oz2TJBXrQei@^a*a-X4e!C^lp(knQ4!X1` z4DQ8{S2FJ%-VEf-cDuw{uo;)@bikPS>eh9~Hi){=eb2kc`$p=~DMuu|sTTiXCwlnVHg1X@vlwqU{j@=h?x8Kh~yrzAoA`d_apZ0sw#-1ndJ0h(`hf zY(Q~+7spRr@A2k`Y$WZaW@L|`lzLP$)sJNy4>k=u6>VEX`2afYd9Yo|*1a+k3=7?1 z#>Ok&G)^kKU7jYN;h)ucN9rE$17e|kDMtloM@bX9!|eI|==JgJiRrg9xaE$Dm88|Y zwI`e8ohbmO3a>`xa6jeko#SgFpbnjqs3jID_UT}bd*sp$o*Rxd8DBC9w_g8Q-Ol^8 z;F(6pLoPvXO};ZUa_AqCD5hF$Sc2c}-DLT+-R-QI?4{h52l{2D<@joeC9k&C!j76z z?a9E(DdOVfGW`#+z4{MpVkZLD~Ugo`adE2PD+P_`$-eQLfjxght$Lqo&7!Y@S(560NXQIgrxG}1q22>`B* z-RGB{@Sh&$7aV^l)ms!SDm0Z{RHLo4nJb|FA91+JNl$ec9aX3BRoeG z-85jAh?~v@k>E5MoP&Xn<0gktT3|E`ME=zyjSzr12M!z85%AHFfRTs0Q&5`q0q+@c zCk?g_VGj`cLQTx@rLnE6_KSkb(nSBnO{}Ag>pKOB2Zb_C-4<43AxZb^nXxI+k4iI& zf61(1M;L!IbN+;?toVvPIU8r@kouNz00D!lvIqhGh5$dz#WW#4RVDJveqD|N zgdis*VTgTMNHCBr!agbNYS#cAq9}%&T`&6%8H0`1IrQ?94knJ=B92Wqs4#VB~>=g@!)Ov zD&f0`Onj z0J)*}*lb>G2ZtnD3x||G0RT8;JQQ>+ApL~~;0aK&Adfz#qo3n{T>t0Z-+YUl&@_lJ zHXH^KhQT00l=K(cKM3nF1Q7YBb6+j(Gi3+?n~s*O$7tcue{BIA8v0Xxx*%Ww1^tJK z5yEE^NEH6x!t~k5TW)L!N^*n%4FV4B@R=OIq5m`kVA0b5f&zFsXhR{P*gygL_<+`0 xp%2AX5{>N_$3PoP$o#mefu4n^#m9t;ntJ%TxOw6Y2>|f-00g1WOray}e*rj$Zdd>S literal 0 HcmV?d00001 diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index 007349ab551..2424b33a5dc 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -3040,3 +3040,14 @@ def test_parquet_reader_multiindex(): def test_parquet_reader_engine_error(): with pytest.raises(ValueError): cudf.read_parquet(BytesIO(), engine="abc") + + +def test_parquet_reader_zstd_huff_tables(datadir): + # Ensure that this zstd-compressed file does not overrun buffers. The + # problem was fixed in nvcomp 3.0.6. + # See https://github.com/rapidsai/cudf/issues/15096 + fname = datadir / "zstd_huff_tables_bug.parquet" + + expected = pa.parquet.read_table(fname).to_pandas() + actual = cudf.read_parquet(fname) + assert_eq(actual, expected) From c32725d53ab1f83a2337df3b6c548bf38eeec700 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 27 Feb 2024 15:40:34 -0500 Subject: [PATCH 305/384] Remove offsets_begin() call from nvtext::generate_ngrams (#15077) Removes call to `strings_column_view::offsets_begin()` call from `nvtext::generate_ngrams()`. A future PR will deprecate the `offsets_begin()` function which hardcodes to int32 type. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Mike Wilson (https://github.com/hyperbolic2346) URL: https://github.com/rapidsai/cudf/pull/15077 --- cpp/src/text/generate_ngrams.cu | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cpp/src/text/generate_ngrams.cu b/cpp/src/text/generate_ngrams.cu index 433237bbf81..fafb2f18b80 100644 --- a/cpp/src/text/generate_ngrams.cu +++ b/cpp/src/text/generate_ngrams.cu @@ -103,11 +103,8 @@ std::unique_ptr generate_ngrams(cudf::strings_column_view const& s // first create a new offsets vector removing nulls and empty strings from the input column std::unique_ptr non_empty_offsets_column = [&] { - cudf::column_view offsets_view(cudf::data_type{cudf::type_id::INT32}, - strings_count + 1, - strings.offsets_begin(), - nullptr, - 0); + cudf::column_view offsets_view( + strings.offsets().type(), strings_count + 1, strings.offsets().head(), nullptr, 0); auto table_offsets = cudf::detail::copy_if( cudf::table_view({offsets_view}), [d_strings, strings_count] __device__(cudf::size_type idx) { From 1719cda0b18bf3f15426f827fc49e23f0ec3bd40 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 27 Feb 2024 15:41:11 -0500 Subject: [PATCH 306/384] Remove calls to strings_column_view::offsets_begin() (#15112) Removes calls to `cudf::strings_column_view::offsets_begin()` since the result cannot have a hardcoded integer type. The goal is to deprecate this member function in this release. Follow on changes may be required to further enable large strings support to these functions. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/15112 --- cpp/examples/strings/custom_prealloc.cu | 2 +- cpp/src/transform/row_conversion.cu | 33 +++++++++++++------------ cpp/tests/io/json_type_cast_test.cu | 33 +++++++++++-------------- 3 files changed, 33 insertions(+), 35 deletions(-) diff --git a/cpp/examples/strings/custom_prealloc.cu b/cpp/examples/strings/custom_prealloc.cu index 93194899fe1..27b553731f8 100644 --- a/cpp/examples/strings/custom_prealloc.cu +++ b/cpp/examples/strings/custom_prealloc.cu @@ -98,7 +98,7 @@ std::unique_ptr redact_strings(cudf::column_view const& names, nvtxRangePushA("redact_strings"); auto const scv = cudf::strings_column_view(names); - auto const offsets = scv.offsets_begin(); + auto const offsets = scv.offsets().begin(); // create working memory to hold the output of each string auto working_memory = rmm::device_uvector(scv.chars_size(stream), stream); diff --git a/cpp/src/transform/row_conversion.cu b/cpp/src/transform/row_conversion.cu index 361a3610afa..32faa097d0e 100644 --- a/cpp/src/transform/row_conversion.cu +++ b/cpp/src/transform/row_conversion.cu @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -212,7 +213,7 @@ struct batch_data { * @return pair of device vector of size_types of the row sizes of the table and a device vector of * offsets into the string column */ -std::pair, rmm::device_uvector> +std::pair, rmm::device_uvector> build_string_row_offsets(table_view const& tbl, size_type fixed_width_and_validity_size, rmm::cuda_stream_view stream) @@ -222,20 +223,20 @@ build_string_row_offsets(table_view const& tbl, thrust::uninitialized_fill(rmm::exec_policy(stream), d_row_sizes.begin(), d_row_sizes.end(), 0); auto d_offsets_iterators = [&]() { - std::vector offsets_iterators; - auto offsets_iter = thrust::make_transform_iterator( - tbl.begin(), [](auto const& col) -> strings_column_view::offset_iterator { - if (!is_fixed_width(col.type())) { - CUDF_EXPECTS(col.type().id() == type_id::STRING, "only string columns are supported!"); - return strings_column_view(col).offsets_begin(); - } else { - return nullptr; - } + std::vector offsets_iterators; + auto itr = thrust::make_transform_iterator( + tbl.begin(), [](auto const& col) -> cudf::detail::input_offsetalator { + return cudf::detail::offsetalator_factory::make_input_iterator( + strings_column_view(col).offsets(), col.offset()); }); - std::copy_if(offsets_iter, - offsets_iter + tbl.num_columns(), - std::back_inserter(offsets_iterators), - [](auto const& offset_ptr) { return offset_ptr != nullptr; }); + auto stencil = thrust::make_transform_iterator( + tbl.begin(), [](auto const& col) -> bool { return !is_fixed_width(col.type()); }); + thrust::copy_if(thrust::host, + itr, + itr + tbl.num_columns(), + stencil, + std::back_inserter(offsets_iterators), + thrust::identity{}); return make_device_uvector_sync( offsets_iterators, stream, rmm::mr::get_current_device_resource()); }(); @@ -858,7 +859,7 @@ CUDF_KERNEL void copy_strings_to_rows(size_type const num_rows, size_type const num_variable_columns, int8_t const** variable_input_data, size_type const* variable_col_output_offsets, - size_type const** variable_col_offsets, + cudf::detail::input_offsetalator* variable_col_offsets, size_type fixed_width_row_size, RowOffsetFunctor row_offsets, size_type const batch_row_offset, @@ -1844,7 +1845,7 @@ std::vector> convert_to_rows( batch_data& batch_info, offsetFunctor offset_functor, column_info_s const& column_info, - std::optional> variable_width_offsets, + std::optional> variable_width_offsets, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { diff --git a/cpp/tests/io/json_type_cast_test.cu b/cpp/tests/io/json_type_cast_test.cu index 8a541022ab0..fe430010f4b 100644 --- a/cpp/tests/io/json_type_cast_test.cu +++ b/cpp/tests/io/json_type_cast_test.cu @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -34,6 +35,8 @@ #include +#include + #include #include #include @@ -43,25 +46,15 @@ using namespace cudf::test::iterators; struct JSONTypeCastTest : public cudf::test::BaseFixture {}; namespace { -struct offsets_to_length { - __device__ cudf::size_type operator()(thrust::tuple const& p) - { - return thrust::get<1>(p) - thrust::get<0>(p); - } -}; /// Returns length of each string in the column auto string_offset_to_length(cudf::strings_column_view const& column, rmm::cuda_stream_view stream) { - auto offsets_begin = column.offsets_begin(); - auto offsets_pair = - thrust::make_zip_iterator(thrust::make_tuple(offsets_begin, thrust::next(offsets_begin))); rmm::device_uvector svs_length(column.size(), stream); - thrust::transform(rmm::exec_policy(cudf::get_default_stream()), - offsets_pair, - offsets_pair + column.size(), - svs_length.begin(), - offsets_to_length{}); + auto itr = + cudf::detail::offsetalator_factory::make_input_iterator(column.offsets(), column.offset()); + thrust::adjacent_difference( + rmm::exec_policy(stream), itr + 1, itr + column.size() + 1, svs_length.begin()); return svs_length; } } // namespace @@ -96,7 +89,8 @@ TEST_F(JSONTypeCastTest, String) auto str_col = cudf::io::json::detail::parse_data( column.chars_begin(stream), - thrust::make_zip_iterator(thrust::make_tuple(column.offsets_begin(), svs_length.begin())), + thrust::make_zip_iterator( + thrust::make_tuple(column.offsets().begin(), svs_length.begin())), column.size(), type, std::move(null_mask), @@ -129,7 +123,8 @@ TEST_F(JSONTypeCastTest, Int) auto col = cudf::io::json::detail::parse_data( column.chars_begin(stream), - thrust::make_zip_iterator(thrust::make_tuple(column.offsets_begin(), svs_length.begin())), + thrust::make_zip_iterator( + thrust::make_tuple(column.offsets().begin(), svs_length.begin())), column.size(), type, std::move(null_mask), @@ -169,7 +164,8 @@ TEST_F(JSONTypeCastTest, StringEscapes) auto col = cudf::io::json::detail::parse_data( column.chars_begin(stream), - thrust::make_zip_iterator(thrust::make_tuple(column.offsets_begin(), svs_length.begin())), + thrust::make_zip_iterator( + thrust::make_tuple(column.offsets().begin(), svs_length.begin())), column.size(), type, std::move(null_mask), @@ -238,7 +234,8 @@ TEST_F(JSONTypeCastTest, ErrorNulls) auto str_col = cudf::io::json::detail::parse_data( column.chars_begin(stream), - thrust::make_zip_iterator(thrust::make_tuple(column.offsets_begin(), svs_length.begin())), + thrust::make_zip_iterator( + thrust::make_tuple(column.offsets().begin(), svs_length.begin())), column.size(), type, std::move(null_mask), From ab2eb58be36e1140157e61aa65838670d97820b7 Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Wed, 28 Feb 2024 08:49:44 -0600 Subject: [PATCH 307/384] Add java option to keep quotes for JSON reads (#15146) Plumbs through the option to enable returning quotes with strings when reading JSON. Authors: - Robert (Bobby) Evans (https://github.com/revans2) Approvers: - Jason Lowe (https://github.com/jlowe) - Bradley Dice (https://github.com/bdice) --- .../main/java/ai/rapids/cudf/JSONOptions.java | 17 ++++++++++++++ java/src/main/java/ai/rapids/cudf/Table.java | 22 ++++++++++++++----- java/src/main/native/src/TableJni.cpp | 19 +++++++++++----- .../test/java/ai/rapids/cudf/TableTest.java | 19 ++++++++++++++++ 4 files changed, 65 insertions(+), 12 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/JSONOptions.java b/java/src/main/java/ai/rapids/cudf/JSONOptions.java index 35165c18c7a..62496e32f7a 100644 --- a/java/src/main/java/ai/rapids/cudf/JSONOptions.java +++ b/java/src/main/java/ai/rapids/cudf/JSONOptions.java @@ -32,6 +32,7 @@ public final class JSONOptions extends ColumnFilterOptions { private final boolean recoverWithNull; private final boolean normalizeSingleQuotes; private final boolean mixedTypesAsStrings; + private final boolean keepStringQuotes; private JSONOptions(Builder builder) { super(builder); @@ -40,6 +41,7 @@ private JSONOptions(Builder builder) { recoverWithNull = builder.recoverWithNull; normalizeSingleQuotes = builder.normalizeSingleQuotes; mixedTypesAsStrings = builder.mixedTypesAsStrings; + keepStringQuotes = builder.keepQuotes; } public boolean isDayFirst() { @@ -63,6 +65,10 @@ public boolean isMixedTypesAsStrings() { return mixedTypesAsStrings; } + public boolean keepStringQuotes() { + return keepStringQuotes; + } + @Override String[] getIncludeColumnNames() { throw new UnsupportedOperationException("JSON reader didn't support column prune"); @@ -80,6 +86,7 @@ public static final class Builder extends ColumnFilterOptions.Builder(lines)) .recovery_mode(recovery_mode) .normalize_single_quotes(static_cast(normalize_single_quotes)) + .keep_quotes(keep_quotes) .mixed_types_as_string(mixed_types_as_string); auto result = @@ -1459,7 +1461,8 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readAndInferJSONFromDataSource JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readAndInferJSON( JNIEnv *env, jclass, jlong buffer, jlong buffer_length, jboolean day_first, jboolean lines, - jboolean recover_with_null, jboolean normalize_single_quotes, jboolean mixed_types_as_string) { + jboolean recover_with_null, jboolean normalize_single_quotes, jboolean mixed_types_as_string, + jboolean keep_quotes) { JNI_NULL_CHECK(env, buffer, "buffer cannot be null", 0); if (buffer_length <= 0) { @@ -1481,6 +1484,7 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readAndInferJSON( .lines(static_cast(lines)) .recovery_mode(recovery_mode) .normalize_single_quotes(static_cast(normalize_single_quotes)) + .keep_quotes(keep_quotes) .mixed_types_as_string(mixed_types_as_string); auto result = @@ -1569,7 +1573,8 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_TableWithMeta_releaseTable(JNIE JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readJSONFromDataSource( JNIEnv *env, jclass, jintArray j_num_children, jobjectArray col_names, jintArray j_types, jintArray j_scales, jboolean day_first, jboolean lines, jboolean recover_with_null, - jboolean normalize_single_quotes, jboolean mixed_types_as_string, jlong ds_handle) { + jboolean normalize_single_quotes, jboolean mixed_types_as_string, jboolean keep_quotes, + jlong ds_handle) { JNI_NULL_CHECK(env, ds_handle, "no data source handle given", 0); @@ -1601,7 +1606,8 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readJSONFromDataSource( .lines(static_cast(lines)) .recovery_mode(recovery_mode) .normalize_single_quotes(static_cast(normalize_single_quotes)) - .mixed_types_as_string(mixed_types_as_string); + .mixed_types_as_string(mixed_types_as_string) + .keep_quotes(keep_quotes); if (!n_types.is_null()) { if (n_types.size() != n_scales.size()) { @@ -1640,7 +1646,7 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readJSON( JNIEnv *env, jclass, jintArray j_num_children, jobjectArray col_names, jintArray j_types, jintArray j_scales, jstring inputfilepath, jlong buffer, jlong buffer_length, jboolean day_first, jboolean lines, jboolean recover_with_null, - jboolean normalize_single_quotes, jboolean mixed_types_as_string) { + jboolean normalize_single_quotes, jboolean mixed_types_as_string, jboolean keep_quotes) { bool read_buffer = true; if (buffer == 0) { @@ -1687,7 +1693,8 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readJSON( .lines(static_cast(lines)) .recovery_mode(recovery_mode) .normalize_single_quotes(static_cast(normalize_single_quotes)) - .mixed_types_as_string(mixed_types_as_string); + .mixed_types_as_string(mixed_types_as_string) + .keep_quotes(keep_quotes); if (!n_types.is_null()) { if (n_types.size() != n_scales.size()) { diff --git a/java/src/test/java/ai/rapids/cudf/TableTest.java b/java/src/test/java/ai/rapids/cudf/TableTest.java index e270c4a5183..efdb6f4bb1b 100644 --- a/java/src/test/java/ai/rapids/cudf/TableTest.java +++ b/java/src/test/java/ai/rapids/cudf/TableTest.java @@ -349,6 +349,25 @@ void testReadSingleQuotesJSONFile() throws IOException { } } + @Test + void testReadSingleQuotesJSONFileKeepQuotes() throws IOException { + Schema schema = Schema.builder() + .column(DType.STRING, "A") + .build(); + JSONOptions opts = JSONOptions.builder() + .withLines(true) + .withNormalizeSingleQuotes(true) + .withKeepQuotes(true) + .build(); + try (Table expected = new Table.TestBuilder() + .column("\"TEST\"\"", "\"TESTER'\"") // Note that escapes are also processed + .build(); + MultiBufferDataSource source = sourceFrom(TEST_JSON_SINGLE_QUOTES_FILE); + Table table = Table.readJSON(schema, opts, source)) { + assertTablesAreEqual(expected, table); + } + } + private static final byte[] NESTED_JSON_DATA_BUFFER = ("{\"a\":{\"c\":\"C1\"}}\n" + "{\"a\":{\"c\":\"C2\", \"b\":\"B2\"}}\n" + "{\"d\":[1,2,3]}\n" + From 990ef0f87708c8e3e338b8f0148b0d6d7b6f18c9 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Wed, 28 Feb 2024 08:51:00 -0600 Subject: [PATCH 308/384] JNI bindings for distinct_hash_join (#15019) Adds Java bindings to the distinct hash join functionality added in #14990. Authors: - Jason Lowe (https://github.com/jlowe) Approvers: - Jim Brennan (https://github.com/jbrennan333) - Nghia Truong (https://github.com/ttnghia) --- java/src/main/java/ai/rapids/cudf/Table.java | 105 +++++++++++++++-- java/src/main/native/src/TableJni.cpp | 28 ++++- .../test/java/ai/rapids/cudf/TableTest.java | 111 +++++++++++++++++- 3 files changed, 231 insertions(+), 13 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/Table.java b/java/src/main/java/ai/rapids/cudf/Table.java index 1356c93c64d..c562e08b4c8 100644 --- a/java/src/main/java/ai/rapids/cudf/Table.java +++ b/java/src/main/java/ai/rapids/cudf/Table.java @@ -626,6 +626,9 @@ private static native long[] leftHashJoinGatherMapsWithCount(long leftTable, lon private static native long[] innerJoinGatherMaps(long leftKeys, long rightKeys, boolean compareNullsEqual) throws CudfException; + private static native long[] innerDistinctJoinGatherMaps(long leftKeys, long rightKeys, + boolean compareNullsEqual) throws CudfException; + private static native long innerJoinRowCount(long table, long hashJoin) throws CudfException; private static native long[] innerHashJoinGatherMaps(long table, long hashJoin) throws CudfException; @@ -2920,7 +2923,9 @@ private static GatherMap[] buildJoinGatherMaps(long[] gatherMapData) { * the table argument represents the key columns from the right table. Two {@link GatherMap} * instances will be returned that can be used to gather the left and right tables, * respectively, to produce the result of the left join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * @param rightKeys join key columns from the right table * @param compareNullsEqual true if null key values should match otherwise false * @return left and right table gather maps @@ -2956,7 +2961,9 @@ public long leftJoinRowCount(HashJoin rightHash) { * the {@link HashJoin} argument has been constructed from the key columns from the right table. * Two {@link GatherMap} instances will be returned that can be used to gather the left and right * tables, respectively, to produce the result of the left join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * @param rightHash hash table built from join key columns from the right table * @return left and right table gather maps */ @@ -2975,11 +2982,15 @@ public GatherMap[] leftJoinGatherMaps(HashJoin rightHash) { * the {@link HashJoin} argument has been constructed from the key columns from the right table. * Two {@link GatherMap} instances will be returned that can be used to gather the left and right * tables, respectively, to produce the result of the left join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * This interface allows passing an output row count that was previously computed from * {@link #leftJoinRowCount(HashJoin)}. + * * WARNING: Passing a row count that is smaller than the actual row count will result * in undefined behavior. + * * @param rightHash hash table built from join key columns from the right table * @param outputRowCount number of output rows in the join result * @return left and right table gather maps @@ -3013,7 +3024,9 @@ public long conditionalLeftJoinRowCount(Table rightTable, CompiledExpression con * the columns from the left table, and the table argument represents the columns from the * right table. Two {@link GatherMap} instances will be returned that can be used to gather * the left and right tables, respectively, to produce the result of the left join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * @param rightTable the right side table of the join in the join * @param condition conditional expression to evaluate during the join * @return left and right table gather maps @@ -3032,11 +3045,15 @@ public GatherMap[] conditionalLeftJoinGatherMaps(Table rightTable, * the columns from the left table, and the table argument represents the columns from the * right table. Two {@link GatherMap} instances will be returned that can be used to gather * the left and right tables, respectively, to produce the result of the left join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * This interface allows passing an output row count that was previously computed from * {@link #conditionalLeftJoinRowCount(Table, CompiledExpression)}. + * * WARNING: Passing a row count that is smaller than the actual row count will result * in undefined behavior. + * * @param rightTable the right side table of the join in the join * @param condition conditional expression to evaluate during the join * @param outputRowCount number of output rows in the join result @@ -3085,7 +3102,9 @@ public static MixedJoinSize mixedLeftJoinSize(Table leftKeys, Table rightKeys, * assumed to be a logical AND of the equality condition and inequality condition. * Two {@link GatherMap} instances will be returned that can be used to gather * the left and right tables, respectively, to produce the result of the left join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * @param leftKeys the left table's key columns for the equality condition * @param rightKeys the right table's key columns for the equality condition * @param leftConditional the left table's columns needed to evaluate the inequality condition @@ -3112,10 +3131,13 @@ public static GatherMap[] mixedLeftJoinGatherMaps(Table leftKeys, Table rightKey * assumed to be a logical AND of the equality condition and inequality condition. * Two {@link GatherMap} instances will be returned that can be used to gather * the left and right tables, respectively, to produce the result of the left join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * This interface allows passing the size result from * {@link #mixedLeftJoinSize(Table, Table, Table, Table, CompiledExpression, NullEquality)} * when the output size was computed previously. + * * @param leftKeys the left table's key columns for the equality condition * @param rightKeys the right table's key columns for the equality condition * @param leftConditional the left table's columns needed to evaluate the inequality condition @@ -3145,14 +3167,16 @@ public static GatherMap[] mixedLeftJoinGatherMaps(Table leftKeys, Table rightKey * the table argument represents the key columns from the right table. Two {@link GatherMap} * instances will be returned that can be used to gather the left and right tables, * respectively, to produce the result of the inner join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * @param rightKeys join key columns from the right table * @param compareNullsEqual true if null key values should match otherwise false * @return left and right table gather maps */ public GatherMap[] innerJoinGatherMaps(Table rightKeys, boolean compareNullsEqual) { if (getNumberOfColumns() != rightKeys.getNumberOfColumns()) { - throw new IllegalArgumentException("column count mismatch, this: " + getNumberOfColumns() + + throw new IllegalArgumentException("Column count mismatch, this: " + getNumberOfColumns() + "rightKeys: " + rightKeys.getNumberOfColumns()); } long[] gatherMapData = @@ -3160,6 +3184,30 @@ public GatherMap[] innerJoinGatherMaps(Table rightKeys, boolean compareNullsEqua return buildJoinGatherMaps(gatherMapData); } + /** + * Computes the gather maps that can be used to manifest the result of an inner equi-join between + * two tables where the right table is guaranteed to not contain any duplicated join keys. It is + * assumed this table instance holds the key columns from the left table, and the table argument + * represents the key columns from the right table. Two {@link GatherMap} instances will be + * returned that can be used to gather the left and right tables, respectively, to produce the + * result of the inner join. + * + * It is the responsibility of the caller to close the resulting gather map instances. + * + * @param rightKeys join key columns from the right table + * @param compareNullsEqual true if null key values should match otherwise false + * @return left and right table gather maps + */ + public GatherMap[] innerDistinctJoinGatherMaps(Table rightKeys, boolean compareNullsEqual) { + if (getNumberOfColumns() != rightKeys.getNumberOfColumns()) { + throw new IllegalArgumentException("Column count mismatch, this: " + getNumberOfColumns() + + "rightKeys: " + rightKeys.getNumberOfColumns()); + } + long[] gatherMapData = + innerDistinctJoinGatherMaps(getNativeView(), rightKeys.getNativeView(), compareNullsEqual); + return buildJoinGatherMaps(gatherMapData); + } + /** * Computes the number of rows resulting from an inner equi-join between two tables. * @param otherHash hash table built from join key columns from the other table @@ -3167,7 +3215,7 @@ public GatherMap[] innerJoinGatherMaps(Table rightKeys, boolean compareNullsEqua */ public long innerJoinRowCount(HashJoin otherHash) { if (getNumberOfColumns() != otherHash.getNumberOfColumns()) { - throw new IllegalArgumentException("column count mismatch, this: " + getNumberOfColumns() + + throw new IllegalArgumentException("Column count mismatch, this: " + getNumberOfColumns() + "otherKeys: " + otherHash.getNumberOfColumns()); } return innerJoinRowCount(getNativeView(), otherHash.getNativeView()); @@ -3179,13 +3227,15 @@ public long innerJoinRowCount(HashJoin otherHash) { * the {@link HashJoin} argument has been constructed from the key columns from the right table. * Two {@link GatherMap} instances will be returned that can be used to gather the left and right * tables, respectively, to produce the result of the inner join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * @param rightHash hash table built from join key columns from the right table * @return left and right table gather maps */ public GatherMap[] innerJoinGatherMaps(HashJoin rightHash) { if (getNumberOfColumns() != rightHash.getNumberOfColumns()) { - throw new IllegalArgumentException("column count mismatch, this: " + getNumberOfColumns() + + throw new IllegalArgumentException("Column count mismatch, this: " + getNumberOfColumns() + "rightKeys: " + rightHash.getNumberOfColumns()); } long[] gatherMapData = innerHashJoinGatherMaps(getNativeView(), rightHash.getNativeView()); @@ -3198,18 +3248,22 @@ public GatherMap[] innerJoinGatherMaps(HashJoin rightHash) { * the {@link HashJoin} argument has been constructed from the key columns from the right table. * Two {@link GatherMap} instances will be returned that can be used to gather the left and right * tables, respectively, to produce the result of the inner join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * This interface allows passing an output row count that was previously computed from * {@link #innerJoinRowCount(HashJoin)}. + * * WARNING: Passing a row count that is smaller than the actual row count will result * in undefined behavior. + * * @param rightHash hash table built from join key columns from the right table * @param outputRowCount number of output rows in the join result * @return left and right table gather maps */ public GatherMap[] innerJoinGatherMaps(HashJoin rightHash, long outputRowCount) { if (getNumberOfColumns() != rightHash.getNumberOfColumns()) { - throw new IllegalArgumentException("column count mismatch, this: " + getNumberOfColumns() + + throw new IllegalArgumentException("Column count mismatch, this: " + getNumberOfColumns() + "rightKeys: " + rightHash.getNumberOfColumns()); } long[] gatherMapData = innerHashJoinGatherMapsWithCount(getNativeView(), @@ -3237,7 +3291,9 @@ public long conditionalInnerJoinRowCount(Table rightTable, * the columns from the left table, and the table argument represents the columns from the * right table. Two {@link GatherMap} instances will be returned that can be used to gather * the left and right tables, respectively, to produce the result of the inner join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * @param rightTable the right side table of the join * @param condition conditional expression to evaluate during the join * @return left and right table gather maps @@ -3256,11 +3312,15 @@ public GatherMap[] conditionalInnerJoinGatherMaps(Table rightTable, * the columns from the left table, and the table argument represents the columns from the * right table. Two {@link GatherMap} instances will be returned that can be used to gather * the left and right tables, respectively, to produce the result of the inner join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * This interface allows passing an output row count that was previously computed from * {@link #conditionalInnerJoinRowCount(Table, CompiledExpression)}. + * * WARNING: Passing a row count that is smaller than the actual row count will result * in undefined behavior. + * * @param rightTable the right side table of the join in the join * @param condition conditional expression to evaluate during the join * @param outputRowCount number of output rows in the join result @@ -3309,7 +3369,9 @@ public static MixedJoinSize mixedInnerJoinSize(Table leftKeys, Table rightKeys, * assumed to be a logical AND of the equality condition and inequality condition. * Two {@link GatherMap} instances will be returned that can be used to gather * the left and right tables, respectively, to produce the result of the inner join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * @param leftKeys the left table's key columns for the equality condition * @param rightKeys the right table's key columns for the equality condition * @param leftConditional the left table's columns needed to evaluate the inequality condition @@ -3336,10 +3398,13 @@ public static GatherMap[] mixedInnerJoinGatherMaps(Table leftKeys, Table rightKe * assumed to be a logical AND of the equality condition and inequality condition. * Two {@link GatherMap} instances will be returned that can be used to gather * the left and right tables, respectively, to produce the result of the inner join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * This interface allows passing the size result from * {@link #mixedInnerJoinSize(Table, Table, Table, Table, CompiledExpression, NullEquality)} * when the output size was computed previously. + * * @param leftKeys the left table's key columns for the equality condition * @param rightKeys the right table's key columns for the equality condition * @param leftConditional the left table's columns needed to evaluate the inequality condition @@ -3369,14 +3434,16 @@ public static GatherMap[] mixedInnerJoinGatherMaps(Table leftKeys, Table rightKe * the table argument represents the key columns from the right table. Two {@link GatherMap} * instances will be returned that can be used to gather the left and right tables, * respectively, to produce the result of the full join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * @param rightKeys join key columns from the right table * @param compareNullsEqual true if null key values should match otherwise false * @return left and right table gather maps */ public GatherMap[] fullJoinGatherMaps(Table rightKeys, boolean compareNullsEqual) { if (getNumberOfColumns() != rightKeys.getNumberOfColumns()) { - throw new IllegalArgumentException("column count mismatch, this: " + getNumberOfColumns() + + throw new IllegalArgumentException("Column count mismatch, this: " + getNumberOfColumns() + "rightKeys: " + rightKeys.getNumberOfColumns()); } long[] gatherMapData = @@ -3396,7 +3463,7 @@ public GatherMap[] fullJoinGatherMaps(Table rightKeys, boolean compareNullsEqual */ public long fullJoinRowCount(HashJoin rightHash) { if (getNumberOfColumns() != rightHash.getNumberOfColumns()) { - throw new IllegalArgumentException("column count mismatch, this: " + getNumberOfColumns() + + throw new IllegalArgumentException("Column count mismatch, this: " + getNumberOfColumns() + "rightKeys: " + rightHash.getNumberOfColumns()); } return fullJoinRowCount(getNativeView(), rightHash.getNativeView()); @@ -3408,13 +3475,15 @@ public long fullJoinRowCount(HashJoin rightHash) { * the {@link HashJoin} argument has been constructed from the key columns from the right table. * Two {@link GatherMap} instances will be returned that can be used to gather the left and right * tables, respectively, to produce the result of the full join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * @param rightHash hash table built from join key columns from the right table * @return left and right table gather maps */ public GatherMap[] fullJoinGatherMaps(HashJoin rightHash) { if (getNumberOfColumns() != rightHash.getNumberOfColumns()) { - throw new IllegalArgumentException("column count mismatch, this: " + getNumberOfColumns() + + throw new IllegalArgumentException("Column count mismatch, this: " + getNumberOfColumns() + "rightKeys: " + rightHash.getNumberOfColumns()); } long[] gatherMapData = fullHashJoinGatherMaps(getNativeView(), rightHash.getNativeView()); @@ -3427,7 +3496,9 @@ public GatherMap[] fullJoinGatherMaps(HashJoin rightHash) { * the {@link HashJoin} argument has been constructed from the key columns from the right table. * Two {@link GatherMap} instances will be returned that can be used to gather the left and right * tables, respectively, to produce the result of the full join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * This interface allows passing an output row count that was previously computed from * {@link #fullJoinRowCount(HashJoin)}. * WARNING: Passing a row count that is smaller than the actual row count will result @@ -3438,7 +3509,7 @@ public GatherMap[] fullJoinGatherMaps(HashJoin rightHash) { */ public GatherMap[] fullJoinGatherMaps(HashJoin rightHash, long outputRowCount) { if (getNumberOfColumns() != rightHash.getNumberOfColumns()) { - throw new IllegalArgumentException("column count mismatch, this: " + getNumberOfColumns() + + throw new IllegalArgumentException("Column count mismatch, this: " + getNumberOfColumns() + "rightKeys: " + rightHash.getNumberOfColumns()); } long[] gatherMapData = fullHashJoinGatherMapsWithCount(getNativeView(), @@ -3452,7 +3523,9 @@ public GatherMap[] fullJoinGatherMaps(HashJoin rightHash, long outputRowCount) { * the columns from the left table, and the table argument represents the columns from the * right table. Two {@link GatherMap} instances will be returned that can be used to gather * the left and right tables, respectively, to produce the result of the full join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * @param rightTable the right side table of the join * @param condition conditional expression to evaluate during the join * @return left and right table gather maps @@ -3471,7 +3544,9 @@ public GatherMap[] conditionalFullJoinGatherMaps(Table rightTable, * assumed to be a logical AND of the equality condition and inequality condition. * Two {@link GatherMap} instances will be returned that can be used to gather * the left and right tables, respectively, to produce the result of the full join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * @param leftKeys the left table's key columns for the equality condition * @param rightKeys the right table's key columns for the equality condition * @param leftConditional the left table's columns needed to evaluate the inequality condition @@ -3512,7 +3587,7 @@ private static GatherMap buildSemiJoinGatherMap(long[] gatherMapData) { */ public GatherMap leftSemiJoinGatherMap(Table rightKeys, boolean compareNullsEqual) { if (getNumberOfColumns() != rightKeys.getNumberOfColumns()) { - throw new IllegalArgumentException("column count mismatch, this: " + getNumberOfColumns() + + throw new IllegalArgumentException("Column count mismatch, this: " + getNumberOfColumns() + "rightKeys: " + rightKeys.getNumberOfColumns()); } long[] gatherMapData = @@ -3612,7 +3687,9 @@ public static MixedJoinSize mixedLeftSemiJoinSize(Table leftKeys, Table rightKey * assumed to be a logical AND of the equality condition and inequality condition. * A {@link GatherMap} instance will be returned that can be used to gather * the left table to produce the result of the left semi join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * @param leftKeys the left table's key columns for the equality condition * @param rightKeys the right table's key columns for the equality condition * @param leftConditional the left table's columns needed to evaluate the inequality condition @@ -3639,10 +3716,13 @@ public static GatherMap mixedLeftSemiJoinGatherMap(Table leftKeys, Table rightKe * assumed to be a logical AND of the equality condition and inequality condition. * A {@link GatherMap} instance will be returned that can be used to gather * the left table to produce the result of the left semi join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * This interface allows passing the size result from * {@link #mixedLeftSemiJoinSize(Table, Table, Table, Table, CompiledExpression, NullEquality)} * when the output size was computed previously. + * * @param leftKeys the left table's key columns for the equality condition * @param rightKeys the right table's key columns for the equality condition * @param leftConditional the left table's columns needed to evaluate the inequality condition @@ -3679,7 +3759,7 @@ public static GatherMap mixedLeftSemiJoinGatherMap(Table leftKeys, Table rightKe */ public GatherMap leftAntiJoinGatherMap(Table rightKeys, boolean compareNullsEqual) { if (getNumberOfColumns() != rightKeys.getNumberOfColumns()) { - throw new IllegalArgumentException("column count mismatch, this: " + getNumberOfColumns() + + throw new IllegalArgumentException("Column count mismatch, this: " + getNumberOfColumns() + "rightKeys: " + rightKeys.getNumberOfColumns()); } long[] gatherMapData = @@ -3779,7 +3859,9 @@ public static MixedJoinSize mixedLeftAntiJoinSize(Table leftKeys, Table rightKey * assumed to be a logical AND of the equality condition and inequality condition. * A {@link GatherMap} instance will be returned that can be used to gather * the left table to produce the result of the left anti join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * @param leftKeys the left table's key columns for the equality condition * @param rightKeys the right table's key columns for the equality condition * @param leftConditional the left table's columns needed to evaluate the inequality condition @@ -3806,10 +3888,13 @@ public static GatherMap mixedLeftAntiJoinGatherMap(Table leftKeys, Table rightKe * assumed to be a logical AND of the equality condition and inequality condition. * A {@link GatherMap} instance will be returned that can be used to gather * the left table to produce the result of the left anti join. + * * It is the responsibility of the caller to close the resulting gather map instances. + * * This interface allows passing the size result from * {@link #mixedLeftAntiJoinSize(Table, Table, Table, Table, CompiledExpression, NullEquality)} * when the output size was computed previously. + * * @param leftKeys the left table's key columns for the equality condition * @param rightKeys the right table's key columns for the equality condition * @param leftConditional the left table's columns needed to evaluate the inequality condition diff --git a/java/src/main/native/src/TableJni.cpp b/java/src/main/native/src/TableJni.cpp index 8585761788e..84f1174fd3f 100644 --- a/java/src/main/native/src/TableJni.cpp +++ b/java/src/main/native/src/TableJni.cpp @@ -702,9 +702,9 @@ jlongArray gather_maps_to_java(JNIEnv *env, jlongArray gather_map_to_java(JNIEnv *env, std::unique_ptr> map) { // release the underlying device buffer to Java - auto gather_map_buffer = std::make_unique(map->release()); cudf::jni::native_jlongArray result(env, 3); - result[0] = static_cast(gather_map_buffer->size()); + result[0] = static_cast(map->size() * sizeof(cudf::size_type)); + auto gather_map_buffer = std::make_unique(map->release()); result[1] = ptr_as_jlong(gather_map_buffer->data()); result[2] = release_as_jlong(gather_map_buffer); return result.get_jArray(); @@ -2557,6 +2557,30 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_Table_innerJoinGatherMaps( }); } +JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_Table_innerDistinctJoinGatherMaps( + JNIEnv *env, jclass, jlong j_left_keys, jlong j_right_keys, jboolean compare_nulls_equal) { + return cudf::jni::join_gather_maps( + env, j_left_keys, j_right_keys, compare_nulls_equal, + [](cudf::table_view const &left, cudf::table_view const &right, cudf::null_equality nulleq) { + auto has_nulls = cudf::has_nested_nulls(left) || cudf::has_nested_nulls(right) ? + cudf::nullable_join::YES : + cudf::nullable_join::NO; + std::pair>, + std::unique_ptr>> + maps; + if (cudf::detail::has_nested_columns(right)) { + cudf::distinct_hash_join hash(right, left, has_nulls, nulleq); + maps = hash.inner_join(); + } else { + cudf::distinct_hash_join hash(right, left, has_nulls, nulleq); + maps = hash.inner_join(); + } + // Unique join returns {right map, left map} but all the other joins + // return {left map, right map}. Swap here to make it consistent. + return std::make_pair(std::move(maps.second), std::move(maps.first)); + }); +} + JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_innerJoinRowCount(JNIEnv *env, jclass, jlong j_left_table, jlong j_right_hash_join) { diff --git a/java/src/test/java/ai/rapids/cudf/TableTest.java b/java/src/test/java/ai/rapids/cudf/TableTest.java index efdb6f4bb1b..6f0b2b51f4c 100644 --- a/java/src/test/java/ai/rapids/cudf/TableTest.java +++ b/java/src/test/java/ai/rapids/cudf/TableTest.java @@ -33,7 +33,6 @@ import com.google.common.base.Charsets; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import org.apache.avro.SchemaBuilder; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.parquet.hadoop.ParquetFileReader; @@ -2104,6 +2103,116 @@ void testInnerJoinGatherMapsNulls() { } } + private void checkInnerDistinctJoin(Table leftKeys, Table rightKeys, Table expected, + boolean compareNullsEqual) { + GatherMap[] maps = leftKeys.innerDistinctJoinGatherMaps(rightKeys, compareNullsEqual); + try { + verifyJoinGatherMaps(maps, expected); + } finally { + for (GatherMap map : maps) { + map.close(); + } + } + } + + @Test + void testInnerDistinctJoinGatherMaps() { + try (Table leftKeys = new Table.TestBuilder().column(2, 3, 9, 0, 1, 7, 4, 6, 5, 8, 6).build(); + Table rightKeys = new Table.TestBuilder().column(6, 5, 9, 8, 10, 32).build(); + Table expected = new Table.TestBuilder() + .column(2, 7, 8, 9, 10) // left + .column(2, 0, 1, 3, 0) // right + .build()) { + checkInnerDistinctJoin(leftKeys, rightKeys, expected, false); + } + } + + @Test + void testInnerDistinctJoinGatherMapsWithNested() { + StructType structType = new StructType(false, + new BasicType(false, DType.STRING), + new BasicType(false, DType.INT32)); + StructData[] leftData = new StructData[]{ + new StructData("abc", 1), + new StructData("xyz", 1), + new StructData("abc", 2), + new StructData("xyz", 2), + new StructData("abc", 1), + new StructData("abc", 3), + new StructData("xyz", 3) + }; + StructData[] rightData = new StructData[]{ + new StructData("abc", 1), + new StructData("xyz", 4), + new StructData("xyz", 2), + new StructData("abc", -1), + }; + try (Table leftKeys = new Table.TestBuilder().column(structType, leftData).build(); + Table rightKeys = new Table.TestBuilder().column(structType, rightData).build(); + Table expected = new Table.TestBuilder() + .column(0, 3, 4) + .column(0, 2, 0) + .build()) { + checkInnerDistinctJoin(leftKeys, rightKeys, expected, false); + } + } + + @Test + void testInnerDistinctJoinGatherMapsNullsEqual() { + try (Table leftKeys = new Table.TestBuilder() + .column(2, 3, 9, 0, 1, 7, 4, null, null, 8) + .build(); + Table rightKeys = new Table.TestBuilder() + .column(null, 9, 8, 10, 32) + .build(); + Table expected = new Table.TestBuilder() + .column(2, 7, 8, 9) // left + .column(1, 0, 0, 2) // right + .build()) { + checkInnerDistinctJoin(leftKeys, rightKeys, expected, true); + } + } + + @Test + void testInnerDistinctJoinGatherMapsWithNestedNullsEqual() { + StructType structType = new StructType(true, + new BasicType(true, DType.STRING), + new BasicType(true, DType.INT32)); + StructData[] leftData = new StructData[]{ + new StructData("abc", 1), + null, + new StructData("xyz", 1), + new StructData("abc", 2), + new StructData("xyz", null), + null, + new StructData("abc", 1), + new StructData("abc", 3), + new StructData("xyz", 3), + new StructData(null, null), + new StructData(null, 1) + }; + StructData[] rightData = new StructData[]{ + null, + new StructData("abc", 1), + new StructData("xyz", 4), + new StructData("xyz", 2), + new StructData(null, null), + new StructData(null, 2), + new StructData(null, 1), + new StructData("xyz", null), + new StructData("abc", null), + new StructData("abc", -1) + }; + try (Table leftKeys = new Table.TestBuilder().column(structType, leftData).build(); + Table rightKeys = new Table.TestBuilder().column(structType, rightData).build(); + Table expected = new Table.TestBuilder() + .column(0, 1, 4, 5, 6, 9, 10) + .column(1, 0, 7, 0, 1, 4, 6) + .build()) { + checkInnerDistinctJoin(leftKeys, rightKeys, expected, true); + } + } + @Test void testInnerHashJoinGatherMaps() { try (Table leftKeys = new Table.TestBuilder().column(2, 3, 9, 0, 1, 7, 4, 6, 5, 8).build(); From 8526e6d5b21361465d1c72ecbea64d3d2d9bf849 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 28 Feb 2024 09:55:50 -0600 Subject: [PATCH 309/384] Drop python-snappy from dependencies. (#15161) Previously `python-snappy` was a test dependency. It does not appear that we rely on this directly, as there are no instances of `import snappy`. Recently, pandas also dropped this dependency: https://github.com/pandas-dev/pandas/pull/54633 More generally, we can refactor the dependency list to use `pandas[all]` now that we require pandas 2. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Kyle Edwards (https://github.com/KyleFromNVIDIA) - Vyas Ramasubramani (https://github.com/vyasr) - GALI PREM SAGAR (https://github.com/galipremsagar) - Ray Douglass (https://github.com/raydouglass) --- .../all_cuda-118_arch-x86_64.yaml | 1 - .../all_cuda-122_arch-x86_64.yaml | 1 - dependencies.yaml | 46 +------------------ python/cudf/pyproject.toml | 44 +----------------- 4 files changed, 3 insertions(+), 89 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index dc78bf68dda..79b786fe012 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -79,7 +79,6 @@ dependencies: - pytest-xdist - pytest<8 - python-confluent-kafka>=1.9.0,<1.10.0a0 -- python-snappy>=0.6.0 - python>=3.9,<3.11 - pytorch<1.12.0 - rapids-dask-dependency==24.4.* diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index 627cfa7667c..66a4ee57238 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -77,7 +77,6 @@ dependencies: - pytest-xdist - pytest<8 - python-confluent-kafka>=1.9.0,<1.10.0a0 -- python-snappy>=0.6.0 - python>=3.9,<3.11 - pytorch<1.12.0 - rapids-dask-dependency==24.4.* diff --git a/dependencies.yaml b/dependencies.yaml index 4011bd764e1..4281e907862 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -612,7 +612,6 @@ dependencies: - hypothesis - pytest-benchmark - pytest-cases>=3.8.2 - - python-snappy>=0.6.0 - scipy - output_types: conda packages: @@ -712,49 +711,8 @@ dependencies: packages: # dependencies to run pandas tests # https://github.com/pandas-dev/pandas/blob/main/environment.yml - # TODO: When pandas 2.0 is the minimum version, can just specify pandas[all] - - beautifulsoup4 - - blosc - - brotlipy - - boto3 - - botocore>=1.24.21 - - bottleneck - - fastparquet - - flask - - fsspec - - html5lib - - hypothesis - - gcsfs - - ipython - - jinja2 - - lxml - - matplotlib - - moto - - numba - - numexpr - - openpyxl - - odfpy - - py - - psycopg2-binary - - pyarrow - - pymysql - - pyreadstat - - pytest-asyncio - - pytest-reportlog - - python-snappy - - pytest-timeout - - pyxlsb - - s3fs - - scipy - - sqlalchemy - - tables - - pandas-gbq - - tabulate - - xarray - - xlrd - - xlsxwriter - - xlwt - - zstandard + # pandas[all] includes all of the required dependencies + - pandas[all] test_python_cudf_pandas: common: - output_types: pyproject diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index ef3b439bdf4..590786f2414 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -62,55 +62,13 @@ test = [ "pytest-cov", "pytest-xdist", "pytest<8", - "python-snappy>=0.6.0", "scipy", "tokenizers==0.13.1", "transformers==4.24.0", "tzdata", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. pandas-tests = [ - "beautifulsoup4", - "blosc", - "boto3", - "botocore>=1.24.21", - "bottleneck", - "brotlipy", - "fastparquet", - "flask", - "fsspec", - "gcsfs", - "html5lib", - "hypothesis", - "ipython", - "jinja2", - "lxml", - "matplotlib", - "moto", - "numba", - "numexpr", - "odfpy", - "openpyxl", - "pandas-gbq", - "psycopg2-binary", - "py", - "pyarrow", - "pymysql", - "pyreadstat", - "pytest-asyncio", - "pytest-reportlog", - "pytest-timeout", - "python-snappy", - "pyxlsb", - "s3fs", - "scipy", - "sqlalchemy", - "tables", - "tabulate", - "xarray", - "xlrd", - "xlsxwriter", - "xlwt", - "zstandard", + "pandas[all]", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. cudf-pandas-tests = [ "ipython", From 896b5bced6597e81f3a9e96e5b6bcc72cb364e68 Mon Sep 17 00:00:00 2001 From: Paul Mattione <156858817+pmattione-nvidia@users.noreply.github.com> Date: Wed, 28 Feb 2024 15:20:22 -0500 Subject: [PATCH 310/384] Compile-time ipow computation with array lookup (#15110) Compile-time ipow() computation with array lookup. Results in up to 8% speed improvement for decimal64 -> double conversions. Improvement is negligible for other conversions but is not worse. New benchmark test will be in a separate PR. Fix fixed_point -> string conversion test. Also fix rounding comments. Closes #9346 Authors: - Paul Mattione (https://github.com/pmattione-nvidia) Approvers: - Yunsong Wang (https://github.com/PointKernel) - Shruti Shivakumar (https://github.com/shrshi) URL: https://github.com/rapidsai/cudf/pull/15110 --- cpp/include/cudf/fixed_point/fixed_point.hpp | 63 ++++++++++++++------ cpp/include/cudf/round.hpp | 7 ++- cpp/tests/strings/fixed_point_tests.cpp | 5 +- 3 files changed, 53 insertions(+), 22 deletions(-) diff --git a/cpp/include/cudf/fixed_point/fixed_point.hpp b/cpp/include/cudf/fixed_point/fixed_point.hpp index a8a681f181e..542e2b3c5c8 100644 --- a/cpp/include/cudf/fixed_point/fixed_point.hpp +++ b/cpp/include/cudf/fixed_point/fixed_point.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -82,12 +83,43 @@ constexpr inline auto is_supported_construction_value_type() // Helper functions for `fixed_point` type namespace detail { + /** - * @brief A function for integer exponentiation by squaring + * @brief Recursively computes integer exponentiation * - * https://simple.wikipedia.org/wiki/Exponentiation_by_squaring
- * Note: this is the iterative equivalent of the recursive definition (faster)
- * Quick-bench: http://quick-bench.com/Wg7o7HYQC9FW5M0CO0wQAjSwP_Y + * @note This is intended to be run at compile time + * + * @tparam Rep Representation type for return type + * @tparam Base The base to be exponentiated + * @param exp The exponent to be used for exponentiation + * @return Result of `Base` to the power of `exponent` of type `Rep` + */ +template +CUDF_HOST_DEVICE inline constexpr Rep get_power(int32_t exp) +{ + // Compute power recursively + return (exp > 0) ? Rep(Base) * get_power(exp - 1) : 1; +} + +/** + * @brief Implementation of integer exponentiation by array lookup + * + * @tparam Rep Representation type for return type + * @tparam Base The base to be exponentiated + * @tparam Exponents The exponents for the array entries + * @param exponent The exponent to be used for exponentiation + * @return Result of `Base` to the power of `exponent` of type `Rep` + */ +template +CUDF_HOST_DEVICE inline Rep ipow_impl(int32_t exponent, cuda::std::index_sequence) +{ + // Compute powers at compile time, storing into array + static constexpr Rep powers[] = {get_power(Exponents)...}; + return powers[exponent]; +} + +/** + * @brief A function for integer exponentiation by array lookup * * @tparam Rep Representation type for return type * @tparam Base The base to be exponentiated @@ -102,19 +134,16 @@ template = 0 && "integer exponentiation with negative exponent is not possible."); - if (exponent == 0) { return static_cast(1); } - - auto extra = static_cast(1); - auto square = static_cast(Base); - while (exponent > 1) { - if (exponent & 1 /* odd */) { - extra *= square; - exponent -= 1; - } - exponent /= 2; - square *= square; + if constexpr (Base == numeric::Radix::BASE_2) { + return static_cast(1) << exponent; + } else { // BASE_10 + // Build index sequence for building power array at compile time + static constexpr auto max_exp = cuda::std::numeric_limits::digits10; + static constexpr auto exponents = cuda::std::make_index_sequence{}; + + // Get compile-time result + return ipow_impl(Base)>(exponent, exponents); } - return square * extra; } /** @brief Function that performs a `right shift` scale "times" on the `val` diff --git a/cpp/include/cudf/round.hpp b/cpp/include/cudf/round.hpp index 030d3d42773..ee088628b94 100644 --- a/cpp/include/cudf/round.hpp +++ b/cpp/include/cudf/round.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,8 +32,9 @@ namespace cudf { /** * @brief Different rounding methods for `cudf::round` * - * Info on HALF_UP rounding: https://en.wikipedia.org/wiki/Rounding#Round_half_up - * Info on HALF_EVEN rounding: https://en.wikipedia.org/wiki/Rounding#Round_half_to_even + * Info on HALF_EVEN rounding: https://en.wikipedia.org/wiki/Rounding#Rounding_half_to_even + * Info on HALF_UP rounding: https://en.wikipedia.org/wiki/Rounding#Rounding_half_away_from_zero + * Note: HALF_UP means up in MAGNITUDE: Away from zero! Because of how Java and python define it */ enum class rounding_method : int32_t { HALF_UP, HALF_EVEN }; diff --git a/cpp/tests/strings/fixed_point_tests.cpp b/cpp/tests/strings/fixed_point_tests.cpp index 0a1c004d0a0..9205207cc53 100644 --- a/cpp/tests/strings/fixed_point_tests.cpp +++ b/cpp/tests/strings/fixed_point_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -324,7 +324,8 @@ TEST_F(StringsConvertTest, DISABLED_FixedPointStringConversionOperator) { auto const max = cuda::std::numeric_limits<__int128_t>::max(); - auto const x = numeric::decimal128{max, numeric::scale_type{-10}}; + // Must use scaled_integer, else shift (multiply) is undefined behavior (integer overflow) + auto const x = numeric::decimal128(numeric::scaled_integer{max, numeric::scale_type{-10}}); EXPECT_EQ(static_cast(x), "17014118346046923173168730371.5884105727"); auto const y = numeric::decimal128{max, numeric::scale_type{10}}; From 3adfddcfa2cdac4acb16a50916442763a1d8a78b Mon Sep 17 00:00:00 2001 From: Jim Brennan Date: Wed, 28 Feb 2024 15:24:30 -0600 Subject: [PATCH 311/384] Make HostColumnVector.DataType accessor methods public (#15157) * Make HostColumnVector.DataType accessor methods public Signed-off-by: Jim Brennan * add accessors for StructData * update copyrights --------- Signed-off-by: Jim Brennan --- .../java/ai/rapids/cudf/HostColumnVector.java | 46 +++++++++++-------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/HostColumnVector.java b/java/src/main/java/ai/rapids/cudf/HostColumnVector.java index 3e4baf962bc..e64c428ecbb 100644 --- a/java/src/main/java/ai/rapids/cudf/HostColumnVector.java +++ b/java/src/main/java/ai/rapids/cudf/HostColumnVector.java @@ -1,6 +1,6 @@ /* * - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -1179,12 +1179,12 @@ public final ColumnBuilder appendNull() { private ColumnBuilder append(StructData structData) { assert type.isNestedType(); if (type.equals(DType.STRUCT)) { - if (structData == null || structData.dataRecord == null) { + if (structData == null || structData.isNull()) { return appendNull(); } else { for (int i = 0; i < structData.getNumFields(); i++) { ColumnBuilder childBuilder = childBuilders.get(i); - appendChildOrNull(childBuilder, structData.dataRecord.get(i)); + appendChildOrNull(childBuilder, structData.getField(i)); } endStruct(); } @@ -2077,10 +2077,10 @@ public String toString() { } public static abstract class DataType { - abstract DType getType(); - abstract boolean isNullable(); - abstract DataType getChild(int index); - abstract int getNumChildren(); + public abstract DType getType(); + public abstract boolean isNullable(); + public abstract DataType getChild(int index); + public abstract int getNumChildren(); } public static class ListType extends HostColumnVector.DataType { @@ -2093,17 +2093,17 @@ public ListType(boolean isNullable, DataType child) { } @Override - DType getType() { + public DType getType() { return DType.LIST; } @Override - boolean isNullable() { + public boolean isNullable() { return isNullable; } @Override - HostColumnVector.DataType getChild(int index) { + public HostColumnVector.DataType getChild(int index) { if (index > 0) { return null; } @@ -2111,7 +2111,7 @@ HostColumnVector.DataType getChild(int index) { } @Override - int getNumChildren() { + public int getNumChildren() { return 1; } } @@ -2134,6 +2134,14 @@ public int getNumFields() { return 0; } } + + public boolean isNull() { + return (this.dataRecord == null); + } + + public Object getField(int index) { + return this.dataRecord.get(index); + } } public static class StructType extends HostColumnVector.DataType { @@ -2150,22 +2158,22 @@ public StructType(boolean isNullable, DataType... children) { } @Override - DType getType() { + public DType getType() { return DType.STRUCT; } @Override - boolean isNullable() { + public boolean isNullable() { return isNullable; } @Override - HostColumnVector.DataType getChild(int index) { + public HostColumnVector.DataType getChild(int index) { return children.get(index); } @Override - int getNumChildren() { + public int getNumChildren() { return children.size(); } } @@ -2180,22 +2188,22 @@ public BasicType(boolean isNullable, DType type) { } @Override - DType getType() { + public DType getType() { return type; } @Override - boolean isNullable() { + public boolean isNullable() { return isNullable; } @Override - HostColumnVector.DataType getChild(int index) { + public HostColumnVector.DataType getChild(int index) { return null; } @Override - int getNumChildren() { + public int getNumChildren() { return 0; } } From 8507b3dfe44794cd549222598320d9cf25c6e34c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 28 Feb 2024 18:48:50 -0600 Subject: [PATCH 312/384] [ci] update matrix filters for dask-cudf builds (#15174) --- .github/workflows/build.yaml | 2 +- .github/workflows/pr.yaml | 6 +++--- .github/workflows/test.yaml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1c68b3504e0..e60c47fae2b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -92,7 +92,7 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.11" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d7f47f628d6..9e11993048f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -128,7 +128,7 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.11" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) build_type: pull-request script: "ci/build_wheel_dask_cudf.sh" wheel-tests-dask-cudf: @@ -136,7 +136,7 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.11" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) build_type: pull-request script: ci/test_wheel_dask_cudf.sh devcontainer: @@ -152,7 +152,7 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.11" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) build_type: pull-request script: ci/cudf_pandas_scripts/run_tests.sh # pandas-tests: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index da733f51779..e66b2e1f872 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -99,7 +99,7 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.11" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} From b670af6b55f03e3d273d5c94ab0988378c1fa907 Mon Sep 17 00:00:00 2001 From: "Richard (Rick) Zamora" Date: Thu, 29 Feb 2024 00:38:51 -0600 Subject: [PATCH 313/384] Avoid dict normalization in ``__dask_tokenize__`` (#15187) There are currently [CI failures](https://github.com/rapidsai/cudf/actions/runs/8089269486/job/22105880070?pr=15181#step:7:1050) that seem to be caused by non-deterministic `dict` normalization in `Frame.__dask_tokenize__`. This PR avoids normalizing that dictionary. Authors: - Richard (Rick) Zamora (https://github.com/rjzamora) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/15187 --- python/cudf/cudf/core/frame.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index 79005193b4e..809bdb4e6d1 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -1958,7 +1958,7 @@ def __dask_tokenize__(self): return [ type(self), - normalize_token(self._dtypes), + str(self._dtypes), normalize_token(self.to_pandas()), ] From f7e486043c30810625fe2d13f5b20d60f90b8d2e Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 29 Feb 2024 00:19:15 -0800 Subject: [PATCH 314/384] Enable creation of columns from scalar (#15181) This PR enables creation of pylibcudf columns from scalar values. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/15181 --- python/cudf/cudf/_lib/pylibcudf/column.pyx | 26 ++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/python/cudf/cudf/_lib/pylibcudf/column.pyx b/python/cudf/cudf/_lib/pylibcudf/column.pyx index 2a7215099d5..62a83efa3e2 100644 --- a/python/cudf/cudf/_lib/pylibcudf/column.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/column.pyx @@ -1,14 +1,18 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. +from cython.operator cimport dereference from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from rmm._lib.device_buffer cimport DeviceBuffer from cudf._lib.cpp.column.column cimport column, column_contents +from cudf._lib.cpp.column.column_factories cimport make_column_from_scalar +from cudf._lib.cpp.scalar.scalar cimport scalar from cudf._lib.cpp.types cimport size_type from .gpumemoryview cimport gpumemoryview +from .scalar cimport Scalar from .types cimport DataType, type_id from .utils cimport int_to_bitmask_ptr, int_to_void_ptr @@ -196,6 +200,28 @@ cdef class Column: children, ) + @staticmethod + def from_scalar(Scalar slr, size_type size): + """Create a Column from a Scalar. + + Parameters + ---------- + slr : Scalar + The scalar to create a column from. + size : size_type + The number of elements in the column. + + Returns + ------- + Column + A Column containing the scalar repeated `size` times. + """ + cdef const scalar* c_scalar = slr.get() + cdef unique_ptr[column] c_result + with nogil: + c_result = move(make_column_from_scalar(dereference(c_scalar), size)) + return Column.from_libcudf(move(c_result)) + cpdef DataType type(self): """The type of data in the column.""" return self._data_type From 1a3b7890e1f110e93082308546eccbeae8a4784a Mon Sep 17 00:00:00 2001 From: jakirkham Date: Thu, 29 Feb 2024 05:53:40 -0800 Subject: [PATCH 315/384] Dynamically set version in RAPIDS doc builds (#15101) Following up on issue ( https://github.com/rapidsai/build-planning/issues/15 ), drop RAPIDS version hard-coding in doc builds. Authors: - https://github.com/jakirkham Approvers: - Bradley Dice (https://github.com/bdice) - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/cudf/pull/15101 --- ci/build_docs.sh | 3 ++- ci/release/update-version.sh | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 2b55a9db8af..529eaeae696 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -3,6 +3,8 @@ set -euo pipefail +export RAPIDS_VERSION_NUMBER="$(rapids-generate-version)" + rapids-logger "Create test conda environment" . /opt/conda/etc/profile.d/conda.sh @@ -27,7 +29,6 @@ rapids-mamba-retry install \ --channel "${PYTHON_CHANNEL}" \ libcudf cudf dask-cudf -export RAPIDS_VERSION_NUMBER="24.04" export RAPIDS_DOCS_DIR="$(mktemp -d)" rapids-logger "Build CPP docs" diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 1186b02f244..811e7825363 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -86,7 +86,6 @@ for FILE in .github/workflows/*.yaml; do sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}" sed_runner "s/dask-cuda.git@branch-[^\"\s]\+/dask-cuda.git@branch-${NEXT_SHORT_TAG}/g" ${FILE}; done -sed_runner "s/RAPIDS_VERSION_NUMBER=\".*/RAPIDS_VERSION_NUMBER=\"${NEXT_SHORT_TAG}\"/g" ci/build_docs.sh # Java files NEXT_FULL_JAVA_TAG="${NEXT_SHORT_TAG}.${PATCH_PEP440}-SNAPSHOT" From 15f11e10ac76baaac2fd702aab9bdf30dde07d6b Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Thu, 29 Feb 2024 10:10:07 -0500 Subject: [PATCH 316/384] Remove unneeded script parameters in test_cpp_memcheck.sh (#15158) Fixes error introduced in #14992 in `test_cpp_memcheck.sh` Extra line of parameters removed from the call to `run_cudf_memcheck_ctests.sh` Authors: - David Wendt (https://github.com/davidwendt) - https://github.com/jakirkham Approvers: - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/cudf/pull/15158 --- ci/test_cpp_memcheck.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/test_cpp_memcheck.sh b/ci/test_cpp_memcheck.sh index 0233c2b55f8..fda11c64155 100755 --- a/ci/test_cpp_memcheck.sh +++ b/ci/test_cpp_memcheck.sh @@ -8,9 +8,7 @@ source ./ci/test_cpp_common.sh rapids-logger "Memcheck gtests with rmm_mode=cuda" -./ci/run_cudf_memcheck_ctests.sh \ - --gtest_output=xml:"${RAPIDS_TESTS_DIR}${test_name}.xml" \ - && EXITCODE=$? || EXITCODE=$?; +./ci/run_cudf_memcheck_ctests.sh && EXITCODE=$? || EXITCODE=$?; rapids-logger "Test script exiting with value: $EXITCODE" exit ${EXITCODE} From 50630b2011b37f39d1e9255456153550cf40d470 Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Thu, 29 Feb 2024 15:13:20 +0000 Subject: [PATCH 317/384] Implement stable version of `cudf::sort` (#15066) Adds an implementation of `cudf::stable_sort`. While here, cleans up a few small issues around stream-passing and memory resource usage in the detail APIs of some of the sort functions. - Closes #15065 Authors: - Lawrence Mitchell (https://github.com/wence-) Approvers: - David Wendt (https://github.com/davidwendt) - Yunsong Wang (https://github.com/PointKernel) URL: https://github.com/rapidsai/cudf/pull/15066 --- cpp/include/cudf/detail/sorting.hpp | 13 ++- cpp/include/cudf/sorting.hpp | 35 +++---- cpp/src/sort/common_sort_impl.cuh | 101 +++++++++++++++++++++ cpp/src/sort/segmented_sort_impl.cuh | 11 +-- cpp/src/sort/sort.cu | 40 ++------ cpp/src/sort/sort_column.cu | 15 +-- cpp/src/sort/sort_column_impl.cuh | 14 +-- cpp/src/sort/sort_impl.cuh | 11 ++- cpp/src/sort/stable_sort.cu | 34 ++++++- cpp/src/sort/stable_sort_column.cu | 15 +-- cpp/tests/sort/stable_sort_tests.cpp | 131 +++++++++++++++------------ 11 files changed, 274 insertions(+), 146 deletions(-) create mode 100644 cpp/src/sort/common_sort_impl.cuh diff --git a/cpp/include/cudf/detail/sorting.hpp b/cpp/include/cudf/detail/sorting.hpp index 8f92b66d5fa..97cc054da57 100644 --- a/cpp/include/cudf/detail/sorting.hpp +++ b/cpp/include/cudf/detail/sorting.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -150,5 +150,16 @@ std::unique_ptr
sort(table_view const& values, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr); +/** + * @copydoc cudf::stable_sort + * + * @param stream CUDA stream used for device memory operations and kernel launches. + */ +std::unique_ptr
stable_sort(table_view const& values, + std::vector const& column_order, + std::vector const& null_precedence, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr); + } // namespace detail } // namespace cudf diff --git a/cpp/include/cudf/sorting.hpp b/cpp/include/cudf/sorting.hpp index e4e803b2d3c..42bcb5da8e3 100644 --- a/cpp/include/cudf/sorting.hpp +++ b/cpp/include/cudf/sorting.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -115,6 +115,18 @@ std::unique_ptr
sort( rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()); +/** + * @brief Performs a stable lexicographic sort of the rows of a table + * + * @copydoc cudf::sort + */ +std::unique_ptr
stable_sort( + table_view const& input, + std::vector const& column_order = {}, + std::vector const& null_precedence = {}, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()); + /** * @brief Performs a key-value sort. * @@ -148,26 +160,7 @@ std::unique_ptr
sort_by_key( /** * @brief Performs a key-value stable sort. * - * Creates a new table that reorders the rows of `values` according to the - * lexicographic ordering of the rows of `keys`. - * - * The order of equivalent elements is guaranteed to be preserved. - * - * @throws cudf::logic_error if `values.num_rows() != keys.num_rows()`. - * - * @param values The table to reorder - * @param keys The table that determines the ordering - * @param column_order The desired order for each column in `keys`. Size must be - * equal to `keys.num_columns()` or empty. If empty, all columns are sorted in - * ascending order. - * @param null_precedence The desired order of a null element compared to other - * elements for each column in `keys`. Size must be equal to - * `keys.num_columns()` or empty. If empty, all columns will be sorted with - * `null_order::BEFORE`. - * @param stream CUDA stream used for device memory operations and kernel launches - * @param mr Device memory resource used to allocate the returned table's device memory - * @return The reordering of `values` determined by the lexicographic order of - * the rows of `keys`. + * @copydoc cudf::sort_by_key */ std::unique_ptr
stable_sort_by_key( table_view const& values, diff --git a/cpp/src/sort/common_sort_impl.cuh b/cpp/src/sort/common_sort_impl.cuh new file mode 100644 index 00000000000..745e2717304 --- /dev/null +++ b/cpp/src/sort/common_sort_impl.cuh @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include + +#include +#include + +#include + +namespace cudf { +namespace detail { + +/** + * @brief The enum specifying which sorting method to use (stable or unstable). + */ +enum class sort_method : bool { STABLE, UNSTABLE }; + +/** + * @brief Functor performs a fast-path, in-place sort on eligible columns + * + * @tparam method Whether to use a stable or unstable sort. + */ +template +struct inplace_column_sort_fn { + /** + * @brief Check if fast-path, in-place sort is available for the given column + * + * @param column to check + * @return true if fast-path sort is available, false otherwise. + */ + static bool is_usable(column_view const& column) + { + return !column.has_nulls() && cudf::is_fixed_width(column.type()) && + !cudf::is_floating_point(column.type()); + } + /** + * @brief Check if fast-path, in-place sort is available for the given table + * + * @param table to check + * @return true if fast-path sort is available, false otherwise. + */ + static bool is_usable(table_view const& table) + { + return table.num_columns() == 1 && is_usable(table.column(0)); + } + + /** + * @brief Fast-path sort a column in place + * + * Precondition, is_usable(column) returned true + * + * @tparam T column data type. + * @param col Column to sort, modified in place. + * @param order Ascending or descending sort order. + * @param stream CUDA stream used for device memory operations and kernel launches + * + */ + template ()>* = nullptr> + void operator()(mutable_column_view& col, order order, rmm::cuda_stream_view stream) const + { + auto const do_sort = [&](auto const cmp) { + if constexpr (method == sort_method::STABLE) { + thrust::stable_sort(rmm::exec_policy(stream), col.begin(), col.end(), cmp); + } else { + thrust::sort(rmm::exec_policy(stream), col.begin(), col.end(), cmp); + } + }; + if (order == order::ASCENDING) { + do_sort(thrust::less()); + } else { + do_sort(thrust::greater()); + } + } + + template ()>* = nullptr> + void operator()(mutable_column_view&, order, rmm::cuda_stream_view) const + { + CUDF_FAIL("Column type must be relationally comparable and fixed-width"); + } +}; + +} // namespace detail +} // namespace cudf diff --git a/cpp/src/sort/segmented_sort_impl.cuh b/cpp/src/sort/segmented_sort_impl.cuh index 5d11bf055f1..796e178fecd 100644 --- a/cpp/src/sort/segmented_sort_impl.cuh +++ b/cpp/src/sort/segmented_sort_impl.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,10 @@ * limitations under the License. */ +#pragma once + +#include "common_sort_impl.cuh" + #include #include #include @@ -29,11 +33,6 @@ namespace cudf { namespace detail { -/** - * @brief The enum specifying which sorting method to use (stable or unstable). - */ -enum class sort_method { STABLE, UNSTABLE }; - /** * @brief Functor performs faster segmented sort on eligible columns */ diff --git a/cpp/src/sort/sort.cu b/cpp/src/sort/sort.cu index 46edae798d4..adffc06ab93 100644 --- a/cpp/src/sort/sort.cu +++ b/cpp/src/sort/sort.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ * limitations under the License. */ +#include "common_sort_impl.cuh" #include "sort_impl.cuh" #include @@ -37,7 +38,7 @@ std::unique_ptr sorted_order(table_view const& input, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { - return sorted_order(input, column_order, null_precedence, stream, mr); + return sorted_order(input, column_order, null_precedence, stream, mr); } std::unique_ptr
sort_by_key(table_view const& values, @@ -61,47 +62,24 @@ std::unique_ptr
sort_by_key(table_view const& values, mr); } -struct inplace_column_sort_fn { - template ()>* = nullptr> - void operator()(mutable_column_view& col, bool ascending, rmm::cuda_stream_view stream) const - { - CUDF_EXPECTS(!col.has_nulls(), "Nulls not supported for in-place sort"); - if (ascending) { - thrust::sort(rmm::exec_policy(stream), col.begin(), col.end(), thrust::less()); - } else { - thrust::sort(rmm::exec_policy(stream), col.begin(), col.end(), thrust::greater()); - } - } - - template ()>* = nullptr> - void operator()(mutable_column_view&, bool, rmm::cuda_stream_view) const - { - CUDF_FAIL("Column type must be relationally comparable and fixed-width"); - } -}; - std::unique_ptr
sort(table_view const& input, std::vector const& column_order, std::vector const& null_precedence, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { - CUDF_FUNC_RANGE(); // fast-path sort conditions: single, non-floating-point, fixed-width column with no nulls - if (input.num_columns() == 1 && !input.column(0).has_nulls() && - cudf::is_fixed_width(input.column(0).type()) && - !cudf::is_floating_point(input.column(0).type())) { - auto output = std::make_unique(input.column(0), stream, mr); - auto view = output->mutable_view(); - bool ascending = (column_order.empty() ? true : column_order.front() == order::ASCENDING); + if (inplace_column_sort_fn::is_usable(input)) { + auto output = std::make_unique(input.column(0), stream, mr); + auto view = output->mutable_view(); + auto order = (column_order.empty() ? order::ASCENDING : column_order.front()); cudf::type_dispatcher( - output->type(), inplace_column_sort_fn{}, view, ascending, stream); + output->type(), inplace_column_sort_fn{}, view, order, stream); std::vector> columns; columns.emplace_back(std::move(output)); return std::make_unique
(std::move(columns)); } - return detail::sort_by_key( - input, input, column_order, null_precedence, cudf::get_default_stream(), mr); + return detail::sort_by_key(input, input, column_order, null_precedence, stream, mr); } } // namespace detail diff --git a/cpp/src/sort/sort_column.cu b/cpp/src/sort/sort_column.cu index 9df04251e93..7db44476988 100644 --- a/cpp/src/sort/sort_column.cu +++ b/cpp/src/sort/sort_column.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ * limitations under the License. */ +#include "common_sort_impl.cuh" #include "sort_column_impl.cuh" #include @@ -30,11 +31,11 @@ namespace detail { * sorted_order(column_view&,order,null_order,rmm::cuda_stream_view,rmm::mr::device_memory_resource*) */ template <> -std::unique_ptr sorted_order(column_view const& input, - order column_order, - null_order null_precedence, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) +std::unique_ptr sorted_order(column_view const& input, + order column_order, + null_order null_precedence, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) { auto sorted_indices = cudf::make_numeric_column( data_type(type_to_id()), input.size(), mask_state::UNALLOCATED, stream, mr); @@ -42,7 +43,7 @@ std::unique_ptr sorted_order(column_view const& input, thrust::sequence( rmm::exec_policy(stream), indices_view.begin(), indices_view.end(), 0); cudf::type_dispatcher(input.type(), - column_sorted_order_fn{}, + column_sorted_order_fn{}, input, indices_view, column_order == order::ASCENDING, diff --git a/cpp/src/sort/sort_column_impl.cuh b/cpp/src/sort/sort_column_impl.cuh index 5abc6bdfadf..7af24f22b67 100644 --- a/cpp/src/sort/sort_column_impl.cuh +++ b/cpp/src/sort/sort_column_impl.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,8 @@ #pragma once +#include "common_sort_impl.cuh" + #include #include #include @@ -36,7 +38,7 @@ namespace detail { * This API offers fast sorting for primitive types. It cannot handle nested types and will not * consider `NaN` as equivalent to other `NaN`. * - * @tparam stable Whether to use stable sort + * @tparam method Whether to use stable sort * @param input Column to sort. The column data is not modified. * @param column_order Ascending or descending sort order * @param null_precedence How null rows are to be ordered @@ -45,7 +47,7 @@ namespace detail { * @param mr Device memory resource used to allocate the returned column's device memory * @return Sorted indices for the input column. */ -template +template std::unique_ptr sorted_order(column_view const& input, order column_order, null_order null_precedence, @@ -78,7 +80,7 @@ struct simple_comparator { null_order null_precedence{}; }; -template +template struct column_sorted_order_fn { /** * @brief Compile time check for allowing faster sort. @@ -121,7 +123,7 @@ struct column_sorted_order_fn { auto const do_sort = [&](auto const comp) { // Compiling `thrust::*sort*` APIs is expensive. // Thus, we should optimize that by using constexpr condition to only compile what we need. - if constexpr (stable) { + if constexpr (method == sort_method::STABLE) { thrust::stable_sort_by_key(rmm::exec_policy(stream), d_col.begin(), d_col.end(), @@ -165,7 +167,7 @@ struct column_sorted_order_fn { auto comp = simple_comparator{*keys, input.has_nulls(), ascending, null_precedence}; // Compiling `thrust::*sort*` APIs is expensive. // Thus, we should optimize that by using constexpr condition to only compile what we need. - if constexpr (stable) { + if constexpr (method == sort_method::STABLE) { thrust::stable_sort( rmm::exec_policy(stream), indices.begin(), indices.end(), comp); } else { diff --git a/cpp/src/sort/sort_impl.cuh b/cpp/src/sort/sort_impl.cuh index 5fae8db1a70..e0331d65053 100644 --- a/cpp/src/sort/sort_impl.cuh +++ b/cpp/src/sort/sort_impl.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ #pragma once +#include "common_sort_impl.cuh" #include "sort_column_impl.cuh" #include @@ -30,7 +31,7 @@ namespace detail { * @tparam stable Whether to use stable sort * @param stream CUDA stream used for device memory operations and kernel launches */ -template +template std::unique_ptr sorted_order(table_view input, std::vector const& column_order, std::vector const& null_precedence, @@ -39,7 +40,7 @@ std::unique_ptr sorted_order(table_view input, { if (input.num_rows() == 0 or input.num_columns() == 0) { return cudf::make_numeric_column( - data_type(type_to_id()), 0, mask_state::UNALLOCATED, stream); + data_type(type_to_id()), 0, mask_state::UNALLOCATED, stream, mr); } if (not column_order.empty()) { @@ -57,7 +58,7 @@ std::unique_ptr sorted_order(table_view input, auto const single_col = input.column(0); auto const col_order = column_order.empty() ? order::ASCENDING : column_order.front(); auto const null_prec = null_precedence.empty() ? null_order::BEFORE : null_precedence.front(); - return sorted_order(single_col, col_order, null_prec, stream, mr); + return sorted_order(single_col, col_order, null_prec, stream, mr); } std::unique_ptr sorted_indices = cudf::make_numeric_column( @@ -71,7 +72,7 @@ std::unique_ptr sorted_order(table_view input, auto const do_sort = [&](auto const comparator) { // Compiling `thrust::*sort*` APIs is expensive. // Thus, we should optimize that by using constexpr condition to only compile what we need. - if constexpr (stable) { + if constexpr (method == sort_method::STABLE) { thrust::stable_sort(rmm::exec_policy(stream), mutable_indices_view.begin(), mutable_indices_view.end(), diff --git a/cpp/src/sort/stable_sort.cu b/cpp/src/sort/stable_sort.cu index cf602dcf1a9..0bfe2cfef16 100644 --- a/cpp/src/sort/stable_sort.cu +++ b/cpp/src/sort/stable_sort.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ * limitations under the License. */ +#include "common_sort_impl.cuh" #include "sort_impl.cuh" #include @@ -34,7 +35,26 @@ std::unique_ptr stable_sorted_order(table_view const& input, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { - return sorted_order(input, column_order, null_precedence, stream, mr); + return sorted_order(input, column_order, null_precedence, stream, mr); +} + +std::unique_ptr
stable_sort(table_view const& input, + std::vector const& column_order, + std::vector const& null_precedence, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) +{ + if (inplace_column_sort_fn::is_usable(input)) { + auto output = std::make_unique(input.column(0), stream, mr); + auto view = output->mutable_view(); + auto order = (column_order.empty() ? order::ASCENDING : column_order.front()); + cudf::type_dispatcher( + output->type(), inplace_column_sort_fn{}, view, order, stream); + std::vector> columns; + columns.emplace_back(std::move(output)); + return std::make_unique
(std::move(columns)); + } + return detail::stable_sort_by_key(input, input, column_order, null_precedence, stream, mr); } std::unique_ptr
stable_sort_by_key(table_view const& values, @@ -69,6 +89,16 @@ std::unique_ptr stable_sorted_order(table_view const& input, return detail::stable_sorted_order(input, column_order, null_precedence, stream, mr); } +std::unique_ptr
stable_sort(table_view const& input, + std::vector const& column_order, + std::vector const& null_precedence, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) +{ + CUDF_FUNC_RANGE(); + return detail::stable_sort(input, column_order, null_precedence, stream, mr); +} + std::unique_ptr
stable_sort_by_key(table_view const& values, table_view const& keys, std::vector const& column_order, diff --git a/cpp/src/sort/stable_sort_column.cu b/cpp/src/sort/stable_sort_column.cu index be519ead951..25a6c92034a 100644 --- a/cpp/src/sort/stable_sort_column.cu +++ b/cpp/src/sort/stable_sort_column.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ * limitations under the License. */ +#include "common_sort_impl.cuh" #include "sort_column_impl.cuh" #include @@ -30,11 +31,11 @@ namespace detail { * sorted_order(column_view&,order,null_order,rmm::cuda_stream_view,rmm::mr::device_memory_resource*) */ template <> -std::unique_ptr sorted_order(column_view const& input, - order column_order, - null_order null_precedence, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) +std::unique_ptr sorted_order(column_view const& input, + order column_order, + null_order null_precedence, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) { auto sorted_indices = cudf::make_numeric_column( data_type(type_to_id()), input.size(), mask_state::UNALLOCATED, stream, mr); @@ -42,7 +43,7 @@ std::unique_ptr sorted_order(column_view const& input, thrust::sequence( rmm::exec_policy(stream), indices_view.begin(), indices_view.end(), 0); cudf::type_dispatcher(input.type(), - column_sorted_order_fn{}, + column_sorted_order_fn{}, input, indices_view, column_order == order::ASCENDING, diff --git a/cpp/tests/sort/stable_sort_tests.cpp b/cpp/tests/sort/stable_sort_tests.cpp index 71520ef007b..341f8317004 100644 --- a/cpp/tests/sort/stable_sort_tests.cpp +++ b/cpp/tests/sort/stable_sort_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,12 +34,14 @@ void run_stable_sort_test(cudf::table_view input, cudf::column_view expected_sorted_indices, std::vector column_order = {}, - std::vector null_precedence = {}) + std::vector null_precedence = {}, + bool by_key = true) { - auto got_sort_by_key_table = cudf::sort_by_key(input, input, column_order, null_precedence); - auto expected_sort_by_key_table = cudf::gather(input, expected_sorted_indices); + auto got = by_key ? cudf::stable_sort_by_key(input, input, column_order, null_precedence) + : cudf::stable_sort(input, column_order, null_precedence); + auto expected = cudf::gather(input, expected_sorted_indices); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected_sort_by_key_table->view(), got_sort_by_key_table->view()); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected->view(), got->view()); } using TestTypes = cudf::test::Concat col3{{10, 40, 70, 10, 2, 10}, {1, 1, 0, 1, 1, 1}}; cudf::table_view input{{col1, col2, col3}}; - cudf::test::fixed_width_column_wrapper expected{{1, 0, 3, 5, 4, 2}}; std::vector column_order{ cudf::order::ASCENDING, cudf::order::ASCENDING, cudf::order::DESCENDING}; std::vector null_precedence{ cudf::null_order::AFTER, cudf::null_order::AFTER, cudf::null_order::AFTER}; + auto expected = std::is_same_v + // All the bools are true, and therefore don't affect sort order, + // so this is just the sort order of the nullable string column + ? cudf::test::fixed_width_column_wrapper{{0, 3, 5, 1, 4, 2}} + : cudf::test::fixed_width_column_wrapper{{1, 0, 3, 5, 4, 2}}; auto got = cudf::stable_sorted_order(input, column_order, null_precedence); - if (not std::is_same_v) { - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, got->view()); - - run_stable_sort_test(input, expected, column_order, null_precedence); - } else { - // for bools only validate that the null element landed at the back, since - // the rest of the values are equivalent and yields random sorted order. - auto to_host = [](cudf::column_view const& col) { - thrust::host_vector h_data(col.size()); - CUDF_CUDA_TRY(cudaMemcpy( - h_data.data(), col.data(), h_data.size() * sizeof(int32_t), cudaMemcpyDefault)); - return h_data; - }; - thrust::host_vector h_exp = to_host(expected); - thrust::host_vector h_got = to_host(got->view()); - EXPECT_EQ(h_exp[h_exp.size() - 1], h_got[h_got.size() - 1]); - - cudf::test::fixed_width_column_wrapper expected_for_bool{{0, 3, 5, 1, 4, 2}}; - run_stable_sort_test(input, expected_for_bool, column_order, null_precedence); - } + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, got->view()); + + run_stable_sort_test(input, expected, column_order, null_precedence, false); + run_stable_sort_test(input, expected, column_order, null_precedence, true); +} + +TYPED_TEST(StableSort, SingleColumnNoNull) +{ + // This test exercises the "fast-path" single column sort. + using T = TypeParam; + // 0 1 2 3 4 5 6 7 8 9 + cudf::test::fixed_width_column_wrapper col{{7, 1, -2, 5, 1, 0, 1, -2, 0, 5}}; + cudf::table_view input{{col}}; + std::vector column_order{cudf::order::ASCENDING}; + auto expected = + std::is_same_v + ? cudf::test::fixed_width_column_wrapper{{8, 5, 0, 1, 2, 3, 4, 6, 7, 9}} + : std::is_unsigned_v + ? cudf::test::fixed_width_column_wrapper{{5, 8, 1, 4, 6, 3, 9, 0, 2, 7}} + : cudf::test::fixed_width_column_wrapper{{2, 7, 5, 8, 1, 4, 6, 3, 9, 0}}; + run_stable_sort_test(input, expected, column_order, {}, false); + run_stable_sort_test(input, expected, column_order, {}, true); +} + +TYPED_TEST(StableSort, SingleColumnWithNull) +{ + using T = TypeParam; + // 0 1 2 3 4 5 6 7 8 9 + cudf::test::fixed_width_column_wrapper col{{7, 1, -2, 5, 1, 0, 1, -2, 0, 5}, + {1, 1, 0, 0, 1, 0, 1, 0, 1, 0}}; + cudf::table_view input{{col}}; + std::vector column_order{cudf::order::ASCENDING}; + std::vector null_precedence{cudf::null_order::BEFORE}; + auto expected = + std::is_same_v + ? cudf::test::fixed_width_column_wrapper{{5, 2, 3, 7, 9, 8, 0, 1, 4, 6}} + : std::is_unsigned_v + ? cudf::test::fixed_width_column_wrapper{{5, 3, 9, 2, 7, 8, 1, 4, 6, 0}} + : cudf::test::fixed_width_column_wrapper{{2, 7, 5, 3, 9, 8, 1, 4, 6, 0}}; + run_stable_sort_test(input, expected, column_order, {}, false); + run_stable_sort_test(input, expected, column_order, {}, true); } TYPED_TEST(StableSort, WithNullMin) @@ -117,32 +144,19 @@ TYPED_TEST(StableSort, WithNullMin) cudf::test::fixed_width_column_wrapper col3{{10, 40, 70, 10, 2}, {1, 1, 0, 1, 1}}; cudf::table_view input{{col1, col2, col3}}; - cudf::test::fixed_width_column_wrapper expected{{2, 1, 0, 3, 4}}; std::vector column_order{ cudf::order::ASCENDING, cudf::order::ASCENDING, cudf::order::DESCENDING}; + auto expected = std::is_same_v + // All the bools are true, and therefore don't affect sort order, + // so this is just the sort order of the string column + ? cudf::test::fixed_width_column_wrapper{{2, 0, 3, 1, 4}} + : cudf::test::fixed_width_column_wrapper{{2, 1, 0, 3, 4}}; + auto got = cudf::stable_sorted_order(input, column_order); - auto got = cudf::stable_sorted_order(input, column_order); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, got->view()); - if (!std::is_same_v) { - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, got->view()); - - run_stable_sort_test(input, expected, column_order); - } else { - // for bools only validate that the null element landed at the front, since - // the rest of the values are equivalent and yields random sorted order. - auto to_host = [](cudf::column_view const& col) { - thrust::host_vector h_data(col.size()); - CUDF_CUDA_TRY(cudaMemcpy( - h_data.data(), col.data(), h_data.size() * sizeof(int32_t), cudaMemcpyDefault)); - return h_data; - }; - thrust::host_vector h_exp = to_host(expected); - thrust::host_vector h_got = to_host(got->view()); - EXPECT_EQ(h_exp.front(), h_got.front()); - - cudf::test::fixed_width_column_wrapper expected_for_bool{{2, 0, 3, 1, 4}}; - run_stable_sort_test(input, expected_for_bool, column_order); - } + run_stable_sort_test(input, expected, column_order, {}, false); + run_stable_sort_test(input, expected, column_order, {}, true); } TYPED_TEST(StableSort, WithAllValid) @@ -154,22 +168,19 @@ TYPED_TEST(StableSort, WithAllValid) cudf::test::fixed_width_column_wrapper col3{{10, 40, 70, 10, 2}}; cudf::table_view input{{col1, col2, col3}}; - cudf::test::fixed_width_column_wrapper expected{{2, 1, 0, 3, 4}}; std::vector column_order{ cudf::order::ASCENDING, cudf::order::ASCENDING, cudf::order::DESCENDING}; + auto expected = std::is_same_v + // All the bools are true, and therefore don't affect sort order, + // so this is just the sort order of the string column + ? cudf::test::fixed_width_column_wrapper{{2, 0, 3, 1, 4}} + : cudf::test::fixed_width_column_wrapper{{2, 1, 0, 3, 4}}; + auto got = cudf::stable_sorted_order(input, column_order); - auto got = cudf::stable_sorted_order(input, column_order); - - // Skip validating bools order. Valid true bools are all - // equivalent, and yield random order after thrust::sort - if (!std::is_same_v) { - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, got->view()); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, got->view()); - run_stable_sort_test(input, expected, column_order); - } else { - cudf::test::fixed_width_column_wrapper expected_for_bool{{2, 0, 3, 1, 4}}; - run_stable_sort_test(input, expected_for_bool, column_order); - } + run_stable_sort_test(input, expected, column_order, {}, false); + run_stable_sort_test(input, expected, column_order, {}, true); } TYPED_TEST(StableSort, MisMatchInColumnOrderSize) From efc4edfa9dcb30d63379ad23bef23ca330d5bcdf Mon Sep 17 00:00:00 2001 From: Yunsong Wang Date: Thu, 29 Feb 2024 09:04:06 -0800 Subject: [PATCH 318/384] Fix memcheck error in distinct inner join (#15164) Closes #15156 Fixes the invalid global read introduced by #14990 and simplifies the logic. Authors: - Yunsong Wang (https://github.com/PointKernel) Approvers: - Bradley Dice (https://github.com/bdice) - David Wendt (https://github.com/davidwendt) URL: https://github.com/rapidsai/cudf/pull/15164 --- cpp/src/join/distinct_hash_join.cu | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/cpp/src/join/distinct_hash_join.cu b/cpp/src/join/distinct_hash_join.cu index 7c834d1a96b..981a7bf0dea 100644 --- a/cpp/src/join/distinct_hash_join.cu +++ b/cpp/src/join/distinct_hash_join.cu @@ -205,18 +205,14 @@ CUDF_KERNEL void distinct_join_probe_kernel(Iter iter, cudf::size_type buffer_size = 0; while (idx - block.thread_rank() < n) { // the whole thread block falls into the same iteration - cudf::size_type thread_count{0}; - cudf::size_type build_idx{0}; - if (idx < n) { - auto const found = hash_table.find(*(iter + idx)); - thread_count = found != hash_table.end(); - build_idx = static_cast(found->second); - } + auto const found = idx < n ? hash_table.find(*(iter + idx)) : hash_table.end(); + auto const has_match = found != hash_table.end(); // Use a whole-block scan to calculate the output location cudf::size_type offset; cudf::size_type block_count; - block_scan(block_scan_temp_storage).ExclusiveSum(thread_count, offset, block_count); + block_scan(block_scan_temp_storage) + .ExclusiveSum(static_cast(has_match), offset, block_count); if (buffer_size + block_count > buffer_capacity) { flush_buffer(block, buffer_size, buffer, counter, build_indices, probe_indices); @@ -224,8 +220,9 @@ CUDF_KERNEL void distinct_join_probe_kernel(Iter iter, buffer_size = 0; } - if (thread_count == 1) { - buffer[buffer_size + offset] = cuco::pair{build_idx, static_cast(idx)}; + if (has_match) { + buffer[buffer_size + offset] = cuco::pair{static_cast(found->second), + static_cast(idx)}; } buffer_size += block_count; block.sync(); From b7d9335dc716e731c4fa820e77409b2bb0734eb8 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Thu, 29 Feb 2024 20:27:52 +0100 Subject: [PATCH 319/384] Document how cuDF is pronounced (#14753) Document in `README.md` and sphinx landing pages how cuDF is pronounced. It is known people may pronounce cuDF in ways that aren't how it was conceived, such as "see-you-dee-ef". The correct way to pronounce is not documented anywhere so people who have never heard it from someone knowledgeable aren't able to know for sure, and thus this should be clearly documented. Authors: - Peter Andreas Entschev (https://github.com/pentschev) Approvers: - David Wendt (https://github.com/davidwendt) - Nghia Truong (https://github.com/ttnghia) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/14753 --- README.md | 5 +++-- docs/cudf/source/index.rst | 12 ++++++------ docs/dask_cudf/source/index.rst | 5 +++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index a64e39452ec..599e194bc1a 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,9 @@ ## 📢 cuDF can now be used as a no-code-change accelerator for pandas! To learn more, see [here](https://rapids.ai/cudf-pandas/)! -cuDF is a GPU DataFrame library for loading joining, aggregating, -filtering, and otherwise manipulating data. cuDF leverages +cuDF (pronounced "KOO-dee-eff") is a GPU DataFrame library +for loading, joining, aggregating, filtering, and otherwise +manipulating data. cuDF leverages [libcudf](https://docs.rapids.ai/api/libcudf/stable/), a blazing-fast C++/CUDA dataframe library and the [Apache Arrow](https://arrow.apache.org/) columnar format to provide a diff --git a/docs/cudf/source/index.rst b/docs/cudf/source/index.rst index 3765b560a7f..3b8dfa5fe01 100644 --- a/docs/cudf/source/index.rst +++ b/docs/cudf/source/index.rst @@ -5,12 +5,12 @@ Welcome to the cuDF documentation! :width: 300px :align: center -**cuDF** is a Python GPU DataFrame library (built on the `Apache Arrow -`_ columnar memory format) for loading, joining, -aggregating, filtering, and otherwise manipulating data. cuDF also provides a -pandas-like API that will be familiar to data engineers & data scientists, so -they can use it to easily accelerate their workflows without going into -the details of CUDA programming. +**cuDF** (pronounced "KOO-dee-eff") is a Python GPU DataFrame library (built +on the `Apache Arrow `_ columnar memory format) +for loading, joining, aggregating, filtering, and otherwise manipulating data. +cuDF also provides a pandas-like API that will be familiar to data engineers +& data scientists, so they can use it to easily accelerate their workflows +without going into the details of CUDA programming. ``cudf.pandas`` is built on cuDF and accelerates pandas code on the GPU. It supports 100% of the pandas API, using the GPU for diff --git a/docs/dask_cudf/source/index.rst b/docs/dask_cudf/source/index.rst index 0442ab0929a..9a216690384 100644 --- a/docs/dask_cudf/source/index.rst +++ b/docs/dask_cudf/source/index.rst @@ -6,8 +6,9 @@ Welcome to dask-cudf's documentation! ===================================== -Dask-cuDF is an extension library for the `Dask `__ -parallel computing framework that provides a `cuDF +**Dask-cuDF** (pronounced "DASK KOO-dee-eff") is an extension +library for the `Dask `__ parallel computing +framework that provides a `cuDF `__-backed distributed dataframe with the same API as `Dask dataframes `__. From 08e3c96e482ead102cd06b99a0bbdfef2735e0bd Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Thu, 29 Feb 2024 12:01:53 -0800 Subject: [PATCH 320/384] Eliminate duplicate allocation of nested string columns (#15061) Issue https://github.com/rapidsai/cudf/issues/14965 Authors: - Vukasin Milovanovic (https://github.com/vuule) Approvers: - Bradley Dice (https://github.com/bdice) - Nghia Truong (https://github.com/ttnghia) - Mark Harris (https://github.com/harrism) URL: https://github.com/rapidsai/cudf/pull/15061 --- cpp/src/io/parquet/reader_impl.cpp | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 26d810a3337..93fc6bd6bb5 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -60,7 +60,7 @@ void reader::impl::decode_page_data(size_t skip_rows, size_t num_rows) // TODO: This step is somewhat redundant if size info has already been calculated (nested schema, // chunked reader). auto const has_strings = (kernel_mask & STRINGS_MASK) != 0; - std::vector col_sizes(_input_columns.size(), 0L); + std::vector col_string_sizes(_input_columns.size(), 0L); if (has_strings) { ComputePageStringSizes(subpass.pages, pass.chunks, @@ -71,10 +71,10 @@ void reader::impl::decode_page_data(size_t skip_rows, size_t num_rows) kernel_mask, _stream); - col_sizes = calculate_page_string_offsets(); + col_string_sizes = calculate_page_string_offsets(); // check for overflow - if (std::any_of(col_sizes.cbegin(), col_sizes.cend(), [](size_t sz) { + if (std::any_of(col_string_sizes.cbegin(), col_string_sizes.cend(), [](std::size_t sz) { return sz > std::numeric_limits::max(); })) { CUDF_FAIL("String column exceeds the column size limit", std::overflow_error); @@ -157,8 +157,9 @@ void reader::impl::decode_page_data(size_t skip_rows, size_t num_rows) valids[idx] = out_buf.null_mask(); data[idx] = out_buf.data(); // only do string buffer for leaf - if (out_buf.string_size() == 0 && col_sizes[pass.chunks[c].src_col_index] > 0) { - out_buf.create_string_data(col_sizes[pass.chunks[c].src_col_index], _stream); + if (idx == max_depth - 1 and out_buf.string_size() == 0 and + col_string_sizes[pass.chunks[c].src_col_index] > 0) { + out_buf.create_string_data(col_string_sizes[pass.chunks[c].src_col_index], _stream); } if (has_strings) { str_data[idx] = out_buf.string_data(); } out_buf.user_data |= @@ -272,21 +273,21 @@ void reader::impl::decode_page_data(size_t skip_rows, size_t num_rows) auto const& child = (*cols)[input_col.nesting[l_idx + 1]]; // the final offset for a list at level N is the size of it's child - int const offset = child.type.id() == type_id::LIST ? child.size - 1 : child.size; - CUDF_CUDA_TRY(cudaMemcpyAsync(static_cast(out_buf.data()) + (out_buf.size - 1), + size_type const offset = child.type.id() == type_id::LIST ? child.size - 1 : child.size; + CUDF_CUDA_TRY(cudaMemcpyAsync(static_cast(out_buf.data()) + (out_buf.size - 1), &offset, - sizeof(offset), + sizeof(size_type), cudaMemcpyDefault, _stream.value())); out_buf.user_data |= PARQUET_COLUMN_BUFFER_FLAG_LIST_TERMINATED; } else if (out_buf.type.id() == type_id::STRING) { // need to cap off the string offsets column - size_type const sz = static_cast(col_sizes[idx]); - cudaMemcpyAsync(static_cast(out_buf.data()) + out_buf.size, - &sz, - sizeof(size_type), - cudaMemcpyDefault, - _stream.value()); + auto const sz = static_cast(col_string_sizes[idx]); + CUDF_CUDA_TRY(cudaMemcpyAsync(static_cast(out_buf.data()) + out_buf.size, + &sz, + sizeof(size_type), + cudaMemcpyDefault, + _stream.value())); } } } From a9e41e73505876b171ca620c52a8638dae9896fd Mon Sep 17 00:00:00 2001 From: nvdbaranec <56695930+nvdbaranec@users.noreply.github.com> Date: Thu, 29 Feb 2024 14:07:29 -0600 Subject: [PATCH 321/384] Performance optimizations for parquet sub-rowgroup reader. (#15020) This PR implements a basket of optimizations for the parquet reader to bring non-chunked reads close to par following the merge of the sub-rowgroup reader. The primary culprit for the performance hit was that in the case where we perform no splits, we were making a full copy of all of the pages into the subpass struct (including a pinned memory allocation). This is unnecessary because we can just represent the pages in the subpass as a span that wraps the existing pages in the pass. In addition, several `hostdevice_vector`s used for work that could be done entirely device-side were converted to `rmm::device_uvector`. Finally, I converted a number of functions that were taking hostdevice_vectors to use spans instead and added some missing operators to the `hostdevice_vector` class itself. This PR doesn't recover all the time (there is some new work that we have to do in all cases) but it takes out most of the sting. A sample of some of the benchmarks that were most notably affected: ``` Original Time Sub-rowgroup-implementation This PR parquet_read_decode Int, device buffer 0 29260860778 26373181343 28121328587 Int, device buffer 1 30692134492 27474241282 29495189226 parquet_read_chunks Int, device buffer 33895028252 29986276949 32293548191 Float, device buffer 57055985251 49640274260 55795392897 ``` Authors: - https://github.com/nvdbaranec Approvers: - Mike Wilson (https://github.com/hyperbolic2346) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/15020 --- cpp/src/io/parquet/decode_preprocess.cu | 4 +- cpp/src/io/parquet/page_data.cu | 8 +- cpp/src/io/parquet/page_delta_decode.cu | 12 +- cpp/src/io/parquet/page_hdr.cu | 2 +- cpp/src/io/parquet/page_string_decode.cu | 24 +-- cpp/src/io/parquet/parquet_gpu.hpp | 32 ++- cpp/src/io/parquet/reader_impl_chunking.cu | 170 +++++++++++----- cpp/src/io/parquet/reader_impl_chunking.hpp | 18 +- cpp/src/io/parquet/reader_impl_preprocess.cu | 202 +++++++++++-------- cpp/src/io/utilities/hostdevice_span.hpp | 40 +++- 10 files changed, 333 insertions(+), 179 deletions(-) diff --git a/cpp/src/io/parquet/decode_preprocess.cu b/cpp/src/io/parquet/decode_preprocess.cu index fea4777af43..862dedf6200 100644 --- a/cpp/src/io/parquet/decode_preprocess.cu +++ b/cpp/src/io/parquet/decode_preprocess.cu @@ -385,8 +385,8 @@ CUDF_KERNEL void __launch_bounds__(preprocess_block_size) /** * @copydoc cudf::io::parquet::gpu::ComputePageSizes */ -void ComputePageSizes(cudf::detail::hostdevice_vector& pages, - cudf::detail::hostdevice_vector const& chunks, +void ComputePageSizes(cudf::detail::hostdevice_span pages, + cudf::detail::hostdevice_span chunks, size_t min_row, size_t num_rows, bool compute_num_rows, diff --git a/cpp/src/io/parquet/page_data.cu b/cpp/src/io/parquet/page_data.cu index 2a9f2d56755..79154851cc7 100644 --- a/cpp/src/io/parquet/page_data.cu +++ b/cpp/src/io/parquet/page_data.cu @@ -609,11 +609,11 @@ struct mask_tform { } // anonymous namespace -uint32_t GetAggregatedDecodeKernelMask(cudf::detail::hostdevice_vector& pages, +uint32_t GetAggregatedDecodeKernelMask(cudf::detail::hostdevice_span pages, rmm::cuda_stream_view stream) { // determine which kernels to invoke - auto mask_iter = thrust::make_transform_iterator(pages.d_begin(), mask_tform{}); + auto mask_iter = thrust::make_transform_iterator(pages.device_begin(), mask_tform{}); return thrust::reduce( rmm::exec_policy(stream), mask_iter, mask_iter + pages.size(), 0U, thrust::bit_or{}); } @@ -621,8 +621,8 @@ uint32_t GetAggregatedDecodeKernelMask(cudf::detail::hostdevice_vector /** * @copydoc cudf::io::parquet::detail::DecodePageData */ -void __host__ DecodePageData(cudf::detail::hostdevice_vector& pages, - cudf::detail::hostdevice_vector const& chunks, +void __host__ DecodePageData(cudf::detail::hostdevice_span pages, + cudf::detail::hostdevice_span chunks, size_t num_rows, size_t min_row, int level_type_size, diff --git a/cpp/src/io/parquet/page_delta_decode.cu b/cpp/src/io/parquet/page_delta_decode.cu index ebad1434c7f..c68b6a32c8b 100644 --- a/cpp/src/io/parquet/page_delta_decode.cu +++ b/cpp/src/io/parquet/page_delta_decode.cu @@ -745,8 +745,8 @@ CUDF_KERNEL void __launch_bounds__(decode_block_size) /** * @copydoc cudf::io::parquet::detail::DecodeDeltaBinary */ -void DecodeDeltaBinary(cudf::detail::hostdevice_vector& pages, - cudf::detail::hostdevice_vector const& chunks, +void DecodeDeltaBinary(cudf::detail::hostdevice_span pages, + cudf::detail::hostdevice_span chunks, size_t num_rows, size_t min_row, int level_type_size, @@ -770,8 +770,8 @@ void DecodeDeltaBinary(cudf::detail::hostdevice_vector& pages, /** * @copydoc cudf::io::parquet::gpu::DecodeDeltaByteArray */ -void DecodeDeltaByteArray(cudf::detail::hostdevice_vector& pages, - cudf::detail::hostdevice_vector const& chunks, +void DecodeDeltaByteArray(cudf::detail::hostdevice_span pages, + cudf::detail::hostdevice_span chunks, size_t num_rows, size_t min_row, int level_type_size, @@ -795,8 +795,8 @@ void DecodeDeltaByteArray(cudf::detail::hostdevice_vector& pages, /** * @copydoc cudf::io::parquet::gpu::DecodeDeltaByteArray */ -void DecodeDeltaLengthByteArray(cudf::detail::hostdevice_vector& pages, - cudf::detail::hostdevice_vector const& chunks, +void DecodeDeltaLengthByteArray(cudf::detail::hostdevice_span pages, + cudf::detail::hostdevice_span chunks, size_t num_rows, size_t min_row, int level_type_size, diff --git a/cpp/src/io/parquet/page_hdr.cu b/cpp/src/io/parquet/page_hdr.cu index a15ccf328de..0dae0724823 100644 --- a/cpp/src/io/parquet/page_hdr.cu +++ b/cpp/src/io/parquet/page_hdr.cu @@ -396,7 +396,7 @@ void __launch_bounds__(128) gpuDecodePageHeaders(ColumnChunkDesc* chunks, } num_values = bs->ck.num_values; page_info = chunk_pages ? chunk_pages[chunk].pages : nullptr; - max_num_pages = page_info ? bs->ck.max_num_pages : 0; + max_num_pages = page_info ? (bs->ck.num_data_pages + bs->ck.num_dict_pages) : 0; values_found = 0; __syncwarp(); while (values_found < num_values && bs->cur < bs->end) { diff --git a/cpp/src/io/parquet/page_string_decode.cu b/cpp/src/io/parquet/page_string_decode.cu index 5cd8205b4ba..101bd34f09f 100644 --- a/cpp/src/io/parquet/page_string_decode.cu +++ b/cpp/src/io/parquet/page_string_decode.cu @@ -1112,8 +1112,8 @@ struct page_tform_functor { /** * @copydoc cudf::io::parquet::detail::ComputePageStringSizes */ -void ComputePageStringSizes(cudf::detail::hostdevice_vector& pages, - cudf::detail::hostdevice_vector const& chunks, +void ComputePageStringSizes(cudf::detail::hostdevice_span pages, + cudf::detail::hostdevice_span chunks, rmm::device_uvector& temp_string_buf, size_t min_row, size_t num_rows, @@ -1157,7 +1157,7 @@ void ComputePageStringSizes(cudf::detail::hostdevice_vector& pages, // check for needed temp space for DELTA_BYTE_ARRAY auto const need_sizes = thrust::any_of( - rmm::exec_policy(stream), pages.d_begin(), pages.d_end(), [] __device__(auto& page) { + rmm::exec_policy(stream), pages.device_begin(), pages.device_end(), [] __device__(auto& page) { return page.temp_string_size != 0; }); @@ -1165,8 +1165,8 @@ void ComputePageStringSizes(cudf::detail::hostdevice_vector& pages, // sum up all of the temp_string_sizes auto const page_sizes = [] __device__(PageInfo const& page) { return page.temp_string_size; }; auto const total_size = thrust::transform_reduce(rmm::exec_policy(stream), - pages.d_begin(), - pages.d_end(), + pages.device_begin(), + pages.device_end(), page_sizes, 0L, thrust::plus{}); @@ -1175,8 +1175,8 @@ void ComputePageStringSizes(cudf::detail::hostdevice_vector& pages, // page's chunk of the temp buffer rmm::device_uvector page_string_offsets(pages.size(), stream); thrust::transform_exclusive_scan(rmm::exec_policy_nosync(stream), - pages.d_begin(), - pages.d_end(), + pages.device_begin(), + pages.device_end(), page_string_offsets.begin(), page_sizes, 0L, @@ -1187,10 +1187,10 @@ void ComputePageStringSizes(cudf::detail::hostdevice_vector& pages, // now use the offsets array to set each page's temp_string_buf pointers thrust::transform(rmm::exec_policy_nosync(stream), - pages.d_begin(), - pages.d_end(), + pages.device_begin(), + pages.device_end(), page_string_offsets.begin(), - pages.d_begin(), + pages.device_begin(), page_tform_functor{temp_string_buf.data()}); } } @@ -1198,8 +1198,8 @@ void ComputePageStringSizes(cudf::detail::hostdevice_vector& pages, /** * @copydoc cudf::io::parquet::detail::DecodeStringPageData */ -void __host__ DecodeStringPageData(cudf::detail::hostdevice_vector& pages, - cudf::detail::hostdevice_vector const& chunks, +void __host__ DecodeStringPageData(cudf::detail::hostdevice_span pages, + cudf::detail::hostdevice_span chunks, size_t num_rows, size_t min_row, int level_type_size, diff --git a/cpp/src/io/parquet/parquet_gpu.hpp b/cpp/src/io/parquet/parquet_gpu.hpp index 64e1c199779..86d6ec42c04 100644 --- a/cpp/src/io/parquet/parquet_gpu.hpp +++ b/cpp/src/io/parquet/parquet_gpu.hpp @@ -388,7 +388,6 @@ struct ColumnChunkDesc { level_bits{def_level_bits_, rep_level_bits_}, num_data_pages(0), num_dict_pages(0), - max_num_pages(0), dict_page(nullptr), str_dict_index(nullptr), valid_map_base{nullptr}, @@ -417,7 +416,6 @@ struct ColumnChunkDesc { level_bits[level_type::NUM_LEVEL_TYPES]{}; // bits to encode max definition/repetition levels int32_t num_data_pages{}; // number of data pages int32_t num_dict_pages{}; // number of dictionary pages - int32_t max_num_pages{}; // size of page_info array PageInfo const* dict_page{}; string_index_pair* str_dict_index{}; // index for string dictionary bitmask_type** valid_map_base{}; // base pointers of valid bit map for this column @@ -644,7 +642,7 @@ void BuildStringDictionaryIndex(ColumnChunkDesc* chunks, * @param[in] stream CUDA stream to use * @return Bitwise OR of all page `kernel_mask` values */ -uint32_t GetAggregatedDecodeKernelMask(cudf::detail::hostdevice_vector& pages, +uint32_t GetAggregatedDecodeKernelMask(cudf::detail::hostdevice_span pages, rmm::cuda_stream_view stream); /** @@ -671,8 +669,8 @@ uint32_t GetAggregatedDecodeKernelMask(cudf::detail::hostdevice_vector * @param level_type_size Size in bytes of the type for level decoding * @param stream CUDA stream to use */ -void ComputePageSizes(cudf::detail::hostdevice_vector& pages, - cudf::detail::hostdevice_vector const& chunks, +void ComputePageSizes(cudf::detail::hostdevice_span pages, + cudf::detail::hostdevice_span chunks, size_t min_row, size_t num_rows, bool compute_num_rows, @@ -697,8 +695,8 @@ void ComputePageSizes(cudf::detail::hostdevice_vector& pages, * @param[in] kernel_mask Mask of kernels to run * @param[in] stream CUDA stream to use */ -void ComputePageStringSizes(cudf::detail::hostdevice_vector& pages, - cudf::detail::hostdevice_vector const& chunks, +void ComputePageStringSizes(cudf::detail::hostdevice_span pages, + cudf::detail::hostdevice_span chunks, rmm::device_uvector& temp_string_buf, size_t min_row, size_t num_rows, @@ -720,8 +718,8 @@ void ComputePageStringSizes(cudf::detail::hostdevice_vector& pages, * @param[out] error_code Error code for kernel failures * @param[in] stream CUDA stream to use */ -void DecodePageData(cudf::detail::hostdevice_vector& pages, - cudf::detail::hostdevice_vector const& chunks, +void DecodePageData(cudf::detail::hostdevice_span pages, + cudf::detail::hostdevice_span chunks, size_t num_rows, size_t min_row, int level_type_size, @@ -742,8 +740,8 @@ void DecodePageData(cudf::detail::hostdevice_vector& pages, * @param[out] error_code Error code for kernel failures * @param[in] stream CUDA stream to use */ -void DecodeStringPageData(cudf::detail::hostdevice_vector& pages, - cudf::detail::hostdevice_vector const& chunks, +void DecodeStringPageData(cudf::detail::hostdevice_span pages, + cudf::detail::hostdevice_span chunks, size_t num_rows, size_t min_row, int level_type_size, @@ -764,8 +762,8 @@ void DecodeStringPageData(cudf::detail::hostdevice_vector& pages, * @param[out] error_code Error code for kernel failures * @param[in] stream CUDA stream to use */ -void DecodeDeltaBinary(cudf::detail::hostdevice_vector& pages, - cudf::detail::hostdevice_vector const& chunks, +void DecodeDeltaBinary(cudf::detail::hostdevice_span pages, + cudf::detail::hostdevice_span chunks, size_t num_rows, size_t min_row, int level_type_size, @@ -786,8 +784,8 @@ void DecodeDeltaBinary(cudf::detail::hostdevice_vector& pages, * @param[out] error_code Error code for kernel failures * @param[in] stream CUDA stream to use */ -void DecodeDeltaByteArray(cudf::detail::hostdevice_vector& pages, - cudf::detail::hostdevice_vector const& chunks, +void DecodeDeltaByteArray(cudf::detail::hostdevice_span pages, + cudf::detail::hostdevice_span chunks, size_t num_rows, size_t min_row, int level_type_size, @@ -808,8 +806,8 @@ void DecodeDeltaByteArray(cudf::detail::hostdevice_vector& pages, * @param[out] error_code Error code for kernel failures * @param[in] stream CUDA stream to use */ -void DecodeDeltaLengthByteArray(cudf::detail::hostdevice_vector& pages, - cudf::detail::hostdevice_vector const& chunks, +void DecodeDeltaLengthByteArray(cudf::detail::hostdevice_span pages, + cudf::detail::hostdevice_span chunks, size_t num_rows, size_t min_row, int level_type_size, diff --git a/cpp/src/io/parquet/reader_impl_chunking.cu b/cpp/src/io/parquet/reader_impl_chunking.cu index a7af20f5d7c..b05318d3a91 100644 --- a/cpp/src/io/parquet/reader_impl_chunking.cu +++ b/cpp/src/io/parquet/reader_impl_chunking.cu @@ -21,6 +21,7 @@ #include "reader_impl_chunking.hpp" #include +#include #include #include @@ -32,6 +33,7 @@ #include #include #include +#include #include #include @@ -549,8 +551,64 @@ struct get_page_span { } }; +/** + * @brief Return the span of page indices for a given column index + + */ +struct get_page_span_by_column { + cudf::device_span page_offsets; + + __device__ page_span operator()(size_t i) const + { + return {static_cast(page_offsets[i]), static_cast(page_offsets[i + 1])}; + } +}; + +/** + * @brief Return the size of a span + * + */ struct get_span_size { - __device__ size_t operator()(page_span const& s) const { return s.end - s.start; } + CUDF_HOST_DEVICE size_t operator()(page_span const& s) const { return s.end - s.start; } +}; + +/** + * @brief Return the size of a span in an array of spans, handling out-of-bounds indices. + * + */ +struct get_span_size_by_index { + cudf::device_span page_indices; + + __device__ size_t operator()(size_t i) const + { + return i >= page_indices.size() ? 0 : page_indices[i].end - page_indices[i].start; + } +}; + +/** + * @brief Copy page from appropriate source location (as defined by page_offsets) to the destination + * location, and store the index mapping. + */ +struct copy_subpass_page { + cudf::device_span src_pages; + cudf::device_span dst_pages; + cudf::device_span page_src_index; + cudf::device_span page_offsets; + cudf::device_span page_indices; + + __device__ void operator()(size_t i) const + { + auto const index = + thrust::lower_bound(thrust::seq, page_offsets.begin(), page_offsets.end(), i) - + page_offsets.begin(); + auto const col_index = page_offsets[index] == i ? index : index - 1; + // index within the pages for the column + auto const col_page_index = i - page_offsets[col_index]; + auto const src_page_index = page_indices[col_index].start + col_page_index; + + dst_pages[i] = src_pages[src_page_index]; + page_src_index[i] = src_page_index; + } }; /** @@ -575,7 +633,7 @@ struct get_span_size { * expected memory usage (including scratch space) * */ -std::tuple, size_t, size_t> compute_next_subpass( +std::tuple, size_t, size_t> compute_next_subpass( device_span c_info, device_span pages, device_span page_offsets, @@ -618,9 +676,8 @@ std::tuple, size_t, size_t> compute_next_subpass( size_t const total_pages = thrust::reduce(rmm::exec_policy(stream), page_count_iter, page_count_iter + num_columns); - return {cudf::detail::make_std_vector_sync(page_bounds, stream), - total_pages, - h_aggregated_info[end_index].size_bytes - cumulative_size}; + return { + std::move(page_bounds), total_pages, h_aggregated_info[end_index].size_bytes - cumulative_size}; } std::vector compute_page_splits_by_row(device_span c_info, @@ -674,11 +731,13 @@ std::vector compute_page_splits_by_row(device_span const& chunks, - cudf::detail::hostdevice_vector& pages, + cudf::detail::hostdevice_span chunks, + cudf::detail::hostdevice_span pages, bool dict_pages, rmm::cuda_stream_view stream) { + CUDF_FUNC_RANGE(); + auto for_each_codec_page = [&](Compression codec, std::function const& f) { for (size_t p = 0; p < pages.size(); p++) { if (chunks[pages[p].chunk_idx].codec == codec && @@ -715,8 +774,8 @@ std::vector compute_page_splits_by_row(device_span pages, std::optional expected_row_count, rmm::cuda_stream_view stream) { + CUDF_FUNC_RANGE(); + // sum row counts for all non-dictionary, non-list columns. other columns will be indicated as 0 rmm::device_uvector row_counts(pages.size(), stream); // worst case: num keys == num pages @@ -1221,7 +1282,9 @@ void reader::impl::setup_next_pass(bool uses_custom_row_bounds) // if we are doing subpass reading, generate more accurate num_row estimates for list columns. // this helps us to generate more accurate subpass splits. - if (_input_pass_read_limit != 0) { generate_list_column_row_count_estimates(); } + if (pass.has_compressed_data && _input_pass_read_limit != 0) { + generate_list_column_row_count_estimates(); + } #if defined(PARQUET_CHUNK_LOGGING) printf("Pass: row_groups(%'lu), chunks(%'lu), pages(%'lu)\n", @@ -1266,21 +1329,21 @@ void reader::impl::setup_next_subpass(bool uses_custom_row_bounds) ? min_subpass_size : _input_pass_read_limit - pass.base_mem_size; + // page_indices is an array of spans where each element N is the + // indices into the pass.pages array that represents the subset of pages + // for column N to use for the subpass. auto [page_indices, total_pages, total_expected_size] = - [&]() -> std::tuple, size_t, size_t> { - // special case: if we contain no compressed data, or if we have no input limit, we can always - // just do 1 subpass since what we already have loaded is all the temporary memory we will ever - // use. + [&]() -> std::tuple, size_t, size_t> { if (!pass.has_compressed_data || _input_pass_read_limit == 0) { - std::vector page_indices; - page_indices.reserve(num_columns); + rmm::device_uvector page_indices( + num_columns, _stream, rmm::mr::get_current_device_resource()); auto iter = thrust::make_counting_iterator(0); - std::transform( - iter, iter + num_columns, std::back_inserter(page_indices), [&](size_t i) -> page_span { - return {static_cast(pass.page_offsets[i]), - static_cast(pass.page_offsets[i + 1])}; - }); - return {page_indices, pass.pages.size(), 0}; + thrust::transform(rmm::exec_policy_nosync(_stream), + iter, + iter + num_columns, + page_indices.begin(), + get_page_span_by_column{pass.page_offsets}); + return {std::move(page_indices), pass.pages.size(), size_t{0}}; } // otherwise we have to look forward and choose a batch of pages @@ -1319,37 +1382,50 @@ void reader::impl::setup_next_subpass(bool uses_custom_row_bounds) _stream); }(); - // fill out the subpass struct - subpass.pages = cudf::detail::hostdevice_vector(0, total_pages, _stream); - subpass.page_src_index = - cudf::detail::hostdevice_vector(total_pages, total_pages, _stream); - // copy the appropriate subset of pages from each column - size_t page_count = 0; - for (size_t c_idx = 0; c_idx < num_columns; c_idx++) { - auto const num_column_pages = page_indices[c_idx].end - page_indices[c_idx].start; - subpass.column_page_count.push_back(num_column_pages); - std::copy(pass.pages.begin() + page_indices[c_idx].start, - pass.pages.begin() + page_indices[c_idx].end, - std::back_inserter(subpass.pages)); - - // mapping back to original pages in the pass - thrust::sequence(thrust::host, - subpass.page_src_index.begin() + page_count, - subpass.page_src_index.begin() + page_count + num_column_pages, - page_indices[c_idx].start); - page_count += num_column_pages; + // check to see if we are processing the entire pass (enabling us to skip a bunch of work) + subpass.single_subpass = total_pages == pass.pages.size(); + + // in the single pass case, no page copying is necessary - just use what's in the pass itself + if (subpass.single_subpass) { + subpass.pages = pass.pages; + } + // copy the appropriate subset of pages from each column and store the mapping back to the source + // (pass) pages + else { + subpass.page_buf = cudf::detail::hostdevice_vector(total_pages, total_pages, _stream); + subpass.page_src_index = rmm::device_uvector(total_pages, _stream); + auto iter = thrust::make_counting_iterator(0); + rmm::device_uvector dst_offsets(num_columns + 1, _stream); + thrust::transform_exclusive_scan(rmm::exec_policy_nosync(_stream), + iter, + iter + num_columns + 1, + dst_offsets.begin(), + get_span_size_by_index{page_indices}, + 0, + thrust::plus{}); + thrust::for_each( + rmm::exec_policy_nosync(_stream), + iter, + iter + total_pages, + copy_subpass_page{ + pass.pages, subpass.page_buf, subpass.page_src_index, dst_offsets, page_indices}); + subpass.pages = subpass.page_buf; } - // print_hostdevice_vector(subpass.page_src_index); + + std::vector h_spans = cudf::detail::make_std_vector_async(page_indices, _stream); + subpass.pages.device_to_host_async(_stream); + + _stream.synchronize(); + + subpass.column_page_count = std::vector(num_columns); + std::transform( + h_spans.begin(), h_spans.end(), subpass.column_page_count.begin(), get_span_size{}); // decompress the data for the pages in this subpass. if (pass.has_compressed_data) { subpass.decomp_page_data = decompress_page_data(pass.chunks, subpass.pages, false, _stream); } - subpass.pages.host_to_device_async(_stream); - subpass.page_src_index.host_to_device_async(_stream); - _stream.synchronize(); - // buffers needed by the decode kernels { // nesting information (sizes, etc) stored -per page- @@ -1541,7 +1617,7 @@ void reader::impl::compute_output_chunks_for_subpass() // generate row_indices and cumulative output sizes for all pages rmm::device_uvector c_info(subpass.pages.size(), _stream); auto page_input = - thrust::make_transform_iterator(subpass.pages.d_begin(), get_page_output_size{}); + thrust::make_transform_iterator(subpass.pages.device_begin(), get_page_output_size{}); auto page_keys = make_page_key_iterator(subpass.pages); thrust::inclusive_scan_by_key(rmm::exec_policy_nosync(_stream), page_keys, diff --git a/cpp/src/io/parquet/reader_impl_chunking.hpp b/cpp/src/io/parquet/reader_impl_chunking.hpp index a9cf0e94ec8..b959c793011 100644 --- a/cpp/src/io/parquet/reader_impl_chunking.hpp +++ b/cpp/src/io/parquet/reader_impl_chunking.hpp @@ -69,9 +69,17 @@ struct subpass_intermediate_data { rmm::device_buffer decomp_page_data; rmm::device_buffer level_decode_data{}; - cudf::detail::hostdevice_vector pages{}; + cudf::detail::hostdevice_span pages{}; + + // optimization. if the single_subpass flag is set, it means we will only be doing + // one subpass for the entire pass. this allows us to skip various pieces of work + // during processing. notably, page_buf will not be allocated to hold a compacted + // copy of the pages specific to the subpass. + bool single_subpass{false}; + cudf::detail::hostdevice_vector page_buf{}; + // for each page in the subpass, the index of our source page in the pass - cudf::detail::hostdevice_vector page_src_index{}; + rmm::device_uvector page_src_index{0, cudf::get_default_stream()}; // for each column in the file (indexed by _input_columns.size()) // the number of associated pages for this subpass std::vector column_page_count; @@ -111,10 +119,10 @@ struct pass_intermediate_data { // 1 1 1 1 1 2 2 2 // // page_offsets would be 0, 5, 8 - cudf::detail::hostdevice_vector page_offsets{}; + rmm::device_uvector page_offsets{0, cudf::get_default_stream()}; - rmm::device_buffer decomp_dict_data{0, rmm::cuda_stream_default}; - rmm::device_uvector str_dict_index{0, rmm::cuda_stream_default}; + rmm::device_buffer decomp_dict_data{0, cudf::get_default_stream()}; + rmm::device_uvector str_dict_index{0, cudf::get_default_stream()}; int level_type_size{0}; diff --git a/cpp/src/io/parquet/reader_impl_preprocess.cu b/cpp/src/io/parquet/reader_impl_preprocess.cu index 48ff32038b3..c524547c4d7 100644 --- a/cpp/src/io/parquet/reader_impl_preprocess.cu +++ b/cpp/src/io/parquet/reader_impl_preprocess.cu @@ -18,6 +18,7 @@ #include "reader_impl.hpp" #include +#include #include #include @@ -37,6 +38,7 @@ #include #include #include +#include #include #include @@ -350,6 +352,7 @@ std::string encoding_to_string(Encoding encoding) } return result; } + /** * @brief Create a readable string for the user that will list out all unsupported encodings found. * @@ -368,6 +371,73 @@ std::string encoding_to_string(Encoding encoding) return encoding_bitmask_to_str(unsupported); } +/** + * @brief Sort pages in chunk/schema order + * + * @param unsorted_pages The unsorted pages + * @param chunks The chunks associated with the pages + * @param stream CUDA stream used for device memory operations and kernel launches + * @returns The sorted vector of pages + */ +cudf::detail::hostdevice_vector sort_pages(device_span unsorted_pages, + device_span chunks, + rmm::cuda_stream_view stream) +{ + CUDF_FUNC_RANGE(); + + // sort the pages in chunk/schema order. we use chunk.src_col_index instead of + // chunk.src_col_schema because the user may have reordered them (reading columns, "a" and "b" but + // returning them as "b" and "a") + // + // ordering of pages is by input column schema, repeated across row groups. so + // if we had 3 columns, each with 2 pages, and 1 row group, our schema values might look like + // + // 1, 1, 2, 2, 3, 3 + // + // However, if we had more than one row group, the pattern would be + // + // 1, 1, 2, 2, 3, 3, 1, 1, 2, 2, 3, 3 + // ^ row group 0 | + // ^ row group 1 + // + // To process pages by key (exclusive_scan_by_key, reduce_by_key, etc), the ordering we actually + // want is + // + // 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3 + // + // We also need to preserve key-relative page ordering, so we need to use a stable sort. + rmm::device_uvector page_keys{unsorted_pages.size(), stream}; + thrust::transform( + rmm::exec_policy_nosync(stream), + unsorted_pages.begin(), + unsorted_pages.end(), + page_keys.begin(), + cuda::proclaim_return_type([chunks = chunks.begin()] __device__(PageInfo const& page) { + return chunks[page.chunk_idx].src_col_index; + })); + // we are doing this by sorting indices first and then transforming the output because nvcc + // started generating kernels using too much shared memory when trying to sort the pages + // directly. + rmm::device_uvector sort_indices(unsorted_pages.size(), stream); + thrust::sequence(rmm::exec_policy_nosync(stream), sort_indices.begin(), sort_indices.end(), 0); + thrust::stable_sort_by_key(rmm::exec_policy_nosync(stream), + page_keys.begin(), + page_keys.end(), + sort_indices.begin(), + thrust::less()); + auto pass_pages = + cudf::detail::hostdevice_vector(unsorted_pages.size(), unsorted_pages.size(), stream); + thrust::transform( + rmm::exec_policy_nosync(stream), + sort_indices.begin(), + sort_indices.end(), + pass_pages.d_begin(), + cuda::proclaim_return_type([unsorted_pages = unsorted_pages.begin()] __device__( + int32_t i) { return unsorted_pages[i]; })); + stream.synchronize(); + return pass_pages; +} + /** * @brief Decode the page information for a given pass. * @@ -377,21 +447,35 @@ void decode_page_headers(pass_intermediate_data& pass, device_span unsorted_pages, rmm::cuda_stream_view stream) { - cudf::detail::hostdevice_vector chunk_page_info(pass.chunks.size(), stream); - - // IMPORTANT : if you change how pages are stored within a chunk (dist pages, then data pages), - // please update preprocess_nested_columns to reflect this. - for (size_t c = 0, page_count = 0; c < pass.chunks.size(); c++) { - pass.chunks[c].max_num_pages = pass.chunks[c].num_data_pages + pass.chunks[c].num_dict_pages; - chunk_page_info[c].pages = &unsorted_pages[page_count]; - page_count += pass.chunks[c].max_num_pages; - } + CUDF_FUNC_RANGE(); + + auto iter = thrust::make_counting_iterator(0); + rmm::device_uvector chunk_page_counts(pass.chunks.size() + 1, stream); + thrust::transform_exclusive_scan( + rmm::exec_policy_nosync(stream), + iter, + iter + pass.chunks.size() + 1, + chunk_page_counts.begin(), + cuda::proclaim_return_type( + [chunks = pass.chunks.d_begin(), num_chunks = pass.chunks.size()] __device__(size_t i) { + return static_cast( + i >= num_chunks ? 0 : chunks[i].num_data_pages + chunks[i].num_dict_pages); + }), + 0, + thrust::plus{}); + rmm::device_uvector d_chunk_page_info(pass.chunks.size(), stream); + thrust::for_each(rmm::exec_policy_nosync(stream), + iter, + iter + pass.chunks.size(), + [cpi = d_chunk_page_info.begin(), + chunk_page_counts = chunk_page_counts.begin(), + unsorted_pages = unsorted_pages.begin()] __device__(size_t i) { + cpi[i].pages = &unsorted_pages[chunk_page_counts[i]]; + }); kernel_error error_code(stream); - pass.chunks.host_to_device_async(stream); - chunk_page_info.host_to_device_async(stream); - DecodePageHeaders(pass.chunks.device_ptr(), - chunk_page_info.device_ptr(), + DecodePageHeaders(pass.chunks.d_begin(), + d_chunk_page_info.begin(), pass.chunks.size(), error_code.data(), stream); @@ -421,56 +505,8 @@ void decode_page_headers(pass_intermediate_data& pass, thrust::maximum()); pass.level_type_size = std::max(1, cudf::util::div_rounding_up_safe(max_level_bits, 8)); - // sort the pages in chunk/schema order. we use chunk.src_col_index instead of - // chunk.src_col_schema because the user may have reordered them (reading columns, "a" and "b" but - // returning them as "b" and "a") - // - // ordering of pages is by input column schema, repeated across row groups. so - // if we had 3 columns, each with 2 pages, and 1 row group, our schema values might look like - // - // 1, 1, 2, 2, 3, 3 - // - // However, if we had more than one row group, the pattern would be - // - // 1, 1, 2, 2, 3, 3, 1, 1, 2, 2, 3, 3 - // ^ row group 0 | - // ^ row group 1 - // - // To process pages by key (exclusive_scan_by_key, reduce_by_key, etc), the ordering we actually - // want is - // - // 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3 - // - // We also need to preserve key-relative page ordering, so we need to use a stable sort. - { - rmm::device_uvector page_keys{unsorted_pages.size(), stream}; - thrust::transform(rmm::exec_policy_nosync(stream), - unsorted_pages.begin(), - unsorted_pages.end(), - page_keys.begin(), - [chunks = pass.chunks.d_begin()] __device__(PageInfo const& page) { - return chunks[page.chunk_idx].src_col_index; - }); - // we are doing this by sorting indices first and then transforming the output because nvcc - // started generating kernels using too much shared memory when trying to sort the pages - // directly. - rmm::device_uvector sort_indices(unsorted_pages.size(), stream); - thrust::sequence(rmm::exec_policy_nosync(stream), sort_indices.begin(), sort_indices.end(), 0); - thrust::stable_sort_by_key(rmm::exec_policy_nosync(stream), - page_keys.begin(), - page_keys.end(), - sort_indices.begin(), - thrust::less()); - pass.pages = cudf::detail::hostdevice_vector( - unsorted_pages.size(), unsorted_pages.size(), stream); - thrust::transform(rmm::exec_policy_nosync(stream), - sort_indices.begin(), - sort_indices.end(), - pass.pages.d_begin(), - [unsorted_pages = unsorted_pages.begin()] __device__(int32_t i) { - return unsorted_pages[i]; - }); - } + // sort the pages in chunk/schema order. + pass.pages = sort_pages(unsorted_pages, pass.chunks, stream); // compute offsets to each group of input pages. // page_keys: 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3 @@ -486,11 +522,11 @@ void decode_page_headers(pass_intermediate_data& pass, page_counts.begin()) .second; auto const num_page_counts = page_counts_end - page_counts.begin(); - pass.page_offsets = cudf::detail::hostdevice_vector(num_page_counts + 1, stream); + pass.page_offsets = rmm::device_uvector(num_page_counts + 1, stream); thrust::exclusive_scan(rmm::exec_policy_nosync(stream), page_counts.begin(), page_counts.begin() + num_page_counts + 1, - pass.page_offsets.d_begin()); + pass.page_offsets.begin()); // setup dict_page for each chunk if necessary thrust::for_each(rmm::exec_policy_nosync(stream), @@ -502,7 +538,6 @@ void decode_page_headers(pass_intermediate_data& pass, } }); - pass.page_offsets.device_to_host_async(stream); pass.pages.device_to_host_async(stream); pass.chunks.device_to_host_async(stream); stream.synchronize(); @@ -589,6 +624,8 @@ struct set_final_row_count { void reader::impl::build_string_dict_indices() { + CUDF_FUNC_RANGE(); + auto& pass = *_pass_itm_data; // compute number of indices per chunk and a summed total @@ -1229,12 +1266,16 @@ void reader::impl::preprocess_subpass_pages(bool uses_custom_row_bounds, size_t _stream); } - // copy our now-correct row counts back to the base pages stored in the pass. auto iter = thrust::make_counting_iterator(0); - thrust::for_each(rmm::exec_policy_nosync(_stream), - iter, - iter + subpass.pages.size(), - update_pass_num_rows{pass.pages, subpass.pages, subpass.page_src_index}); + + // copy our now-correct row counts back to the base pages stored in the pass. + // only need to do this if we are not processing the whole pass in one subpass + if (!subpass.single_subpass) { + thrust::for_each(rmm::exec_policy_nosync(_stream), + iter, + iter + subpass.pages.size(), + update_pass_num_rows{pass.pages, subpass.pages, subpass.page_src_index}); + } // computes: // PageInfo::chunk_row (the chunk-relative row index) for all pages in the pass. The start_row @@ -1250,14 +1291,17 @@ void reader::impl::preprocess_subpass_pages(bool uses_custom_row_bounds, size_t chunk_row_output_iter{pass.pages.device_ptr()}); // copy chunk row into the subpass pages - thrust::for_each(rmm::exec_policy_nosync(_stream), - iter, - iter + subpass.pages.size(), - update_subpass_chunk_row{pass.pages, subpass.pages, subpass.page_src_index}); + // only need to do this if we are not processing the whole pass in one subpass + if (!subpass.single_subpass) { + thrust::for_each(rmm::exec_policy_nosync(_stream), + iter, + iter + subpass.pages.size(), + update_subpass_chunk_row{pass.pages, subpass.pages, subpass.page_src_index}); + } // retrieve pages back pass.pages.device_to_host_async(_stream); - subpass.pages.device_to_host_async(_stream); + if (!subpass.single_subpass) { subpass.pages.device_to_host_async(_stream); } _stream.synchronize(); // at this point we have an accurate row count so we can compute how many rows we will actually be @@ -1382,7 +1426,7 @@ void reader::impl::allocate_columns(size_t skip_rows, size_t num_rows, bool uses thrust::make_counting_iterator(num_keys), size_input.begin(), get_page_nesting_size{ - d_cols_info.data(), max_depth, subpass.pages.size(), subpass.pages.d_begin()}); + d_cols_info.data(), max_depth, subpass.pages.size(), subpass.pages.device_begin()}); auto const reduction_keys = cudf::detail::make_counting_transform_iterator(0, get_reduction_key{subpass.pages.size()}); cudf::detail::hostdevice_vector sizes{_input_columns.size() * max_depth, _stream}; @@ -1402,7 +1446,7 @@ void reader::impl::allocate_columns(size_t skip_rows, size_t num_rows, bool uses reduction_keys + num_keys, size_input.cbegin(), start_offset_output_iterator{ - subpass.pages.d_begin(), 0, d_cols_info.data(), max_depth, subpass.pages.size()}); + subpass.pages.device_begin(), 0, d_cols_info.data(), max_depth, subpass.pages.size()}); sizes.device_to_host_sync(_stream); for (size_type idx = 0; idx < static_cast(_input_columns.size()); idx++) { @@ -1442,7 +1486,7 @@ std::vector reader::impl::calculate_page_string_offsets() rmm::device_uvector d_col_sizes(col_sizes.size(), _stream); // use page_index to fetch page string sizes in the proper order - auto val_iter = thrust::make_transform_iterator(subpass.pages.d_begin(), + auto val_iter = thrust::make_transform_iterator(subpass.pages.device_begin(), page_to_string_size{pass.chunks.d_begin()}); // do scan by key to calculate string offsets for each page diff --git a/cpp/src/io/utilities/hostdevice_span.hpp b/cpp/src/io/utilities/hostdevice_span.hpp index 539e8e84e59..ec5e0410bc0 100644 --- a/cpp/src/io/utilities/hostdevice_span.hpp +++ b/cpp/src/io/utilities/hostdevice_span.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,7 +33,30 @@ class hostdevice_span { hostdevice_span(hostdevice_span&&) = default; ///< Move constructor hostdevice_span(T* cpu_data, T* gpu_data, size_t size) - : _size(size), _host_data(cpu_data), _device_data(gpu_data) + : _size(size), _device_data(gpu_data), _host_data(cpu_data) + { + } + + /// Constructor from container + /// @param in The container to construct the span from + template ().host_ptr())> (*)[], + T (*)[]>>* = nullptr> + constexpr hostdevice_span(C& in) : hostdevice_span(in.host_ptr(), in.device_ptr(), in.size()) + { + } + + /// Constructor from const container + /// @param in The container to construct the span from + template ().host_ptr())> (*)[], + T (*)[]>>* = nullptr> + constexpr hostdevice_span(C const& in) + : hostdevice_span(in.host_ptr(), in.device_ptr(), in.size()) { } @@ -50,10 +73,15 @@ class hostdevice_span { * @tparam T The device span type. * @return A typed device span of the hostdevice view's data. */ - [[nodiscard]] operator cudf::device_span() const - { - return cudf::device_span(_device_data, size()); - } + [[nodiscard]] operator cudf::device_span() { return {_device_data, size()}; } + + /** + * @brief Converts a hostdevice view into a device span of const data. + * + * @tparam T The device span type. + * @return A const typed device span of the hostdevice view's data. + */ + [[nodiscard]] operator cudf::device_span() const { return {_device_data, size()}; } /** * @brief Returns the underlying device data. From 200fc0b35216c01235103e491d5217b932670ebc Mon Sep 17 00:00:00 2001 From: Yunsong Wang Date: Thu, 29 Feb 2024 13:25:35 -0800 Subject: [PATCH 322/384] Use cuco::static_set in the hash-based groupby (#14813) Depends on https://github.com/rapidsai/cudf/pull/14849 Contributes to #12261 This PR migrates hash groupby to use the new `cuco::static_set` data structure. It doesn't change any existing libcudf behavior but uncovers the fact that the cudf python `value_counts` doesn't guarantee output orders thus the PR becomes a breaking change. Authors: - Yunsong Wang (https://github.com/PointKernel) Approvers: - David Wendt (https://github.com/davidwendt) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/14813 --- cpp/benchmarks/groupby/group_max.cpp | 7 +- cpp/benchmarks/groupby/group_struct_keys.cpp | 9 +- cpp/include/cudf/detail/cuco_helpers.hpp | 5 + cpp/src/groupby/hash/groupby.cu | 123 ++++++++---------- cpp/src/groupby/hash/groupby_kernels.cuh | 47 +++---- cpp/src/groupby/hash/multi_pass_kernels.cuh | 13 +- .../source/user_guide/pandas-comparison.md | 2 +- python/cudf/cudf/core/dataframe.py | 4 +- python/cudf/cudf/core/groupby/groupby.py | 28 ++-- python/cudf/cudf/tests/test_groupby.py | 16 ++- 10 files changed, 125 insertions(+), 129 deletions(-) diff --git a/cpp/benchmarks/groupby/group_max.cpp b/cpp/benchmarks/groupby/group_max.cpp index e65c37f001d..b7b330f02e5 100644 --- a/cpp/benchmarks/groupby/group_max.cpp +++ b/cpp/benchmarks/groupby/group_max.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include +#include #include @@ -50,9 +51,13 @@ void bench_groupby_max(nvbench::state& state, nvbench::type_list) requests[0].values = vals->view(); requests[0].aggregations.push_back(cudf::make_max_aggregation()); + auto const mem_stats_logger = cudf::memory_stats_logger(); state.set_cuda_stream(nvbench::make_cuda_stream_view(cudf::get_default_stream().value())); state.exec(nvbench::exec_tag::sync, [&](nvbench::launch& launch) { auto const result = gb_obj.aggregate(requests); }); + + state.add_buffer_size( + mem_stats_logger.peak_memory_usage(), "peak_memory_usage", "peak_memory_usage"); } NVBENCH_BENCH_TYPES(bench_groupby_max, diff --git a/cpp/benchmarks/groupby/group_struct_keys.cpp b/cpp/benchmarks/groupby/group_struct_keys.cpp index 44a12c1c30e..cadd9c2d137 100644 --- a/cpp/benchmarks/groupby/group_struct_keys.cpp +++ b/cpp/benchmarks/groupby/group_struct_keys.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ */ #include +#include #include @@ -80,11 +81,15 @@ void bench_groupby_struct_keys(nvbench::state& state) requests[0].aggregations.push_back(cudf::make_min_aggregation()); // Set up nvbench default stream - auto stream = cudf::get_default_stream(); + auto const mem_stats_logger = cudf::memory_stats_logger(); + auto stream = cudf::get_default_stream(); state.set_cuda_stream(nvbench::make_cuda_stream_view(stream.value())); state.exec(nvbench::exec_tag::sync, [&](nvbench::launch& launch) { auto const result = gb_obj.aggregate(requests); }); + + state.add_buffer_size( + mem_stats_logger.peak_memory_usage(), "peak_memory_usage", "peak_memory_usage"); } NVBENCH_BENCH(bench_groupby_struct_keys) diff --git a/cpp/include/cudf/detail/cuco_helpers.hpp b/cpp/include/cudf/detail/cuco_helpers.hpp index 506f6475637..dca5a39bece 100644 --- a/cpp/include/cudf/detail/cuco_helpers.hpp +++ b/cpp/include/cudf/detail/cuco_helpers.hpp @@ -16,11 +16,16 @@ #pragma once +#include + #include #include namespace cudf::detail { +/// Sentinel value for `cudf::size_type` +static cudf::size_type constexpr CUDF_SIZE_TYPE_SENTINEL = -1; + /// Default load factor for cuco data structures static double constexpr CUCO_DESIRED_LOAD_FACTOR = 0.5; diff --git a/cpp/src/groupby/hash/groupby.cu b/cpp/src/groupby/hash/groupby.cu index 7b85dd02c10..acc1b087510 100644 --- a/cpp/src/groupby/hash/groupby.cu +++ b/cpp/src/groupby/hash/groupby.cu @@ -22,23 +22,19 @@ #include #include #include -#include #include #include #include #include +#include #include #include #include -#include #include -#include -#include #include #include #include #include -#include #include #include #include @@ -49,12 +45,9 @@ #include -#include -#include -#include +#include #include #include -#include #include #include @@ -66,15 +59,12 @@ namespace detail { namespace hash { namespace { -// TODO: replace it with `cuco::static_map` -// https://github.com/rapidsai/cudf/issues/10401 -template -using map_type = concurrent_unordered_map< - cudf::size_type, - cudf::size_type, +// TODO: similar to `contains_table`, using larger CG size like 2 or 4 for nested +// types and `cg_size = 1`for flat data to improve performance +using probing_scheme_type = cuco::linear_probing< + 1, ///< Number of threads used to handle each input key cudf::experimental::row::hash::device_row_hasher, - ComparatorType>; + cudf::nullate::DYNAMIC>>; /** * @brief List of aggregation operations that can be computed with a hash-based @@ -190,14 +180,14 @@ class groupby_simple_aggregations_collector final } }; -template +template class hash_compound_agg_finalizer final : public cudf::detail::aggregation_finalizer { column_view col; data_type result_type; cudf::detail::result_cache* sparse_results; cudf::detail::result_cache* dense_results; device_span gather_map; - map_type const& map; + SetType set; bitmask_type const* __restrict__ row_bitmask; rmm::cuda_stream_view stream; rmm::mr::device_memory_resource* mr; @@ -209,7 +199,7 @@ class hash_compound_agg_finalizer final : public cudf::detail::aggregation_final cudf::detail::result_cache* sparse_results, cudf::detail::result_cache* dense_results, device_span gather_map, - map_type const& map, + SetType set, bitmask_type const* row_bitmask, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) @@ -217,7 +207,7 @@ class hash_compound_agg_finalizer final : public cudf::detail::aggregation_final sparse_results(sparse_results), dense_results(dense_results), gather_map(gather_map), - map(map), + set(set), row_bitmask(row_bitmask), stream(stream), mr(mr) @@ -340,8 +330,8 @@ class hash_compound_agg_finalizer final : public cudf::detail::aggregation_final rmm::exec_policy(stream), thrust::make_counting_iterator(0), col.size(), - ::cudf::detail::var_hash_functor>{ - map, row_bitmask, *var_result_view, *values_view, *sum_view, *count_view, agg._ddof}); + ::cudf::detail::var_hash_functor{ + set, row_bitmask, *var_result_view, *values_view, *sum_view, *count_view, agg._ddof}); sparse_results->add_result(col, agg, std::move(var_result)); dense_results->add_result(col, agg, to_dense_agg_result(agg)); } @@ -398,13 +388,13 @@ flatten_single_pass_aggs(host_span requests) * * @see groupby_null_templated() */ -template +template void sparse_to_dense_results(table_view const& keys, host_span requests, cudf::detail::result_cache* sparse_results, cudf::detail::result_cache* dense_results, device_span gather_map, - map_type const& map, + SetType set, bool keys_have_nulls, null_policy include_null_keys, rmm::cuda_stream_view stream, @@ -423,7 +413,7 @@ void sparse_to_dense_results(table_view const& keys, // Given an aggregation, this will get the result from sparse_results and // convert and return dense, compacted result auto finalizer = hash_compound_agg_finalizer( - col, sparse_results, dense_results, gather_map, map, row_bitmask_ptr, stream, mr); + col, sparse_results, dense_results, gather_map, set, row_bitmask_ptr, stream, mr); for (auto&& agg : agg_v) { agg->finalize(finalizer); } @@ -467,11 +457,11 @@ auto create_sparse_results_table(table_view const& flattened_values, * @brief Computes all aggregations from `requests` that require a single pass * over the data and stores the results in `sparse_results` */ -template +template void compute_single_pass_aggs(table_view const& keys, host_span requests, cudf::detail::result_cache* sparse_results, - map_type& map, + SetType set, bool keys_have_nulls, null_policy include_null_keys, rmm::cuda_stream_view stream) @@ -494,16 +484,16 @@ void compute_single_pass_aggs(table_view const& keys, ? cudf::detail::bitmask_and(keys, stream, rmm::mr::get_current_device_resource()).first : rmm::device_buffer{}; - thrust::for_each_n(rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - keys.num_rows(), - hash::compute_single_pass_aggs_fn>{ - map, - *d_values, - *d_sparse_table, - d_aggs.data(), - static_cast(row_bitmask.data()), - skip_key_rows_with_nulls}); + thrust::for_each_n( + rmm::exec_policy(stream), + thrust::make_counting_iterator(0), + keys.num_rows(), + hash::compute_single_pass_aggs_fn{set, + *d_values, + *d_sparse_table, + d_aggs.data(), + static_cast(row_bitmask.data()), + skip_key_rows_with_nulls}); // Add results back to sparse_results cache auto sparse_result_cols = sparse_table.release(); for (size_t i = 0; i < aggs.size(); i++) { @@ -517,23 +507,15 @@ void compute_single_pass_aggs(table_view const& keys, * @brief Computes and returns a device vector containing all populated keys in * `map`. */ -template -rmm::device_uvector extract_populated_keys(map_type const& map, +template +rmm::device_uvector extract_populated_keys(SetType const& key_set, size_type num_keys, rmm::cuda_stream_view stream) { rmm::device_uvector populated_keys(num_keys, stream); + auto const keys_end = key_set.retrieve_all(populated_keys.begin(), stream.value()); - auto const get_key = cuda::proclaim_return_type::key_type>( - [] __device__(auto const& element) { return element.first; }); // first = key - auto const key_used = [unused = map.get_unused_key()] __device__(auto key) { - return key != unused; - }; - auto const key_itr = thrust::make_transform_iterator(map.data(), get_key); - auto const end_it = cudf::detail::copy_if_safe( - key_itr, key_itr + map.capacity(), populated_keys.begin(), key_used, stream); - - populated_keys.resize(std::distance(populated_keys.begin(), end_it), stream); + populated_keys.resize(std::distance(populated_keys.begin(), keys_end), stream); return populated_keys; } @@ -580,30 +562,33 @@ std::unique_ptr
groupby(table_view const& keys, auto const row_hash = cudf::experimental::row::hash::row_hasher{std::move(preprocessed_keys)}; auto const d_row_hash = row_hash.device_hasher(has_null); - size_type constexpr unused_key{std::numeric_limits::max()}; - size_type constexpr unused_value{std::numeric_limits::max()}; - // Cache of sparse results where the location of aggregate value in each - // column is indexed by the hash map + // column is indexed by the hash set cudf::detail::result_cache sparse_results(requests.size()); auto const comparator_helper = [&](auto const d_key_equal) { - using allocator_type = typename map_type::allocator_type; - - auto const map = map_type::create(compute_hash_table_size(num_keys), - stream, - unused_key, - unused_value, - d_row_hash, - d_key_equal, - allocator_type()); - // Compute all single pass aggs first - compute_single_pass_aggs( - keys, requests, &sparse_results, *map, keys_have_nulls, include_null_keys, stream); + auto const set = cuco::static_set{num_keys, + 0.5, // desired load factor + cuco::empty_key{cudf::detail::CUDF_SIZE_TYPE_SENTINEL}, + d_key_equal, + probing_scheme_type{d_row_hash}, + cuco::thread_scope_device, + cuco::storage<1>{}, + cudf::detail::cuco_allocator{stream}, + stream.value()}; - // Extract the populated indices from the hash map and create a gather map. + // Compute all single pass aggs first + compute_single_pass_aggs(keys, + requests, + &sparse_results, + set.ref(cuco::insert_and_find), + keys_have_nulls, + include_null_keys, + stream); + + // Extract the populated indices from the hash set and create a gather map. // Gathering using this map from sparse results will give dense results. - auto gather_map = extract_populated_keys(*map, keys.num_rows(), stream); + auto gather_map = extract_populated_keys(set, keys.num_rows(), stream); // Compact all results from sparse_results and insert into cache sparse_to_dense_results(keys, @@ -611,7 +596,7 @@ std::unique_ptr
groupby(table_view const& keys, &sparse_results, cache, gather_map, - *map, + set.ref(cuco::find), keys_have_nulls, include_null_keys, stream, diff --git a/cpp/src/groupby/hash/groupby_kernels.cuh b/cpp/src/groupby/hash/groupby_kernels.cuh index 4dfb191480b..9abfe22950a 100644 --- a/cpp/src/groupby/hash/groupby_kernels.cuh +++ b/cpp/src/groupby/hash/groupby_kernels.cuh @@ -30,42 +30,34 @@ namespace groupby { namespace detail { namespace hash { /** - * @brief Compute single-pass aggregations and store results into a sparse - * `output_values` table, and populate `map` with indices of unique keys + * @brief Computes single-pass aggregations and store results into a sparse `output_values` table, + * and populate `set` with indices of unique keys * - * The hash map is built by inserting every row `i` from the `keys` and - * `values` tables as a single (key,value) pair. When the pair is inserted, if - * the key was not already present in the map, then the corresponding value is - * simply copied to the output. If the key was already present in the map, - * then the inserted `values` row is aggregated with the existing row. This - * aggregation is done for every element `j` in the row by applying aggregation - * operation `j` between the new and existing element. + * The hash set is built by inserting every row index `i` from the `keys` and `values` tables. If + * the index was not present in the set, insert they index and then copy it to the output. If the + * key was already present in the set, then the inserted index is aggregated with the existing row. + * This aggregation is done for every element `j` in the row by applying aggregation operation `j` + * between the new and existing element. * * Instead of storing the entire rows from `input_keys` and `input_values` in - * the hashmap, we instead store the row indices. For example, when inserting - * row at index `i` from `input_keys` into the hash map, the value `i` is what - * gets stored for the hash map's "key". It is assumed the `map` was constructed + * the hashset, we instead store the row indices. For example, when inserting + * row at index `i` from `input_keys` into the hash set, the value `i` is what + * gets stored for the hash set's "key". It is assumed the `set` was constructed * with a custom comparator that uses these row indices to check for equality * between key rows. For example, comparing two keys `k0` and `k1` will compare * the two rows `input_keys[k0] ?= input_keys[k1]` * - * Likewise, we store the row indices for the hash maps "values". These indices - * index into the `output_values` table. For a given key `k` (which is an index - * into `input_keys`), the corresponding value `v` indexes into `output_values` - * and stores the result of aggregating rows from `input_values` from rows of - * `input_keys` equivalent to the row at `k`. - * * The exact size of the result is not known a priori, but can be upper bounded * by the number of rows in `input_keys` & `input_values`. Therefore, it is * assumed `output_values` has sufficient storage for an equivalent number of * rows. In this way, after all rows are aggregated, `output_values` will likely * be "sparse", meaning that not all rows contain the result of an aggregation. * - * @tparam Map The type of the hash map + * @tparam SetType The type of the hash set device ref */ -template +template struct compute_single_pass_aggs_fn { - Map map; + SetType set; table_device_view input_values; mutable_table_device_view output_values; aggregation::Kind const* __restrict__ aggs; @@ -75,9 +67,9 @@ struct compute_single_pass_aggs_fn { /** * @brief Construct a new compute_single_pass_aggs_fn functor object * - * @param map Hash map object to insert key,value pairs into. + * @param set_ref Hash set object to insert key,value pairs into. * @param input_values The table whose rows will be aggregated in the values - * of the hash map + * of the hash set * @param output_values Table that stores the results of aggregating rows of * `input_values`. * @param aggs The set of aggregation operations to perform across the @@ -88,13 +80,13 @@ struct compute_single_pass_aggs_fn { * null values should be skipped. It `true`, it is assumed `row_bitmask` is a * bitmask where bit `i` indicates the presence of a null value in row `i`. */ - compute_single_pass_aggs_fn(Map map, + compute_single_pass_aggs_fn(SetType set, table_device_view input_values, mutable_table_device_view output_values, aggregation::Kind const* aggs, bitmask_type const* row_bitmask, bool skip_rows_with_nulls) - : map(map), + : set(set), input_values(input_values), output_values(output_values), aggs(aggs), @@ -106,10 +98,9 @@ struct compute_single_pass_aggs_fn { __device__ void operator()(size_type i) { if (not skip_rows_with_nulls or cudf::bit_is_set(row_bitmask, i)) { - auto result = map.insert(thrust::make_pair(i, i)); + auto const result = set.insert_and_find(i); - cudf::detail::aggregate_row( - output_values, result.first->second, input_values, i, aggs); + cudf::detail::aggregate_row(output_values, *result.first, input_values, i, aggs); } } }; diff --git a/cpp/src/groupby/hash/multi_pass_kernels.cuh b/cpp/src/groupby/hash/multi_pass_kernels.cuh index 4bc73631732..7043eafdc10 100644 --- a/cpp/src/groupby/hash/multi_pass_kernels.cuh +++ b/cpp/src/groupby/hash/multi_pass_kernels.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,23 +31,23 @@ namespace cudf { namespace detail { -template +template struct var_hash_functor { - Map const map; + SetType set; bitmask_type const* __restrict__ row_bitmask; mutable_column_device_view target; column_device_view source; column_device_view sum; column_device_view count; size_type ddof; - var_hash_functor(Map const map, + var_hash_functor(SetType set, bitmask_type const* row_bitmask, mutable_column_device_view target, column_device_view source, column_device_view sum, column_device_view count, size_type ddof) - : map(map), + : set(set), row_bitmask(row_bitmask), target(target), source(source), @@ -96,8 +96,7 @@ struct var_hash_functor { __device__ inline void operator()(size_type source_index) { if (row_bitmask == nullptr or cudf::bit_is_set(row_bitmask, source_index)) { - auto result = map.find(source_index); - auto target_index = result->second; + auto const target_index = *set.find(source_index); auto col = source; auto source_type = source.type(); diff --git a/docs/cudf/source/user_guide/pandas-comparison.md b/docs/cudf/source/user_guide/pandas-comparison.md index 03ce58ea9e3..549d91b771a 100644 --- a/docs/cudf/source/user_guide/pandas-comparison.md +++ b/docs/cudf/source/user_guide/pandas-comparison.md @@ -87,7 +87,7 @@ using `.from_arrow()` or `.from_pandas()`. ## Result ordering -By default, `join` (or `merge`) and `groupby` operations in cuDF +By default, `join` (or `merge`), `value_counts` and `groupby` operations in cuDF do *not* guarantee output ordering. Compare the results obtained from Pandas and cuDF below: diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 9b4a79c6841..a0e1a041342 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -7688,10 +7688,10 @@ def value_counts( dog 4 0 cat 4 0 ant 6 0 - >>> df.value_counts() + >>> df.value_counts().sort_index() num_legs num_wings - 4 0 2 2 2 1 + 4 0 2 6 0 1 Name: count, dtype: int64 """ diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 9612349a607..e4370be304a 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -109,11 +109,11 @@ def _is_row_of(chunk, obj): Parrot 30.0 Parrot 20.0 Name: Max Speed, dtype: float64 ->>> ser.groupby(level=0).mean() +>>> ser.groupby(level=0, sort=True).mean() Falcon 370.0 Parrot 25.0 Name: Max Speed, dtype: float64 ->>> ser.groupby(ser > 100).mean() +>>> ser.groupby(ser > 100, sort=True).mean() Max Speed False 25.0 True 370.0 @@ -133,7 +133,7 @@ def _is_row_of(chunk, obj): 1 Falcon 370.0 2 Parrot 24.0 3 Parrot 26.0 ->>> df.groupby(['Animal']).mean() +>>> df.groupby(['Animal'], sort=True).mean() Max Speed Animal Falcon 375.0 @@ -151,22 +151,22 @@ def _is_row_of(chunk, obj): Wild 350.0 Parrot Captive 30.0 Wild 20.0 ->>> df.groupby(level=0).mean() +>>> df.groupby(level=0, sort=True).mean() Max Speed Animal Falcon 370.0 Parrot 25.0 ->>> df.groupby(level="Type").mean() +>>> df.groupby(level="Type", sort=True).mean() Max Speed Type -Wild 185.0 Captive 210.0 +Wild 185.0 >>> df = cudf.DataFrame({{'A': 'a a b'.split(), ... 'B': [1,2,3], ... 'C': [4,6,5]}}) ->>> g1 = df.groupby('A', group_keys=False) ->>> g2 = df.groupby('A', group_keys=True) +>>> g1 = df.groupby('A', group_keys=False, sort=True) +>>> g2 = df.groupby('A', group_keys=True, sort=True) Notice that ``g1`` have ``g2`` have two groups, ``a`` and ``b``, and only differ in their ``group_keys`` argument. Calling `apply` in various ways, @@ -539,11 +539,11 @@ def agg(self, func): ... 'b': [1, 2, 3], ... 'c': [2, 2, 1] ... }) - >>> a.groupby('a').agg('sum') + >>> a.groupby('a', sort=True).agg('sum') b c a - 2 3 1 1 3 4 + 2 3 1 Specifying a list of aggregations to perform on each column. @@ -553,12 +553,12 @@ def agg(self, func): ... 'b': [1, 2, 3], ... 'c': [2, 2, 1] ... }) - >>> a.groupby('a').agg(['sum', 'min']) + >>> a.groupby('a', sort=True).agg(['sum', 'min']) b c sum min sum min a - 2 3 3 1 1 1 3 1 4 2 + 2 3 3 1 1 Using a dict to specify aggregations to perform per column. @@ -568,12 +568,12 @@ def agg(self, func): ... 'b': [1, 2, 3], ... 'c': [2, 2, 1] ... }) - >>> a.groupby('a').agg({'a': 'max', 'b': ['min', 'mean']}) + >>> a.groupby('a', sort=True).agg({'a': 'max', 'b': ['min', 'mean']}) a b max min mean a - 2 2 3 3.0 1 1 1 1.5 + 2 2 3 3.0 Using lambdas/callables to specify aggregations taking parameters. diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index 63e0cf98b27..f856bbedca2 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -55,12 +55,12 @@ def assert_groupby_results_equal( if isinstance(expect, (pd.DataFrame, cudf.DataFrame)): expect = expect.sort_values(by=by).reset_index(drop=True) else: - expect = expect.sort_values().reset_index(drop=True) + expect = expect.sort_values(by=by).reset_index(drop=True) if isinstance(got, cudf.DataFrame): got = got.sort_values(by=by).reset_index(drop=True) else: - got = got.sort_values().reset_index(drop=True) + got = got.sort_values(by=by).reset_index(drop=True) assert_eq(expect, got, **kwargs) @@ -179,7 +179,7 @@ def test_groupby_agg_min_max_dictlist(nelem): def test_groupby_as_index_single_agg(pdf, gdf, as_index): gdf = gdf.groupby("y", as_index=as_index).agg({"x": "mean"}) pdf = pdf.groupby("y", as_index=as_index).agg({"x": "mean"}) - assert_groupby_results_equal(pdf, gdf) + assert_groupby_results_equal(pdf, gdf, as_index=as_index, by="y") @pytest.mark.parametrize("engine", ["cudf", "jit"]) @@ -190,7 +190,7 @@ def test_groupby_as_index_apply(pdf, gdf, as_index, engine): ) kwargs = {"func": lambda df: df["x"].mean(), "include_groups": False} pdf = pdf.groupby("y", as_index=as_index).apply(**kwargs) - assert_groupby_results_equal(pdf, gdf) + assert_groupby_results_equal(pdf, gdf, as_index=as_index, by="y") @pytest.mark.parametrize("as_index", [True, False]) @@ -3714,7 +3714,13 @@ def test_group_by_value_counts(normalize, sort, ascending, dropna, as_index): # TODO: Remove `check_names=False` once testing against `pandas>=2.0.0` assert_groupby_results_equal( - actual, expected, check_names=False, check_index_type=False + actual, + expected, + check_names=False, + check_index_type=False, + as_index=as_index, + by=["gender", "education"], + sort=sort, ) From c1e26a63d33563190f452047e548f24fb47a63bf Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Thu, 29 Feb 2024 17:15:17 -0500 Subject: [PATCH 323/384] Fix cudf::test::to_host to handle both offset types for strings columns (#15073) The `cudf::test::to_host` function is updated to handle int32 and int64 offset types for strings columns when copying data to host memory. This function is used with `cudf::test::print()` as well. Also moved the function from the header `column_utilities.hpp` to the `column_utilities.cu` file. And moved the specialization for of `to_host` for fixed-point types from the header to `.cu` as well. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Karthikeyan (https://github.com/karthikeyann) URL: https://github.com/rapidsai/cudf/pull/15073 --- cpp/include/cudf_test/column_utilities.hpp | 43 +------------ cpp/tests/utilities/column_utilities.cu | 75 ++++++++++++++++++++++ 2 files changed, 78 insertions(+), 40 deletions(-) diff --git a/cpp/include/cudf_test/column_utilities.hpp b/cpp/include/cudf_test/column_utilities.hpp index 49d5098f823..cbfd7a5e45c 100644 --- a/cpp/include/cudf_test/column_utilities.hpp +++ b/cpp/include/cudf_test/column_utilities.hpp @@ -194,23 +194,7 @@ std::pair, std::vector> to_host(column_view * `column_view`'s data, and second is the column's bitmask. */ template ()>* = nullptr> -std::pair, std::vector> to_host(column_view c) -{ - using namespace numeric; - using Rep = typename T::rep; - - auto host_rep_types = thrust::host_vector(c.size()); - - CUDF_CUDA_TRY( - cudaMemcpy(host_rep_types.data(), c.begin(), c.size() * sizeof(Rep), cudaMemcpyDefault)); - - auto to_fp = [&](Rep val) { return T{scaled_integer{val, scale_type{c.type().scale()}}}; }; - auto begin = thrust::make_transform_iterator(std::cbegin(host_rep_types), to_fp); - auto const host_fixed_points = thrust::host_vector(begin, begin + c.size()); - - return {host_fixed_points, bitmask_to_host(c)}; -} -//! @endcond +std::pair, std::vector> to_host(column_view c); /** * @brief Copies the data and bitmask of a `column_view` of strings @@ -223,29 +207,8 @@ std::pair, std::vector> to_host(column_view * and second is the column's bitmask. */ template <> -inline std::pair, std::vector> to_host(column_view c) -{ - thrust::host_vector host_data(c.size()); - auto stream = cudf::get_default_stream(); - if (c.size() > c.null_count()) { - auto const scv = strings_column_view(c); - auto const h_chars = cudf::detail::make_std_vector_sync( - cudf::device_span(scv.chars_begin(stream), scv.chars_size(stream)), stream); - auto const h_offsets = cudf::detail::make_std_vector_sync( - cudf::device_span(scv.offsets().data() + scv.offset(), - scv.size() + 1), - stream); - - // build std::string vector from chars and offsets - std::transform( - std::begin(h_offsets), - std::end(h_offsets) - 1, - std::begin(h_offsets) + 1, - host_data.begin(), - [&](auto start, auto end) { return std::string(h_chars.data() + start, end - start); }); - } - return {std::move(host_data), bitmask_to_host(c)}; -} +std::pair, std::vector> to_host(column_view c); +//! @endcond } // namespace cudf::test diff --git a/cpp/tests/utilities/column_utilities.cu b/cpp/tests/utilities/column_utilities.cu index 018c6aeec2c..a556a8702bd 100644 --- a/cpp/tests/utilities/column_utilities.cu +++ b/cpp/tests/utilities/column_utilities.cu @@ -940,5 +940,80 @@ bool validate_host_masks(std::vector const& expected_mask, }); } +template ()>*> +std::pair, std::vector> to_host(column_view c) +{ + using namespace numeric; + using Rep = typename T::rep; + + auto host_rep_types = thrust::host_vector(c.size()); + + CUDF_CUDA_TRY( + cudaMemcpy(host_rep_types.data(), c.begin(), c.size() * sizeof(Rep), cudaMemcpyDefault)); + + auto to_fp = [&](Rep val) { return T{scaled_integer{val, scale_type{c.type().scale()}}}; }; + auto begin = thrust::make_transform_iterator(std::cbegin(host_rep_types), to_fp); + auto const host_fixed_points = thrust::host_vector(begin, begin + c.size()); + + return {host_fixed_points, bitmask_to_host(c)}; +} + +template std::pair, std::vector> to_host( + column_view c); +template std::pair, std::vector> to_host( + column_view c); +template std::pair, std::vector> to_host( + column_view c); + +namespace { +struct strings_to_host_fn { + template || + std::is_same_v>* = nullptr> + void operator()(thrust::host_vector& host_data, + char const* chars, + cudf::column_view const& offsets, + rmm::cuda_stream_view stream) + { + auto const h_offsets = cudf::detail::make_std_vector_sync( + cudf::device_span(offsets.data(), offsets.size()), stream); + // build std::string vector from chars and offsets + std::transform(std::begin(h_offsets), + std::end(h_offsets) - 1, + std::begin(h_offsets) + 1, + host_data.begin(), + [&](auto start, auto end) { return std::string(chars + start, end - start); }); + } + + template && + !std::is_same_v>* = nullptr> + void operator()(thrust::host_vector&, + char const*, + cudf::column_view const&, + rmm::cuda_stream_view) + { + CUDF_FAIL("invalid offsets type"); + } +}; +} // namespace + +template <> +std::pair, std::vector> to_host(column_view c) +{ + thrust::host_vector host_data(c.size()); + auto stream = cudf::get_default_stream(); + if (c.size() > c.null_count()) { + auto const scv = strings_column_view(c); + auto const h_chars = cudf::detail::make_std_vector_sync( + cudf::device_span(scv.chars_begin(stream), scv.chars_size(stream)), stream); + auto offsets = + cudf::slice(scv.offsets(), {scv.offset(), scv.offset() + scv.size() + 1}).front(); + cudf::type_dispatcher( + offsets.type(), strings_to_host_fn{}, host_data, h_chars.data(), offsets, stream); + } + return {std::move(host_data), bitmask_to_host(c)}; +} + } // namespace test } // namespace cudf From a4f1118f23cc7cfdb7e3d03abf7726740ff52af7 Mon Sep 17 00:00:00 2001 From: Suraj Aralihalli Date: Thu, 29 Feb 2024 14:21:11 -0800 Subject: [PATCH 324/384] Resolve path parsing issues in `get_json_object` (#15082) This PR addresses a parsing issue related to JSONPath by implementing distinct parsing rules for values inside and outside brackets. For instance, in `{ "A.B": 2, "'A": { "B'": 3 } }`, `$.'A.B'` differs from `$['A.B']`. (See [Assertible JSON Path Documentation](https://assertible.com/docs/guide/json-path)) The fix ensures accurate parsing of JSONPath values containing quotes. For example in `{ "A.B": 2, "'A": { "B'": 3 } }` | JSONPath | Before Fix | Spark | After Fix | |---------------|-------------------------------------------------------|----------------------|---------------------| | $.'A.B' | 2 | 3 | 3 | | $.'A | CUDF_FAIL("Encountered invalid JSONPath input string")| {"B'": 3} | {"B'": 3} | Resolves [12483](https://github.com/rapidsai/cudf/issues/12483). Authors: - Suraj Aralihalli (https://github.com/SurajAralihalli) - Nghia Truong (https://github.com/ttnghia) Approvers: - Robert (Bobby) Evans (https://github.com/revans2) - Mike Wilson (https://github.com/hyperbolic2346) - Nghia Truong (https://github.com/ttnghia) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/15082 --- cpp/src/json/json_path.cu | 24 +++++++++--- cpp/tests/json/json_tests.cpp | 38 +++++++++++++++++++ java/src/main/native/src/ColumnViewJni.cpp | 10 ++++- .../java/ai/rapids/cudf/ColumnVectorTest.java | 16 ++++++++ 4 files changed, 81 insertions(+), 7 deletions(-) diff --git a/cpp/src/json/json_path.cu b/cpp/src/json/json_path.cu index 25f136e2336..ff42d9c8620 100644 --- a/cpp/src/json/json_path.cu +++ b/cpp/src/json/json_path.cu @@ -521,6 +521,14 @@ struct path_operator { int index{-1}; // index for subscript operator }; +/** + * @brief Enum to specify whether parsing values enclosed within brackets, like `['book']`. + */ +enum class bracket_state : bool { + INSIDE, ///< Parsing inside brackets + OUTSIDE ///< Parsing outside brackets +}; + /** * @brief Parsing class that holds the current state of the JSONPath string to be parsed * and provides functions for navigating through it. This is only called on the host @@ -541,7 +549,7 @@ class path_state : private parser { case '.': { path_operator op; string_view term{".[", 2}; - if (parse_path_name(op.name, term)) { + if (parse_path_name(op.name, term, bracket_state::OUTSIDE)) { // this is another potential use case for __SPARK_BEHAVIORS / configurability // Spark currently only handles the wildcard operator inside [*], it does // not handle .* @@ -564,7 +572,7 @@ class path_state : private parser { path_operator op; string_view term{"]", 1}; bool const is_string = *pos == '\''; - if (parse_path_name(op.name, term)) { + if (parse_path_name(op.name, term, bracket_state::INSIDE)) { pos++; if (op.name.size_bytes() == 1 && op.name.data()[0] == '*') { op.type = path_operator_type::CHILD_WILDCARD; @@ -600,7 +608,8 @@ class path_state : private parser { private: cudf::io::parse_options_view json_opts{',', '\n', '\"', '.'}; - bool parse_path_name(string_view& name, string_view const& terminators) + // b_state is set to INSIDE while parsing values enclosed within [ ], otherwise OUTSIDE + bool parse_path_name(string_view& name, string_view const& terminators, bracket_state b_state) { switch (*pos) { case '*': @@ -609,8 +618,11 @@ class path_state : private parser { break; case '\'': - if (parse_string(name, false, '\'') != parse_result::SUCCESS) { return false; } - break; + if (b_state == bracket_state::INSIDE) { + if (parse_string(name, false, '\'') != parse_result::SUCCESS) { return false; } + break; + } + // if not inside the [ ] -> go to default default: { size_t const chars_left = input_len - (pos - input); @@ -656,7 +668,7 @@ std::pair>, int> build_comma do { op = p_state.get_next_operator(); if (op.type == path_operator_type::ERROR) { - CUDF_FAIL("Encountered invalid JSONPath input string"); + CUDF_FAIL("Encountered invalid JSONPath input string", std::invalid_argument); } if (op.type == path_operator_type::CHILD_WILDCARD) { max_stack_depth++; } // convert pointer to device pointer diff --git a/cpp/tests/json/json_tests.cpp b/cpp/tests/json/json_tests.cpp index 0894472dcc3..6c9050becc1 100644 --- a/cpp/tests/json/json_tests.cpp +++ b/cpp/tests/json/json_tests.cpp @@ -588,6 +588,15 @@ TEST_F(JsonPathTests, GetJsonObjectIllegalQuery) }; EXPECT_THROW(query(), std::invalid_argument); } + + { + auto const input = cudf::test::strings_column_wrapper{R"({"a": "b"})"}; + auto const json_path = std::string{"${a}"}; + auto const query = [&]() { + auto const result = cudf::get_json_object(cudf::strings_column_view(input), json_path); + }; + EXPECT_THROW(query(), std::invalid_argument); + } } // queries that are legal, but reference invalid parts of the input @@ -1018,4 +1027,33 @@ TEST_F(JsonPathTests, MissingFieldsAsNulls) do_test("$.tup[*].a.x", "[\"5\"]", "[null,null,null,\"5\"]"); } +TEST_F(JsonPathTests, QueriesContainingQuotes) +{ + std::string input_string = R"({"AB": 1, "A.B": 2, "'A": {"B'": 3}, "A": {"B": 4} })"; + + auto do_test = [&input_string](auto const& json_path_string, + auto const& expected_string, + bool const& expect_null = false) { + auto const input = cudf::test::strings_column_wrapper{input_string}; + auto const json_path = std::string{json_path_string}; + cudf::get_json_object_options options; + options.set_allow_single_quotes(true); + auto const result = cudf::get_json_object(cudf::strings_column_view(input), json_path, options); + auto const expected = + cudf::test::strings_column_wrapper{std::initializer_list{expected_string}, + std::initializer_list{!expect_null}}; + + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(*result, expected); + }; + + // Set 1 + do_test(R"($.AB)", "1"); + do_test(R"($['A.B'])", "2"); + do_test(R"($.'A.B')", "3"); + do_test(R"($.A.B)", "4"); + + // Set 2 + do_test(R"($.'A)", R"({"B'": 3})"); +} + CUDF_TEST_PROGRAM_MAIN() diff --git a/java/src/main/native/src/ColumnViewJni.cpp b/java/src/main/native/src/ColumnViewJni.cpp index 1c4eb8a83ab..dd3859a4160 100644 --- a/java/src/main/native/src/ColumnViewJni.cpp +++ b/java/src/main/native/src/ColumnViewJni.cpp @@ -2452,7 +2452,15 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_getJSONObject( options.set_allow_single_quotes(allow_single_quotes); options.set_strip_quotes_from_single_strings(strip_quotes_from_single_strings); options.set_missing_fields_as_nulls(missing_fields_as_nulls); - return release_as_jlong(cudf::get_json_object(n_strings_col_view, *n_scalar_path, options)); + auto result_col_ptr = [&]() { + try { + return cudf::get_json_object(n_strings_col_view, *n_scalar_path, options); + } catch (std::invalid_argument const &err) { + auto const null_scalar = cudf::string_scalar(std::string(""), false); + return cudf::make_column_from_scalar(null_scalar, n_strings_col_view.size()); + } catch (...) { throw; } + }(); + return release_as_jlong(result_col_ptr); } CATCH_STD(env, 0) } diff --git a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java index 75573046af2..bac4d1e4b3e 100644 --- a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java +++ b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java @@ -6405,6 +6405,22 @@ void testGetJSONObjectWithSingleQuotes() { } } +@Test +void testGetJSONObjectWithInvalidQueries() { + String jsonString = "{" + + "\'a\': \'A\"\'" + + "}"; + + GetJsonObjectOptions options = GetJsonObjectOptions.builder().allowSingleQuotes(true).build(); + try (ColumnVector json = ColumnVector.fromStrings(jsonString, jsonString); + Scalar nullString = Scalar.fromString(null); + ColumnVector expectedAuthors = ColumnVector.fromScalar(nullString, 2); + Scalar path = Scalar.fromString("."); + ColumnVector gotAuthors = json.getJSONObject(path, options)) { + assertColumnsAreEqual(expectedAuthors, gotAuthors); + } +} + @Test void testMakeStructEmpty() { final int numRows = 10; From e96ff74fc020c06ee47a76e47f3fff2555531d32 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 29 Feb 2024 17:52:24 -0600 Subject: [PATCH 325/384] Add support for Python 3.11, require NumPy 1.23+ (#15111) Contributes to https://github.com/rapidsai/build-planning/issues/3 This PR adds support for Python 3.11. It also bumps uses of `NumPy` to `numpy>=1.23`, see https://github.com/rapidsai/build-planning/issues/3#issuecomment-1967952280. ## Notes for Reviewers This is part of ongoing work to add Python 3.11 support across RAPIDS. The Python 3.11 CI workflows introduced in https://github.com/rapidsai/shared-workflows/pull/176 are *optional*... they are not yet required to run successfully for PRs to be merged. This PR can be merged once all jobs are running successfully (including the non-required jobs for Python 3.11). The CI logs should be verified that the jobs are building and testing with Python 3.11. See https://github.com/rapidsai/shared-workflows/pull/176 for more details. *(created with [rapids-reviser](https://github.com/rapidsai/rapids-reviser))* Authors: - James Lamb (https://github.com/jameslamb) - Kyle Edwards (https://github.com/KyleFromNVIDIA) - https://github.com/jakirkham - Bradley Dice (https://github.com/bdice) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Bradley Dice (https://github.com/bdice) - Kyle Edwards (https://github.com/KyleFromNVIDIA) - Jake Awe (https://github.com/AyodeAwe) - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15111 --- README.md | 2 +- .../all_cuda-118_arch-x86_64.yaml | 11 ++++----- .../all_cuda-122_arch-x86_64.yaml | 11 ++++----- conda/recipes/cudf/meta.yaml | 3 ++- dependencies.yaml | 24 ++++++++++++------- .../cudf/tests/test_cuda_array_interface.py | 13 +++++----- python/cudf/cudf/tests/test_string.py | 2 +- .../cudf/tests/text/test_subword_tokenizer.py | 3 ++- python/cudf/pyproject.toml | 9 +++---- python/cudf_kafka/pyproject.toml | 2 +- python/custreamz/pyproject.toml | 1 + python/dask_cudf/pyproject.toml | 3 ++- 12 files changed, 47 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 599e194bc1a..8f9e57ff3ad 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ cuDF can be installed with conda (via [miniconda](https://docs.conda.io/projects ```bash conda install -c rapidsai -c conda-forge -c nvidia \ - cudf=24.04 python=3.10 cuda-version=11.8 + cudf=24.04 python=3.11 cuda-version=12.2 ``` We also provide [nightly Conda packages](https://anaconda.org/rapidsai-nightly) built from the HEAD diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 79b786fe012..c12e88f1c0f 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -4,7 +4,6 @@ channels: - rapidsai - rapidsai-nightly - dask/label/dev -- pytorch - conda-forge - nvidia dependencies: @@ -59,7 +58,7 @@ dependencies: - ninja - notebook - numba>=0.57 -- numpy>=1.21 +- numpy>=1.23 - numpydoc - nvcc_linux-64=11.8 - nvcomp==3.0.6 @@ -79,8 +78,8 @@ dependencies: - pytest-xdist - pytest<8 - python-confluent-kafka>=1.9.0,<1.10.0a0 -- python>=3.9,<3.11 -- pytorch<1.12.0 +- python>=3.9,<3.12 +- pytorch>=2.1.0 - rapids-dask-dependency==24.4.* - rich - rmm==24.4.* @@ -96,8 +95,8 @@ dependencies: - sphinxcontrib-websupport - streamz - sysroot_linux-64==2.17 -- tokenizers==0.13.1 -- transformers==4.24.0 +- tokenizers==0.15.2 +- transformers==4.38.1 - typing_extensions>=4.0.0 - zlib>=1.2.13 - pip: diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index 66a4ee57238..e773812967d 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -4,7 +4,6 @@ channels: - rapidsai - rapidsai-nightly - dask/label/dev -- pytorch - conda-forge - nvidia dependencies: @@ -58,7 +57,7 @@ dependencies: - ninja - notebook - numba>=0.57 -- numpy>=1.21 +- numpy>=1.23 - numpydoc - nvcomp==3.0.6 - nvtx>=0.2.1 @@ -77,8 +76,8 @@ dependencies: - pytest-xdist - pytest<8 - python-confluent-kafka>=1.9.0,<1.10.0a0 -- python>=3.9,<3.11 -- pytorch<1.12.0 +- python>=3.9,<3.12 +- pytorch>=2.1.0 - rapids-dask-dependency==24.4.* - rich - rmm==24.4.* @@ -94,8 +93,8 @@ dependencies: - sphinxcontrib-websupport - streamz - sysroot_linux-64==2.17 -- tokenizers==0.13.1 -- transformers==4.24.0 +- tokenizers==0.15.2 +- transformers==4.38.1 - typing_extensions>=4.0.0 - zlib>=1.2.13 - pip: diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 80920dc7b5f..6a85fadaa48 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -65,6 +65,7 @@ requirements: - scikit-build-core >=0.7.0 - setuptools - dlpack >=0.5,<0.6.0a0 + - numpy 1.23 - pyarrow ==14.0.2.* - libcudf ={{ version }} - rmm ={{ minor_version }} @@ -83,7 +84,7 @@ requirements: - pandas >=2.0,<2.2.2dev0 - cupy >=12.0.0 - numba >=0.57 - - numpy >=1.21 + - {{ pin_compatible('numpy', max_pin='x') }} - {{ pin_compatible('pyarrow', max_pin='x') }} - libcudf ={{ version }} - {{ pin_compatible('rmm', max_pin='x.x') }} diff --git a/dependencies.yaml b/dependencies.yaml index 4281e907862..a83a03b571b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -188,7 +188,6 @@ channels: - rapidsai - rapidsai-nightly - dask/label/dev - - pytorch - conda-forge - nvidia dependencies: @@ -258,13 +257,17 @@ dependencies: - *cmake_ver - cython>=3.0.3 - *ninja - - &numpy numpy>=1.21 # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. - pyarrow==14.0.2.* - output_types: conda packages: - scikit-build-core>=0.7.0 + - output_types: pyproject + packages: + # Hard pin the patch version used during the build. + # Sync with conda build constraint & wheel run constraint. + - numpy==1.23.* - output_types: [requirements, pyproject] packages: - scikit-build-core[pyproject]>=0.7.0 @@ -488,15 +491,19 @@ dependencies: py: "3.10" packages: - python=3.10 + - matrix: + py: "3.11" + packages: + - python=3.11 - matrix: packages: - - python>=3.9,<3.11 + - python>=3.9,<3.12 run_common: common: - output_types: [conda, requirements, pyproject] packages: - fsspec>=0.6.0 - - *numpy + - numpy>=1.23 - pandas>=2.0,<2.2.2dev0 run_cudf: common: @@ -624,8 +631,8 @@ dependencies: - output_types: pyproject packages: - msgpack - - &tokenizers tokenizers==0.13.1 - - &transformers transformers==4.24.0 + - &tokenizers tokenizers==0.15.2 + - &transformers transformers==4.38.1 - tzdata specific: - output_types: conda @@ -633,9 +640,8 @@ dependencies: - matrix: arch: x86_64 packages: - # Currently, CUDA builds of pytorch do not exist for aarch64. We require - # version <1.12.0 because newer versions use nvidia::cuda-toolkit. - - pytorch<1.12.0 + # Currently, CUDA + aarch64 builds of pytorch do not exist on conda-forge. + - pytorch>=2.1.0 # We only install these on x86_64 to avoid pulling pytorch as a # dependency of transformers. - *tokenizers diff --git a/python/cudf/cudf/tests/test_cuda_array_interface.py b/python/cudf/cudf/tests/test_cuda_array_interface.py index a9d11922943..1f20152172b 100644 --- a/python/cudf/cudf/tests/test_cuda_array_interface.py +++ b/python/cudf/cudf/tests/test_cuda_array_interface.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2023, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. import types from contextlib import ExitStack as does_not_raise @@ -193,10 +193,11 @@ def test_cuda_array_interface_pytorch(): assert_eq(got, cudf.Series(buffer, dtype=np.bool_)) - index = cudf.Index([], dtype="float64") - tensor = torch.tensor(index) - got = cudf.Index(tensor) - assert_eq(got, index) + # TODO: This test fails with PyTorch 2. Is it still expected to be valid? + # index = cudf.Index([], dtype="float64") + # tensor = torch.tensor(index) + # got = cudf.Index(tensor) + # assert_eq(got, index) index = cudf.core.index.RangeIndex(start=0, stop=100) tensor = torch.tensor(index) @@ -212,7 +213,7 @@ def test_cuda_array_interface_pytorch(): str_series = cudf.Series(["a", "g"]) - with pytest.raises(NotImplementedError): + with pytest.raises(AttributeError): str_series.__cuda_array_interface__ cat_series = str_series.astype("category") diff --git a/python/cudf/cudf/tests/test_string.py b/python/cudf/cudf/tests/test_string.py index a9ba80a395d..de771a56e77 100644 --- a/python/cudf/cudf/tests/test_string.py +++ b/python/cudf/cudf/tests/test_string.py @@ -891,7 +891,7 @@ def test_string_repeat(data, repeats): ) @pytest.mark.parametrize("repl", ["qwerty", "", " "]) @pytest.mark.parametrize("case,case_raise", [(None, 0), (True, 1), (False, 1)]) -@pytest.mark.parametrize("flags,flags_raise", [(0, 0), (1, 1)]) +@pytest.mark.parametrize("flags,flags_raise", [(0, 0), (re.U, 1)]) def test_string_replace( ps_gs, pat, repl, case, case_raise, flags, flags_raise, regex ): diff --git a/python/cudf/cudf/tests/text/test_subword_tokenizer.py b/python/cudf/cudf/tests/text/test_subword_tokenizer.py index ac17daa8601..b21edc0477f 100644 --- a/python/cudf/cudf/tests/text/test_subword_tokenizer.py +++ b/python/cudf/cudf/tests/text/test_subword_tokenizer.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. import os import cupy @@ -27,6 +27,7 @@ def assert_equal_tokenization_outputs(hf_output, cudf_output): ) +@pytest.mark.skip(reason="segfaults") @pytest.mark.parametrize("seq_len", [32, 64]) @pytest.mark.parametrize("stride", [0, 15, 30]) @pytest.mark.parametrize("add_special_tokens", [True, False]) diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 590786f2414..5afd82220a4 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -6,7 +6,7 @@ requires = [ "cmake>=3.26.4", "cython>=3.0.3", "ninja", - "numpy>=1.21", + "numpy==1.23.*", "protoc-wheel", "pyarrow==14.0.2.*", "rmm==24.4.*", @@ -30,7 +30,7 @@ dependencies = [ "cupy-cuda11x>=12.0.0", "fsspec>=0.6.0", "numba>=0.57", - "numpy>=1.21", + "numpy>=1.23", "nvtx>=0.2.1", "packaging", "pandas>=2.0,<2.2.2dev0", @@ -49,6 +49,7 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ] [project.optional-dependencies] @@ -63,8 +64,8 @@ test = [ "pytest-xdist", "pytest<8", "scipy", - "tokenizers==0.13.1", - "transformers==4.24.0", + "tokenizers==0.15.2", + "transformers==4.38.1", "tzdata", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. pandas-tests = [ diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index 216d83940ce..7369b99aaf4 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -6,7 +6,7 @@ requires = [ "cmake>=3.26.4", "cython>=3.0.3", "ninja", - "numpy>=1.21", + "numpy==1.23.*", "pyarrow==14.0.2.*", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/custreamz/pyproject.toml b/python/custreamz/pyproject.toml index 12b0356c9c1..ccaa2543cc3 100644 --- a/python/custreamz/pyproject.toml +++ b/python/custreamz/pyproject.toml @@ -32,6 +32,7 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ] [project.optional-dependencies] diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index 5d4ea429d5f..4ecfc4f3f85 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "cudf==24.4.*", "cupy-cuda11x>=12.0.0", "fsspec>=0.6.0", - "numpy>=1.21", + "numpy>=1.23", "pandas>=2.0,<2.2.2dev0", "rapids-dask-dependency==24.4.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -33,6 +33,7 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ] [project.entry-points."dask.dataframe.backends"] From 56a3b8f6516f830d836b50cc0d93ae67c4db9613 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Thu, 29 Feb 2024 18:02:53 -0800 Subject: [PATCH 326/384] Fix chunked reads of Parquet delta encoded pages (#14921) The chunked Parquet reader currently does not properly estimate the sizes of string pages that are delta encoded. This PR modifies `gpuDecodeTotalPageStringSize()` to take into account the new encodings. Authors: - Ed Seidl (https://github.com/etseidl) - Mike Wilson (https://github.com/hyperbolic2346) - Vukasin Milovanovic (https://github.com/vuule) Approvers: - Mike Wilson (https://github.com/hyperbolic2346) - https://github.com/nvdbaranec URL: https://github.com/rapidsai/cudf/pull/14921 --- cpp/src/io/parquet/decode_preprocess.cu | 136 ++++++++++++++++++-- cpp/src/io/parquet/page_decode.cuh | 1 + cpp/src/io/parquet/page_string_decode.cu | 1 + cpp/tests/io/parquet_chunked_reader_test.cu | 109 ++++++++++++++-- 4 files changed, 223 insertions(+), 24 deletions(-) diff --git a/cpp/src/io/parquet/decode_preprocess.cu b/cpp/src/io/parquet/decode_preprocess.cu index 862dedf6200..19c398c5965 100644 --- a/cpp/src/io/parquet/decode_preprocess.cu +++ b/cpp/src/io/parquet/decode_preprocess.cu @@ -14,6 +14,7 @@ * limitations under the License. */ +#include "delta_binary.cuh" #include "io/utilities/column_buffer.hpp" #include "page_decode.cuh" @@ -40,26 +41,139 @@ constexpr int rolling_buf_size = LEVEL_DECODE_BUF_SIZE; using unused_state_buf = page_state_buffers_s<0, 0, 0>; /** + * @brief Calculate string bytes for DELTA_LENGTH_BYTE_ARRAY encoded pages + * + * Result is valid only on thread 0. + * + * @param s The local page info + * @param t Thread index + */ +__device__ size_type gpuDeltaLengthPageStringSize(page_state_s* s, int t) +{ + if (t == 0) { + // find the beginning of char data + delta_binary_decoder string_lengths; + auto const* string_start = string_lengths.find_end_of_block(s->data_start, s->data_end); + // distance is size of string data + return static_cast(std::distance(string_start, s->data_end)); + } + return 0; +} + +/** + * @brief Calculate string bytes for DELTA_BYTE_ARRAY encoded pages + * + * This expects all threads in the thread block (preprocess_block_size). + * + * @param s The local page info + * @param t Thread index + */ +__device__ size_type gpuDeltaPageStringSize(page_state_s* s, int t) +{ + using cudf::detail::warp_size; + using WarpReduce = cub::WarpReduce; + __shared__ typename WarpReduce::TempStorage temp_storage[2]; + + __shared__ __align__(16) delta_binary_decoder prefixes; + __shared__ __align__(16) delta_binary_decoder suffixes; + + int const lane_id = t % warp_size; + int const warp_id = t / warp_size; + + if (t == 0) { + auto const* suffix_start = prefixes.find_end_of_block(s->data_start, s->data_end); + suffixes.init_binary_block(suffix_start, s->data_end); + } + __syncthreads(); + + // two warps will traverse the prefixes and suffixes and sum them up + auto const db = t < warp_size ? &prefixes : t < 2 * warp_size ? &suffixes : nullptr; + + size_t total_bytes = 0; + if (db != nullptr) { + // initialize with first value (which is stored in last_value) + if (lane_id == 0) { total_bytes = db->last_value; } + + uleb128_t lane_sum = 0; + while (db->current_value_idx < db->num_encoded_values(true)) { + // calculate values for current mini-block + db->calc_mini_block_values(lane_id); + + // get per lane sum for mini-block + for (uint32_t i = 0; i < db->values_per_mb; i += warp_size) { + uint32_t const idx = db->current_value_idx + i + lane_id; + if (idx < db->value_count) { + lane_sum += db->value[rolling_index(idx)]; + } + } + + if (lane_id == 0) { db->setup_next_mini_block(true); } + __syncwarp(); + } + + // get sum for warp. + // note: warp_sum will only be valid on lane 0. + auto const warp_sum = WarpReduce(temp_storage[warp_id]).Sum(lane_sum); + + if (lane_id == 0) { total_bytes += warp_sum; } + } + __syncthreads(); + + // now sum up total_bytes from the two warps. result is only valid on thread 0. + auto const final_bytes = + cudf::detail::single_lane_block_sum_reduce(total_bytes); + + return static_cast(final_bytes); +} + +/** + * @brief Calculate the number of string bytes in the page. * * This function expects the dictionary position to be at 0 and will traverse - * the entire thing. + * the entire thing (for plain and dictionary encoding). * - * Operates on a single warp only. Expects t < 32 + * This expects all threads in the thread block (preprocess_block_size). Result is only + * valid on thread 0. * * @param s The local page info * @param t Thread index */ __device__ size_type gpuDecodeTotalPageStringSize(page_state_s* s, int t) { + using cudf::detail::warp_size; size_type target_pos = s->num_input_values; size_type str_len = 0; - if (s->dict_base) { - auto const [new_target_pos, len] = - gpuDecodeDictionaryIndices(s, nullptr, target_pos, t); - target_pos = new_target_pos; - str_len = len; - } else if ((s->col.data_type & 7) == BYTE_ARRAY) { - str_len = gpuInitStringDescriptors(s, nullptr, target_pos, t); + switch (s->page.encoding) { + case Encoding::PLAIN_DICTIONARY: + case Encoding::RLE_DICTIONARY: + if (t < warp_size && s->dict_base) { + auto const [new_target_pos, len] = + gpuDecodeDictionaryIndices(s, nullptr, target_pos, t); + target_pos = new_target_pos; + str_len = len; + } + break; + + case Encoding::PLAIN: + // For V2 headers, we know how many values are present, so can skip an expensive scan. + if ((s->page.flags & PAGEINFO_FLAGS_V2) != 0) { + auto const num_values = s->page.num_input_values - s->page.num_nulls; + str_len = s->dict_size - sizeof(int) * num_values; + } + // For V1, the choice is an overestimate (s->dict_size), or an exact number that's + // expensive to compute. For now we're going with the latter. + else { + str_len = gpuInitStringDescriptors(s, nullptr, target_pos, t); + } + break; + + case Encoding::DELTA_LENGTH_BYTE_ARRAY: str_len = gpuDeltaLengthPageStringSize(s, t); break; + + case Encoding::DELTA_BYTE_ARRAY: str_len = gpuDeltaPageStringSize(s, t); break; + + default: + // not a valid string encoding, so just return 0 + break; } if (!t) { s->dict_pos = target_pos; } return str_len; @@ -348,9 +462,9 @@ CUDF_KERNEL void __launch_bounds__(preprocess_block_size) } // retrieve total string size. - // TODO: make this block-based instead of just 1 warp if (compute_string_sizes) { - if (t < 32) { s->page.str_bytes = gpuDecodeTotalPageStringSize(s, t); } + auto const str_bytes = gpuDecodeTotalPageStringSize(s, t); + if (t == 0) { s->page.str_bytes = str_bytes; } } // update output results: diff --git a/cpp/src/io/parquet/page_decode.cuh b/cpp/src/io/parquet/page_decode.cuh index 4353e079496..cf3e1911496 100644 --- a/cpp/src/io/parquet/page_decode.cuh +++ b/cpp/src/io/parquet/page_decode.cuh @@ -1291,6 +1291,7 @@ inline __device__ bool setupLocalPageInfo(page_state_s* const s, s->dict_bits = 0; s->dict_base = nullptr; s->dict_size = 0; + s->dict_val = 0; // NOTE: if additional encodings are supported in the future, modifications must // be made to is_supported_encoding() in reader_impl_preprocess.cu switch (s->page.encoding) { diff --git a/cpp/src/io/parquet/page_string_decode.cu b/cpp/src/io/parquet/page_string_decode.cu index 101bd34f09f..b63f96fda46 100644 --- a/cpp/src/io/parquet/page_string_decode.cu +++ b/cpp/src/io/parquet/page_string_decode.cu @@ -549,6 +549,7 @@ __device__ thrust::pair totalDeltaByteArraySize(uint8_t const* d // get sum for warp. // note: warp_sum will only be valid on lane 0. auto const warp_sum = WarpReduce(temp_storage[warp_id]).Sum(lane_sum); + __syncwarp(); auto const warp_max = WarpReduce(temp_storage[warp_id]).Reduce(lane_max, cub::Max()); if (lane_id == 0) { diff --git a/cpp/tests/io/parquet_chunked_reader_test.cu b/cpp/tests/io/parquet_chunked_reader_test.cu index ea6d65a8c14..2c992677a65 100644 --- a/cpp/tests/io/parquet_chunked_reader_test.cu +++ b/cpp/tests/io/parquet_chunked_reader_test.cu @@ -62,6 +62,7 @@ using int32s_lists_col = cudf::test::lists_column_wrapper; auto write_file(std::vector>& input_columns, std::string const& filename, bool nullable, + bool delta_encoding, std::size_t max_page_size_bytes = cudf::io::default_max_page_size_bytes, std::size_t max_page_size_rows = cudf::io::default_max_page_size_rows) { @@ -86,14 +87,22 @@ auto write_file(std::vector>& input_columns, } auto input_table = std::make_unique(std::move(input_columns)); - auto filepath = - temp_env->get_temp_filepath(nullable ? filename + "_nullable.parquet" : filename + ".parquet"); + auto file_name = filename; + if (nullable) { file_name = file_name + "_nullable"; } + if (delta_encoding) { file_name = file_name + "_delta"; } + auto const filepath = temp_env->get_temp_filepath(file_name + ".parquet"); + + auto const dict_policy = + delta_encoding ? cudf::io::dictionary_policy::NEVER : cudf::io::dictionary_policy::ALWAYS; + auto const v2_headers = delta_encoding; auto const write_opts = cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, *input_table) .max_page_size_bytes(max_page_size_bytes) .max_page_size_rows(max_page_size_rows) .max_page_fragment_size(cudf::io::default_max_page_fragment_size) + .dictionary_policy(dict_policy) + .write_v2_headers(v2_headers) .build(); cudf::io::write_parquet(write_opts); @@ -140,7 +149,7 @@ TEST_F(ParquetChunkedReaderTest, TestChunkedReadNoData) input_columns.emplace_back(int32s_col{}.release()); input_columns.emplace_back(int64s_col{}.release()); - auto const [expected, filepath] = write_file(input_columns, "chunked_read_empty", false); + auto const [expected, filepath] = write_file(input_columns, "chunked_read_empty", false, false); auto const [result, num_chunks] = chunked_read(filepath, 1'000); EXPECT_EQ(num_chunks, 1); EXPECT_EQ(result->num_rows(), 0); @@ -152,24 +161,38 @@ TEST_F(ParquetChunkedReaderTest, TestChunkedReadSimpleData) { auto constexpr num_rows = 40'000; - auto const generate_input = [num_rows](bool nullable) { + auto const generate_input = [num_rows](bool nullable, bool use_delta) { std::vector> input_columns; auto const value_iter = thrust::make_counting_iterator(0); input_columns.emplace_back(int32s_col(value_iter, value_iter + num_rows).release()); input_columns.emplace_back(int64s_col(value_iter, value_iter + num_rows).release()); - return write_file(input_columns, "chunked_read_simple", nullable); + return write_file(input_columns, "chunked_read_simple", nullable, false); }; { - auto const [expected, filepath] = generate_input(false); + auto const [expected, filepath] = generate_input(false, false); + auto const [result, num_chunks] = chunked_read(filepath, 240'000); + EXPECT_EQ(num_chunks, 2); + CUDF_TEST_EXPECT_TABLES_EQUAL(*expected, *result); + } + + { + auto const [expected, filepath] = generate_input(false, true); + auto const [result, num_chunks] = chunked_read(filepath, 240'000); + EXPECT_EQ(num_chunks, 2); + CUDF_TEST_EXPECT_TABLES_EQUAL(*expected, *result); + } + + { + auto const [expected, filepath] = generate_input(true, false); auto const [result, num_chunks] = chunked_read(filepath, 240'000); EXPECT_EQ(num_chunks, 2); CUDF_TEST_EXPECT_TABLES_EQUAL(*expected, *result); } { - auto const [expected, filepath] = generate_input(true); + auto const [expected, filepath] = generate_input(true, true); auto const [result, num_chunks] = chunked_read(filepath, 240'000); EXPECT_EQ(num_chunks, 2); CUDF_TEST_EXPECT_TABLES_EQUAL(*expected, *result); @@ -186,7 +209,8 @@ TEST_F(ParquetChunkedReaderTest, TestChunkedReadBoundaryCases) std::vector> input_columns; auto const value_iter = thrust::make_counting_iterator(0); input_columns.emplace_back(int32s_col(value_iter, value_iter + num_rows).release()); - return write_file(input_columns, "chunked_read_simple_boundary", false /*nullable*/); + return write_file( + input_columns, "chunked_read_simple_boundary", false /*nullable*/, false /*delta_encoding*/); }(); // Test with zero limit: everything will be read in one chunk @@ -264,7 +288,7 @@ TEST_F(ParquetChunkedReaderTest, TestChunkedReadWithString) { auto constexpr num_rows = 60'000; - auto const generate_input = [num_rows](bool nullable) { + auto const generate_input = [num_rows](bool nullable, bool use_delta) { std::vector> input_columns; auto const value_iter = thrust::make_counting_iterator(0); @@ -296,13 +320,16 @@ TEST_F(ParquetChunkedReaderTest, TestChunkedReadWithString) return write_file(input_columns, "chunked_read_with_strings", nullable, + use_delta, 512 * 1024, // 512KB per page 20000 // 20k rows per page ); }; - auto const [expected_no_null, filepath_no_null] = generate_input(false); - auto const [expected_with_nulls, filepath_with_nulls] = generate_input(true); + auto const [expected_no_null, filepath_no_null] = generate_input(false, false); + auto const [expected_with_nulls, filepath_with_nulls] = generate_input(true, false); + auto const [expected_no_null_delta, filepath_no_null_delta] = generate_input(false, true); + auto const [expected_with_nulls_delta, filepath_with_nulls_delta] = generate_input(true, true); // Test with zero limit: everything will be read in one chunk { @@ -315,6 +342,16 @@ TEST_F(ParquetChunkedReaderTest, TestChunkedReadWithString) EXPECT_EQ(num_chunks, 1); CUDF_TEST_EXPECT_TABLES_EQUAL(*expected_with_nulls, *result); } + { + auto const [result, num_chunks] = chunked_read(filepath_no_null_delta, 0); + EXPECT_EQ(num_chunks, 1); + CUDF_TEST_EXPECT_TABLES_EQUAL(*expected_no_null_delta, *result); + } + { + auto const [result, num_chunks] = chunked_read(filepath_with_nulls_delta, 0); + EXPECT_EQ(num_chunks, 1); + CUDF_TEST_EXPECT_TABLES_EQUAL(*expected_with_nulls_delta, *result); + } // Test with a very small limit: 1 byte { @@ -327,6 +364,16 @@ TEST_F(ParquetChunkedReaderTest, TestChunkedReadWithString) EXPECT_EQ(num_chunks, 3); CUDF_TEST_EXPECT_TABLES_EQUAL(*expected_with_nulls, *result); } + { + auto const [result, num_chunks] = chunked_read(filepath_no_null_delta, 1); + EXPECT_EQ(num_chunks, 3); + CUDF_TEST_EXPECT_TABLES_EQUAL(*expected_no_null_delta, *result); + } + { + auto const [result, num_chunks] = chunked_read(filepath_with_nulls_delta, 1); + EXPECT_EQ(num_chunks, 3); + CUDF_TEST_EXPECT_TABLES_EQUAL(*expected_with_nulls_delta, *result); + } // Test with a very large limit { @@ -339,6 +386,16 @@ TEST_F(ParquetChunkedReaderTest, TestChunkedReadWithString) EXPECT_EQ(num_chunks, 1); CUDF_TEST_EXPECT_TABLES_EQUAL(*expected_with_nulls, *result); } + { + auto const [result, num_chunks] = chunked_read(filepath_no_null_delta, 2L << 40); + EXPECT_EQ(num_chunks, 1); + CUDF_TEST_EXPECT_TABLES_EQUAL(*expected_no_null_delta, *result); + } + { + auto const [result, num_chunks] = chunked_read(filepath_with_nulls_delta, 2L << 40); + EXPECT_EQ(num_chunks, 1); + CUDF_TEST_EXPECT_TABLES_EQUAL(*expected_with_nulls_delta, *result); + } // Other tests: @@ -352,6 +409,16 @@ TEST_F(ParquetChunkedReaderTest, TestChunkedReadWithString) EXPECT_EQ(num_chunks, 2); CUDF_TEST_EXPECT_TABLES_EQUAL(*expected_with_nulls, *result); } + { + auto const [result, num_chunks] = chunked_read(filepath_no_null_delta, 500'000); + EXPECT_EQ(num_chunks, 2); + CUDF_TEST_EXPECT_TABLES_EQUAL(*expected_no_null_delta, *result); + } + { + auto const [result, num_chunks] = chunked_read(filepath_with_nulls_delta, 500'000); + EXPECT_EQ(num_chunks, 2); + CUDF_TEST_EXPECT_TABLES_EQUAL(*expected_with_nulls_delta, *result); + } { auto const [result, num_chunks] = chunked_read(filepath_no_null, 1'000'000); @@ -363,13 +430,23 @@ TEST_F(ParquetChunkedReaderTest, TestChunkedReadWithString) EXPECT_EQ(num_chunks, 1); CUDF_TEST_EXPECT_TABLES_EQUAL(*expected_with_nulls, *result); } + { + auto const [result, num_chunks] = chunked_read(filepath_no_null_delta, 1'000'000); + EXPECT_EQ(num_chunks, 1); + CUDF_TEST_EXPECT_TABLES_EQUAL(*expected_no_null_delta, *result); + } + { + auto const [result, num_chunks] = chunked_read(filepath_with_nulls_delta, 1'000'000); + EXPECT_EQ(num_chunks, 1); + CUDF_TEST_EXPECT_TABLES_EQUAL(*expected_with_nulls_delta, *result); + } } TEST_F(ParquetChunkedReaderTest, TestChunkedReadWithStringPrecise) { auto constexpr num_rows = 60'000; - auto const generate_input = [num_rows](bool nullable) { + auto const generate_input = [num_rows](bool nullable, bool use_delta) { std::vector> input_columns; // strings Page total bytes cumulative @@ -388,12 +465,13 @@ TEST_F(ParquetChunkedReaderTest, TestChunkedReadWithStringPrecise) return write_file(input_columns, "chunked_read_with_strings_precise", nullable, + use_delta, 512 * 1024, // 512KB per page 20000 // 20k rows per page ); }; - auto const [expected_no_null, filepath_no_null] = generate_input(false); + auto const [expected_no_null, filepath_no_null] = generate_input(false, false); // a chunk limit of 1 byte less than 2 pages should force it to produce 3 chunks: // each 1 page in size @@ -434,6 +512,7 @@ TEST_F(ParquetChunkedReaderTest, TestChunkedReadWithStructs) return write_file(input_columns, "chunked_read_with_structs", nullable, + false /*delta_encoding*/, 512 * 1024, // 512KB per page 20000 // 20k rows per page ); @@ -515,6 +594,7 @@ TEST_F(ParquetChunkedReaderTest, TestChunkedReadWithListsNoNulls) return write_file(input_columns, "chunked_read_with_lists_no_null", false /*nullable*/, + false /*delta_encoding*/, 512 * 1024, // 512KB per page 20000 // 20k rows per page ); @@ -597,6 +677,7 @@ TEST_F(ParquetChunkedReaderTest, TestChunkedReadWithListsHavingNulls) return write_file(input_columns, "chunked_read_with_lists_nulls", true /*nullable*/, + false /*delta_encoding*/, 512 * 1024, // 512KB per page 20000 // 20k rows per page ); @@ -685,6 +766,7 @@ TEST_F(ParquetChunkedReaderTest, TestChunkedReadWithStructsOfLists) return write_file(input_columns, "chunked_read_with_structs_of_lists", nullable, + false /*delta_encoding*/, 512 * 1024, // 512KB per page 20000 // 20k rows per page ); @@ -825,6 +907,7 @@ TEST_F(ParquetChunkedReaderTest, TestChunkedReadWithListsOfStructs) return write_file(input_columns, "chunked_read_with_lists_of_structs", nullable, + false /*delta_encoding*/, 512 * 1024, // 512KB per page 20000 // 20k rows per page ); From 3b228e2c6d3ec39fcba553c63d53a56760dc1ca6 Mon Sep 17 00:00:00 2001 From: Nghia Truong <7416935+ttnghia@users.noreply.github.com> Date: Fri, 1 Mar 2024 11:22:32 -0700 Subject: [PATCH 327/384] Implement `segmented_row_bit_count` for computing row sizes by segments of rows (#15169) This implements `cudf::segmented_bit_count`, a version of `cudf::row_bit_count` with adding `segment_length` parameter to the interface. With the new parameter, `segmented_bit_count` allows to compute aggregate sizes for each "segment" of rows instead of computing size for each row. Currently, only fixed-length segments are supported. Authors: - Nghia Truong (https://github.com/ttnghia) Approvers: - https://github.com/nvdbaranec - Vukasin Milovanovic (https://github.com/vuule) URL: https://github.com/rapidsai/cudf/pull/15169 --- cpp/include/cudf/detail/transform.hpp | 12 +- cpp/include/cudf/transform.hpp | 25 +- cpp/src/transform/row_bit_count.cu | 95 ++++-- cpp/tests/CMakeLists.txt | 1 + cpp/tests/transform/row_bit_count_test.cu | 300 +++++++++--------- .../transform/segmented_row_bit_count_test.cu | 251 +++++++++++++++ 6 files changed, 503 insertions(+), 181 deletions(-) create mode 100644 cpp/tests/transform/segmented_row_bit_count_test.cu diff --git a/cpp/include/cudf/detail/transform.hpp b/cpp/include/cudf/detail/transform.hpp index 215ad50aed6..965fea84860 100644 --- a/cpp/include/cudf/detail/transform.hpp +++ b/cpp/include/cudf/detail/transform.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -100,5 +100,15 @@ std::unique_ptr row_bit_count(table_view const& t, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr); +/** + * @copydoc cudf::segmented_row_bit_count + * + * @param stream CUDA stream used for device memory operations and kernel launches. + */ +std::unique_ptr segmented_row_bit_count(table_view const& t, + size_type segment_length, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr); + } // namespace detail } // namespace cudf diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index 412fe17ef26..49ec3d7c0d5 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -224,5 +224,28 @@ std::unique_ptr row_bit_count( table_view const& t, rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()); +/** + * @brief Returns an approximate cumulative size in bits of all columns in the `table_view` for + * each segment of rows. + * + * This is similar to counting bit size per row for the input table in `cudf::row_bit_count`, + * except that row sizes are accumulated by segments. + * + * Currently, only fixed-length segments are supported. In case the input table has number of rows + * not divisible by `segment_length`, its last segment is considered as shorter than the others. + * + * @throw std::invalid_argument if the input `segment_length` is non-positive or larger than the + * number of rows in the input table. + * + * @param t The table view to perform the computation on + * @param segment_length The number of rows in each segment for which the total size is computed + * @param mr Device memory resource used to allocate the returned columns' device memory + * @return A 32-bit integer column containing the bit counts for each segment of rows + */ +std::unique_ptr segmented_row_bit_count( + table_view const& t, + size_type segment_length, + rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()); + /** @} */ // end of group } // namespace cudf diff --git a/cpp/src/transform/row_bit_count.cu b/cpp/src/transform/row_bit_count.cu index eda8ec7a463..78bd558501b 100644 --- a/cpp/src/transform/row_bit_count.cu +++ b/cpp/src/transform/row_bit_count.cu @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -31,8 +32,10 @@ #include #include -#include +#include +#include #include +#include namespace cudf { namespace detail { @@ -398,26 +401,32 @@ __device__ size_type row_size_functor::operator()(column_device_vie * @param cols An span of column_device_views representing a column hierarchy * @param info An span of column_info structs corresponding the elements in `cols` * @param output Output span of size (# rows) where per-row bit sizes are stored + * @param segment_length The number of rows in each segment for which the total size is computed * @param max_branch_depth Maximum depth of the span stack needed per-thread */ -CUDF_KERNEL void compute_row_sizes(device_span cols, - device_span info, - device_span output, - size_type max_branch_depth) +CUDF_KERNEL void compute_segment_sizes(device_span cols, + device_span info, + device_span output, + size_type segment_length, + size_type max_branch_depth) { extern __shared__ row_span thread_branch_stacks[]; int const tid = threadIdx.x + blockIdx.x * blockDim.x; - auto const num_rows = output.size(); - if (tid >= num_rows) { return; } + auto const num_segments = static_cast(output.size()); + if (tid >= num_segments) { return; } // my_branch_stack points to the last span prior to branching. a branch occurs only // when we are inside of a list contained within a struct column. row_span* my_branch_stack = thread_branch_stacks + (threadIdx.x * max_branch_depth); size_type branch_depth{0}; - // current row span - always starts at 1 row. - row_span cur_span{tid, tid + 1}; + // current row span - always starts at spanning over `segment_length` rows. + auto const num_rows = cols[0].size(); + auto const get_default_row_span = [=] { + return row_span{tid * segment_length, cuda::std::min((tid + 1) * segment_length, num_rows)}; + }; + auto cur_span = get_default_row_span(); // output size size_type& size = output[tid]; @@ -444,7 +453,7 @@ CUDF_KERNEL void compute_row_sizes(device_span cols, if (info[idx].depth == 0) { branch_depth = 0; last_branch_depth = 0; - cur_span = row_span{tid, tid + 1}; + cur_span = get_default_row_span(); } // add the contributing size of this row @@ -465,17 +474,18 @@ CUDF_KERNEL void compute_row_sizes(device_span cols, } // anonymous namespace -/** - * @copydoc cudf::detail::row_bit_count - * - */ -std::unique_ptr row_bit_count(table_view const& t, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) +std::unique_ptr segmented_row_bit_count(table_view const& t, + size_type segment_length, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) { - // no rows + // If there is no rows, segment_length will not be checked. if (t.num_rows() <= 0) { return cudf::make_empty_column(type_id::INT32); } + CUDF_EXPECTS(segment_length >= 1 && segment_length <= t.num_rows(), + "Invalid segment length.", + std::invalid_argument); + // flatten the hierarchy and determine some information about it. std::vector cols; std::vector info; @@ -484,17 +494,28 @@ std::unique_ptr row_bit_count(table_view const& t, CUDF_EXPECTS(info.size() == cols.size(), "Size/info mismatch"); // create output buffer and view - auto output = cudf::make_fixed_width_column( - data_type{type_id::INT32}, t.num_rows(), mask_state::UNALLOCATED, stream, mr); + auto const num_segments = cudf::util::div_rounding_up_safe(t.num_rows(), segment_length); + auto output = cudf::make_fixed_width_column( + data_type{type_id::INT32}, num_segments, mask_state::UNALLOCATED, stream, mr); mutable_column_view mcv = output->mutable_view(); // simple case. if we have no complex types (lists, strings, etc), the per-row size is already // trivially computed if (h_info.complex_type_count <= 0) { - thrust::fill(rmm::exec_policy(stream), - mcv.begin(), - mcv.end(), - h_info.simple_per_row_size); + thrust::tabulate( + rmm::exec_policy_nosync(stream), + mcv.begin(), + mcv.end(), + cuda::proclaim_return_type( + [segment_length, + num_rows = t.num_rows(), + per_row_size = h_info.simple_per_row_size] __device__(size_type const segment_idx) { + // Since the number of rows may not divisible by segment_length, + // the last segment may be shorter than the others. + auto const current_length = + cuda::std::min(segment_length, num_rows - segment_length * segment_idx); + return per_row_size * current_length; + })); return output; } @@ -523,22 +544,34 @@ std::unique_ptr row_bit_count(table_view const& t, // should we be aborting if we reach some extremely small block size, or just if we hit 0? CUDF_EXPECTS(block_size > 0, "Encountered a column hierarchy too complex for row_bit_count"); - cudf::detail::grid_1d grid{t.num_rows(), block_size, 1}; - compute_row_sizes<<>>( + cudf::detail::grid_1d grid{num_segments, block_size, 1}; + compute_segment_sizes<<>>( {std::get<1>(d_cols), cols.size()}, {d_info.data(), info.size()}, - {mcv.data(), static_cast(t.num_rows())}, + {mcv.data(), static_cast(mcv.size())}, + segment_length, h_info.max_branch_depth); return output; } +std::unique_ptr row_bit_count(table_view const& t, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) +{ + return segmented_row_bit_count(t, 1, stream, mr); +} + } // namespace detail -/** - * @copydoc cudf::row_bit_count - * - */ +std::unique_ptr segmented_row_bit_count(table_view const& t, + size_type segment_length, + rmm::mr::device_memory_resource* mr) +{ + CUDF_FUNC_RANGE(); + return detail::segmented_row_bit_count(t, segment_length, cudf::get_default_stream(), mr); +} + std::unique_ptr row_bit_count(table_view const& t, rmm::mr::device_memory_resource* mr) { CUDF_FUNC_RANGE(); diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 3e377b07eee..93443b04bd5 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -259,6 +259,7 @@ ConfigureTest( transform/mask_to_bools_test.cpp transform/bools_to_mask_test.cpp transform/row_bit_count_test.cu + transform/segmented_row_bit_count_test.cu transform/one_hot_encode_tests.cpp ) diff --git a/cpp/tests/transform/row_bit_count_test.cu b/cpp/tests/transform/row_bit_count_test.cu index 236407e62f3..01a042130d6 100644 --- a/cpp/tests/transform/row_bit_count_test.cu +++ b/cpp/tests/transform/row_bit_count_test.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,6 +35,148 @@ #include +namespace row_bit_count_test { + +template +std::pair, std::unique_ptr> build_list_column() +{ + using LCW = cudf::test::lists_column_wrapper; + constexpr cudf::size_type type_size = sizeof(cudf::device_storage_type_t) * CHAR_BIT; + + // { + // {{1, 2}, {3, 4, 5}}, + // {{}}, + // {LCW{10}}, + // {{6, 7, 8}, {9}}, + // {{-1, -2}, {-3, -4}}, + // {{-5, -6, -7}, {-8, -9}} + // } + cudf::test::fixed_width_column_wrapper values{ + 1, 2, 3, 4, 5, 10, 6, 7, 8, 9, -1, -2, -3, -4, -5, -6, -7, -8, -9}; + cudf::test::fixed_width_column_wrapper inner_offsets{ + 0, 2, 5, 6, 9, 10, 12, 14, 17, 19}; + auto inner_list = cudf::make_lists_column(9, inner_offsets.release(), values.release(), 0, {}); + cudf::test::fixed_width_column_wrapper outer_offsets{0, 2, 2, 3, 5, 7, 9}; + auto list = cudf::make_lists_column(6, outer_offsets.release(), std::move(inner_list), 0, {}); + + // expected size = (num rows at level 1 + num_rows at level 2) + # values in the leaf + cudf::test::fixed_width_column_wrapper expected{ + ((4 + 8) * CHAR_BIT) + (type_size * 5), + ((4 + 0) * CHAR_BIT) + (type_size * 0), + ((4 + 4) * CHAR_BIT) + (type_size * 1), + ((4 + 8) * CHAR_BIT) + (type_size * 4), + ((4 + 8) * CHAR_BIT) + (type_size * 4), + ((4 + 8) * CHAR_BIT) + (type_size * 5)}; + + return {std::move(list), expected.release()}; +} + +std::pair, std::unique_ptr> build_struct_column() +{ + std::vector struct_validity{0, 1, 1, 1, 1, 0}; + std::vector strings{"abc", "def", "", "z", "bananas", "daïs"}; + + cudf::test::fixed_width_column_wrapper col0{0, 1, 2, 3, 4, 5}; + cudf::test::fixed_width_column_wrapper col1{{8, 9, 10, 11, 12, 13}, {1, 0, 1, 1, 1, 1}}; + cudf::test::strings_column_wrapper col2(strings.begin(), strings.end()); + + // creating a struct column will cause all child columns to be promoted to have validity + cudf::test::structs_column_wrapper struct_col({col0, col1, col2}, struct_validity); + + // expect (1 offset (4 bytes) + (length of string if row is valid) + 1 validity bit) + + // (1 float + 1 validity bit) + + // (1 int16_t + 1 validity bit) + + // (1 validity bit) + cudf::test::fixed_width_column_wrapper expected_sizes{84, 108, 84, 92, 140, 84}; + + return {struct_col.release(), expected_sizes.release()}; +} + +std::unique_ptr build_nested_column1(std::vector const& struct_validity) +{ + // tests the "branching" case -> list ...>>> + + // List, float, int16> + + // Inner list column + cudf::test::lists_column_wrapper list{{1, 2, 3, 4, 5}, + {6, 7, 8}, + {33, 34, 35, 36, 37, 38, 39}, + {-1, -2}, + {-10, -11, -1, -20}, + {40, 41, 42}, + {100, 200, 300}, + {-100, -200, -300}}; + + // floats + std::vector ages{5, 10, 15, 20, 4, 75, 16, -16}; + std::vector ages_validity = {1, 1, 1, 1, 0, 1, 0, 1}; + auto ages_column = + cudf::test::fixed_width_column_wrapper(ages.begin(), ages.end(), ages_validity.begin()); + + // int16 values + std::vector vals{-1, -2, -3, 1, 2, 3, 8, 9}; + auto i16_column = cudf::test::fixed_width_column_wrapper(vals.begin(), vals.end()); + + // Assemble struct column + auto struct_column = + cudf::test::structs_column_wrapper({list, ages_column, i16_column}, struct_validity); + + // wrap in a list + std::vector outer_offsets{0, 1, 1, 3, 6, 7, 8}; + cudf::test::fixed_width_column_wrapper outer_offsets_col(outer_offsets.begin(), + outer_offsets.end()); + auto const size = static_cast(outer_offsets_col).size() - 1; + + // Each struct (list child) has size: + // (1 offset (4 bytes) + (list size if row is valid) + 1 validity bit) + + // (1 float + 1 validity bit) + + // (1 int16_t + 1 validity bit) + + // (1 validity bit) + // Each top level list has size: + // 1 offset (4 bytes) + (list size if row is valid). + + return cudf::make_lists_column(static_cast(size), + outer_offsets_col.release(), + struct_column.release(), + 0, + rmm::device_buffer{}); +} + +std::unique_ptr build_nested_column2(std::vector const& struct_validity) +{ + // List>, Struct>> + + // Inner list column + // clang-format off + cudf::test::lists_column_wrapper list{ + {{1, 2, 3, 4, 5}, {2, 3}}, + {{6, 7, 8}, {8, 9}}, + {{1, 2}, {3, 4, 5}, {33, 34, 35, 36, 37, 38, 39}}}; + // clang-format on + + // Inner struct + std::vector vals{-1, -2, -3}; + auto i16_column = cudf::test::fixed_width_column_wrapper(vals.begin(), vals.end()); + auto inner_struct = cudf::test::structs_column_wrapper({i16_column}); + + // outer struct + auto outer_struct = cudf::test::structs_column_wrapper({list, inner_struct}, struct_validity); + + // wrap in a list + std::vector outer_offsets{0, 1, 1, 3}; + cudf::test::fixed_width_column_wrapper outer_offsets_col(outer_offsets.begin(), + outer_offsets.end()); + auto const size = static_cast(outer_offsets_col).size() - 1; + return cudf::make_lists_column(static_cast(size), + outer_offsets_col.release(), + outer_struct.release(), + 0, + rmm::device_buffer{}); +} + +} // namespace row_bit_count_test + template struct RowBitCountTyped : public cudf::test::BaseFixture {}; @@ -82,45 +224,11 @@ TYPED_TEST(RowBitCountTyped, SimpleTypesWithNulls) CUDF_TEST_EXPECT_COLUMNS_EQUAL(*expected, *result); } -template -std::pair, std::unique_ptr> build_list_column() -{ - using LCW = cudf::test::lists_column_wrapper; - constexpr cudf::size_type type_size = sizeof(cudf::device_storage_type_t) * CHAR_BIT; - - // { - // {{1, 2}, {3, 4, 5}}, - // {{}}, - // {LCW{10}}, - // {{6, 7, 8}, {9}}, - // {{-1, -2}, {-3, -4}}, - // {{-5, -6, -7}, {-8, -9}} - // } - cudf::test::fixed_width_column_wrapper values{ - 1, 2, 3, 4, 5, 10, 6, 7, 8, 9, -1, -2, -3, -4, -5, -6, -7, -8, -9}; - cudf::test::fixed_width_column_wrapper inner_offsets{ - 0, 2, 5, 6, 9, 10, 12, 14, 17, 19}; - auto inner_list = cudf::make_lists_column(9, inner_offsets.release(), values.release(), 0, {}); - cudf::test::fixed_width_column_wrapper outer_offsets{0, 2, 2, 3, 5, 7, 9}; - auto list = cudf::make_lists_column(6, outer_offsets.release(), std::move(inner_list), 0, {}); - - // expected size = (num rows at level 1 + num_rows at level 2) + # values in the leaf - cudf::test::fixed_width_column_wrapper expected{ - ((4 + 8) * CHAR_BIT) + (type_size * 5), - ((4 + 0) * CHAR_BIT) + (type_size * 0), - ((4 + 4) * CHAR_BIT) + (type_size * 1), - ((4 + 8) * CHAR_BIT) + (type_size * 4), - ((4 + 8) * CHAR_BIT) + (type_size * 4), - ((4 + 8) * CHAR_BIT) + (type_size * 5)}; - - return {std::move(list), expected.release()}; -} - TYPED_TEST(RowBitCountTyped, Lists) { using T = TypeParam; - auto [col, expected_sizes] = build_list_column(); + auto [col, expected_sizes] = row_bit_count_test::build_list_column(); cudf::table_view t({*col}); auto result = cudf::row_bit_count(t); @@ -272,27 +380,6 @@ TEST_F(RowBitCount, StructsWithLists_RowsExceedingASingleBlock) CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(row_bit_counts->view(), expected_row_bit_counts->view()); } -std::pair, std::unique_ptr> build_struct_column() -{ - std::vector struct_validity{0, 1, 1, 1, 1, 0}; - std::vector strings{"abc", "def", "", "z", "bananas", "daïs"}; - - cudf::test::fixed_width_column_wrapper col0{0, 1, 2, 3, 4, 5}; - cudf::test::fixed_width_column_wrapper col1{{8, 9, 10, 11, 12, 13}, {1, 0, 1, 1, 1, 1}}; - cudf::test::strings_column_wrapper col2(strings.begin(), strings.end()); - - // creating a struct column will cause all child columns to be promoted to have validity - cudf::test::structs_column_wrapper struct_col({col0, col1, col2}, struct_validity); - - // expect (1 offset (4 bytes) + (length of string if row is valid) + 1 validity bit) + - // (1 float + 1 validity bit) + - // (1 int16_t + 1 validity bit) + - // (1 validity bit) - cudf::test::fixed_width_column_wrapper expected_sizes{84, 108, 84, 92, 140, 84}; - - return {struct_col.release(), expected_sizes.release()}; -} - TEST_F(RowBitCount, StructsNoNulls) { std::vector strings{"abc", "daïs", "", "z", "bananas", "warp"}; @@ -319,7 +406,7 @@ TEST_F(RowBitCount, StructsNoNulls) TEST_F(RowBitCount, StructsNulls) { - auto [struct_col, expected_sizes] = build_struct_column(); + auto [struct_col, expected_sizes] = row_bit_count_test::build_struct_column(); cudf::table_view t({*struct_col}); auto result = cudf::row_bit_count(t); @@ -346,101 +433,18 @@ TEST_F(RowBitCount, StructsNested) CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, *result); } -std::unique_ptr build_nested_column1(std::vector const& struct_validity) -{ - // tests the "branching" case -> list ...>>> - - // List, float, int16> - - // Inner list column - cudf::test::lists_column_wrapper list{{1, 2, 3, 4, 5}, - {6, 7, 8}, - {33, 34, 35, 36, 37, 38, 39}, - {-1, -2}, - {-10, -11, -1, -20}, - {40, 41, 42}, - {100, 200, 300}, - {-100, -200, -300}}; - - // floats - std::vector ages{5, 10, 15, 20, 4, 75, 16, -16}; - std::vector ages_validity = {1, 1, 1, 1, 0, 1, 0, 1}; - auto ages_column = - cudf::test::fixed_width_column_wrapper(ages.begin(), ages.end(), ages_validity.begin()); - - // int16 values - std::vector vals{-1, -2, -3, 1, 2, 3, 8, 9}; - auto i16_column = cudf::test::fixed_width_column_wrapper(vals.begin(), vals.end()); - - // Assemble struct column - auto struct_column = - cudf::test::structs_column_wrapper({list, ages_column, i16_column}, struct_validity); - - // wrap in a list - std::vector outer_offsets{0, 1, 1, 3, 6, 7, 8}; - cudf::test::fixed_width_column_wrapper outer_offsets_col(outer_offsets.begin(), - outer_offsets.end()); - auto const size = static_cast(outer_offsets_col).size() - 1; - - // Each struct (list child) has size: - // (1 offset (4 bytes) + (list size if row is valid) + 1 validity bit) + - // (1 float + 1 validity bit) + - // (1 int16_t + 1 validity bit) + - // (1 validity bit) - // Each top level list has size: - // 1 offset (4 bytes) + (list size if row is valid). - - return cudf::make_lists_column(static_cast(size), - outer_offsets_col.release(), - struct_column.release(), - 0, - rmm::device_buffer{}); -} - -std::unique_ptr build_nested_column2(std::vector const& struct_validity) -{ - // List>, Struct>> - - // Inner list column - // clang-format off - cudf::test::lists_column_wrapper list{ - {{1, 2, 3, 4, 5}, {2, 3}}, - {{6, 7, 8}, {8, 9}}, - {{1, 2}, {3, 4, 5}, {33, 34, 35, 36, 37, 38, 39}}}; - // clang-format on - - // Inner struct - std::vector vals{-1, -2, -3}; - auto i16_column = cudf::test::fixed_width_column_wrapper(vals.begin(), vals.end()); - auto inner_struct = cudf::test::structs_column_wrapper({i16_column}); - - // outer struct - auto outer_struct = cudf::test::structs_column_wrapper({list, inner_struct}, struct_validity); - - // wrap in a list - std::vector outer_offsets{0, 1, 1, 3}; - cudf::test::fixed_width_column_wrapper outer_offsets_col(outer_offsets.begin(), - outer_offsets.end()); - auto const size = static_cast(outer_offsets_col).size() - 1; - return make_lists_column(static_cast(size), - outer_offsets_col.release(), - outer_struct.release(), - 0, - rmm::device_buffer{}); -} - TEST_F(RowBitCount, NestedTypes) { // List, float, List, int16> { - auto const col_no_nulls = build_nested_column1({1, 1, 1, 1, 1, 1, 1, 1}); + auto const col_no_nulls = row_bit_count_test::build_nested_column1({1, 1, 1, 1, 1, 1, 1, 1}); auto const expected_sizes_no_nulls = cudf::test::fixed_width_column_wrapper{276, 32, 520, 572, 212, 212} .release(); cudf::table_view no_nulls_t({*col_no_nulls}); auto no_nulls_result = cudf::row_bit_count(no_nulls_t); - auto const col_nulls = build_nested_column1({0, 0, 1, 1, 1, 1, 1, 1}); + auto const col_nulls = row_bit_count_test::build_nested_column1({0, 0, 1, 1, 1, 1, 1, 1}); auto const expected_sizes_with_nulls = cudf::test::fixed_width_column_wrapper{116, 32, 424, 572, 212, 212} .release(); @@ -469,11 +473,11 @@ TEST_F(RowBitCount, NestedTypes) // List>, Struct>> { - auto col_no_nulls = build_nested_column2({1, 1, 1}); + auto col_no_nulls = row_bit_count_test::build_nested_column2({1, 1, 1}); cudf::table_view no_nulls_t({*col_no_nulls}); auto no_nulls_result = cudf::row_bit_count(no_nulls_t); - auto col_nulls = build_nested_column2({1, 0, 1}); + auto col_nulls = row_bit_count_test::build_nested_column2({1, 0, 1}); cudf::table_view nulls_t({*col_nulls}); auto nulls_result = cudf::row_bit_count(nulls_t); @@ -597,15 +601,15 @@ struct sum_functor { TEST_F(RowBitCount, Table) { // complex nested column - auto col0 = build_nested_column1({1, 1, 1, 1, 1, 1, 1, 1}); + auto col0 = row_bit_count_test::build_nested_column1({1, 1, 1, 1, 1, 1, 1, 1}); auto col0_sizes = cudf::test::fixed_width_column_wrapper{276, 32, 520, 572, 212, 212}.release(); // struct column - auto [col1, col1_sizes] = build_struct_column(); + auto [col1, col1_sizes] = row_bit_count_test::build_struct_column(); // list column - auto [col2, col2_sizes] = build_list_column(); + auto [col2, col2_sizes] = row_bit_count_test::build_list_column(); cudf::table_view t({*col0, *col1, *col2}); auto result = cudf::row_bit_count(t); diff --git a/cpp/tests/transform/segmented_row_bit_count_test.cu b/cpp/tests/transform/segmented_row_bit_count_test.cu new file mode 100644 index 00000000000..652b9053582 --- /dev/null +++ b/cpp/tests/transform/segmented_row_bit_count_test.cu @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +// Reuse function defined in `row_bit_count_test.cu`. +namespace row_bit_count_test { +template +std::pair, std::unique_ptr> build_list_column(); +std::pair, std::unique_ptr> build_struct_column(); +std::unique_ptr build_nested_column1(std::vector const& struct_validity); +std::unique_ptr build_nested_column2(std::vector const& struct_validity); +} // namespace row_bit_count_test + +namespace { + +// Compute row bit count, then sum up sizes for each segment of rows. +std::pair, std::unique_ptr> +compute_segmented_row_bit_count(cudf::table_view const& input, cudf::size_type segment_length) +{ + // The expected values are computed with the assumption that + // the outputs of `cudf::row_bit_count` are correct. + // This should be fine as they are verified by their own unit tests in `row_bit_count_test.cu`. + auto const row_sizes = cudf::row_bit_count(input); + auto const num_segments = cudf::util::div_rounding_up_safe(row_sizes->size(), segment_length); + auto expected = + cudf::make_fixed_width_column(cudf::data_type{cudf::type_id::INT32}, num_segments); + + thrust::transform( + rmm::exec_policy(cudf::get_default_stream()), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(num_segments), + expected->mutable_view().begin(), + cuda::proclaim_return_type( + [segment_length, + num_segments, + num_rows = row_sizes->size(), + d_sizes = row_sizes->view().begin()] __device__(auto const segment_idx) { + // Since the number of rows may not divisible by segment_length, + // the last segment may be shorter than the others. + auto const size_begin = d_sizes + segment_idx * segment_length; + auto const size_end = std::min(size_begin + segment_length, d_sizes + num_rows); + return thrust::reduce(thrust::seq, size_begin, size_end); + })); + + auto actual = cudf::segmented_row_bit_count(input, segment_length); + return {std::move(expected), std::move(actual)}; +} + +} // namespace + +struct SegmentedRowBitCount : public cudf::test::BaseFixture {}; + +TEST_F(SegmentedRowBitCount, Lists) +{ + auto const col = std::get<0>(row_bit_count_test::build_list_column()); + auto const input = cudf::table_view({*col}); + + auto constexpr segment_length = 3; + auto const [expected, actual] = compute_segmented_row_bit_count(input, segment_length); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*expected, *actual); +} + +TEST_F(SegmentedRowBitCount, StringsWithNulls) +{ + // clang-format off + std::vector const strings { "daïs", "def", "", "z", "bananas", "warp", "", "zing" }; + std::vector const valids { 1, 0, 0, 1, 0, 1, 1, 1 }; + // clang-format on + cudf::test::strings_column_wrapper const col(strings.begin(), strings.end(), valids.begin()); + auto const input = cudf::table_view({col}); + + auto constexpr segment_length = 2; + auto const [expected, actual] = compute_segmented_row_bit_count(input, segment_length); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*expected, *actual); +} + +TEST_F(SegmentedRowBitCount, StructsWithNulls) +{ + auto const col = std::get<0>(row_bit_count_test::build_struct_column()); + auto const input = cudf::table_view({*col}); + + auto constexpr segment_length = 2; + auto const [expected, actual] = compute_segmented_row_bit_count(input, segment_length); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*expected, *actual); +} + +TEST_F(SegmentedRowBitCount, NestedTypes) +{ + auto constexpr segment_length = 2; + + { + // List, float, List, int16> + auto const col = row_bit_count_test::build_nested_column1({1, 1, 1, 1, 1, 1, 1, 1}); + auto const input = cudf::table_view({*col}); + auto const [expected, actual] = compute_segmented_row_bit_count(input, segment_length); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*expected, *actual); + } + { + // List, float, List, int16> + auto const col = row_bit_count_test::build_nested_column1({0, 0, 1, 1, 1, 1, 1, 1}); + auto const input = cudf::table_view({*col}); + auto const [expected, actual] = compute_segmented_row_bit_count(input, segment_length); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*expected, *actual); + } + + { + // List>, Struct>> + auto const col = row_bit_count_test::build_nested_column2({1, 1, 1}); + auto const input = cudf::table_view({*col}); + auto const [expected, actual] = compute_segmented_row_bit_count(input, segment_length); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*expected, *actual); + } + { + // List>, Struct>> + auto const col = row_bit_count_test::build_nested_column2({1, 0, 1}); + auto const input = cudf::table_view({*col}); + auto const [expected, actual] = compute_segmented_row_bit_count(input, segment_length); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*expected, *actual); + } +} + +TEST_F(SegmentedRowBitCount, NestedTypesTable) +{ + auto const col0 = row_bit_count_test::build_nested_column1({1, 1, 1, 1, 1, 1, 1, 1}); + auto const col1 = std::get<0>(row_bit_count_test::build_struct_column()); + auto const col2 = std::get<0>(row_bit_count_test::build_list_column()); + auto const input = cudf::table_view({*col0, *col1, *col2}); + + { + auto const segment_length = 2; + auto const [expected, actual] = compute_segmented_row_bit_count(input, segment_length); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*expected, *actual); + } + + { + auto const segment_length = 4; + auto const [expected, actual] = compute_segmented_row_bit_count(input, segment_length); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*expected, *actual); + } + + { + auto const segment_length = 5; + auto const [expected, actual] = compute_segmented_row_bit_count(input, segment_length); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*expected, *actual); + } +} + +TEST_F(SegmentedRowBitCount, EmptyInput) +{ + { + auto const input = cudf::table_view{}; + { + auto const result = cudf::segmented_row_bit_count(input, 0); + EXPECT_TRUE(result != nullptr && result->size() == 0); + } + { + auto const result = cudf::segmented_row_bit_count(input, 1000); + EXPECT_TRUE(result != nullptr && result->size() == 0); + } + } + + { + auto const strings = cudf::make_empty_column(cudf::type_id::STRING); + auto const ints = cudf::make_empty_column(cudf::type_id::INT32); + auto const input = cudf::table_view{{*strings, *ints}}; + { + auto const result = cudf::segmented_row_bit_count(input, 0); + EXPECT_TRUE(result != nullptr && result->size() == 0); + } + { + auto const result = cudf::segmented_row_bit_count(input, 1000); + EXPECT_TRUE(result != nullptr && result->size() == 0); + } + } +} + +TEST_F(SegmentedRowBitCount, InvalidSegment) +{ + auto const col = cudf::make_fixed_width_column(cudf::data_type{cudf::type_to_id()}, 16); + auto const input = cudf::table_view({*col}); + + EXPECT_NO_THROW(cudf::segmented_row_bit_count(input, 1)); + EXPECT_NO_THROW(cudf::segmented_row_bit_count(input, input.num_rows())); + EXPECT_THROW(cudf::segmented_row_bit_count(input, -1), std::invalid_argument); + EXPECT_THROW(cudf::segmented_row_bit_count(input, 0), std::invalid_argument); + EXPECT_THROW(cudf::segmented_row_bit_count(input, input.num_rows() + 1), std::invalid_argument); + EXPECT_THROW(cudf::segmented_row_bit_count(input, 1000), std::invalid_argument); +} + +TEST_F(SegmentedRowBitCount, EdgeCases) +{ + auto const col0 = row_bit_count_test::build_nested_column1({1, 1, 1, 1, 1, 1, 1, 1}); + auto const col1 = std::get<0>(row_bit_count_test::build_struct_column()); + auto const col2 = std::get<0>(row_bit_count_test::build_list_column()); + auto const input = cudf::table_view({*col0, *col1, *col2}); + + { + auto const segment_length = 1; + auto const [expected, actual] = compute_segmented_row_bit_count(input, segment_length); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*expected, *actual); + } + + { + EXPECT_EQ(input.num_rows(), 6); + auto const segment_length = 4; // input.num_rows()==6, not divisible by segment_length . + auto const [expected, actual] = compute_segmented_row_bit_count(input, segment_length); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*expected, *actual); + } + + { + auto const segment_length = input.num_rows(); + auto const [expected, actual] = compute_segmented_row_bit_count(input, segment_length); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*expected, *actual); + } +} From d3e49f644be2475bffe0ee779c4d171be938b3af Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Fri, 1 Mar 2024 16:42:34 -0500 Subject: [PATCH 328/384] Fix includes for row_operators.cuh (#15194) Simple change removes the `cudf/sorting.hpp` include from `row_operators.cuh`. Found this while waiting for recompiles to finish. Changes to `sorting.hpp` seemed to cause more recompiling than expected. Also took the opportunity to change the `include ` to `include ` Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Paul Mattione (https://github.com/pmattione-nvidia) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/15194 --- cpp/include/cudf/table/row_operators.cuh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cpp/include/cudf/table/row_operators.cuh b/cpp/include/cudf/table/row_operators.cuh index 4806f96c934..0e57d24f4b3 100644 --- a/cpp/include/cudf/table/row_operators.cuh +++ b/cpp/include/cudf/table/row_operators.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,19 +20,16 @@ #include #include #include -#include #include #include #include +#include #include #include #include -#include #include -#include - namespace cudf { /** @@ -470,7 +467,9 @@ class element_hasher { template ())> __device__ hash_value_type operator()(column_device_view col, size_type row_index) const { - if (has_nulls && col.is_null(row_index)) { return std::numeric_limits::max(); } + if (has_nulls && col.is_null(row_index)) { + return cuda::std::numeric_limits::max(); + } return hash_function{}(col.element(row_index)); } @@ -554,7 +553,7 @@ class element_hasher_with_seed { private: uint32_t _seed{DEFAULT_HASH_SEED}; - hash_value_type _null_hash{std::numeric_limits::max()}; + hash_value_type _null_hash{cuda::std::numeric_limits::max()}; Nullate _has_nulls; }; From f911ce8c784e55c4dbfc997fdf67236eb4842e35 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Fri, 1 Mar 2024 16:42:52 -0500 Subject: [PATCH 329/384] Change make_strings_children to return uvector (#15171) Changes the `cudf::strings::detail::make_strings_children` utility to return a `rmm::device_uvector` instead of a chars column. This further helps enable large strings support by not storing chars in a column. This is an internal utility and so is non-breaking for any public APIs. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Yunsong Wang (https://github.com/PointKernel) URL: https://github.com/rapidsai/cudf/pull/15171 --- cpp/benchmarks/json/json.cu | 3 +- .../cudf/strings/detail/strings_children.cuh | 11 +++--- cpp/src/io/csv/writer_impl.cu | 4 +-- cpp/src/io/json/write_json.cu | 4 +-- cpp/src/lists/interleave_columns.cu | 4 +-- cpp/src/replace/clamp.cu | 4 +-- cpp/src/strings/capitalize.cu | 4 +-- cpp/src/strings/case.cu | 2 +- cpp/src/strings/char_types/char_types.cu | 4 +-- cpp/src/strings/combine/concatenate.cu | 18 ++++------ cpp/src/strings/combine/join.cu | 35 ++++++++++--------- cpp/src/strings/combine/join_list_elements.cu | 18 ++++------ cpp/src/strings/convert/convert_booleans.cu | 2 +- cpp/src/strings/convert/convert_datetime.cu | 18 +++++----- cpp/src/strings/convert/convert_durations.cu | 2 +- .../strings/convert/convert_fixed_point.cu | 2 +- cpp/src/strings/convert/convert_floats.cu | 2 +- cpp/src/strings/convert/convert_hex.cu | 4 +-- cpp/src/strings/convert/convert_integers.cu | 2 +- cpp/src/strings/convert/convert_ipv4.cu | 6 ++-- cpp/src/strings/convert/convert_lists.cu | 9 ++--- cpp/src/strings/convert/convert_urls.cu | 4 +-- cpp/src/strings/filling/fill.cu | 9 ++--- cpp/src/strings/filter_chars.cu | 4 +-- cpp/src/strings/padding.cu | 8 ++--- cpp/src/strings/regex/utilities.cuh | 5 ++- cpp/src/strings/repeat_strings.cu | 13 +++---- cpp/src/strings/replace/backref_re.cu | 6 ++-- cpp/src/strings/replace/multi.cu | 4 +-- cpp/src/strings/replace/multi_re.cu | 4 +-- cpp/src/strings/replace/replace.cu | 4 +-- cpp/src/strings/replace/replace_re.cu | 4 +-- cpp/src/strings/replace/replace_slice.cu | 4 +-- cpp/src/strings/slice.cu | 2 +- cpp/src/strings/translate.cu | 4 +-- cpp/src/text/detokenize.cu | 9 ++--- cpp/src/text/generate_ngrams.cu | 18 ++++------ cpp/src/text/normalize.cu | 8 ++--- cpp/src/text/replace.cu | 8 ++--- 39 files changed, 123 insertions(+), 153 deletions(-) diff --git a/cpp/benchmarks/json/json.cu b/cpp/benchmarks/json/json.cu index 020c8e413b3..a54d7d48dc4 100644 --- a/cpp/benchmarks/json/json.cu +++ b/cpp/benchmarks/json/json.cu @@ -179,8 +179,7 @@ auto build_json_string_column(int desired_bytes, int num_rows) desired_bytes, num_rows, {*d_books, *d_bicycles}, *d_book_pct, *d_misc_order, *d_store_order}; auto [offsets, chars] = cudf::strings::detail::make_strings_children( jb, num_rows, cudf::get_default_stream(), rmm::mr::get_current_device_resource()); - return cudf::make_strings_column( - num_rows, std::move(offsets), std::move(chars->release().data.release()[0]), 0, {}); + return cudf::make_strings_column(num_rows, std::move(offsets), chars.release(), 0, {}); } void BM_case(benchmark::State& state, std::string query_arg) diff --git a/cpp/include/cudf/strings/detail/strings_children.cuh b/cpp/include/cudf/strings/detail/strings_children.cuh index 8e2b6055a5c..49c4be88ca5 100644 --- a/cpp/include/cudf/strings/detail/strings_children.cuh +++ b/cpp/include/cudf/strings/detail/strings_children.cuh @@ -34,7 +34,7 @@ namespace strings { namespace detail { /** - * @brief Creates child offsets and chars columns by applying the template function that + * @brief Creates child offsets and chars data by applying the template function that * can be used for computing the output size of each string as well as create the output * * @throws std::overflow_error if the output strings column exceeds the column size limit @@ -49,7 +49,7 @@ namespace detail { * @param strings_count Number of strings. * @param stream CUDA stream used for device memory operations and kernel launches. * @param mr Device memory resource used to allocate the returned columns' device memory. - * @return offsets child column and chars child column for a strings column + * @return Offsets child column and chars data for a strings column */ template auto make_strings_children(SizeAndExecuteFunction size_and_exec_fn, @@ -84,18 +84,17 @@ auto make_strings_children(SizeAndExecuteFunction size_and_exec_fn, std::overflow_error); // Now build the chars column - std::unique_ptr chars_column = - create_chars_child_column(static_cast(bytes), stream, mr); + rmm::device_uvector chars(bytes, stream, mr); // Execute the function fn again to fill the chars column. // Note that if the output chars column has zero size, the function fn should not be called to // avoid accidentally overwriting the offsets. if (bytes > 0) { - size_and_exec_fn.d_chars = chars_column->mutable_view().template data(); + size_and_exec_fn.d_chars = chars.data(); for_each_fn(size_and_exec_fn); } - return std::pair(std::move(offsets_column), std::move(chars_column)); + return std::pair(std::move(offsets_column), std::move(chars)); } /** diff --git a/cpp/src/io/csv/writer_impl.cu b/cpp/src/io/csv/writer_impl.cu index cedcd97e44e..c143d258448 100644 --- a/cpp/src/io/csv/writer_impl.cu +++ b/cpp/src/io/csv/writer_impl.cu @@ -180,12 +180,12 @@ struct column_to_strings_fn { auto d_column = column_device_view::create(column_v, stream_); escape_strings_fn fn{*d_column, delimiter.value(stream_)}; - auto [offsets_column, chars_column] = + auto [offsets_column, chars] = cudf::strings::detail::make_strings_children(fn, column_v.size(), stream_, mr_); return make_strings_column(column_v.size(), std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), column_v.null_count(), cudf::detail::copy_bitmask(column_v, stream_, mr_)); } diff --git a/cpp/src/io/json/write_json.cu b/cpp/src/io/json/write_json.cu index 8c5b309244d..8c3aceeefd4 100644 --- a/cpp/src/io/json/write_json.cu +++ b/cpp/src/io/json/write_json.cu @@ -169,12 +169,12 @@ struct escape_strings_fn { rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { - auto [offsets_column, chars_column] = + auto [offsets_column, chars] = cudf::strings::detail::make_strings_children(*this, column_v.size(), stream, mr); return make_strings_column(column_v.size(), std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), column_v.null_count(), cudf::detail::copy_bitmask(column_v, stream, mr)); } diff --git a/cpp/src/lists/interleave_columns.cu b/cpp/src/lists/interleave_columns.cu index fe5e1e677ca..478b6c9a209 100644 --- a/cpp/src/lists/interleave_columns.cu +++ b/cpp/src/lists/interleave_columns.cu @@ -228,7 +228,7 @@ struct interleave_list_entries_impl(data_has_null_mask ? num_output_entries : 0, stream); comp_fn.d_validities = validities.data(); - auto [offsets_column, chars_column] = cudf::strings::detail::make_strings_children( + auto [offsets_column, chars] = cudf::strings::detail::make_strings_children( comp_fn, num_output_lists, num_output_entries, stream, mr); auto [null_mask, null_count] = @@ -236,7 +236,7 @@ struct interleave_list_entries_implrelease().data.release()[0]), + chars.release(), null_count, std::move(null_mask)); } diff --git a/cpp/src/replace/clamp.cu b/cpp/src/replace/clamp.cu index 43358a3b165..3cd1fdd20a2 100644 --- a/cpp/src/replace/clamp.cu +++ b/cpp/src/replace/clamp.cu @@ -100,12 +100,12 @@ std::unique_ptr clamp_string_column(strings_column_view const& inp auto fn = clamp_strings_fn{ d_input, lo_itr, lo_replace_itr, hi_itr, hi_replace_itr}; - auto [offsets_column, chars_column] = + auto [offsets_column, chars] = cudf::strings::detail::make_strings_children(fn, input.size(), stream, mr); return make_strings_column(input.size(), std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), input.null_count(), std::move(cudf::detail::copy_bitmask(input.parent(), stream, mr))); } diff --git a/cpp/src/strings/capitalize.cu b/cpp/src/strings/capitalize.cu index 3b99093a89f..3889bd31b4d 100644 --- a/cpp/src/strings/capitalize.cu +++ b/cpp/src/strings/capitalize.cu @@ -229,12 +229,12 @@ std::unique_ptr capitalizer(CapitalFn cfn, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { - auto [offsets_column, chars_column] = + auto [offsets_column, chars] = cudf::strings::detail::make_strings_children(cfn, input.size(), stream, mr); return make_strings_column(input.size(), std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), input.null_count(), cudf::detail::copy_bitmask(input.parent(), stream, mr)); } diff --git a/cpp/src/strings/case.cu b/cpp/src/strings/case.cu index b3bf0e2a787..8d8930013cf 100644 --- a/cpp/src/strings/case.cu +++ b/cpp/src/strings/case.cu @@ -217,7 +217,7 @@ std::unique_ptr convert_case(strings_column_view const& input, cudf::strings::detail::make_strings_children(converter, input.size(), stream, mr); return make_strings_column(input.size(), std::move(offsets), - std::move(chars->release().data.release()[0]), + chars.release(), input.null_count(), cudf::detail::copy_bitmask(input.parent(), stream, mr)); } diff --git a/cpp/src/strings/char_types/char_types.cu b/cpp/src/strings/char_types/char_types.cu index 9c2a2701227..b8c0dfd27e6 100644 --- a/cpp/src/strings/char_types/char_types.cu +++ b/cpp/src/strings/char_types/char_types.cu @@ -200,13 +200,13 @@ std::unique_ptr filter_characters_of_type(strings_column_view const& str rmm::device_buffer null_mask = cudf::detail::copy_bitmask(strings.parent(), stream, mr); // this utility calls filterer to build the offsets and chars columns - auto [offsets_column, chars_column] = + auto [offsets_column, chars] = cudf::strings::detail::make_strings_children(filterer, strings_count, stream, mr); // return new strings column return make_strings_column(strings_count, std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), strings.null_count(), std::move(null_mask)); } diff --git a/cpp/src/strings/combine/concatenate.cu b/cpp/src/strings/combine/concatenate.cu index a48e84eac0c..14f530971f5 100644 --- a/cpp/src/strings/combine/concatenate.cu +++ b/cpp/src/strings/combine/concatenate.cu @@ -142,7 +142,7 @@ std::unique_ptr concatenate(table_view const& strings_columns, // Create device views from the strings columns. auto d_table = table_device_view::create(strings_columns, stream); concat_strings_fn fn{*d_table, d_separator, d_narep, separate_nulls}; - auto [offsets_column, chars_column] = make_strings_children(fn, strings_count, stream, mr); + auto [offsets_column, chars] = make_strings_children(fn, strings_count, stream, mr); // create resulting null mask auto [null_mask, null_count] = cudf::detail::valid_if( @@ -156,11 +156,8 @@ std::unique_ptr concatenate(table_view const& strings_columns, stream, mr); - return make_strings_column(strings_count, - std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), - null_count, - std::move(null_mask)); + return make_strings_column( + strings_count, std::move(offsets_column), chars.release(), null_count, std::move(null_mask)); } namespace { @@ -237,7 +234,7 @@ std::unique_ptr concatenate(table_view const& strings_columns, multi_separator_concat_fn mscf{ *d_table, separator_col_view, separator_rep, col_rep, separate_nulls}; - auto [offsets_column, chars_column] = make_strings_children(mscf, strings_count, stream, mr); + auto [offsets_column, chars] = make_strings_children(mscf, strings_count, stream, mr); // Create resulting null mask auto [null_mask, null_count] = cudf::detail::valid_if( @@ -252,11 +249,8 @@ std::unique_ptr concatenate(table_view const& strings_columns, stream, mr); - return make_strings_column(strings_count, - std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), - null_count, - std::move(null_mask)); + return make_strings_column( + strings_count, std::move(offsets_column), chars.release(), null_count, std::move(null_mask)); } } // namespace detail diff --git a/cpp/src/strings/combine/join.cu b/cpp/src/strings/combine/join.cu index 0e0d6e437a7..c6290ceb6c2 100644 --- a/cpp/src/strings/combine/join.cu +++ b/cpp/src/strings/combine/join.cu @@ -142,28 +142,34 @@ std::unique_ptr join_strings(strings_column_view const& input, auto d_strings = column_device_view::create(input.parent(), stream); - auto chars_column = [&] { + auto chars = [&] { // build the strings column and commandeer the chars column if ((input.size() == input.null_count()) || ((input.chars_size(stream) / (input.size() - input.null_count())) <= AVG_CHAR_BYTES_THRESHOLD)) { - return std::get<1>( - make_strings_children(join_fn{*d_strings, d_separator, d_narep}, input.size(), stream, mr)); + return std::get<1>(make_strings_children( + join_fn{*d_strings, d_separator, d_narep}, input.size(), stream, mr)) + .release(); } // dynamically feeds index pairs to build the output auto indices = cudf::detail::make_counting_transform_iterator( 0, join_gather_fn{*d_strings, d_separator, d_narep}); - auto joined_col = make_strings_column(indices, indices + (input.size() * 2), stream, mr); - auto chars_data = joined_col->release().data; - auto const chars_size = chars_data->size(); - return std::make_unique( - data_type{type_id::INT8}, chars_size, std::move(*chars_data), rmm::device_buffer{}, 0); + auto joined_col = make_strings_column(indices, indices + (input.size() * 2), stream, mr); + auto chars_data = joined_col->release().data; + return std::move(*chars_data); }(); // build the offsets: single string output has offsets [0,chars-size] - auto offsets = cudf::detail::make_device_uvector_async( - std::vector({0, chars_column->size()}), stream, mr); - auto offsets_column = std::make_unique(std::move(offsets), rmm::device_buffer{}, 0); + auto offsets_column = [&] { + if (chars.size() < static_cast(get_offset64_threshold())) { + auto offsets32 = cudf::detail::make_device_uvector_async( + std::vector({0, static_cast(chars.size())}), stream, mr); + return std::make_unique(std::move(offsets32), rmm::device_buffer{}, 0); + } + auto offsets64 = cudf::detail::make_device_uvector_async( + std::vector({0L, static_cast(chars.size())}), stream, mr); + return std::make_unique(std::move(offsets64), rmm::device_buffer{}, 0); + }(); // build the null mask: only one output row so it is either all-valid or all-null auto const null_count = @@ -173,11 +179,8 @@ std::unique_ptr join_strings(strings_column_view const& input, : rmm::device_buffer{0, stream, mr}; // perhaps this return a string_scalar instead of a single-row column - return make_strings_column(1, - std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), - null_count, - std::move(null_mask)); + return make_strings_column( + 1, std::move(offsets_column), std::move(chars), null_count, std::move(null_mask)); } } // namespace detail diff --git a/cpp/src/strings/combine/join_list_elements.cu b/cpp/src/strings/combine/join_list_elements.cu index 619f5feba15..170e621e05c 100644 --- a/cpp/src/strings/combine/join_list_elements.cu +++ b/cpp/src/strings/combine/join_list_elements.cu @@ -207,7 +207,7 @@ std::unique_ptr join_list_elements(lists_column_view const& lists_string separate_nulls, empty_list_policy}; - auto [offsets_column, chars_column] = make_strings_children(comp_fn, num_rows, stream, mr); + auto [offsets_column, chars] = make_strings_children(comp_fn, num_rows, stream, mr); auto [null_mask, null_count] = cudf::detail::valid_if(thrust::counting_iterator(0), thrust::counting_iterator(num_rows), @@ -215,11 +215,8 @@ std::unique_ptr join_list_elements(lists_column_view const& lists_string stream, mr); - return make_strings_column(num_rows, - std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), - null_count, - std::move(null_mask)); + return make_strings_column( + num_rows, std::move(offsets_column), chars.release(), null_count, std::move(null_mask)); } namespace { @@ -285,7 +282,7 @@ std::unique_ptr join_list_elements(lists_column_view const& lists_string separate_nulls, empty_list_policy}; - auto [offsets_column, chars_column] = make_strings_children(comp_fn, num_rows, stream, mr); + auto [offsets_column, chars] = make_strings_children(comp_fn, num_rows, stream, mr); auto [null_mask, null_count] = cudf::detail::valid_if(thrust::counting_iterator(0), thrust::counting_iterator(num_rows), @@ -293,11 +290,8 @@ std::unique_ptr join_list_elements(lists_column_view const& lists_string stream, mr); - return make_strings_column(num_rows, - std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), - null_count, - std::move(null_mask)); + return make_strings_column( + num_rows, std::move(offsets_column), chars.release(), null_count, std::move(null_mask)); } } // namespace detail diff --git a/cpp/src/strings/convert/convert_booleans.cu b/cpp/src/strings/convert/convert_booleans.cu index 4fe0be7883f..d1de345a709 100644 --- a/cpp/src/strings/convert/convert_booleans.cu +++ b/cpp/src/strings/convert/convert_booleans.cu @@ -147,7 +147,7 @@ std::unique_ptr from_booleans(column_view const& booleans, return make_strings_column(strings_count, std::move(offsets), - std::move(chars->release().data.release()[0]), + chars.release(), booleans.null_count(), std::move(null_mask)); } diff --git a/cpp/src/strings/convert/convert_datetime.cu b/cpp/src/strings/convert/convert_datetime.cu index b7a662b0b76..f54eb082959 100644 --- a/cpp/src/strings/convert/convert_datetime.cu +++ b/cpp/src/strings/convert/convert_datetime.cu @@ -1099,7 +1099,7 @@ struct datetime_formatter_fn { }; // -using strings_children = std::pair, std::unique_ptr>; +using strings_children = std::pair, rmm::device_uvector>; struct dispatch_from_timestamps_fn { template ()>* = nullptr> strings_children operator()(column_device_view const& d_timestamps, @@ -1148,17 +1148,17 @@ std::unique_ptr from_timestamps(column_view const& timestamps, auto const d_timestamps = column_device_view::create(timestamps, stream); // dispatcher is called to handle the different timestamp types - auto [offsets_column, chars_column] = cudf::type_dispatcher(timestamps.type(), - dispatch_from_timestamps_fn(), - *d_timestamps, - *d_names, - d_format_items, - stream, - mr); + auto [offsets_column, chars] = cudf::type_dispatcher(timestamps.type(), + dispatch_from_timestamps_fn(), + *d_timestamps, + *d_names, + d_format_items, + stream, + mr); return make_strings_column(timestamps.size(), std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), timestamps.null_count(), cudf::detail::copy_bitmask(timestamps, stream, mr)); } diff --git a/cpp/src/strings/convert/convert_durations.cu b/cpp/src/strings/convert/convert_durations.cu index 9a58926539c..8076c5c484b 100644 --- a/cpp/src/strings/convert/convert_durations.cu +++ b/cpp/src/strings/convert/convert_durations.cu @@ -422,7 +422,7 @@ struct dispatch_from_durations_fn { return make_strings_column(strings_count, std::move(offsets), - std::move(chars->release().data.release()[0]), + chars.release(), durations.null_count(), std::move(null_mask)); } diff --git a/cpp/src/strings/convert/convert_fixed_point.cu b/cpp/src/strings/convert/convert_fixed_point.cu index c59952834d6..fb8ebf55ef1 100644 --- a/cpp/src/strings/convert/convert_fixed_point.cu +++ b/cpp/src/strings/convert/convert_fixed_point.cu @@ -248,7 +248,7 @@ struct dispatch_from_fixed_point_fn { return make_strings_column(input.size(), std::move(offsets), - std::move(chars->release().data.release()[0]), + chars.release(), input.null_count(), cudf::detail::copy_bitmask(input, stream, mr)); } diff --git a/cpp/src/strings/convert/convert_floats.cu b/cpp/src/strings/convert/convert_floats.cu index c56e723de8e..df019ca236a 100644 --- a/cpp/src/strings/convert/convert_floats.cu +++ b/cpp/src/strings/convert/convert_floats.cu @@ -408,7 +408,7 @@ struct dispatch_from_floats_fn { return make_strings_column(strings_count, std::move(offsets), - std::move(chars->release().data.release()[0]), + chars.release(), floats.null_count(), std::move(null_mask)); } diff --git a/cpp/src/strings/convert/convert_hex.cu b/cpp/src/strings/convert/convert_hex.cu index 68cff214507..332bc9837c1 100644 --- a/cpp/src/strings/convert/convert_hex.cu +++ b/cpp/src/strings/convert/convert_hex.cu @@ -180,12 +180,12 @@ struct dispatch_integers_to_hex_fn { { auto const d_column = column_device_view::create(input, stream); - auto [offsets_column, chars_column] = cudf::strings::detail::make_strings_children( + auto [offsets_column, chars] = cudf::strings::detail::make_strings_children( integer_to_hex_fn{*d_column}, input.size(), stream, mr); return make_strings_column(input.size(), std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), input.null_count(), cudf::detail::copy_bitmask(input, stream, mr)); } diff --git a/cpp/src/strings/convert/convert_integers.cu b/cpp/src/strings/convert/convert_integers.cu index 364cb534d2f..eb2e9c28134 100644 --- a/cpp/src/strings/convert/convert_integers.cu +++ b/cpp/src/strings/convert/convert_integers.cu @@ -367,7 +367,7 @@ struct dispatch_from_integers_fn { return make_strings_column(strings_count, std::move(offsets), - std::move(chars->release().data.release()[0]), + chars.release(), integers.null_count(), std::move(null_mask)); } diff --git a/cpp/src/strings/convert/convert_ipv4.cu b/cpp/src/strings/convert/convert_ipv4.cu index e07be26a23c..ce7f98067ef 100644 --- a/cpp/src/strings/convert/convert_ipv4.cu +++ b/cpp/src/strings/convert/convert_ipv4.cu @@ -165,13 +165,13 @@ std::unique_ptr integers_to_ipv4(column_view const& integers, CUDF_EXPECTS(integers.type().id() == type_id::INT64, "Input column must be type_id::INT64 type"); - auto d_column = column_device_view::create(integers, stream); - auto [offsets_column, chars_column] = cudf::strings::detail::make_strings_children( + auto d_column = column_device_view::create(integers, stream); + auto [offsets_column, chars] = cudf::strings::detail::make_strings_children( integers_to_ipv4_fn{*d_column}, integers.size(), stream, mr); return make_strings_column(integers.size(), std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), integers.null_count(), cudf::detail::copy_bitmask(integers, stream, mr)); } diff --git a/cpp/src/strings/convert/convert_lists.cu b/cpp/src/strings/convert/convert_lists.cu index 1f22aea284b..d6c24b6981b 100644 --- a/cpp/src/strings/convert/convert_lists.cu +++ b/cpp/src/strings/convert/convert_lists.cu @@ -216,17 +216,14 @@ std::unique_ptr format_list_column(lists_column_view const& input, auto const d_separators = column_device_view::create(separators.parent(), stream); auto const d_na_rep = na_rep.value(stream); - auto [offsets_column, chars_column] = cudf::strings::detail::make_strings_children( + auto [offsets_column, chars] = cudf::strings::detail::make_strings_children( format_lists_fn{*d_input, *d_separators, d_na_rep, stack_buffer.data(), depth}, input.size(), stream, mr); - return make_strings_column(input.size(), - std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), - 0, - rmm::device_buffer{}); + return make_strings_column( + input.size(), std::move(offsets_column), chars.release(), 0, rmm::device_buffer{}); } } // namespace detail diff --git a/cpp/src/strings/convert/convert_urls.cu b/cpp/src/strings/convert/convert_urls.cu index b96c799cf4d..f5aeeb8d130 100644 --- a/cpp/src/strings/convert/convert_urls.cu +++ b/cpp/src/strings/convert/convert_urls.cu @@ -131,12 +131,12 @@ std::unique_ptr url_encode(strings_column_view const& input, auto d_column = column_device_view::create(input.parent(), stream); - auto [offsets_column, chars_column] = cudf::strings::detail::make_strings_children( + auto [offsets_column, chars] = cudf::strings::detail::make_strings_children( url_encoder_fn{*d_column}, input.size(), stream, mr); return make_strings_column(input.size(), std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), input.null_count(), cudf::detail::copy_bitmask(input.parent(), stream, mr)); } diff --git a/cpp/src/strings/filling/fill.cu b/cpp/src/strings/filling/fill.cu index d2e3b6f6af3..685c3eec744 100644 --- a/cpp/src/strings/filling/fill.cu +++ b/cpp/src/strings/filling/fill.cu @@ -94,13 +94,10 @@ std::unique_ptr fill(strings_column_view const& input, auto const d_str = is_valid ? d_value.value(stream) : string_view{}; auto fn = fill_fn{d_strings, begin, end, d_str}; - auto [offsets_column, chars_column] = make_strings_children(fn, strings_count, stream, mr); + auto [offsets_column, chars] = make_strings_children(fn, strings_count, stream, mr); - return make_strings_column(strings_count, - std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), - null_count, - std::move(null_mask)); + return make_strings_column( + strings_count, std::move(offsets_column), chars.release(), null_count, std::move(null_mask)); } } // namespace detail diff --git a/cpp/src/strings/filter_chars.cu b/cpp/src/strings/filter_chars.cu index 7a26fc45dcb..aaaa751c3f9 100644 --- a/cpp/src/strings/filter_chars.cu +++ b/cpp/src/strings/filter_chars.cu @@ -139,12 +139,12 @@ std::unique_ptr filter_characters( // this utility calls the strip_fn to build the offsets and chars columns filter_fn ffn{*d_strings, keep_characters, table.begin(), table.end(), d_replacement}; - auto [offsets_column, chars_column] = + auto [offsets_column, chars] = cudf::strings::detail::make_strings_children(ffn, strings.size(), stream, mr); return make_strings_column(strings_count, std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), strings.null_count(), cudf::detail::copy_bitmask(strings.parent(), stream, mr)); } diff --git a/cpp/src/strings/padding.cu b/cpp/src/strings/padding.cu index ec77aea6338..85d47af87f6 100644 --- a/cpp/src/strings/padding.cu +++ b/cpp/src/strings/padding.cu @@ -112,7 +112,7 @@ std::unique_ptr pad(strings_column_view const& input, auto d_strings = column_device_view::create(input.parent(), stream); - auto [offsets_column, chars_column] = [&] { + auto [offsets_column, chars] = [&] { if (side == side_type::LEFT) { auto fn = pad_fn{*d_strings, width, fill_char_size, d_fill_char}; return make_strings_children(fn, input.size(), stream, mr); @@ -126,7 +126,7 @@ std::unique_ptr pad(strings_column_view const& input, return make_strings_column(input.size(), std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), input.null_count(), cudf::detail::copy_bitmask(input.parent(), stream, mr)); } @@ -151,12 +151,12 @@ std::unique_ptr zfill(strings_column_view const& input, if (input.is_empty()) return make_empty_column(type_id::STRING); auto d_strings = column_device_view::create(input.parent(), stream); - auto [offsets_column, chars_column] = + auto [offsets_column, chars] = make_strings_children(zfill_fn{*d_strings, width}, input.size(), stream, mr); return make_strings_column(input.size(), std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), input.null_count(), cudf::detail::copy_bitmask(input.parent(), stream, mr)); } diff --git a/cpp/src/strings/regex/utilities.cuh b/cpp/src/strings/regex/utilities.cuh index d5dd80aba53..ae8211ac916 100644 --- a/cpp/src/strings/regex/utilities.cuh +++ b/cpp/src/strings/regex/utilities.cuh @@ -140,10 +140,9 @@ auto make_strings_children(SizeAndExecuteFunction size_and_exec_fn, std::overflow_error); // Now build the chars column - std::unique_ptr chars = - create_chars_child_column(static_cast(char_bytes), stream, mr); + rmm::device_uvector chars(char_bytes, stream, mr); if (char_bytes > 0) { - size_and_exec_fn.d_chars = chars->mutable_view().template data(); + size_and_exec_fn.d_chars = chars.data(); for_each_kernel<<>>( size_and_exec_fn, d_prog, strings_count); } diff --git a/cpp/src/strings/repeat_strings.cu b/cpp/src/strings/repeat_strings.cu index b4a770f72bd..690a72c098f 100644 --- a/cpp/src/strings/repeat_strings.cu +++ b/cpp/src/strings/repeat_strings.cu @@ -160,11 +160,11 @@ std::unique_ptr repeat_strings(strings_column_view const& input, auto const strings_dv_ptr = column_device_view::create(input.parent(), stream); auto const fn = compute_size_and_repeat_fn{*strings_dv_ptr, repeat_times, input.has_nulls()}; - auto [offsets_column, chars_column] = + auto [offsets_column, chars] = make_strings_children(fn, strings_count * repeat_times, strings_count, stream, mr); return make_strings_column(strings_count, std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), input.null_count(), cudf::detail::copy_bitmask(input.parent(), stream, mr)); } @@ -240,7 +240,7 @@ std::unique_ptr repeat_strings(strings_column_view const& input, input.has_nulls(), repeat_times.has_nulls()}; - auto [offsets_column, chars_column] = make_strings_children(fn, strings_count, stream, mr); + auto [offsets_column, chars] = make_strings_children(fn, strings_count, stream, mr); // We generate new bitmask by AND of the two input columns' bitmasks. // Note that if either of the input columns are nullable, the output column will also be nullable @@ -248,11 +248,8 @@ std::unique_ptr repeat_strings(strings_column_view const& input, auto [null_mask, null_count] = cudf::detail::bitmask_and(table_view{{input.parent(), repeat_times}}, stream, mr); - return make_strings_column(strings_count, - std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), - null_count, - std::move(null_mask)); + return make_strings_column( + strings_count, std::move(offsets_column), chars.release(), null_count, std::move(null_mask)); } } // namespace detail diff --git a/cpp/src/strings/replace/backref_re.cu b/cpp/src/strings/replace/backref_re.cu index bb99dc0644c..8e20db18f43 100644 --- a/cpp/src/strings/replace/backref_re.cu +++ b/cpp/src/strings/replace/backref_re.cu @@ -125,8 +125,8 @@ std::unique_ptr replace_with_backrefs(strings_column_view const& input, auto const d_strings = column_device_view::create(input.parent(), stream); - using BackRefIterator = decltype(backrefs.begin()); - auto [offsets_column, chars_column] = make_strings_children( + using BackRefIterator = decltype(backrefs.begin()); + auto [offsets_column, chars] = make_strings_children( backrefs_fn{*d_strings, d_repl_template, backrefs.begin(), backrefs.end()}, *d_prog, input.size(), @@ -135,7 +135,7 @@ std::unique_ptr replace_with_backrefs(strings_column_view const& input, return make_strings_column(input.size(), std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), input.null_count(), cudf::detail::copy_bitmask(input.parent(), stream, mr)); } diff --git a/cpp/src/strings/replace/multi.cu b/cpp/src/strings/replace/multi.cu index ab35393651f..ffa922d5944 100644 --- a/cpp/src/strings/replace/multi.cu +++ b/cpp/src/strings/replace/multi.cu @@ -453,12 +453,12 @@ std::unique_ptr replace_string_parallel(strings_column_view const& input auto d_targets = column_device_view::create(targets.parent(), stream); auto d_replacements = column_device_view::create(repls.parent(), stream); - auto [offsets_column, chars_column] = cudf::strings::detail::make_strings_children( + auto [offsets_column, chars] = cudf::strings::detail::make_strings_children( replace_multi_fn{*d_strings, *d_targets, *d_replacements}, input.size(), stream, mr); return make_strings_column(input.size(), std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), input.null_count(), cudf::detail::copy_bitmask(input.parent(), stream, mr)); } diff --git a/cpp/src/strings/replace/multi_re.cu b/cpp/src/strings/replace/multi_re.cu index ba122d11e0b..743e5894112 100644 --- a/cpp/src/strings/replace/multi_re.cu +++ b/cpp/src/strings/replace/multi_re.cu @@ -185,7 +185,7 @@ std::unique_ptr replace_re(strings_column_view const& input, auto found_ranges = rmm::device_uvector(d_progs.size() * input.size(), stream); - auto [offsets_column, chars_column] = make_strings_children( + auto [offsets_column, chars] = make_strings_children( replace_multi_regex_fn{*d_strings, d_progs, found_ranges.data(), *d_repls}, input.size(), stream, @@ -193,7 +193,7 @@ std::unique_ptr replace_re(strings_column_view const& input, return make_strings_column(input.size(), std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), input.null_count(), cudf::detail::copy_bitmask(input.parent(), stream, mr)); } diff --git a/cpp/src/strings/replace/replace.cu b/cpp/src/strings/replace/replace.cu index 2d255e57686..c37c64e348c 100644 --- a/cpp/src/strings/replace/replace.cu +++ b/cpp/src/strings/replace/replace.cu @@ -530,12 +530,12 @@ std::unique_ptr replace_row_parallel(strings_column_view const& strings, auto d_strings = column_device_view::create(strings.parent(), stream); // this utility calls the given functor to build the offsets and chars columns - auto [offsets_column, chars_column] = cudf::strings::detail::make_strings_children( + auto [offsets_column, chars] = cudf::strings::detail::make_strings_children( replace_row_parallel_fn{*d_strings, d_target, d_repl, maxrepl}, strings.size(), stream, mr); return make_strings_column(strings.size(), std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), strings.null_count(), cudf::detail::copy_bitmask(strings.parent(), stream, mr)); } diff --git a/cpp/src/strings/replace/replace_re.cu b/cpp/src/strings/replace/replace_re.cu index 500bc0c5bb5..bded196946f 100644 --- a/cpp/src/strings/replace/replace_re.cu +++ b/cpp/src/strings/replace/replace_re.cu @@ -116,12 +116,12 @@ std::unique_ptr replace_re(strings_column_view const& input, auto const d_strings = column_device_view::create(input.parent(), stream); - auto [offsets_column, chars_column] = make_strings_children( + auto [offsets_column, chars] = make_strings_children( replace_regex_fn{*d_strings, d_repl, maxrepl}, *d_prog, input.size(), stream, mr); return make_strings_column(input.size(), std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), input.null_count(), cudf::detail::copy_bitmask(input.parent(), stream, mr)); } diff --git a/cpp/src/strings/replace/replace_slice.cu b/cpp/src/strings/replace/replace_slice.cu index 4321f78d2d5..041801336e6 100644 --- a/cpp/src/strings/replace/replace_slice.cu +++ b/cpp/src/strings/replace/replace_slice.cu @@ -91,12 +91,12 @@ std::unique_ptr replace_slice(strings_column_view const& strings, auto d_strings = column_device_view::create(strings.parent(), stream); // this utility calls the given functor to build the offsets and chars columns - auto [offsets_column, chars_column] = cudf::strings::detail::make_strings_children( + auto [offsets_column, chars] = cudf::strings::detail::make_strings_children( replace_slice_fn{*d_strings, d_repl, start, stop}, strings.size(), stream, mr); return make_strings_column(strings.size(), std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), strings.null_count(), cudf::detail::copy_bitmask(strings.parent(), stream, mr)); } diff --git a/cpp/src/strings/slice.cu b/cpp/src/strings/slice.cu index 1e55986fdb8..98f3c9cae0d 100644 --- a/cpp/src/strings/slice.cu +++ b/cpp/src/strings/slice.cu @@ -209,7 +209,7 @@ std::unique_ptr slice_strings(strings_column_view const& strings, return make_strings_column(strings.size(), std::move(offsets), - std::move(chars->release().data.release()[0]), + chars.release(), strings.null_count(), cudf::detail::copy_bitmask(strings.parent(), stream, mr)); } diff --git a/cpp/src/strings/translate.cu b/cpp/src/strings/translate.cu index 039a8ac8a62..a8603f47226 100644 --- a/cpp/src/strings/translate.cu +++ b/cpp/src/strings/translate.cu @@ -110,12 +110,12 @@ std::unique_ptr translate(strings_column_view const& strings, auto d_strings = column_device_view::create(strings.parent(), stream); - auto [offsets_column, chars_column] = make_strings_children( + auto [offsets_column, chars] = make_strings_children( translate_fn{*d_strings, table.begin(), table.end()}, strings.size(), stream, mr); return make_strings_column(strings.size(), std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), strings.null_count(), cudf::detail::copy_bitmask(strings.parent(), stream, mr)); } diff --git a/cpp/src/text/detokenize.cu b/cpp/src/text/detokenize.cu index a317739e4ca..b9964352c74 100644 --- a/cpp/src/text/detokenize.cu +++ b/cpp/src/text/detokenize.cu @@ -156,18 +156,15 @@ std::unique_ptr detokenize(cudf::strings_column_view const& string cudf::string_view const d_separator(separator.data(), separator.size()); - auto [offsets_column, chars_column] = cudf::strings::detail::make_strings_children( + auto [offsets_column, chars] = cudf::strings::detail::make_strings_children( detokenizer_fn{*strings_column, d_row_map, tokens_offsets.data(), d_separator}, output_count, stream, mr); // make the output strings column from the offsets and chars column - return cudf::make_strings_column(output_count, - std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), - 0, - rmm::device_buffer{}); + return cudf::make_strings_column( + output_count, std::move(offsets_column), chars.release(), 0, rmm::device_buffer{}); } } // namespace detail diff --git a/cpp/src/text/generate_ngrams.cu b/cpp/src/text/generate_ngrams.cu index fafb2f18b80..3290b58101d 100644 --- a/cpp/src/text/generate_ngrams.cu +++ b/cpp/src/text/generate_ngrams.cu @@ -135,15 +135,12 @@ std::unique_ptr generate_ngrams(cudf::strings_column_view const& s // compute the number of strings of ngrams auto const ngrams_count = strings_count - ngrams + 1; - auto [offsets_column, chars_column] = cudf::strings::detail::make_strings_children( + auto [offsets_column, chars] = cudf::strings::detail::make_strings_children( ngram_generator_fn{d_strings, ngrams, d_separator}, ngrams_count, stream, mr); // make the output strings column from the offsets and chars column - return cudf::make_strings_column(ngrams_count, - std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), - 0, - rmm::device_buffer{}); + return cudf::make_strings_column( + ngrams_count, std::move(offsets_column), chars.release(), 0, rmm::device_buffer{}); } } // namespace detail @@ -235,14 +232,11 @@ std::unique_ptr generate_character_ngrams(cudf::strings_column_vie "Insufficient number of characters in each string to generate ngrams"); character_ngram_generator_fn generator{d_strings, ngrams, ngram_offsets.data()}; - auto [offsets_column, chars_column] = cudf::strings::detail::make_strings_children( + auto [offsets_column, chars] = cudf::strings::detail::make_strings_children( generator, strings_count, total_ngrams, stream, mr); - return cudf::make_strings_column(total_ngrams, - std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), - 0, - rmm::device_buffer{}); + return cudf::make_strings_column( + total_ngrams, std::move(offsets_column), chars.release(), 0, rmm::device_buffer{}); } namespace { diff --git a/cpp/src/text/normalize.cu b/cpp/src/text/normalize.cu index 3d98ae59dc0..c06a24382ed 100644 --- a/cpp/src/text/normalize.cu +++ b/cpp/src/text/normalize.cu @@ -182,12 +182,12 @@ std::unique_ptr normalize_spaces(cudf::strings_column_view const& auto d_strings = cudf::column_device_view::create(strings.parent(), stream); // build offsets and children using the normalize_space_fn - auto [offsets_column, chars_column] = cudf::strings::detail::make_strings_children( + auto [offsets_column, chars] = cudf::strings::detail::make_strings_children( normalize_spaces_fn{*d_strings}, strings.size(), stream, mr); return cudf::make_strings_column(strings.size(), std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), strings.null_count(), cudf::detail::copy_bitmask(strings.parent(), stream, mr)); } @@ -224,12 +224,12 @@ std::unique_ptr normalize_characters(cudf::strings_column_view con auto d_strings = cudf::column_device_view::create(strings.parent(), stream); // build offsets and children using the codepoint_to_utf8_fn - auto [offsets_column, chars_column] = cudf::strings::detail::make_strings_children( + auto [offsets_column, chars] = cudf::strings::detail::make_strings_children( codepoint_to_utf8_fn{*d_strings, cp_chars, cp_offsets}, strings.size(), stream, mr); return cudf::make_strings_column(strings.size(), std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), strings.null_count(), cudf::detail::copy_bitmask(strings.parent(), stream, mr)); } diff --git a/cpp/src/text/replace.cu b/cpp/src/text/replace.cu index 1fa0606424c..5aed701c037 100644 --- a/cpp/src/text/replace.cu +++ b/cpp/src/text/replace.cu @@ -228,13 +228,13 @@ std::unique_ptr replace_tokens(cudf::strings_column_view const& st rmm::device_buffer null_mask = cudf::detail::copy_bitmask(strings.parent(), stream, mr); // this utility calls replacer to build the offsets and chars columns - auto [offsets_column, chars_column] = + auto [offsets_column, chars] = cudf::strings::detail::make_strings_children(replacer, strings_count, stream, mr); // return new strings column return cudf::make_strings_column(strings_count, std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), strings.null_count(), std::move(null_mask)); } @@ -261,13 +261,13 @@ std::unique_ptr filter_tokens(cudf::strings_column_view const& str rmm::device_buffer null_mask = cudf::detail::copy_bitmask(strings.parent(), stream, mr); // this utility calls filterer to build the offsets and chars columns - auto [offsets_column, chars_column] = + auto [offsets_column, chars] = cudf::strings::detail::make_strings_children(filterer, strings_count, stream, mr); // return new strings column return cudf::make_strings_column(strings_count, std::move(offsets_column), - std::move(chars_column->release().data.release()[0]), + chars.release(), strings.null_count(), std::move(null_mask)); } From e60aad110efcd94003ad78d0f46ac94e531bd1c0 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 1 Mar 2024 18:22:33 -0800 Subject: [PATCH 330/384] Implement search using pylibcudf (#15166) Contributes to #15162 Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/15166 --- .../user_guide/api_docs/pylibcudf/index.rst | 1 + .../user_guide/api_docs/pylibcudf/search.rst | 6 + .../cudf/cudf/_lib/pylibcudf/CMakeLists.txt | 1 + python/cudf/cudf/_lib/pylibcudf/__init__.pxd | 2 + python/cudf/cudf/_lib/pylibcudf/__init__.py | 2 + python/cudf/cudf/_lib/pylibcudf/search.pxd | 21 ++++ python/cudf/cudf/_lib/pylibcudf/search.pyx | 116 ++++++++++++++++++ python/cudf/cudf/_lib/search.pyx | 91 +++++--------- 8 files changed, 178 insertions(+), 62 deletions(-) create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/search.rst create mode 100644 python/cudf/cudf/_lib/pylibcudf/search.pxd create mode 100644 python/cudf/cudf/_lib/pylibcudf/search.pyx diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst index 73f63ae1343..2e5b3916c65 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst @@ -21,6 +21,7 @@ This page provides API documentation for pylibcudf. reduce rolling scalar + search stream_compaction sorting replace diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/search.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/search.rst new file mode 100644 index 00000000000..aa57bcd9d92 --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/search.rst @@ -0,0 +1,6 @@ +====== +search +====== + +.. automodule:: cudf._lib.pylibcudf.search + :members: diff --git a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt index 68e6765cc49..fd749a5edc1 100644 --- a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt +++ b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt @@ -28,6 +28,7 @@ set(cython_sources replace.pyx rolling.pyx scalar.pyx + search.pyx stream_compaction.pyx sorting.pyx table.pyx diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd index 5ef10fb2ffc..96aa42cc257 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd @@ -14,6 +14,7 @@ from . cimport ( reduce, replace, rolling, + search, sorting, stream_compaction, types, @@ -45,6 +46,7 @@ __all__ = [ "reduce", "replace", "rolling", + "search", "stream_compaction", "sorting", "types", diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.py b/python/cudf/cudf/_lib/pylibcudf/__init__.py index 4689c49fdb1..19cc782dd92 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.py +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.py @@ -13,6 +13,7 @@ reduce, replace, rolling, + search, sorting, stream_compaction, types, @@ -43,6 +44,7 @@ "reduce", "replace", "rolling", + "search", "stream_compaction", "sorting", "types", diff --git a/python/cudf/cudf/_lib/pylibcudf/search.pxd b/python/cudf/cudf/_lib/pylibcudf/search.pxd new file mode 100644 index 00000000000..0faf18b108f --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/search.pxd @@ -0,0 +1,21 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from .column cimport Column +from .table cimport Table + + +cpdef Column lower_bound( + Table haystack, + Table needles, + list column_order, + list null_precedence, +) + +cpdef Column upper_bound( + Table haystack, + Table needles, + list column_order, + list null_precedence, +) + +cpdef Column contains(Column haystack, Column needles) diff --git a/python/cudf/cudf/_lib/pylibcudf/search.pyx b/python/cudf/cudf/_lib/pylibcudf/search.pyx new file mode 100644 index 00000000000..a186167af13 --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/search.pyx @@ -0,0 +1,116 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from libcpp.memory cimport unique_ptr +from libcpp.utility cimport move +from libcpp.vector cimport vector + +from cudf._lib.cpp cimport search as cpp_search +from cudf._lib.cpp.column.column cimport column +from cudf._lib.cpp.types cimport null_order, order + +from .column cimport Column +from .table cimport Table + + +cpdef Column lower_bound( + Table haystack, + Table needles, + list column_order, + list null_precedence, +): + """Find smallest indices in haystack where needles may be inserted to retain order. + + Parameters + ---------- + haystack : Table + The search space. + needles : Table + The values for which to find insertion points. + column_order : List[ColumnOrder] + Whether each column should be sorted in ascending or descending order. + null_precedence : List[NullOrder] + Whether nulls should come before or after non-nulls. + + Returns + ------- + Column + The insertion points + """ + cdef unique_ptr[column] c_result + cdef vector[order] c_orders = column_order + cdef vector[null_order] c_null_precedence = null_precedence + with nogil: + c_result = move( + cpp_search.lower_bound( + haystack.view(), + needles.view(), + c_orders, + c_null_precedence, + ) + ) + return Column.from_libcudf(move(c_result)) + + +cpdef Column upper_bound( + Table haystack, + Table needles, + list column_order, + list null_precedence, +): + """Find largest indices in haystack where needles may be inserted to retain order. + + Parameters + ---------- + haystack : Table + The search space. + needles : Table + The values for which to find insertion points. + column_order : List[ColumnOrder] + Whether each column should be sorted in ascending or descending order. + null_precedence : List[NullOrder] + Whether nulls should come before or after non-nulls. + + Returns + ------- + Column + The insertion points + """ + cdef unique_ptr[column] c_result + cdef vector[order] c_orders = column_order + cdef vector[null_order] c_null_precedence = null_precedence + with nogil: + c_result = move( + cpp_search.upper_bound( + haystack.view(), + needles.view(), + c_orders, + c_null_precedence, + ) + ) + return Column.from_libcudf(move(c_result)) + + +cpdef Column contains(Column haystack, Column needles): + """Check whether needles are present in haystack. + + Parameters + ---------- + haystack : Table + The search space. + needles : Table + The values for which to search. + + Returns + ------- + Column + Boolean indicator for each needle. + """ + cdef unique_ptr[column] c_result + with nogil: + c_result = move( + cpp_search.contains( + haystack.view(), + needles.view(), + ) + ) + return Column.from_libcudf(move(c_result)) diff --git a/python/cudf/cudf/_lib/search.pyx b/python/cudf/cudf/_lib/search.pyx index fef3a08c6d7..1ee73949fd3 100644 --- a/python/cudf/cudf/_lib/search.pyx +++ b/python/cudf/cudf/_lib/search.pyx @@ -1,18 +1,10 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from cudf.core.buffer import acquire_spill_lock -from libcpp.memory cimport unique_ptr -from libcpp.utility cimport move -from libcpp.vector cimport vector - -cimport cudf._lib.cpp.search as cpp_search -cimport cudf._lib.cpp.types as libcudf_types from cudf._lib.column cimport Column -from cudf._lib.cpp.column.column cimport column -from cudf._lib.cpp.column.column_view cimport column_view -from cudf._lib.cpp.table.table_view cimport table_view -from cudf._lib.utils cimport table_view_from_columns + +from cudf._lib import pylibcudf @acquire_spill_lock() @@ -31,50 +23,31 @@ def search_sorted( If 'left', the index of the first suitable location is given. If 'right', return the last such index """ - cdef unique_ptr[column] c_result - cdef vector[libcudf_types.order] c_column_order - cdef vector[libcudf_types.null_order] c_null_precedence - cdef libcudf_types.order c_order - cdef libcudf_types.null_order c_null_order - cdef table_view c_table_data = table_view_from_columns(source) - cdef table_view c_values_data = table_view_from_columns(values) - # Note: We are ignoring index columns here - c_order = (libcudf_types.order.ASCENDING - if ascending - else libcudf_types.order.DESCENDING) - c_null_order = ( - libcudf_types.null_order.AFTER - if na_position=="last" - else libcudf_types.null_order.BEFORE + column_order = [ + pylibcudf.types.Order.ASCENDING + if ascending + else pylibcudf.types.Order.DESCENDING + ] * len(source) + null_precedence = [ + pylibcudf.types.NullOrder.AFTER + if na_position == "last" + else pylibcudf.types.NullOrder.BEFORE + ] * len(source) + + func = getattr( + pylibcudf.search, + "lower_bound" if side == "left" else "upper_bound", ) - c_column_order = vector[libcudf_types.order](len(source), c_order) - c_null_precedence = vector[libcudf_types.null_order]( - len(source), c_null_order + return Column.from_pylibcudf( + func( + pylibcudf.Table([c.to_pylibcudf(mode="read") for c in source]), + pylibcudf.Table([c.to_pylibcudf(mode="read") for c in values]), + column_order, + null_precedence, + ) ) - if side == 'left': - with nogil: - c_result = move( - cpp_search.lower_bound( - c_table_data, - c_values_data, - c_column_order, - c_null_precedence, - ) - ) - elif side == 'right': - with nogil: - c_result = move( - cpp_search.upper_bound( - c_table_data, - c_values_data, - c_column_order, - c_null_precedence, - ) - ) - return Column.from_unique_ptr(move(c_result)) - @acquire_spill_lock() def contains(Column haystack, Column needles): @@ -87,15 +60,9 @@ def contains(Column haystack, Column needles): needles : A column of values to search for """ - cdef unique_ptr[column] c_result - cdef column_view c_haystack = haystack.view() - cdef column_view c_needles = needles.view() - - with nogil: - c_result = move( - cpp_search.contains( - c_haystack, - c_needles, - ) + return Column.from_pylibcudf( + pylibcudf.search.contains( + haystack.to_pylibcudf(mode="read"), + needles.to_pylibcudf(mode="read"), ) - return Column.from_unique_ptr(move(c_result)) + ) From 8dbe7cb12a752c44ce3027b96fc37ab0b0db923d Mon Sep 17 00:00:00 2001 From: Alessandro Bellina Date: Mon, 4 Mar 2024 08:43:02 -0600 Subject: [PATCH 331/384] Disable testChunkedPackTwoPasses for now (#15210) Signed-off-by: Alessandro Bellina Co-authored-by: Nghia Truong <7416935+ttnghia@users.noreply.github.com> --- java/src/test/java/ai/rapids/cudf/TableTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/java/src/test/java/ai/rapids/cudf/TableTest.java b/java/src/test/java/ai/rapids/cudf/TableTest.java index 6f0b2b51f4c..bee8d1cbb88 100644 --- a/java/src/test/java/ai/rapids/cudf/TableTest.java +++ b/java/src/test/java/ai/rapids/cudf/TableTest.java @@ -3739,7 +3739,7 @@ void testChunkedPackBasic() { } } } - +/* @Test void testChunkedPackTwoPasses() { // this test packes ~2MB worth of long into a 1MB bounce buffer @@ -3768,6 +3768,7 @@ void testChunkedPackTwoPasses() { } } } +*/ @Test void testContiguousSplitWithStrings() { From 903dcac6a5341c200c4981c7b9d188897164e89c Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 4 Mar 2024 08:43:13 -0600 Subject: [PATCH 332/384] Fix accessing .columns issue (#15212) --- python/cudf/cudf/_lib/utils.pyx | 4 +- python/cudf/cudf/core/indexed_frame.py | 7 ++- python/cudf/cudf/tests/test_dataframe.py | 55 ++++++++++++------------ 3 files changed, 36 insertions(+), 30 deletions(-) diff --git a/python/cudf/cudf/_lib/utils.pyx b/python/cudf/cudf/_lib/utils.pyx index 896cc55b425..b6637e9df08 100644 --- a/python/cudf/cudf/_lib/utils.pyx +++ b/python/cudf/cudf/_lib/utils.pyx @@ -149,7 +149,9 @@ cpdef generate_pandas_metadata(table, index): col for col in table._columns ], - df=table, + # It is OKAY to do `.head(0).to_pandas()` because + # this method will extract `.columns` metadata only + df=table.head(0).to_pandas(), column_names=col_names, index_levels=index_levels, index_descriptors=index_descriptors, diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 8e43000d0a8..3c6e1e17142 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -2872,6 +2872,8 @@ def _slice(self, arg: slice, keep_index: bool = True) -> Self: self._column_names, None if has_range_index or not keep_index else self._index.names, ) + result._data.label_dtype = self._data.label_dtype + result._data.rangeindex = self._data.rangeindex if keep_index and has_range_index: result.index = self.index[start:stop] @@ -3053,7 +3055,7 @@ def duplicated(self, subset=None, keep="first"): @_cudf_nvtx_annotate def _empty_like(self, keep_index=True) -> Self: - return self._from_columns_like_self( + result = self._from_columns_like_self( libcudf.copying.columns_empty_like( [ *(self._index._data.columns if keep_index else ()), @@ -3063,6 +3065,9 @@ def _empty_like(self, keep_index=True) -> Self: self._column_names, self._index.names if keep_index else None, ) + result._data.label_dtype = self._data.label_dtype + result._data.rangeindex = self._data.rangeindex + return result def _split(self, splits, keep_index=True): if self._num_rows == 0: diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 2084db89909..50b14d532e4 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -3012,43 +3012,31 @@ def test_series_rename(): @pytest.mark.parametrize("data_type", dtypes) @pytest.mark.parametrize("nelem", [0, 100]) def test_head_tail(nelem, data_type): - def check_index_equality(left, right): - assert left.index.equals(right.index) - - def check_values_equality(left, right): - if len(left) == 0 and len(right) == 0: - return None - - np.testing.assert_array_equal(left.to_pandas(), right.to_pandas()) - - def check_frame_series_equality(left, right): - check_index_equality(left, right) - check_values_equality(left, right) - - gdf = cudf.DataFrame( + pdf = pd.DataFrame( { "a": np.random.randint(0, 1000, nelem).astype(data_type), "b": np.random.randint(0, 1000, nelem).astype(data_type), } ) + gdf = cudf.from_pandas(pdf) - check_frame_series_equality(gdf.head(), gdf[:5]) - check_frame_series_equality(gdf.head(3), gdf[:3]) - check_frame_series_equality(gdf.head(-2), gdf[:-2]) - check_frame_series_equality(gdf.head(0), gdf[0:0]) + assert_eq(gdf.head(), pdf.head()) + assert_eq(gdf.head(3), pdf.head(3)) + assert_eq(gdf.head(-2), pdf.head(-2)) + assert_eq(gdf.head(0), pdf.head(0)) - check_frame_series_equality(gdf["a"].head(), gdf["a"][:5]) - check_frame_series_equality(gdf["a"].head(3), gdf["a"][:3]) - check_frame_series_equality(gdf["a"].head(-2), gdf["a"][:-2]) + assert_eq(gdf["a"].head(), pdf["a"].head()) + assert_eq(gdf["a"].head(3), pdf["a"].head(3)) + assert_eq(gdf["a"].head(-2), pdf["a"].head(-2)) - check_frame_series_equality(gdf.tail(), gdf[-5:]) - check_frame_series_equality(gdf.tail(3), gdf[-3:]) - check_frame_series_equality(gdf.tail(-2), gdf[2:]) - check_frame_series_equality(gdf.tail(0), gdf[0:0]) + assert_eq(gdf.tail(), pdf.tail()) + assert_eq(gdf.tail(3), pdf.tail(3)) + assert_eq(gdf.tail(-2), pdf.tail(-2)) + assert_eq(gdf.tail(0), pdf.tail(0)) - check_frame_series_equality(gdf["a"].tail(), gdf["a"][-5:]) - check_frame_series_equality(gdf["a"].tail(3), gdf["a"][-3:]) - check_frame_series_equality(gdf["a"].tail(-2), gdf["a"][2:]) + assert_eq(gdf["a"].tail(), pdf["a"].tail()) + assert_eq(gdf["a"].tail(3), pdf["a"].tail(3)) + assert_eq(gdf["a"].tail(-2), pdf["a"].tail(-2)) def test_tail_for_string(): @@ -4328,6 +4316,17 @@ def test_one_row_head(): assert_eq(head_pdf, head_gdf) +@pytest.mark.parametrize("index", [None, [123], ["a", "b"]]) +def test_no_cols_head(index): + pdf = pd.DataFrame(index=index) + gdf = cudf.from_pandas(pdf) + + head_gdf = gdf.head() + head_pdf = pdf.head() + + assert_eq(head_pdf, head_gdf) + + @pytest.mark.parametrize("dtype", ALL_TYPES) @pytest.mark.parametrize( "np_dtype,pd_dtype", From dbdcc31fe1cbe902d495428da3c68dc59d289dc5 Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Mon, 4 Mar 2024 18:22:49 +0000 Subject: [PATCH 333/384] Expose new stable_sort and finish stream_compaction in pylibcudf (#15175) Completes coverage of `sorting.hpp` and `stream_compaction.hpp` Authors: - Lawrence Mitchell (https://github.com/wence-) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/15175 --- python/cudf/cudf/_lib/cpp/sorting.pxd | 7 +- .../cudf/cudf/_lib/cpp/stream_compaction.pxd | 43 +++- python/cudf/cudf/_lib/pylibcudf/sorting.pxd | 2 + python/cudf/cudf/_lib/pylibcudf/sorting.pyx | 39 +++- .../cudf/_lib/pylibcudf/stream_compaction.pxd | 34 +++- .../cudf/_lib/pylibcudf/stream_compaction.pyx | 185 ++++++++++++++++-- python/cudf/cudf/_lib/stream_compaction.pyx | 1 + 7 files changed, 275 insertions(+), 36 deletions(-) diff --git a/python/cudf/cudf/_lib/cpp/sorting.pxd b/python/cudf/cudf/_lib/cpp/sorting.pxd index 68f01003fe6..86dc0f0de95 100644 --- a/python/cudf/cudf/_lib/cpp/sorting.pxd +++ b/python/cudf/cudf/_lib/cpp/sorting.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp cimport bool from libcpp.memory cimport unique_ptr @@ -68,3 +68,8 @@ cdef extern from "cudf/sorting.hpp" namespace "cudf" nogil: table_view source_table, vector[libcudf_types.order] column_order, vector[libcudf_types.null_order] null_precedence) except + + + cdef unique_ptr[table] stable_sort( + table_view source_table, + vector[libcudf_types.order] column_order, + vector[libcudf_types.null_order] null_precedence) except + diff --git a/python/cudf/cudf/_lib/cpp/stream_compaction.pxd b/python/cudf/cudf/_lib/cpp/stream_compaction.pxd index e8539ecb9c3..55854a9444f 100644 --- a/python/cudf/cudf/_lib/cpp/stream_compaction.pxd +++ b/python/cudf/cudf/_lib/cpp/stream_compaction.pxd @@ -30,21 +30,28 @@ cdef extern from "cudf/stream_compaction.hpp" namespace "cudf" nogil: vector[size_type] keys, size_type keep_threshold) except + + cdef unique_ptr[table] drop_nans(table_view source_table, + vector[size_type] keys, + size_type keep_threshold) except + + cdef unique_ptr[table] apply_boolean_mask( table_view source_table, column_view boolean_mask ) except + - cdef size_type distinct_count( - column_view source_table, - null_policy null_handling, - nan_policy nan_handling) except + + cdef unique_ptr[table] unique( + table_view input, + vector[size_type] keys, + duplicate_keep_option keep, + null_equality nulls_equal, + ) except + - cdef unique_ptr[table] stable_distinct( + cdef unique_ptr[table] distinct( table_view input, vector[size_type] keys, duplicate_keep_option keep, null_equality nulls_equal, + nan_equality nans_equals, ) except + cdef unique_ptr[column] distinct_indices( @@ -53,3 +60,29 @@ cdef extern from "cudf/stream_compaction.hpp" namespace "cudf" nogil: null_equality nulls_equal, nan_equality nans_equal, ) except + + + cdef unique_ptr[table] stable_distinct( + table_view input, + vector[size_type] keys, + duplicate_keep_option keep, + null_equality nulls_equal, + nan_equality nans_equal, + ) except + + + cdef size_type unique_count( + column_view column, + null_policy null_handling, + nan_policy nan_handling) except + + + cdef size_type unique_count( + table_view source_table, + null_policy null_handling) except + + + cdef size_type distinct_count( + column_view column, + null_policy null_handling, + nan_policy nan_handling) except + + + cdef size_type distinct_count( + table_view source_table, + null_policy null_handling) except + diff --git a/python/cudf/cudf/_lib/pylibcudf/sorting.pxd b/python/cudf/cudf/_lib/pylibcudf/sorting.pxd index fb22da0b0fd..3ed241622c0 100644 --- a/python/cudf/cudf/_lib/pylibcudf/sorting.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/sorting.pxd @@ -59,3 +59,5 @@ cpdef Table stable_sort_by_key( ) cpdef Table sort(Table source_table, list column_order, list null_precedence) + +cpdef Table stable_sort(Table source_table, list column_order, list null_precedence) diff --git a/python/cudf/cudf/_lib/pylibcudf/sorting.pyx b/python/cudf/cudf/_lib/pylibcudf/sorting.pyx index 4e73760720a..1668a3efc7c 100644 --- a/python/cudf/cudf/_lib/pylibcudf/sorting.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/sorting.pyx @@ -50,7 +50,8 @@ cpdef Column stable_sorted_order( list column_order, list null_precedence, ): - """Computes the row indices required to sort the table, maintaining input order. + """Computes the row indices required to sort the table, + preserving order of equal elements. Parameters ---------- @@ -206,7 +207,8 @@ cpdef Table stable_segmented_sort_by_key( list column_order, list null_precedence, ): - """Sorts the table by key, within segments, maintaining input order. + """Sorts the table by key preserving order of equal elements, + within segments. Parameters ---------- @@ -287,7 +289,7 @@ cpdef Table stable_sort_by_key( list column_order, list null_precedence, ): - """Sorts the table by key, maintaining input order. + """Sorts the table by key preserving order of equal elements. Parameters ---------- @@ -349,3 +351,34 @@ cpdef Table sort(Table source_table, list column_order, list null_precedence): ) ) return Table.from_libcudf(move(c_result)) + + +cpdef Table stable_sort(Table source_table, list column_order, list null_precedence): + """Sorts the table preserving order of equal elements. + + Parameters + ---------- + source_table : Table + The table to sort. + column_order : List[ColumnOrder] + Whether each column should be sorted in ascending or descending order. + null_precedence : List[NullOrder] + Whether nulls should come before or after non-nulls. + + Returns + ------- + Table + The sorted table. + """ + cdef unique_ptr[table] c_result + cdef vector[order] c_orders = column_order + cdef vector[null_order] c_null_precedence = null_precedence + with nogil: + c_result = move( + cpp_sorting.stable_sort( + source_table.view(), + c_orders, + c_null_precedence, + ) + ) + return Table.from_libcudf(move(c_result)) diff --git a/python/cudf/cudf/_lib/pylibcudf/stream_compaction.pxd b/python/cudf/cudf/_lib/pylibcudf/stream_compaction.pxd index 78adb20021c..29acc21fc05 100644 --- a/python/cudf/cudf/_lib/pylibcudf/stream_compaction.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/stream_compaction.pxd @@ -15,19 +15,21 @@ from .table cimport Table cpdef Table drop_nulls(Table source_table, list keys, size_type keep_threshold) -cpdef Table apply_boolean_mask(Table source_table, Column boolean_mask) +cpdef Table drop_nans(Table source_table, list keys, size_type keep_threshold) -cpdef size_type distinct_count( - Column source_table, - null_policy null_handling, - nan_policy nan_handling +cpdef Table unique( + Table input, + list keys, + duplicate_keep_option keep, + null_equality nulls_equal, ) -cpdef Table stable_distinct( +cpdef Table distinct( Table input, list keys, duplicate_keep_option keep, null_equality nulls_equal, + nan_equality nans_equal, ) cpdef Column distinct_indices( @@ -36,3 +38,23 @@ cpdef Column distinct_indices( null_equality nulls_equal, nan_equality nans_equal, ) + +cpdef Table stable_distinct( + Table input, + list keys, + duplicate_keep_option keep, + null_equality nulls_equal, + nan_equality nans_equal, +) + +cpdef size_type unique_count( + Column column, + null_policy null_handling, + nan_policy nan_handling +) + +cpdef size_type distinct_count( + Column column, + null_policy null_handling, + nan_policy nan_handling +) diff --git a/python/cudf/cudf/_lib/pylibcudf/stream_compaction.pyx b/python/cudf/cudf/_lib/pylibcudf/stream_compaction.pyx index 0357866980a..af7a85d31bf 100644 --- a/python/cudf/cudf/_lib/pylibcudf/stream_compaction.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/stream_compaction.pyx @@ -51,6 +51,34 @@ cpdef Table drop_nulls(Table source_table, list keys, size_type keep_threshold): return Table.from_libcudf(move(c_result)) +cpdef Table drop_nans(Table source_table, list keys, size_type keep_threshold): + """Filters out rows from the input table based on the presence of NaNs. + + Parameters + ---------- + source_table : Table + The input table to filter. + keys : List[size_type] + The list of column indexes to consider for NaN filtering. + keep_threshold : size_type + The minimum number of non-NaNs required to keep a row. + + Returns + ------- + Table + A new table with rows removed based on NaNs. + """ + cdef unique_ptr[table] c_result + cdef vector[size_type] c_keys = keys + with nogil: + c_result = move( + cpp_stream_compaction.drop_nulls( + source_table.view(), c_keys, keep_threshold + ) + ) + return Table.from_libcudf(move(c_result)) + + cpdef Table apply_boolean_mask(Table source_table, Column boolean_mask): """Filters out rows from the input table based on a boolean mask. @@ -76,39 +104,55 @@ cpdef Table apply_boolean_mask(Table source_table, Column boolean_mask): return Table.from_libcudf(move(c_result)) -cpdef size_type distinct_count( - Column source_table, - null_policy null_handling, - nan_policy nan_handling +cpdef Table unique( + Table input, + list keys, + duplicate_keep_option keep, + null_equality nulls_equal, ): - """Returns the number of unique elements in the input column. + """Filter duplicate consecutive rows from the input table. Parameters ---------- - source_table : Column - The input column to count the unique elements of. - null_handling : null_policy - Flag to include or exclude nulls from the count. - nan_handling : nan_policy - Flag to include or exclude NaNs from the count. + input : Table + The input table to filter + keys : list[int] + The list of column indexes to consider for filtering. + keep : duplicate_keep_option + The option to specify which rows to keep in the case of duplicates. + nulls_equal : null_equality + The option to specify how nulls are handled in the comparison. Returns ------- - size_type - The number of unique elements in the input column. + Table + New Table with unique rows from each sequence of equivalent rows + as specified by keep. In the same order as the input table. + + Notes + ----- + If the input columns to be filtered on are sorted, then + unique can produce the same result as stable_distinct, but faster. """ - return cpp_stream_compaction.distinct_count( - source_table.view(), null_handling, nan_handling - ) + cdef unique_ptr[table] c_result + cdef vector[size_type] c_keys = keys + with nogil: + c_result = move( + cpp_stream_compaction.unique( + input.view(), c_keys, keep, nulls_equal + ) + ) + return Table.from_libcudf(move(c_result)) -cpdef Table stable_distinct( +cpdef Table distinct( Table input, list keys, duplicate_keep_option keep, null_equality nulls_equal, + nan_equality nans_equal, ): - """Get the distinct rows from the input table, preserving input order. + """Get the distinct rows from the input table. Parameters ---------- @@ -120,18 +164,21 @@ cpdef Table stable_distinct( The option to specify which rows to keep in the case of duplicates. nulls_equal : null_equality The option to specify how nulls are handled in the comparison. + nans_equal : nan_equality + The option to specify how NaNs are handled in the comparison. Returns ------- Table - A new table with distinct rows from the input table. + A new table with distinct rows from the input table. The + output will not necessarily be in the same order as the input. """ cdef unique_ptr[table] c_result cdef vector[size_type] c_keys = keys with nogil: c_result = move( - cpp_stream_compaction.stable_distinct( - input.view(), c_keys, keep, nulls_equal + cpp_stream_compaction.distinct( + input.view(), c_keys, keep, nulls_equal, nans_equal ) ) return Table.from_libcudf(move(c_result)) @@ -169,3 +216,99 @@ cpdef Column distinct_indices( ) ) return Column.from_libcudf(move(c_result)) + + +cpdef Table stable_distinct( + Table input, + list keys, + duplicate_keep_option keep, + null_equality nulls_equal, + nan_equality nans_equal, +): + """Get the distinct rows from the input table, preserving input order. + + Parameters + ---------- + input : Table + The input table to filter. + keys : list + The list of column indexes to consider for distinct filtering. + keep : duplicate_keep_option + The option to specify which rows to keep in the case of duplicates. + nulls_equal : null_equality + The option to specify how nulls are handled in the comparison. + nans_equal : nan_equality + The option to specify how NaNs are handled in the comparison. + + Returns + ------- + Table + A new table with distinct rows from the input table, preserving + the input table order. + """ + cdef unique_ptr[table] c_result + cdef vector[size_type] c_keys = keys + with nogil: + c_result = move( + cpp_stream_compaction.stable_distinct( + input.view(), c_keys, keep, nulls_equal, nans_equal + ) + ) + return Table.from_libcudf(move(c_result)) + + +cpdef size_type unique_count( + Column source, + null_policy null_handling, + nan_policy nan_handling +): + """Returns the number of unique consecutive elements in the input column. + + Parameters + ---------- + source : Column + The input column to count the unique elements of. + null_handling : null_policy + Flag to include or exclude nulls from the count. + nan_handling : nan_policy + Flag to include or exclude NaNs from the count. + + Returns + ------- + size_type + The number of unique consecutive elements in the input column. + + Notes + ----- + If the input column is sorted, then unique_count can produce the + same result as distinct_count, but faster. + """ + return cpp_stream_compaction.unique_count( + source.view(), null_handling, nan_handling + ) + + +cpdef size_type distinct_count( + Column source, + null_policy null_handling, + nan_policy nan_handling +): + """Returns the number of distinct elements in the input column. + + Parameters + ---------- + source : Column + The input column to count the unique elements of. + null_handling : null_policy + Flag to include or exclude nulls from the count. + nan_handling : nan_policy + Flag to include or exclude NaNs from the count. + + Returns + ------- + size_type + The number of distinct elements in the input column. + """ + return cpp_stream_compaction.distinct_count( + source.view(), null_handling, nan_handling + ) diff --git a/python/cudf/cudf/_lib/stream_compaction.pyx b/python/cudf/cudf/_lib/stream_compaction.pyx index 04883eac559..834f91f48d9 100644 --- a/python/cudf/cudf/_lib/stream_compaction.pyx +++ b/python/cudf/cudf/_lib/stream_compaction.pyx @@ -109,6 +109,7 @@ def drop_duplicates(list columns, keep_option, pylibcudf.types.NullEquality.EQUAL if nulls_are_equal else pylibcudf.types.NullEquality.UNEQUAL, + pylibcudf.types.NanEquality.ALL_EQUAL, ) ) From da113015aade79d78628d00578dff22a4dd5cf35 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 4 Mar 2024 13:17:33 -0600 Subject: [PATCH 334/384] Switch `pytest-xdist` algo to `worksteal` (#15207) This PR switches `pytest-xdist` distribution algorithm to a much more efficient algorithm `worksteal`, that will assign any idle pytest worker to pickup remaining pytests. I see a 25% time savings when this switch is made locally: ``` `loadscope`: == 101421 passed, 2115 skipped, 867 xfailed in 1179.48s (0:19:39) == `worksteal`: == 101423 passed, 2115 skipped, 867 xfailed in 891.79s (0:14:51) == ``` Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Bradley Dice (https://github.com/bdice) - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/cudf/pull/15207 --- ci/test_python_cudf.sh | 6 +++--- ci/test_python_other.sh | 4 ++-- ci/test_wheel_cudf.sh | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/test_python_cudf.sh b/ci/test_python_cudf.sh index ace71bb0b75..bacb54b3896 100755 --- a/ci/test_python_cudf.sh +++ b/ci/test_python_cudf.sh @@ -18,7 +18,7 @@ rapids-logger "pytest cudf" ./ci/run_cudf_pytests.sh \ --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf.xml" \ --numprocesses=8 \ - --dist=loadscope \ + --dist=worksteal \ --cov-config=../.coveragerc \ --cov=cudf \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-coverage.xml" \ @@ -32,7 +32,7 @@ rapids-logger "pytest cudf" rapids-logger "pytest for cudf benchmarks" ./ci/run_cudf_pytest_benchmarks.sh \ --numprocesses=8 \ - --dist=loadscope \ + --dist=worksteal \ --cov-config=.coveragerc \ --cov=cudf \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-benchmark-coverage.xml" \ @@ -41,7 +41,7 @@ rapids-logger "pytest for cudf benchmarks" rapids-logger "pytest for cudf benchmarks using pandas" ./ci/run_cudf_pandas_pytest_benchmarks.sh \ --numprocesses=8 \ - --dist=loadscope \ + --dist=worksteal \ --cov-config=.coveragerc \ --cov=cudf \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-benchmark-pandas-coverage.xml" \ diff --git a/ci/test_python_other.sh b/ci/test_python_other.sh index bc15747b26a..9cdceb295db 100755 --- a/ci/test_python_other.sh +++ b/ci/test_python_other.sh @@ -23,7 +23,7 @@ rapids-logger "pytest dask_cudf" ./ci/run_dask_cudf_pytests.sh \ --junitxml="${RAPIDS_TESTS_DIR}/junit-dask-cudf.xml" \ --numprocesses=8 \ - --dist=loadscope \ + --dist=worksteal \ --cov-config=../.coveragerc \ --cov=dask_cudf \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/dask-cudf-coverage.xml" \ @@ -33,7 +33,7 @@ rapids-logger "pytest custreamz" ./ci/run_custreamz_pytests.sh \ --junitxml="${RAPIDS_TESTS_DIR}/junit-custreamz.xml" \ --numprocesses=8 \ - --dist=loadscope \ + --dist=worksteal \ --cov-config=../.coveragerc \ --cov=custreamz \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/custreamz-coverage.xml" \ diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index b7e8f862ed5..af5779f478a 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -37,7 +37,7 @@ else --cache-clear \ --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf.xml" \ --numprocesses=8 \ - --dist=loadscope \ + --dist=worksteal \ . popd fi From 0ff5a2c59cb62d6b3c473885ebbe883d1aae8c4f Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Mon, 4 Mar 2024 15:20:32 -0500 Subject: [PATCH 335/384] Replace local copyright check with pre-commit-hooks verify-copyright (#14917) The local `copyright.py` script is bug-prone. Replace it with a more robust centralized script from `pre-commit-hooks`. Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) - Bradley Dice (https://github.com/bdice) Approvers: - Bradley Dice (https://github.com/bdice) - Jake Awe (https://github.com/AyodeAwe) - Karthikeyan (https://github.com/karthikeyann) - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/14917 --- .pre-commit-config.yaml | 13 +- ci/checks/copyright.py | 277 ---------------------------------------- 2 files changed, 7 insertions(+), 283 deletions(-) delete mode 100644 ci/checks/copyright.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d302543368e..9235c80bdc9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -126,12 +126,6 @@ repos: - cmakelang==0.6.13 verbose: true require_serial: true - - id: copyright-check - name: copyright-check - entry: python ./ci/checks/copyright.py --git-modified-only --update-current-year - language: python - pass_filenames: false - additional_dependencies: [gitpython] - id: doxygen-check name: doxygen-check entry: ./ci/checks/doxygen.sh @@ -161,6 +155,13 @@ repos: hooks: - id: ruff files: python/.*$ + - repo: https://github.com/rapidsai/pre-commit-hooks + rev: v0.0.1 + hooks: + - id: verify-copyright + exclude: | + (?x) + cpp/include/cudf_test/cxxopts[.]hpp$ default_language_version: diff --git a/ci/checks/copyright.py b/ci/checks/copyright.py deleted file mode 100644 index dd89b092496..00000000000 --- a/ci/checks/copyright.py +++ /dev/null @@ -1,277 +0,0 @@ -# Copyright (c) 2019-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import argparse -import datetime -import os -import re -import sys - -import git - -FilesToCheck = [ - re.compile(r"[.](cmake|cpp|cu|cuh|h|hpp|sh|pxd|py|pyx)$"), - re.compile(r"CMakeLists[.]txt$"), - re.compile(r"CMakeLists_standalone[.]txt$"), - re.compile(r"setup[.]cfg$"), - re.compile(r"meta[.]yaml$"), -] -ExemptFiles = [ - re.compile(r"cpp/include/cudf_test/cxxopts.hpp"), -] - -# this will break starting at year 10000, which is probably OK :) -CheckSimple = re.compile( - r"Copyright *(?:\(c\))? *(\d{4}),? *NVIDIA C(?:ORPORATION|orporation)" -) -CheckDouble = re.compile( - r"Copyright *(?:\(c\))? *(\d{4})-(\d{4}),? *NVIDIA C(?:ORPORATION|orporation)" # noqa: E501 -) - - -def checkThisFile(f): - if isinstance(f, git.Diff): - if f.deleted_file or f.b_blob.size == 0: - return False - f = f.b_path - elif not os.path.exists(f) or os.stat(f).st_size == 0: - # This check covers things like symlinks which point to files that DNE - return False - for exempt in ExemptFiles: - if exempt.search(f): - return False - for checker in FilesToCheck: - if checker.search(f): - return True - return False - - -def modifiedFiles(): - """Get a set of all modified files, as Diff objects. - - The files returned have been modified in git since the merge base of HEAD - and the upstream of the target branch. We return the Diff objects so that - we can read only the staged changes. - """ - repo = git.Repo() - # Use the environment variable TARGET_BRANCH or RAPIDS_BASE_BRANCH (defined in CI) if possible - target_branch = os.environ.get("TARGET_BRANCH", os.environ.get("RAPIDS_BASE_BRANCH")) - if target_branch is None: - # Fall back to the closest branch if not on CI - target_branch = repo.git.describe( - all=True, tags=True, match="branch-*", abbrev=0 - ).lstrip("heads/") - - upstream_target_branch = None - if target_branch in repo.heads: - # Use the tracking branch of the local reference if it exists. This - # returns None if no tracking branch is set. - upstream_target_branch = repo.heads[target_branch].tracking_branch() - if upstream_target_branch is None: - # Fall back to the remote with the newest target_branch. This code - # path is used on CI because the only local branch reference is - # current-pr-branch, and thus target_branch is not in repo.heads. - # This also happens if no tracking branch is defined for the local - # target_branch. We use the remote with the latest commit if - # multiple remotes are defined. - candidate_branches = [ - remote.refs[target_branch] for remote in repo.remotes - if target_branch in remote.refs - ] - if len(candidate_branches) > 0: - upstream_target_branch = sorted( - candidate_branches, - key=lambda branch: branch.commit.committed_datetime, - )[-1] - else: - # If no remotes are defined, try to use the local version of the - # target_branch. If this fails, the repo configuration must be very - # strange and we can fix this script on a case-by-case basis. - upstream_target_branch = repo.heads[target_branch] - merge_base = repo.merge_base("HEAD", upstream_target_branch.commit)[0] - diff = merge_base.diff() - changed_files = {f for f in diff if f.b_path is not None} - return changed_files - - -def getCopyrightYears(line): - res = CheckSimple.search(line) - if res: - return int(res.group(1)), int(res.group(1)) - res = CheckDouble.search(line) - if res: - return int(res.group(1)), int(res.group(2)) - return None, None - - -def replaceCurrentYear(line, start, end): - # first turn a simple regex into double (if applicable). then update years - res = CheckSimple.sub(r"Copyright (c) \1-\1, NVIDIA CORPORATION", line) - res = CheckDouble.sub( - rf"Copyright (c) {start:04d}-{end:04d}, NVIDIA CORPORATION", - res, - ) - return res - - -def checkCopyright(f, update_current_year): - """Checks for copyright headers and their years.""" - errs = [] - thisYear = datetime.datetime.now().year - lineNum = 0 - crFound = False - yearMatched = False - - if isinstance(f, git.Diff): - path = f.b_path - lines = f.b_blob.data_stream.read().decode().splitlines(keepends=True) - else: - path = f - with open(f, encoding="utf-8") as fp: - lines = fp.readlines() - - for line in lines: - lineNum += 1 - start, end = getCopyrightYears(line) - if start is None: - continue - crFound = True - if start > end: - e = [ - path, - lineNum, - "First year after second year in the copyright " - "header (manual fix required)", - None, - ] - errs.append(e) - elif thisYear < start or thisYear > end: - e = [ - path, - lineNum, - "Current year not included in the copyright header", - None, - ] - if thisYear < start: - e[-1] = replaceCurrentYear(line, thisYear, end) - if thisYear > end: - e[-1] = replaceCurrentYear(line, start, thisYear) - errs.append(e) - else: - yearMatched = True - # copyright header itself not found - if not crFound: - e = [ - path, - 0, - "Copyright header missing or formatted incorrectly " - "(manual fix required)", - None, - ] - errs.append(e) - # even if the year matches a copyright header, make the check pass - if yearMatched: - errs = [] - - if update_current_year: - errs_update = [x for x in errs if x[-1] is not None] - if len(errs_update) > 0: - lines_changed = ", ".join(str(x[1]) for x in errs_update) - print(f"File: {path}. Changing line(s) {lines_changed}") - for _, lineNum, __, replacement in errs_update: - lines[lineNum - 1] = replacement - with open(path, "w", encoding="utf-8") as out_file: - out_file.writelines(lines) - - return errs - - -def getAllFilesUnderDir(root, pathFilter=None): - retList = [] - for dirpath, dirnames, filenames in os.walk(root): - for fn in filenames: - filePath = os.path.join(dirpath, fn) - if pathFilter(filePath): - retList.append(filePath) - return retList - - -def checkCopyright_main(): - """ - Checks for copyright headers in all the modified files. In case of local - repo, this script will just look for uncommitted files and in case of CI - it compares between branches "$PR_TARGET_BRANCH" and "current-pr-branch" - """ - retVal = 0 - - argparser = argparse.ArgumentParser( - "Checks for a consistent copyright header in git's modified files" - ) - argparser.add_argument( - "--update-current-year", - dest="update_current_year", - action="store_true", - required=False, - help="If set, " - "update the current year if a header is already " - "present and well formatted.", - ) - argparser.add_argument( - "--git-modified-only", - dest="git_modified_only", - action="store_true", - required=False, - help="If set, " - "only files seen as modified by git will be " - "processed.", - ) - - args, dirs = argparser.parse_known_args() - - if args.git_modified_only: - files = [f for f in modifiedFiles() if checkThisFile(f)] - else: - files = [] - for d in [os.path.abspath(d) for d in dirs]: - if not os.path.isdir(d): - raise ValueError(f"{d} is not a directory.") - files += getAllFilesUnderDir(d, pathFilter=checkThisFile) - - errors = [] - for f in files: - errors += checkCopyright(f, args.update_current_year) - - if len(errors) > 0: - if any(e[-1] is None for e in errors): - print("Copyright headers incomplete in some of the files!") - for e in errors: - print(" %s:%d Issue: %s" % (e[0], e[1], e[2])) - print("") - n_fixable = sum(1 for e in errors if e[-1] is not None) - path_parts = os.path.abspath(__file__).split(os.sep) - file_from_repo = os.sep.join(path_parts[path_parts.index("ci") :]) - if n_fixable > 0 and not args.update_current_year: - print( - f"You can run `python {file_from_repo} --git-modified-only " - "--update-current-year` and stage the results in git to " - f"fix {n_fixable} of these errors.\n" - ) - retVal = 1 - - return retVal - - -if __name__ == "__main__": - sys.exit(checkCopyright_main()) From d158ccdbe651952bd649cb0f17c41467c5209824 Mon Sep 17 00:00:00 2001 From: Shruti Shivakumar Date: Mon, 4 Mar 2024 15:25:51 -0500 Subject: [PATCH 336/384] API for JSON unquoted whitespace normalization (#15033) This work is a follow-up to PR #14931 which provided a proof-of-concept for using the a FST to normalize unquoted whitespaces. This PR implements the pre-processing FST in cuIO and adds a JSON reader option that needs to be set to true to invoke the normalizer. Addresses feature request #14865 Authors: - Shruti Shivakumar (https://github.com/shrshi) - Vukasin Milovanovic (https://github.com/vuule) Approvers: - Robert (Bobby) Evans (https://github.com/revans2) - Vukasin Milovanovic (https://github.com/vuule) - Robert Maynard (https://github.com/robertmaynard) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/15033 --- cpp/CMakeLists.txt | 2 +- cpp/include/cudf/io/detail/json.hpp | 10 + cpp/include/cudf/io/json.hpp | 31 +++ ...normalization.cu => json_normalization.cu} | 142 ++++++++++++- cpp/src/io/json/read_json.cu | 7 + .../io/json_whitespace_normalization_test.cu | 201 ++++-------------- .../main/java/ai/rapids/cudf/JSONOptions.java | 15 ++ java/src/main/java/ai/rapids/cudf/Table.java | 9 + java/src/main/native/src/TableJni.cpp | 27 ++- .../test/java/ai/rapids/cudf/TableTest.java | 49 +++-- java/src/test/resources/whitespaces.json | 5 + 11 files changed, 314 insertions(+), 184 deletions(-) rename cpp/src/io/json/{json_quote_normalization.cu => json_normalization.cu} (57%) create mode 100644 java/src/test/resources/whitespaces.json diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 5fd6cd3544a..c74963be50d 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -376,7 +376,7 @@ add_library( src/io/functions.cpp src/io/json/byte_range_info.cu src/io/json/json_column.cu - src/io/json/json_quote_normalization.cu + src/io/json/json_normalization.cu src/io/json/json_tree.cu src/io/json/nested_json_gpu.cu src/io/json/read_json.cu diff --git a/cpp/include/cudf/io/detail/json.hpp b/cpp/include/cudf/io/detail/json.hpp index 0eb0e17ea10..3f7f7e9bb32 100644 --- a/cpp/include/cudf/io/detail/json.hpp +++ b/cpp/include/cudf/io/detail/json.hpp @@ -63,4 +63,14 @@ rmm::device_uvector normalize_single_quotes(rmm::device_uvector&& in rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr); +/** + * @brief Normalize unquoted whitespace (space and tab characters) using FST + * + * @param inbuf Input device buffer + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource to use for device memory allocation + */ +rmm::device_uvector normalize_whitespace(rmm::device_uvector&& inbuf, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr); } // namespace cudf::io::json::detail diff --git a/cpp/include/cudf/io/json.hpp b/cpp/include/cudf/io/json.hpp index f0c3d48ab7e..593dd044d51 100644 --- a/cpp/include/cudf/io/json.hpp +++ b/cpp/include/cudf/io/json.hpp @@ -118,6 +118,9 @@ class json_reader_options { // Normalize single quotes bool _normalize_single_quotes = false; + // Normalize unquoted spaces and tabs + bool _normalize_whitespace = false; + // Whether to recover after an invalid JSON line json_recovery_mode_t _recovery_mode = json_recovery_mode_t::FAIL; @@ -265,6 +268,13 @@ class json_reader_options { */ bool is_enabled_normalize_single_quotes() const { return _normalize_single_quotes; } + /** + * @brief Whether the reader should normalize unquoted whitespace characters + * + * @returns true if the reader should normalize whitespace, false otherwise + */ + bool is_enabled_normalize_whitespace() const { return _normalize_whitespace; } + /** * @brief Queries the JSON reader's behavior on invalid JSON lines. * @@ -358,6 +368,14 @@ class json_reader_options { */ void enable_normalize_single_quotes(bool val) { _normalize_single_quotes = val; } + /** + * @brief Set whether the reader should enable normalization of unquoted whitespace + * + * @param val Boolean value to indicate whether the reader should normalize unquoted whitespace + * characters i.e. tabs and spaces + */ + void enable_normalize_whitespace(bool val) { _normalize_whitespace = val; } + /** * @brief Specifies the JSON reader's behavior on invalid JSON lines. * @@ -533,6 +551,19 @@ class json_reader_options_builder { return *this; } + /** + * @brief Set whether the reader should normalize unquoted whitespace + * + * @param val Boolean value to indicate whether the reader should normalize unquoted + * whitespace + * @return this for chaining + */ + json_reader_options_builder& normalize_whitespace(bool val) + { + options._normalize_whitespace = val; + return *this; + } + /** * @brief Specifies the JSON reader's behavior on invalid JSON lines. * diff --git a/cpp/src/io/json/json_quote_normalization.cu b/cpp/src/io/json/json_normalization.cu similarity index 57% rename from cpp/src/io/json/json_quote_normalization.cu rename to cpp/src/io/json/json_normalization.cu index a13b6e0b016..86e4da664a8 100644 --- a/cpp/src/io/json/json_quote_normalization.cu +++ b/cpp/src/io/json/json_normalization.cu @@ -32,13 +32,15 @@ namespace cudf::io::json { -using SymbolT = char; -using StateT = char; +// Type used to represent the atomic symbol type used within the finite-state machine +using SymbolT = char; +using StateT = char; + +// Type sufficiently large to index symbols within the input and output (may be unsigned) using SymbolOffsetT = uint32_t; namespace normalize_quotes { -// Type sufficiently large to index symbols within the input and output (may be unsigned) enum class dfa_states : StateT { TT_OOS = 0U, TT_DQS, TT_SQS, TT_DEC, TT_SEC, TT_NUM_STATES }; enum class dfa_symbol_group_id : uint32_t { DOUBLE_QUOTE_CHAR, ///< Quote character SG: " @@ -172,6 +174,116 @@ struct TransduceToNormalizedQuotes { } // namespace normalize_quotes +namespace normalize_whitespace { + +enum class dfa_symbol_group_id : uint32_t { + DOUBLE_QUOTE_CHAR, ///< Quote character SG: " + ESCAPE_CHAR, ///< Escape character SG: '\\' + NEWLINE_CHAR, ///< Newline character SG: '\n' + WHITESPACE_SYMBOLS, ///< Whitespace characters SG: '\t' or ' ' + OTHER_SYMBOLS, ///< SG implicitly matching all other characters + NUM_SYMBOL_GROUPS ///< Total number of symbol groups +}; +// Alias for readability of symbol group ids +constexpr auto NUM_SYMBOL_GROUPS = static_cast(dfa_symbol_group_id::NUM_SYMBOL_GROUPS); +// The i-th string representing all the characters of a symbol group +std::array, NUM_SYMBOL_GROUPS - 1> const wna_sgs{ + {{'"'}, {'\\'}, {'\n'}, {' ', '\t'}}}; + +/** + * -------- FST states --------- + * ----------------------------- + * TT_OOS | Out-of-string state handling whitespace and non-whitespace chars outside double + * | quotes as well as any other character not enclosed by a string. Also handles + * | newline character present within a string + * TT_DQS | Double-quoted string state handling all characters within double quotes except + * | newline character + * TT_DEC | State handling escaped characters inside double-quoted string. Note that this + * | state is necessary to process escaped double-quote characters. Without this + * | state, whitespaces following escaped double quotes inside strings may be removed. + * + * NOTE: An important case NOT handled by this FST is that of whitespace following newline + * characters within a string. Consider the following example + * Input: {"a":"x\n y"} + * FST output: {"a":"x\ny"} + * Expected output: {"a":"x\n y"} + * Such strings are not part of the JSON standard (characters allowed within quotes should + * have ASCII at least 0x20 i.e. space character and above) but may be encountered while + * reading JSON files + */ +enum class dfa_states : StateT { TT_OOS = 0U, TT_DQS, TT_DEC, TT_NUM_STATES }; +// Aliases for readability of the transition table +constexpr auto TT_OOS = dfa_states::TT_OOS; +constexpr auto TT_DQS = dfa_states::TT_DQS; +constexpr auto TT_DEC = dfa_states::TT_DEC; +constexpr auto TT_NUM_STATES = static_cast(dfa_states::TT_NUM_STATES); + +// Transition table +std::array, TT_NUM_STATES> const wna_state_tt{ + {/* IN_STATE " \ \n OTHER */ + /* TT_OOS */ {{TT_DQS, TT_OOS, TT_OOS, TT_OOS, TT_OOS}}, + /* TT_DQS */ {{TT_OOS, TT_DEC, TT_OOS, TT_DQS, TT_DQS}}, + /* TT_DEC */ {{TT_DQS, TT_DQS, TT_DQS, TT_DQS, TT_DQS}}}}; + +// The DFA's starting state +constexpr StateT start_state = static_cast(TT_OOS); + +struct TransduceToNormalizedWS { + /** + * @brief Returns the -th output symbol on the transition (state_id, match_id). + */ + template + constexpr CUDF_HOST_DEVICE SymbolT operator()(StateT const state_id, + SymbolGroupT const match_id, + RelativeOffsetT const relative_offset, + SymbolT const read_symbol) const + { + // -------- TRANSLATION TABLE ------------ + // Let the alphabet set be Sigma + // --------------------------------------- + // ---------- NON-SPECIAL CASES: ---------- + // Output symbol same as input symbol + // state | read_symbol -> output_symbol + // DQS | Sigma -> Sigma + // OOS | Sigma\{,\t} -> Sigma\{,\t} + // DEC | Sigma -> Sigma + // ---------- SPECIAL CASES: -------------- + // Input symbol translates to output symbol + // OOS | {} -> + // OOS | {\t} -> + + // Case when read symbol is a space or tab but is unquoted + // This will be the same condition as in `operator()(state_id, match_id, read_symbol)` function + // However, since there is no output in this case i.e. the count returned by + // operator()(state_id, match_id, read_symbol) is zero, this function is never called. + // So skipping the check for this case. + + // In all other cases, we have an output symbol for the input symbol. + // We simply output the input symbol + return read_symbol; + } + + /** + * @brief Returns the number of output characters for a given transition. + * During whitespace normalization, we always emit one output character i.e., the input + * character, except when we need to remove the space/tab character + */ + template + constexpr CUDF_HOST_DEVICE uint32_t operator()(StateT const state_id, + SymbolGroupT const match_id, + SymbolT const read_symbol) const + { + // Case when read symbol is a space or tab but is unquoted + if (match_id == static_cast(dfa_symbol_group_id::WHITESPACE_SYMBOLS) && + state_id == static_cast(dfa_states::TT_OOS)) { + return 0; + } + return 1; + } +}; + +} // namespace normalize_whitespace + namespace detail { rmm::device_uvector normalize_single_quotes(rmm::device_uvector&& inbuf, @@ -198,5 +310,29 @@ rmm::device_uvector normalize_single_quotes(rmm::device_uvector normalize_whitespace(rmm::device_uvector&& inbuf, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) +{ + auto parser = fst::detail::make_fst( + fst::detail::make_symbol_group_lut(normalize_whitespace::wna_sgs), + fst::detail::make_transition_table(normalize_whitespace::wna_state_tt), + fst::detail::make_translation_functor(normalize_whitespace::TransduceToNormalizedWS{}), + stream); + + rmm::device_uvector outbuf(inbuf.size(), stream, mr); + rmm::device_scalar outbuf_size(stream, mr); + parser.Transduce(inbuf.data(), + static_cast(inbuf.size()), + outbuf.data(), + thrust::make_discard_iterator(), + outbuf_size.data(), + normalize_whitespace::start_state, + stream); + + outbuf.resize(outbuf_size.value(stream), stream); + return outbuf; +} + } // namespace detail } // namespace cudf::io::json diff --git a/cpp/src/io/json/read_json.cu b/cpp/src/io/json/read_json.cu index ba8acf2d47a..506d7b6cddc 100644 --- a/cpp/src/io/json/read_json.cu +++ b/cpp/src/io/json/read_json.cu @@ -235,6 +235,13 @@ table_with_metadata read_json(host_span> sources, normalize_single_quotes(std::move(buffer), stream, rmm::mr::get_current_device_resource()); } + // If input JSON buffer has unquoted spaces and tabs and option to normalize whitespaces is + // enabled, invoke pre-processing FST + if (reader_opts.is_enabled_normalize_whitespace()) { + buffer = + normalize_whitespace(std::move(buffer), stream, rmm::mr::get_current_device_resource()); + } + return device_parse_nested_json(buffer, reader_opts, stream, mr); // For debug purposes, use host_parse_nested_json() } diff --git a/cpp/tests/io/json_whitespace_normalization_test.cu b/cpp/tests/io/json_whitespace_normalization_test.cu index 545d8d2c4f9..336d360063f 100644 --- a/cpp/tests/io/json_whitespace_normalization_test.cu +++ b/cpp/tests/io/json_whitespace_normalization_test.cu @@ -13,177 +13,41 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "io/fst/lookup_tables.cuh" -#include "io/utilities/hostdevice_vector.hpp" - #include #include -#include +#include #include -#include +#include +#include +#include #include +#include -#include #include +#include -#include - -#include #include -namespace { -// Type used to represent the atomic symbol type used within the finite-state machine -using SymbolT = char; -using StateT = char; - -// Type sufficiently large to index symbols within the input and output (may be unsigned) -using SymbolOffsetT = uint32_t; - -enum class dfa_symbol_group_id : uint32_t { - DOUBLE_QUOTE_CHAR, ///< Quote character SG: " - ESCAPE_CHAR, ///< Escape character SG: '\\' - NEWLINE_CHAR, ///< Newline character SG: '\n' - WHITESPACE_SYMBOLS, ///< Whitespace characters SG: '\t' or ' ' - OTHER_SYMBOLS, ///< SG implicitly matching all other characters - NUM_SYMBOL_GROUPS ///< Total number of symbol groups -}; -// Alias for readability of symbol group ids -constexpr auto NUM_SYMBOL_GROUPS = static_cast(dfa_symbol_group_id::NUM_SYMBOL_GROUPS); -// The i-th string representing all the characters of a symbol group -std::array, NUM_SYMBOL_GROUPS - 1> const wna_sgs{ - {{'"'}, {'\\'}, {'\n'}, {' ', '\t'}}}; - -/** - * -------- FST states --------- - * ----------------------------- - * TT_OOS | Out-of-string state handling whitespace and non-whitespace chars outside double - * | quotes as well as any other character not enclosed by a string. Also handles - * | newline character present within a string - * TT_DQS | Double-quoted string state handling all characters within double quotes except - * | newline character - * TT_DEC | State handling escaped characters inside double-quoted string. Note that this - * | state is necessary to process escaped double-quote characters. Without this - * | state, whitespaces following escaped double quotes inside strings may be removed. - * - * NOTE: An important case NOT handled by this FST is that of whitespace following newline - * characters within a string. Consider the following example - * Input: {"a":"x\n y"} - * FST output: {"a":"x\ny"} - * Expected output: {"a":"x\n y"} - */ -enum class dfa_states : StateT { TT_OOS = 0U, TT_DQS, TT_DEC, TT_NUM_STATES }; -// Aliases for readability of the transition table -constexpr auto TT_OOS = dfa_states::TT_OOS; -constexpr auto TT_DQS = dfa_states::TT_DQS; -constexpr auto TT_DEC = dfa_states::TT_DEC; -constexpr auto TT_NUM_STATES = static_cast(dfa_states::TT_NUM_STATES); - -// Transition table -std::array, TT_NUM_STATES> const wna_state_tt{ - {/* IN_STATE " \ \n OTHER */ - /* TT_OOS */ {{TT_DQS, TT_OOS, TT_OOS, TT_OOS, TT_OOS}}, - /* TT_DQS */ {{TT_OOS, TT_DEC, TT_OOS, TT_DQS, TT_DQS}}, - /* TT_DEC */ {{TT_DQS, TT_DQS, TT_DQS, TT_DQS, TT_DQS}}}}; - -// The DFA's starting state -constexpr StateT start_state = static_cast(TT_OOS); - -struct TransduceToNormalizedWS { - /** - * @brief Returns the -th output symbol on the transition (state_id, match_id). - */ - template - constexpr CUDF_HOST_DEVICE SymbolT operator()(StateT const state_id, - SymbolGroupT const match_id, - RelativeOffsetT const relative_offset, - SymbolT const read_symbol) const - { - // -------- TRANSLATION TABLE ------------ - // Let the alphabet set be Sigma - // --------------------------------------- - // ---------- NON-SPECIAL CASES: ---------- - // Output symbol same as input symbol - // state | read_symbol -> output_symbol - // DQS | Sigma -> Sigma - // OOS | Sigma\{,\t} -> Sigma\{,\t} - // DEC | Sigma -> Sigma - // ---------- SPECIAL CASES: -------------- - // Input symbol translates to output symbol - // OOS | {} -> - // OOS | {\t} -> - - // Case when read symbol is a space or tab but is unquoted - // This will be the same condition as in `operator()(state_id, match_id, read_symbol)` function - // However, since there is no output in this case i.e. the count returned by - // operator()(state_id, match_id, read_symbol) is zero, this function is never called. - // So skipping the check for this case. - - // In all other cases, we have an output symbol for the input symbol. - // We simply output the input symbol - return read_symbol; - } - - /** - * @brief Returns the number of output characters for a given transition. - * During whitespace normalization, we always emit one output character i.e., the input - * character, except when we need to remove the space/tab character - */ - template - constexpr CUDF_HOST_DEVICE uint32_t operator()(StateT const state_id, - SymbolGroupT const match_id, - SymbolT const read_symbol) const - { - // Case when read symbol is a space or tab but is unquoted - if (match_id == static_cast(dfa_symbol_group_id::WHITESPACE_SYMBOLS) && - state_id == static_cast(dfa_states::TT_OOS)) { - return 0; - } - return 1; - } -}; -} // namespace - // Base test fixture for tests struct JsonWSNormalizationTest : public cudf::test::BaseFixture {}; -void run_test(std::string const& input, std::string const& output) +void run_test(std::string const& host_input, std::string const& expected_host_output) { - auto parser = cudf::io::fst::detail::make_fst( - cudf::io::fst::detail::make_symbol_group_lut(wna_sgs), - cudf::io::fst::detail::make_transition_table(wna_state_tt), - cudf::io::fst::detail::make_translation_functor(TransduceToNormalizedWS{}), - cudf::test::get_default_stream()); - - auto d_input_scalar = cudf::make_string_scalar(input, cudf::test::get_default_stream()); - auto& d_input = static_cast&>(*d_input_scalar); + auto stream_view = cudf::get_default_stream(); + auto device_input = cudf::detail::make_device_uvector_async( + host_input, stream_view, rmm::mr::get_current_device_resource()); - // Prepare input & output buffers - constexpr std::size_t single_item = 1; - cudf::detail::hostdevice_vector output_gpu(input.size(), - cudf::test::get_default_stream()); - cudf::detail::hostdevice_vector output_gpu_size(single_item, - cudf::test::get_default_stream()); + // Preprocessing FST + auto device_fst_output = cudf::io::json::detail::normalize_whitespace( + std::move(device_input), stream_view, rmm::mr::get_current_device_resource()); - // Allocate device-side temporary storage & run algorithm - parser.Transduce(d_input.data(), - static_cast(d_input.size()), - output_gpu.device_ptr(), - thrust::make_discard_iterator(), - output_gpu_size.device_ptr(), - start_state, - cudf::test::get_default_stream()); + auto const preprocessed_host_output = + cudf::detail::make_std_vector_sync(device_fst_output, stream_view); - // Async copy results from device to host - output_gpu.device_to_host_async(cudf::test::get_default_stream()); - output_gpu_size.device_to_host_async(cudf::test::get_default_stream()); - - // Make sure results have been copied back to host - cudf::test::get_default_stream().synchronize(); - - // Verify results - ASSERT_EQ(output_gpu_size[0], output.size()); - CUDF_TEST_EXPECT_VECTOR_EQUAL(output_gpu, output, output.size()); + ASSERT_EQ(preprocessed_host_output.size(), expected_host_output.size()); + CUDF_TEST_EXPECT_VECTOR_EQUAL( + preprocessed_host_output, expected_host_output, preprocessed_host_output.size()); } TEST_F(JsonWSNormalizationTest, GroundTruth_Spaces) @@ -259,4 +123,33 @@ TEST_F(JsonWSNormalizationTest, GroundTruth_InvalidInput) run_test(input, output); } +TEST_F(JsonWSNormalizationTest, ReadJsonOption) +{ + // When mixed type fields are read as strings, the table read will differ depending the + // value of normalize_whitespace + + // Test input + std::string const host_input = "{ \"a\" : {\"b\" :\t\"c\"}}"; + cudf::io::json_reader_options input_options = + cudf::io::json_reader_options::builder( + cudf::io::source_info{host_input.data(), host_input.size()}) + .lines(true) + .mixed_types_as_string(true) + .normalize_whitespace(true); + + cudf::io::table_with_metadata processed_table = cudf::io::read_json(input_options); + + // Expected table + std::string const expected_input = R"({ "a" : {"b":"c"}})"; + cudf::io::json_reader_options expected_input_options = + cudf::io::json_reader_options::builder( + cudf::io::source_info{expected_input.data(), expected_input.size()}) + .lines(true) + .mixed_types_as_string(true) + .normalize_whitespace(false); + + cudf::io::table_with_metadata expected_table = cudf::io::read_json(expected_input_options); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_table.tbl->view(), processed_table.tbl->view()); +} + CUDF_TEST_PROGRAM_MAIN() diff --git a/java/src/main/java/ai/rapids/cudf/JSONOptions.java b/java/src/main/java/ai/rapids/cudf/JSONOptions.java index 62496e32f7a..b37d0d88ec9 100644 --- a/java/src/main/java/ai/rapids/cudf/JSONOptions.java +++ b/java/src/main/java/ai/rapids/cudf/JSONOptions.java @@ -31,6 +31,7 @@ public final class JSONOptions extends ColumnFilterOptions { private final boolean lines; private final boolean recoverWithNull; private final boolean normalizeSingleQuotes; + private final boolean normalizeWhitespace; private final boolean mixedTypesAsStrings; private final boolean keepStringQuotes; @@ -40,6 +41,7 @@ private JSONOptions(Builder builder) { lines = builder.lines; recoverWithNull = builder.recoverWithNull; normalizeSingleQuotes = builder.normalizeSingleQuotes; + normalizeWhitespace = builder.normalizeWhitespace; mixedTypesAsStrings = builder.mixedTypesAsStrings; keepStringQuotes = builder.keepQuotes; } @@ -61,6 +63,10 @@ public boolean isNormalizeSingleQuotes() { return normalizeSingleQuotes; } + public boolean isNormalizeWhitespace() { + return normalizeWhitespace; + } + public boolean isMixedTypesAsStrings() { return mixedTypesAsStrings; } @@ -84,6 +90,7 @@ public static final class Builder extends ColumnFilterOptions.Builder(lines)) .recovery_mode(recovery_mode) .normalize_single_quotes(static_cast(normalize_single_quotes)) - .keep_quotes(keep_quotes) - .mixed_types_as_string(mixed_types_as_string); + .normalize_whitespace(static_cast(normalize_whitespace)) + .mixed_types_as_string(mixed_types_as_string) + .keep_quotes(keep_quotes); auto result = std::make_unique(cudf::io::read_json(opts.build())); @@ -1461,8 +1462,8 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readAndInferJSONFromDataSource JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readAndInferJSON( JNIEnv *env, jclass, jlong buffer, jlong buffer_length, jboolean day_first, jboolean lines, - jboolean recover_with_null, jboolean normalize_single_quotes, jboolean mixed_types_as_string, - jboolean keep_quotes) { + jboolean recover_with_null, jboolean normalize_single_quotes, jboolean normalize_whitespace, + jboolean mixed_types_as_string, jboolean keep_quotes) { JNI_NULL_CHECK(env, buffer, "buffer cannot be null", 0); if (buffer_length <= 0) { @@ -1484,8 +1485,9 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readAndInferJSON( .lines(static_cast(lines)) .recovery_mode(recovery_mode) .normalize_single_quotes(static_cast(normalize_single_quotes)) - .keep_quotes(keep_quotes) - .mixed_types_as_string(mixed_types_as_string); + .normalize_whitespace(static_cast(normalize_whitespace)) + .mixed_types_as_string(mixed_types_as_string) + .keep_quotes(keep_quotes); auto result = std::make_unique(cudf::io::read_json(opts.build())); @@ -1573,8 +1575,8 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_TableWithMeta_releaseTable(JNIE JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readJSONFromDataSource( JNIEnv *env, jclass, jintArray j_num_children, jobjectArray col_names, jintArray j_types, jintArray j_scales, jboolean day_first, jboolean lines, jboolean recover_with_null, - jboolean normalize_single_quotes, jboolean mixed_types_as_string, jboolean keep_quotes, - jlong ds_handle) { + jboolean normalize_single_quotes, jboolean normalize_whitespace, jboolean mixed_types_as_string, + jboolean keep_quotes, jlong ds_handle) { JNI_NULL_CHECK(env, ds_handle, "no data source handle given", 0); @@ -1606,6 +1608,7 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readJSONFromDataSource( .lines(static_cast(lines)) .recovery_mode(recovery_mode) .normalize_single_quotes(static_cast(normalize_single_quotes)) + .normalize_whitespace(static_cast(normalize_whitespace)) .mixed_types_as_string(mixed_types_as_string) .keep_quotes(keep_quotes); @@ -1646,7 +1649,8 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readJSON( JNIEnv *env, jclass, jintArray j_num_children, jobjectArray col_names, jintArray j_types, jintArray j_scales, jstring inputfilepath, jlong buffer, jlong buffer_length, jboolean day_first, jboolean lines, jboolean recover_with_null, - jboolean normalize_single_quotes, jboolean mixed_types_as_string, jboolean keep_quotes) { + jboolean normalize_single_quotes, jboolean normalize_whitespace, jboolean mixed_types_as_string, + jboolean keep_quotes) { bool read_buffer = true; if (buffer == 0) { @@ -1693,6 +1697,7 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readJSON( .lines(static_cast(lines)) .recovery_mode(recovery_mode) .normalize_single_quotes(static_cast(normalize_single_quotes)) + .normalize_whitespace(static_cast(normalize_whitespace)) .mixed_types_as_string(mixed_types_as_string) .keep_quotes(keep_quotes); diff --git a/java/src/test/java/ai/rapids/cudf/TableTest.java b/java/src/test/java/ai/rapids/cudf/TableTest.java index bee8d1cbb88..3f0470d854a 100644 --- a/java/src/test/java/ai/rapids/cudf/TableTest.java +++ b/java/src/test/java/ai/rapids/cudf/TableTest.java @@ -88,6 +88,7 @@ public class TableTest extends CudfTestBase { private static final File TEST_SIMPLE_JSON_FILE = TestUtils.getResourceAsFile("people.json"); private static final File TEST_JSON_ERROR_FILE = TestUtils.getResourceAsFile("people_with_invalid_lines.json"); private static final File TEST_JSON_SINGLE_QUOTES_FILE = TestUtils.getResourceAsFile("single_quotes.json"); + private static final File TEST_JSON_WHITESPACES_FILE = TestUtils.getResourceAsFile("whitespaces.json"); private static final File TEST_MIXED_TYPE_1_JSON = TestUtils.getResourceAsFile("mixed_types_1.json"); private static final File TEST_MIXED_TYPE_2_JSON = TestUtils.getResourceAsFile("mixed_types_2.json"); @@ -349,6 +350,39 @@ void testReadSingleQuotesJSONFile() throws IOException { } @Test + void testReadSingleQuotesJSONFileFeatureDisabled() throws IOException { + Schema schema = Schema.builder() + .column(DType.STRING, "A") + .build(); + JSONOptions opts = JSONOptions.builder() + .withLines(true) + .withNormalizeSingleQuotes(false) + .build(); + try (MultiBufferDataSource source = sourceFrom(TEST_JSON_SINGLE_QUOTES_FILE)) { + assertThrows(CudfException.class, () -> + Table.readJSON(schema, opts, source)); + } + } + + @Test + void testReadWhitespacesJSONFile() throws IOException { + Schema schema = Schema.builder() + .column(DType.STRING, "a") + .build(); + JSONOptions opts = JSONOptions.builder() + .withLines(true) + .withMixedTypesAsStrings(true) + .withNormalizeWhitespace(true) + .build(); + try (Table expected = new Table.TestBuilder() + .column("b", "50", "[1,2,3,4,5,6,7,8]", "{\"c\":\"d\"}", "b") + .build(); + MultiBufferDataSource source = sourceFrom(TEST_JSON_WHITESPACES_FILE); + Table table = Table.readJSON(schema, opts, source)) { + assertTablesAreEqual(expected, table); + } + } + void testReadSingleQuotesJSONFileKeepQuotes() throws IOException { Schema schema = Schema.builder() .column(DType.STRING, "A") @@ -547,21 +581,6 @@ void testReadMixedType2JSONFile() throws IOException { } } - @Test - void testReadSingleQuotesJSONFileFeatureDisabled() throws IOException { - Schema schema = Schema.builder() - .column(DType.STRING, "A") - .build(); - JSONOptions opts = JSONOptions.builder() - .withLines(true) - .withNormalizeSingleQuotes(false) - .build(); - try (MultiBufferDataSource source = sourceFrom(TEST_JSON_SINGLE_QUOTES_FILE)) { - assertThrows(CudfException.class, () -> - Table.readJSON(schema, opts, source)); - } - } - @Test void testReadJSONFromDataSource() throws IOException { Schema schema = Schema.builder() diff --git a/java/src/test/resources/whitespaces.json b/java/src/test/resources/whitespaces.json new file mode 100644 index 00000000000..f5ddd8cde5f --- /dev/null +++ b/java/src/test/resources/whitespaces.json @@ -0,0 +1,5 @@ +{"a":"b"} + { "a" : "50" } +{"a": [1, 2, 3, 4, 5, 6, 7, 8]} +{"a": {"c": "d"}} +{"a": "b"} From c3cad1d7a0aa799a64ec767edb64686f99be78e6 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 4 Mar 2024 16:22:01 -0600 Subject: [PATCH 337/384] Fix `ListColumn.to_pandas()` to retain `list` type (#15155) Fixes: #14568 This PR fixes `ListColumn.to_pandas()` by calling `ArrowArray.to_pylist()` method to retain `list` type in pandas series. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Matthew Roeschke (https://github.com/mroeschke) - Richard (Rick) Zamora (https://github.com/rjzamora) URL: https://github.com/rapidsai/cudf/pull/15155 --- python/cudf/cudf/core/column/lists.py | 18 ++++++++++++++++++ python/cudf/cudf/tests/test_list.py | 4 +++- .../dask_cudf/dask_cudf/tests/test_groupby.py | 6 +----- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/python/cudf/cudf/core/column/lists.py b/python/cudf/cudf/core/column/lists.py index b2205af34e8..d1bf0b74d3c 100644 --- a/python/cudf/cudf/core/column/lists.py +++ b/python/cudf/cudf/core/column/lists.py @@ -6,6 +6,7 @@ from typing import List, Optional, Sequence, Tuple, Union import numpy as np +import pandas as pd import pyarrow as pa from typing_extensions import Self @@ -288,6 +289,23 @@ def _transform_leaves(self, func, *args, **kwargs) -> Self: ) return lc + def to_pandas( + self, + *, + index: Optional[pd.Index] = None, + nullable: bool = False, + ) -> pd.Series: + # Can't rely on Column.to_pandas implementation for lists. + # Need to perform `to_pylist` to preserve list types. + if nullable: + raise NotImplementedError(f"{nullable=} is not implemented.") + + pd_series = pd.Series(self.to_arrow().to_pylist(), dtype="object") + + if index is not None: + pd_series.index = index + return pd_series + class ListMethods(ColumnMethods): """ diff --git a/python/cudf/cudf/tests/test_list.py b/python/cudf/cudf/tests/test_list.py index 7ae7ae34b97..f04cb8a91a4 100644 --- a/python/cudf/cudf/tests/test_list.py +++ b/python/cudf/cudf/tests/test_list.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. import functools import operator @@ -41,6 +41,8 @@ def test_create_list_series(data): expect = pd.Series(data) got = cudf.Series(data) assert_eq(expect, got) + assert isinstance(got[0], type(expect[0])) + assert isinstance(got.to_pandas()[0], type(expect[0])) @pytest.mark.parametrize( diff --git a/python/dask_cudf/dask_cudf/tests/test_groupby.py b/python/dask_cudf/dask_cudf/tests/test_groupby.py index c8cc6e65fa5..30251b88dea 100644 --- a/python/dask_cudf/dask_cudf/tests/test_groupby.py +++ b/python/dask_cudf/dask_cudf/tests/test_groupby.py @@ -702,13 +702,9 @@ def test_is_supported(arg, supported): def test_groupby_unique_lists(): df = pd.DataFrame({"a": [0, 0, 0, 1, 1, 1], "b": [10, 10, 10, 7, 8, 9]}) - ddf = dd.from_pandas(df, 2) gdf = cudf.from_pandas(df) gddf = dask_cudf.from_cudf(gdf, 2) - dd.assert_eq( - ddf.groupby("a").b.unique().compute(), - gddf.groupby("a").b.unique().compute(), - ) + dd.assert_eq( gdf.groupby("a").b.unique(), gddf.groupby("a").b.unique().compute(), From 4f1315587df1d64c384f018d90d4ef4fe69a96be Mon Sep 17 00:00:00 2001 From: Yunsong Wang Date: Mon, 4 Mar 2024 14:38:53 -0800 Subject: [PATCH 338/384] Update labeler and codeowner configs for CMake files (#15208) When working on #15206, I noticed the `rapids_config.cmake` file was not properly labeled. Based on offline discussions, we also noticed that the file's codeowner was misconfigured as well. This PR updates both github `labeler` and `CODEOWNER` files to properly handle files with `.cmake` extension. Authors: - Yunsong Wang (https://github.com/PointKernel) Approvers: - Bradley Dice (https://github.com/bdice) - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/cudf/pull/15208 --- .github/CODEOWNERS | 1 + .github/labeler.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9578d32d13d..31cfeaf4ca3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -11,6 +11,7 @@ python/dask_cudf/ @rapidsai/cudf-dask-codeowners cpp/CMakeLists.txt @rapidsai/cudf-cmake-codeowners cpp/libcudf_kafka/CMakeLists.txt @rapidsai/cudf-cmake-codeowners **/cmake/ @rapidsai/cudf-cmake-codeowners +*.cmake @rapidsai/cudf-cmake-codeowners #java code owners java/ @rapidsai/cudf-java-codeowners diff --git a/.github/labeler.yml b/.github/labeler.yml index b0b0db9684a..d14344384d1 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -10,6 +10,7 @@ libcudf: CMake: - '**/CMakeLists.txt' - '**/cmake/**' + - '**/*.cmake' cuDF (Java): - 'java/**' From e8c13795709c3561cffcb99b3e435d0b4bb6c397 Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Mon, 4 Mar 2024 15:13:49 -0800 Subject: [PATCH 339/384] Update devcontainers to CUDA Toolkit 12.2 (#15099) Authors: - Paul Taylor (https://github.com/trxcllnt) Approvers: - Jake Awe (https://github.com/AyodeAwe) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/15099 --- .devcontainer/cuda11.8-pip/devcontainer.json | 2 +- .../{cuda12.0-conda => cuda12.2-conda}/devcontainer.json | 6 +++--- .../{cuda12.0-pip => cuda12.2-pip}/devcontainer.json | 8 ++++---- .github/workflows/pr.yaml | 4 +++- 4 files changed, 11 insertions(+), 9 deletions(-) rename .devcontainer/{cuda12.0-conda => cuda12.2-conda}/devcontainer.json (92%) rename .devcontainer/{cuda12.0-pip => cuda12.2-pip}/devcontainer.json (87%) diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json index 84616c25cf2..15b51da8dea 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -5,7 +5,7 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.04-cpp-llvm16-cuda11.8-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.04-cpp-cuda11.8-ubuntu22.04" } }, "hostRequirements": {"gpu": "optional"}, diff --git a/.devcontainer/cuda12.0-conda/devcontainer.json b/.devcontainer/cuda12.2-conda/devcontainer.json similarity index 92% rename from .devcontainer/cuda12.0-conda/devcontainer.json rename to .devcontainer/cuda12.2-conda/devcontainer.json index ef2b34b41a6..31ae8426763 100644 --- a/.devcontainer/cuda12.0-conda/devcontainer.json +++ b/.devcontainer/cuda12.2-conda/devcontainer.json @@ -3,7 +3,7 @@ "context": "${localWorkspaceFolder}/.devcontainer", "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", "args": { - "CUDA": "12.0", + "CUDA": "12.2", "PYTHON_PACKAGE_MANAGER": "conda", "BASE": "rapidsai/devcontainers:24.04-cpp-mambaforge-ubuntu22.04" } @@ -15,7 +15,7 @@ "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.0-envs}"], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.2-envs}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/cudf,type=bind,consistency=consistent", @@ -24,7 +24,7 @@ "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.0-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent" + "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.2-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent" ], "customizations": { "vscode": { diff --git a/.devcontainer/cuda12.0-pip/devcontainer.json b/.devcontainer/cuda12.2-pip/devcontainer.json similarity index 87% rename from .devcontainer/cuda12.0-pip/devcontainer.json rename to .devcontainer/cuda12.2-pip/devcontainer.json index d3257b6cf43..93367527a86 100644 --- a/.devcontainer/cuda12.0-pip/devcontainer.json +++ b/.devcontainer/cuda12.2-pip/devcontainer.json @@ -3,9 +3,9 @@ "context": "${localWorkspaceFolder}/.devcontainer", "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", "args": { - "CUDA": "12.0", + "CUDA": "12.2", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.04-cpp-llvm16-cuda12.0-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.04-cpp-cuda12.2-ubuntu22.04" } }, "hostRequirements": {"gpu": "optional"}, @@ -15,7 +15,7 @@ "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.0-venvs}"], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.2-venvs}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/cudf,type=bind,consistency=consistent", @@ -23,7 +23,7 @@ "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.0-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent" + "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.2-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent" ], "customizations": { "vscode": { diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9e11993048f..4a662ed0f43 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -141,8 +141,10 @@ jobs: script: ci/test_wheel_dask_cudf.sh devcontainer: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location with: + arch: '["amd64"]' + cuda: '["12.2"]' build_command: | sccache -z; build-all -DBUILD_BENCHMARKS=ON --verbose; From f12b8e1b378ae5a4806bce86a1801c2c488097ac Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 4 Mar 2024 13:18:42 -1000 Subject: [PATCH 340/384] Allow to_pandas to return pandas.ArrowDtype (#15182) Adds a `arrow_type: bool` parameter to `to_pandas` to allow the conversion to return `pandas.ArrowDtype` in pandas. (Opens up the dream of cudf to pandas round tripping to happen via arrow formatted data) Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Michael Wang (https://github.com/isVoid) - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15182 --- python/cudf/cudf/core/_base_index.py | 10 +++- python/cudf/cudf/core/column/categorical.py | 8 ++- python/cudf/cudf/core/column/column.py | 21 +++++-- python/cudf/cudf/core/column/datetime.py | 53 ++++++++++++------ python/cudf/cudf/core/column/interval.py | 12 +++- python/cudf/cudf/core/column/numerical.py | 11 +++- python/cudf/cudf/core/column/string.py | 11 +++- python/cudf/cudf/core/column/struct.py | 21 +++++-- python/cudf/cudf/core/column/timedelta.py | 31 +++++++---- python/cudf/cudf/core/dataframe.py | 22 ++++++-- python/cudf/cudf/core/index.py | 61 ++++++++++++++------- python/cudf/cudf/core/multiindex.py | 6 +- python/cudf/cudf/core/series.py | 32 +++++++++-- python/cudf/cudf/tests/test_dataframe.py | 41 ++++++++++++++ python/cudf/cudf/tests/test_index.py | 38 +++++++++++++ python/cudf/cudf/tests/test_multiindex.py | 34 ++++++++++++ python/cudf/cudf/tests/test_series.py | 43 ++++++++++++++- 17 files changed, 382 insertions(+), 73 deletions(-) diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index 58e2241e810..de44f392eef 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -910,7 +910,7 @@ def notna(self): """ raise NotImplementedError - def to_pandas(self, *, nullable: bool = False): + def to_pandas(self, *, nullable: bool = False, arrow_type: bool = False): """ Convert to a Pandas Index. @@ -924,6 +924,12 @@ def to_pandas(self, *, nullable: bool = False): If ``nullable`` is ``False``, the resulting index will either convert null values to ``np.nan`` or ``None`` depending on the dtype. + arrow_type : bool, Default False + Return the Index with a ``pandas.ArrowDtype`` + + Notes + ----- + nullable and arrow_type cannot both be set to ``True`` Examples -------- @@ -937,6 +943,8 @@ def to_pandas(self, *, nullable: bool = False): >>> type(idx) + >>> idx.to_pandas(arrow_type=True) + Index([-3, 10, 15, 20], dtype='int64[pyarrow]') """ raise NotImplementedError diff --git a/python/cudf/cudf/core/column/categorical.py b/python/cudf/cudf/core/column/categorical.py index 9ecd461cf99..4c64e7085c9 100644 --- a/python/cudf/cudf/core/column/categorical.py +++ b/python/cudf/cudf/core/column/categorical.py @@ -770,10 +770,16 @@ def __cuda_array_interface__(self) -> Mapping[str, Any]: ) def to_pandas( - self, *, index: Optional[pd.Index] = None, nullable: bool = False + self, + *, + index: Optional[pd.Index] = None, + nullable: bool = False, + arrow_type: bool = False, ) -> pd.Series: if nullable: raise NotImplementedError(f"{nullable=} is not implemented.") + elif arrow_type: + raise NotImplementedError(f"{arrow_type=} is not implemented.") if self.categories.dtype.kind == "f": new_mask = bools_to_mask(self.notnull()) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index cecdaf70750..be196833f32 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -199,6 +199,7 @@ def to_pandas( *, index: Optional[pd.Index] = None, nullable: bool = False, + arrow_type: bool = False, ) -> pd.Series: """Convert object to pandas type. @@ -206,13 +207,23 @@ def to_pandas( """ # This default implementation does not handle nulls in any meaningful # way - if nullable: + if arrow_type and nullable: + raise ValueError( + f"{arrow_type=} and {nullable=} cannot both be set." + ) + elif nullable: raise NotImplementedError(f"{nullable=} is not implemented.") - pd_series = self.to_arrow().to_pandas() + pa_array = self.to_arrow() + if arrow_type: + return pd.Series( + pd.arrays.ArrowExtensionArray(pa_array), index=index + ) + else: + pd_series = pa_array.to_pandas() - if index is not None: - pd_series.index = index - return pd_series + if index is not None: + pd_series.index = index + return pd_series @property def values_host(self) -> "np.ndarray": diff --git a/python/cudf/cudf/core/column/datetime.py b/python/cudf/cudf/core/column/datetime.py index b03b21a7aba..85f07064c97 100644 --- a/python/cudf/cudf/core/column/datetime.py +++ b/python/cudf/cudf/core/column/datetime.py @@ -318,18 +318,27 @@ def to_pandas( *, index: Optional[pd.Index] = None, nullable: bool = False, + arrow_type: bool = False, ) -> pd.Series: - if nullable: + if arrow_type and nullable: + raise ValueError( + f"{arrow_type=} and {nullable=} cannot both be set." + ) + elif nullable: raise NotImplementedError(f"{nullable=} is not implemented.") - # `copy=True` workaround until following issue is fixed: - # https://issues.apache.org/jira/browse/ARROW-9772 - - return pd.Series( - self.to_arrow(), - copy=True, - dtype=self.dtype, - index=index, - ) + elif arrow_type: + return pd.Series( + pd.arrays.ArrowExtensionArray(self.to_arrow()), index=index + ) + else: + # `copy=True` workaround until following issue is fixed: + # https://issues.apache.org/jira/browse/ARROW-9772 + return pd.Series( + self.to_arrow(), + copy=True, + dtype=self.dtype, + index=index, + ) @property def values(self): @@ -723,15 +732,25 @@ def to_pandas( *, index: Optional[pd.Index] = None, nullable: bool = False, + arrow_type: bool = False, ) -> pd.Series: - if nullable: + if arrow_type and nullable: + raise ValueError( + f"{arrow_type=} and {nullable=} cannot both be set." + ) + elif nullable: raise NotImplementedError(f"{nullable=} is not implemented.") - series = self._local_time.to_pandas().dt.tz_localize( - self.dtype.tz, ambiguous="NaT", nonexistent="NaT" - ) - if index is not None: - series.index = index - return series + elif arrow_type: + return pd.Series( + pd.arrays.ArrowExtensionArray(self.to_arrow()), index=index + ) + else: + series = self._local_time.to_pandas().dt.tz_localize( + self.dtype.tz, ambiguous="NaT", nonexistent="NaT" + ) + if index is not None: + series.index = index + return series def to_arrow(self): return pa.compute.assume_timezone( diff --git a/python/cudf/cudf/core/column/interval.py b/python/cudf/cudf/core/column/interval.py index 5d93fa26298..dcec8957bb2 100644 --- a/python/cudf/cudf/core/column/interval.py +++ b/python/cudf/cudf/core/column/interval.py @@ -105,15 +105,25 @@ def as_interval_column(self, dtype): raise ValueError("dtype must be IntervalDtype") def to_pandas( - self, *, index: Optional[pd.Index] = None, nullable: bool = False + self, + *, + index: Optional[pd.Index] = None, + nullable: bool = False, + arrow_type: bool = False, ) -> pd.Series: # Note: This does not handle null values in the interval column. # However, this exact sequence (calling __from_arrow__ on the output of # self.to_arrow) is currently the best known way to convert interval # types into pandas (trying to convert the underlying numerical columns # directly is problematic), so we're stuck with this for now. + if arrow_type and nullable: + raise ValueError( + f"{arrow_type=} and {nullable=} cannot both be set." + ) if nullable: raise NotImplementedError(f"{nullable=} is not implemented.") + elif arrow_type: + raise NotImplementedError(f"{nullable=} is not implemented.") return pd.Series( self.dtype.to_pandas().__from_arrow__(self.to_arrow()), index=index ) diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index b80dd626066..82d82593c77 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -690,8 +690,17 @@ def to_pandas( *, index: Optional[pd.Index] = None, nullable: bool = False, + arrow_type: bool = False, ) -> pd.Series: - if nullable and self.dtype in np_dtypes_to_pandas_dtypes: + if arrow_type and nullable: + raise ValueError( + f"{arrow_type=} and {nullable=} cannot both be set." + ) + if arrow_type: + return pd.Series( + pd.arrays.ArrowExtensionArray(self.to_arrow()), index=index + ) + elif nullable and self.dtype in np_dtypes_to_pandas_dtypes: pandas_nullable_dtype = np_dtypes_to_pandas_dtypes[self.dtype] arrow_array = self.to_arrow() pandas_array = pandas_nullable_dtype.__from_arrow__(arrow_array) diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index 2373f94ee97..dea60f58690 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -5791,8 +5791,17 @@ def to_pandas( *, index: Optional[pd.Index] = None, nullable: bool = False, + arrow_type: bool = False, ) -> pd.Series: - if nullable: + if arrow_type and nullable: + raise ValueError( + f"{arrow_type=} and {nullable=} cannot both be set." + ) + if arrow_type: + return pd.Series( + pd.arrays.ArrowExtensionArray(self.to_arrow()), index=index + ) + elif nullable: pandas_array = pd.StringDtype().__from_arrow__(self.to_arrow()) pd_series = pd.Series(pandas_array, copy=False) else: diff --git a/python/cudf/cudf/core/column/struct.py b/python/cudf/cudf/core/column/struct.py index 69e9a50956b..1b2ffcc2700 100644 --- a/python/cudf/cudf/core/column/struct.py +++ b/python/cudf/cudf/core/column/struct.py @@ -58,14 +58,27 @@ def to_arrow(self): ) def to_pandas( - self, *, index: Optional[pd.Index] = None, nullable: bool = False + self, + *, + index: Optional[pd.Index] = None, + nullable: bool = False, + arrow_type: bool = False, ) -> pd.Series: # We cannot go via Arrow's `to_pandas` because of the following issue: # https://issues.apache.org/jira/browse/ARROW-12680 - if nullable: + if arrow_type and nullable: + raise ValueError( + f"{arrow_type=} and {nullable=} cannot both be set." + ) + elif nullable: raise NotImplementedError(f"{nullable=} is not implemented.") - - return pd.Series(self.to_arrow().tolist(), dtype="object", index=index) + pa_array = self.to_arrow() + if arrow_type: + return pd.Series( + pd.arrays.ArrowExtensionArray(pa_array), index=index + ) + else: + return pd.Series(pa_array.tolist(), dtype="object", index=index) @cached_property def memory_usage(self): diff --git a/python/cudf/cudf/core/column/timedelta.py b/python/cudf/cudf/core/column/timedelta.py index b911c86fa01..dab2723795e 100644 --- a/python/cudf/cudf/core/column/timedelta.py +++ b/python/cudf/cudf/core/column/timedelta.py @@ -147,20 +147,31 @@ def to_arrow(self) -> pa.Array: ) def to_pandas( - self, *, index: Optional[pd.Index] = None, nullable: bool = False + self, + *, + index: Optional[pd.Index] = None, + nullable: bool = False, + arrow_type: bool = False, ) -> pd.Series: # `copy=True` workaround until following issue is fixed: # https://issues.apache.org/jira/browse/ARROW-9772 - - if nullable: + if arrow_type and nullable: + raise ValueError( + f"{arrow_type=} and {nullable=} cannot both be set." + ) + elif nullable: raise NotImplementedError(f"{nullable=} is not implemented.") - - return pd.Series( - self.to_arrow(), - copy=True, - dtype=self.dtype, - index=index, - ) + elif arrow_type: + return pd.Series( + pd.arrays.ArrowExtensionArray(self.to_arrow()), index=index + ) + else: + return pd.Series( + self.to_arrow(), + copy=True, + dtype=self.dtype, + index=index, + ) def _binaryop(self, other: ColumnBinaryOperand, op: str) -> ColumnBase: reflect, op = self._check_reflected_op(op) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index a0e1a041342..d7d2e1acd85 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -5203,7 +5203,9 @@ def describe( return res @_cudf_nvtx_annotate - def to_pandas(self, *, nullable: bool = False) -> pd.DataFrame: + def to_pandas( + self, *, nullable: bool = False, arrow_type: bool = False + ) -> pd.DataFrame: """ Convert to a Pandas DataFrame. @@ -5218,11 +5220,17 @@ def to_pandas(self, *, nullable: bool = False) -> pd.DataFrame: If ``nullable`` is ``False``, the resulting columns will either convert null values to ``np.nan`` or ``None`` depending on the dtype. + arrow_type : bool, Default False + Return the Index with a ``pandas.ArrowDtype`` Returns ------- out : Pandas DataFrame + Notes + ----- + nullable and arrow_type cannot both be set to ``True`` + Examples -------- >>> import cudf @@ -5236,8 +5244,7 @@ def to_pandas(self, *, nullable: bool = False) -> pd.DataFrame: >>> type(pdf) - ``nullable`` parameter can be used to control - whether dtype can be Pandas Nullable or not: + ``nullable=True`` converts the result to pandas nullable types: >>> df = cudf.DataFrame({'a': [0, None, 2], 'b': [True, False, None]}) >>> df @@ -5265,13 +5272,20 @@ def to_pandas(self, *, nullable: bool = False) -> pd.DataFrame: a float64 b object dtype: object + + ``arrow_type=True`` converts the result to ``pandas.ArrowDtype``: + + >>> df.to_pandas(arrow_type=True).dtypes + a int64[pyarrow] + b bool[pyarrow] + dtype: object """ out_data = {} out_index = self.index.to_pandas() for i, col_key in enumerate(self._data): out_data[i] = self._data[col_key].to_pandas( - index=out_index, nullable=nullable + index=out_index, nullable=nullable, arrow_type=arrow_type ) out_df = pd.DataFrame(out_data, index=out_index) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 1b9893d1256..9d481037ec6 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -483,9 +483,13 @@ def dtype(self): return _maybe_convert_to_default_type(dtype) @_cudf_nvtx_annotate - def to_pandas(self, *, nullable: bool = False) -> pd.RangeIndex: + def to_pandas( + self, *, nullable: bool = False, arrow_type: bool = False + ) -> pd.RangeIndex: if nullable: raise NotImplementedError(f"{nullable=} is not implemented.") + elif arrow_type: + raise NotImplementedError(f"{arrow_type=} is not implemented.") return pd.RangeIndex( start=self._start, stop=self._stop, @@ -1521,9 +1525,12 @@ def _clean_nulls_from_index(self): def any(self): return self._values.any() - def to_pandas(self, *, nullable: bool = False) -> pd.Index: + def to_pandas( + self, *, nullable: bool = False, arrow_type: bool = False + ) -> pd.Index: return pd.Index( - self._values.to_pandas(nullable=nullable), name=self.name + self._values.to_pandas(nullable=nullable, arrow_type=arrow_type), + name=self.name, ) def append(self, other): @@ -2094,18 +2101,26 @@ def isocalendar(self): return cudf.core.tools.datetimes._to_iso_calendar(self) @_cudf_nvtx_annotate - def to_pandas(self, *, nullable: bool = False) -> pd.DatetimeIndex: - if nullable: + def to_pandas( + self, *, nullable: bool = False, arrow_type: bool = False + ) -> pd.DatetimeIndex: + if arrow_type and nullable: + raise ValueError( + f"{arrow_type=} and {nullable=} cannot both be set." + ) + elif nullable: raise NotImplementedError(f"{nullable=} is not implemented.") - freq = ( - self._freq._maybe_as_fast_pandas_offset() - if self._freq is not None - else None - ) - return pd.DatetimeIndex( - self._values.to_pandas(), name=self.name, freq=freq - ) + result = self._values.to_pandas(arrow_type=arrow_type) + if arrow_type: + return pd.Index(result, name=self.name) + else: + freq = ( + self._freq._maybe_as_fast_pandas_offset() + if self._freq is not None + else None + ) + return pd.DatetimeIndex(result, name=self.name, freq=freq) @_cudf_nvtx_annotate def _get_dt_field(self, field): @@ -2426,13 +2441,21 @@ def __getitem__(self, index): return value @_cudf_nvtx_annotate - def to_pandas(self, *, nullable: bool = False) -> pd.TimedeltaIndex: - if nullable: + def to_pandas( + self, *, nullable: bool = False, arrow_type: bool = False + ) -> pd.TimedeltaIndex: + if arrow_type and nullable: + raise ValueError( + f"{arrow_type=} and {nullable=} cannot both be set." + ) + elif nullable: raise NotImplementedError(f"{nullable=} is not implemented.") - return pd.TimedeltaIndex( - self._values.to_pandas(), - name=self.name, - ) + + result = self._values.to_pandas(arrow_type=arrow_type) + if arrow_type: + return pd.Index(result, name=self.name) + else: + return pd.TimedeltaIndex(result, name=self.name) @property # type: ignore @_cudf_nvtx_annotate diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index df1b1ea10cd..70112044f75 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1574,10 +1574,12 @@ def droplevel(self, level=-1): return mi @_cudf_nvtx_annotate - def to_pandas(self, *, nullable: bool = False) -> pd.MultiIndex: + def to_pandas( + self, *, nullable: bool = False, arrow_type: bool = False + ) -> pd.MultiIndex: result = self.to_frame( index=False, name=list(range(self.nlevels)) - ).to_pandas(nullable=nullable) + ).to_pandas(nullable=nullable, arrow_type=arrow_type) return pd.MultiIndex.from_frame(result, names=self.names) @classmethod diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 3f51ecdf7dc..cb5008af3ad 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -1983,10 +1983,14 @@ def any(self, axis=0, bool_only=None, skipna=True, **kwargs): @_cudf_nvtx_annotate def to_pandas( - self, *, index: bool = True, nullable: bool = False + self, + *, + index: bool = True, + nullable: bool = False, + arrow_type: bool = False, ) -> pd.Series: """ - Convert to a Pandas Series. + Convert to a pandas Series. Parameters ---------- @@ -2003,10 +2007,16 @@ def to_pandas( If ``nullable`` is ``False``, the resulting series will either convert null values to ``np.nan`` or ``None`` depending on the dtype. + arrow_type : bool, Default False + Return the Series with a ``pandas.ArrowDtype`` Returns ------- - out : Pandas Series + out : pandas Series + + Notes + ----- + nullable and arrow_type cannot both be set to ``True`` Examples -------- @@ -2021,8 +2031,7 @@ def to_pandas( >>> type(pds) - ``nullable`` parameter can be used to control - whether dtype can be Pandas Nullable or not: + ``nullable=True`` converts the result to pandas nullable types: >>> ser = cudf.Series([10, 20, None, 30]) >>> ser @@ -2043,12 +2052,23 @@ def to_pandas( 2 NaN 3 30.0 dtype: float64 + + ``arrow_type=True`` converts the result to ``pandas.ArrowDtype``: + + >>> ser.to_pandas(arrow_type=True) + 0 10 + 1 20 + 2 + 3 30 + dtype: int64[pyarrow] """ if index is True: index = self.index.to_pandas() else: index = None # type: ignore[assignment] - s = self._column.to_pandas(index=index, nullable=nullable) + s = self._column.to_pandas( + index=index, nullable=nullable, arrow_type=arrow_type + ) s.name = self.name return s diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 50b14d532e4..3143851ddd6 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -10861,3 +10861,44 @@ def test_dataframe_duplicate_index_reindex(): lfunc_args_and_kwargs=([10, 11, 12, 13], {}), rfunc_args_and_kwargs=([10, 11, 12, 13], {}), ) + + +@pytest.mark.parametrize( + "scalar", + [ + 1, + 1.0, + "a", + datetime.datetime(2020, 1, 1), + datetime.timedelta(1), + {"1": 2}, + [1], + decimal.Decimal("1.0"), + ], +) +def test_dataframe_to_pandas_arrow_type_nullable_raises(scalar): + pa_array = pa.array([scalar, None]) + df = cudf.DataFrame({"a": pa_array}) + with pytest.raises(ValueError): + df.to_pandas(nullable=True, arrow_type=True) + + +@pytest.mark.parametrize( + "scalar", + [ + 1, + 1.0, + "a", + datetime.datetime(2020, 1, 1), + datetime.timedelta(1), + {"1": 2}, + [1], + decimal.Decimal("1.0"), + ], +) +def test_dataframe_to_pandas_arrow_type(scalar): + pa_array = pa.array([scalar, None]) + df = cudf.DataFrame({"a": pa_array}) + result = df.to_pandas(arrow_type=True) + expected = pd.DataFrame({"a": pd.arrays.ArrowExtensionArray(pa_array)}) + pd.testing.assert_frame_equal(result, expected) diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index cced05d2217..51e9a3022f4 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -3,6 +3,7 @@ """ Test related to Index """ +import datetime import operator import re @@ -3138,3 +3139,40 @@ def test_from_pandas_rangeindex_return_rangeindex(): def test_index_to_pandas_nullable_notimplemented(idx): with pytest.raises(NotImplementedError): idx.to_pandas(nullable=True) + + +@pytest.mark.parametrize( + "scalar", + [ + 1, + 1.0, + "a", + datetime.datetime(2020, 1, 1), + datetime.timedelta(1), + {"1": 2}, + ], +) +def test_index_to_pandas_arrow_type_nullable_raises(scalar): + pa_array = pa.array([scalar, None]) + idx = cudf.Index(pa_array) + with pytest.raises(ValueError): + idx.to_pandas(nullable=True, arrow_type=True) + + +@pytest.mark.parametrize( + "scalar", + [ + 1, + 1.0, + "a", + datetime.datetime(2020, 1, 1), + datetime.timedelta(1), + {"1": 2}, + ], +) +def test_index_to_pandas_arrow_type(scalar): + pa_array = pa.array([scalar, None]) + idx = cudf.Index(pa_array) + result = idx.to_pandas(arrow_type=True) + expected = pd.Index(pd.arrays.ArrowExtensionArray(pa_array)) + pd.testing.assert_index_equal(result, expected) diff --git a/python/cudf/cudf/tests/test_multiindex.py b/python/cudf/cudf/tests/test_multiindex.py index a13fe333107..4926d79e734 100644 --- a/python/cudf/cudf/tests/test_multiindex.py +++ b/python/cudf/cudf/tests/test_multiindex.py @@ -3,6 +3,7 @@ """ Test related to MultiIndex """ +import datetime import itertools import operator import pickle @@ -13,6 +14,7 @@ import cupy as cp import numpy as np import pandas as pd +import pyarrow as pa import pytest import cudf @@ -2118,3 +2120,35 @@ def test_multiindex_from_arrays(array): def test_multiindex_from_arrays_wrong_arg(arg): with pytest.raises(TypeError): cudf.MultiIndex.from_arrays(arg) + + +@pytest.mark.parametrize( + "scalar", + [ + 1, + 1.0, + "a", + datetime.datetime(2020, 1, 1), + datetime.timedelta(1), + {"1": 2}, + ], +) +def test_index_to_pandas_arrow_type_nullable_raises(scalar): + pa_array = pa.array([scalar, None]) + midx = cudf.MultiIndex(levels=[pa_array], codes=[[0]]) + with pytest.raises(ValueError): + midx.to_pandas(nullable=True, arrow_type=True) + + +@pytest.mark.parametrize( + "scalar", + [1, 1.0, "a", datetime.datetime(2020, 1, 1), datetime.timedelta(1)], +) +def test_index_to_pandas_arrow_type(scalar): + pa_array = pa.array([scalar, None]) + midx = cudf.MultiIndex(levels=[pa_array], codes=[[0]]) + result = midx.to_pandas(arrow_type=True) + expected = pd.MultiIndex( + levels=[pd.arrays.ArrowExtensionArray(pa_array)], codes=[[0]] + ) + pd.testing.assert_index_equal(result, expected) diff --git a/python/cudf/cudf/tests/test_series.py b/python/cudf/cudf/tests/test_series.py index caf8947e3b0..6b5c0406deb 100644 --- a/python/cudf/cudf/tests/test_series.py +++ b/python/cudf/cudf/tests/test_series.py @@ -1,5 +1,5 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. - +import datetime import decimal import hashlib import operator @@ -2708,3 +2708,44 @@ def test_series_from_large_string(): expected = pd.Series(pa_large_string_array) assert_eq(expected, got) + + +@pytest.mark.parametrize( + "scalar", + [ + 1, + 1.0, + "a", + datetime.datetime(2020, 1, 1), + datetime.timedelta(1), + {"1": 2}, + [1], + decimal.Decimal("1.0"), + ], +) +def test_series_to_pandas_arrow_type_nullable_raises(scalar): + pa_array = pa.array([scalar, None]) + ser = cudf.Series(pa_array) + with pytest.raises(ValueError): + ser.to_pandas(nullable=True, arrow_type=True) + + +@pytest.mark.parametrize( + "scalar", + [ + 1, + 1.0, + "a", + datetime.datetime(2020, 1, 1), + datetime.timedelta(1), + {"1": 2}, + [1], + decimal.Decimal("1.0"), + ], +) +def test_series_to_pandas_arrow_type(scalar): + pa_array = pa.array([scalar, None]) + ser = cudf.Series(pa_array) + result = ser.to_pandas(arrow_type=True) + expected = pd.Series(pd.arrays.ArrowExtensionArray(pa_array)) + pd.testing.assert_series_equal(result, expected) From 3571291c533412f8efa4c5d41caa865564b5391b Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 4 Mar 2024 15:04:54 -1000 Subject: [PATCH 341/384] Use as_column instead of full (#14698) Similar to https://github.com/rapidsai/cudf/pull/14689, ensures there's 1 entrypoint to create a column from a scalar. This builds on https://github.com/rapidsai/cudf/pull/14620 Authors: - Matthew Roeschke (https://github.com/mroeschke) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Richard (Rick) Zamora (https://github.com/rjzamora) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/14698 --- python/cudf/cudf/core/column/__init__.py | 1 - python/cudf/cudf/core/column/categorical.py | 12 +-- python/cudf/cudf/core/column/column.py | 100 ++++++-------------- python/cudf/cudf/core/column/decimal.py | 4 +- python/cudf/cudf/core/column/numerical.py | 3 +- python/cudf/cudf/core/column/string.py | 12 ++- python/cudf/cudf/core/column/timedelta.py | 4 +- python/cudf/cudf/core/dataframe.py | 26 +++-- python/cudf/cudf/core/index.py | 6 +- python/cudf/cudf/core/indexed_frame.py | 14 ++- python/cudf/cudf/core/multiindex.py | 8 +- python/cudf/cudf/core/series.py | 5 +- python/cudf/cudf/core/tools/datetimes.py | 4 +- python/cudf/cudf/core/window/rolling.py | 5 +- python/cudf/cudf/io/parquet.py | 14 +-- python/cudf/cudf/tests/test_testing.py | 6 +- python/cudf/cudf/utils/utils.py | 6 +- python/dask_cudf/dask_cudf/backends.py | 6 +- 18 files changed, 101 insertions(+), 135 deletions(-) diff --git a/python/cudf/cudf/core/column/__init__.py b/python/cudf/cudf/core/column/__init__.py index a1c86b617b0..2a46654ccc2 100644 --- a/python/cudf/cudf/core/column/__init__.py +++ b/python/cudf/cudf/core/column/__init__.py @@ -16,7 +16,6 @@ column_empty_like_same_mask, concat_columns, deserialize_columns, - full, serialize_columns, ) from cudf.core.column.datetime import DatetimeColumn # noqa: F401 diff --git a/python/cudf/cudf/core/column/categorical.py b/python/cudf/cudf/core/column/categorical.py index 4c64e7085c9..88bb4521a5b 100644 --- a/python/cudf/cudf/core/column/categorical.py +++ b/python/cudf/cudf/core/column/categorical.py @@ -734,8 +734,8 @@ def normalize_binop_value(self, other: ScalarLike) -> CategoricalColumn: ) return other - ary = column.full( - len(self), self._encode(other), dtype=self.codes.dtype + ary = column.as_column( + self._encode(other), length=len(self), dtype=self.codes.dtype ) return column.build_categorical_column( categories=self.dtype.categories._values, @@ -1444,11 +1444,9 @@ def _create_empty_categorical_column( return column.build_categorical_column( categories=column.as_column(dtype.categories), codes=column.as_column( - column.full( - categorical_column.size, - _DEFAULT_CATEGORICAL_VALUE, - categorical_column.codes.dtype, - ) + _DEFAULT_CATEGORICAL_VALUE, + length=categorical_column.size, + dtype=categorical_column.codes.dtype, ), offset=categorical_column.offset, size=categorical_column.size, diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index be196833f32..8941d111d02 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -58,7 +58,6 @@ infer_dtype, is_bool_dtype, is_datetime64_dtype, - is_decimal_dtype, is_dtype_equal, is_integer_dtype, is_list_dtype, @@ -866,7 +865,7 @@ def isin(self, values: Sequence) -> ColumnBase: except ValueError: # pandas functionally returns all False when cleansing via # typecasting fails - return full(len(self), False, dtype="bool") + return as_column(False, length=len(self), dtype="bool") return lhs._obtain_isin_result(rhs) @@ -893,9 +892,9 @@ def _isin_earlystop(self, rhs: ColumnBase) -> Union[ColumnBase, None]: if self.null_count and rhs.null_count: return self.isnull() else: - return cudf.core.column.full(len(self), False, dtype="bool") + return as_column(False, length=len(self), dtype="bool") elif self.null_count == 0 and (rhs.null_count == len(rhs)): - return cudf.core.column.full(len(self), False, dtype="bool") + return as_column(False, length=len(self), dtype="bool") else: return None @@ -1356,9 +1355,7 @@ def _label_encoding( na_sentinel = cudf.Scalar(-1) def _return_sentinel_column(): - return cudf.core.column.full( - size=len(self), fill_value=na_sentinel, dtype=dtype - ) + return as_column(na_sentinel, dtype=dtype, length=len(self)) if dtype is None: dtype = min_scalar_type(max(len(cats), na_sentinel), 8) @@ -1455,7 +1452,9 @@ def column_empty( elif isinstance(dtype, ListDtype): data = None children = ( - full(row_count + 1, 0, dtype=libcudf.types.size_type_dtype), + as_column( + 0, length=row_count + 1, dtype=libcudf.types.size_type_dtype + ), column_empty(row_count, dtype=dtype.element_type), ) elif isinstance(dtype, CategoricalDtype): @@ -1474,7 +1473,9 @@ def column_empty( elif dtype.kind in "OU" and not isinstance(dtype, DecimalDtype): data = as_buffer(rmm.DeviceBuffer(size=0)) children = ( - full(row_count + 1, 0, dtype=libcudf.types.size_type_dtype), + as_column( + 0, length=row_count + 1, dtype=libcudf.types.size_type_dtype + ), ) else: data = as_buffer(rmm.DeviceBuffer(size=row_count * dtype.itemsize)) @@ -2017,33 +2018,32 @@ def as_column( if dtype is not None: data = data.astype(dtype) - elif isinstance(arbitrary, (pd.Timestamp, pd.Timedelta)): - # This will always treat NaTs as nulls since it's not technically a - # discrete value like NaN - length = length or 1 - data = as_column( - pa.array(pd.Series([arbitrary] * length), from_pandas=True) - ) - if dtype is not None: - data = data.astype(dtype) - - elif np.isscalar(arbitrary) and not isinstance(arbitrary, memoryview): - length = length or 1 + elif is_scalar(arbitrary) and not isinstance(arbitrary, memoryview): + if length is None: + length = 1 + elif length < 0: + raise ValueError(f"{length=} must be >=0.") + if isinstance(arbitrary, pd.Interval): + # No cudf.Scalar support yet + return as_column( + pd.Series([arbitrary] * length), + nan_as_null=nan_as_null, + dtype=dtype, + length=length, + ) if ( - (nan_as_null is True) + nan_as_null is True and isinstance(arbitrary, (np.floating, float)) and np.isnan(arbitrary) ): - arbitrary = None if dtype is None: - dtype = cudf.dtype("float64") - - data = as_column(full(length, arbitrary, dtype=dtype)) - if not nan_as_null and not is_decimal_dtype(data.dtype): - if np.issubdtype(data.dtype, np.floating): - data = data.fillna(np.nan) - elif np.issubdtype(data.dtype, np.datetime64): - data = data.fillna(np.datetime64("NaT")) + dtype = getattr(arbitrary, "dtype", cudf.dtype("float64")) + arbitrary = None + arbitrary = cudf.Scalar(arbitrary, dtype=dtype) + if length == 0: + return column_empty(length, dtype=arbitrary.dtype) + else: + return ColumnBase.from_scalar(arbitrary, length) elif hasattr(arbitrary, "__array_interface__"): # CUDF assumes values are always contiguous @@ -2161,8 +2161,6 @@ def as_column( return as_column( np.asarray(view), dtype=dtype, nan_as_null=nan_as_null ) - elif isinstance(arbitrary, cudf.Scalar): - data = ColumnBase.from_scalar(arbitrary, length if length else 1) else: if dtype is not None: # Arrow throws a type error if the input is of @@ -2505,42 +2503,6 @@ def deserialize_columns(headers: List[dict], frames: List) -> List[ColumnBase]: return columns -def full( - size: int, fill_value: ScalarLike, dtype: Optional[Dtype] = None -) -> ColumnBase: - """ - Returns a column of given size and dtype, filled with a given value. - - Parameters - ---------- - size : int - size of the expected column. - fill_value : scalar - A scalar value to fill a new array. - dtype : default None - Data type specifier. It is inferred from other arguments by default. - - Returns - ------- - Column - - Examples - -------- - >>> import cudf - >>> col = cudf.core.column.full(size=5, fill_value=7, dtype='int8') - >>> col - - >>> cudf.Series(col) - 0 7 - 1 7 - 2 7 - 3 7 - 4 7 - dtype: int8 - """ - return ColumnBase.from_scalar(cudf.Scalar(fill_value, dtype), size) - - def concat_columns(objs: "MutableSequence[ColumnBase]") -> ColumnBase: """Concatenate a sequence of columns.""" if len(objs) == 0: diff --git a/python/cudf/cudf/core/column/decimal.py b/python/cudf/cudf/core/column/decimal.py index 0e90b522f2c..b83a6ded416 100644 --- a/python/cudf/cudf/core/column/decimal.py +++ b/python/cudf/cudf/core/column/decimal.py @@ -69,8 +69,8 @@ def as_string_column( def __pow__(self, other): if isinstance(other, int): if other == 0: - res = cudf.core.column.full( - size=len(self), fill_value=1, dtype=self.dtype + res = cudf.core.column.as_column( + 1, dtype=self.dtype, length=len(self) ) if self.nullable: res = res.set_mask(self.mask) diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index 82d82593c77..8d9da8982ac 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -42,7 +42,6 @@ as_column, build_column, column, - full, string, ) from cudf.core.dtypes import CategoricalDtype @@ -513,7 +512,7 @@ def find_and_replace( ) if len(replacement_col) == 1 and len(to_replace_col) > 1: replacement_col = column.as_column( - full(len(to_replace_col), replacement[0], self.dtype) + replacement[0], length=len(to_replace_col), dtype=self.dtype ) elif len(replacement_col) == 1 and len(to_replace_col) == 0: return self.copy() diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index dea60f58690..e947c9375d7 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -5499,7 +5499,9 @@ def __init__( if len(children) == 0 and size != 0: # all nulls-column: - offsets = column.full(size + 1, 0, dtype=size_type_dtype) + offsets = column.as_column( + 0, length=size + 1, dtype=size_type_dtype + ) children = (offsets,) @@ -5930,8 +5932,8 @@ def _binaryop( "__eq__", "__ne__", }: - return column.full( - len(self), op == "__ne__", dtype="bool" + return column.as_column( + op == "__ne__", length=len(self), dtype="bool" ).set_mask(self.mask) else: return NotImplemented @@ -5940,7 +5942,9 @@ def _binaryop( if isinstance(other, cudf.Scalar): other = cast( StringColumn, - column.full(len(self), other, dtype="object"), + column.as_column( + other, length=len(self), dtype="object" + ), ) # Explicit types are necessary because mypy infers ColumnBase diff --git a/python/cudf/cudf/core/column/timedelta.py b/python/cudf/cudf/core/column/timedelta.py index dab2723795e..ee326b254b9 100644 --- a/python/cudf/cudf/core/column/timedelta.py +++ b/python/cudf/cudf/core/column/timedelta.py @@ -510,7 +510,7 @@ def components(self, index=None) -> "cudf.DataFrame": break for name in keys_list: - res_col = cudf.core.column.full(len(self), 0, dtype="int64") + res_col = column.as_column(0, length=len(self), dtype="int64") if self.nullable: res_col = res_col.set_mask(self.mask) data[name] = res_col @@ -599,7 +599,7 @@ def nanoseconds(self) -> "cudf.core.column.NumericalColumn": # of nanoseconds. if self._time_unit != "ns": - res_col = cudf.core.column.full(len(self), 0, dtype="int64") + res_col = column.as_column(0, length=len(self), dtype="int64") if self.nullable: res_col = res_col.set_mask(self.mask) return cast("cudf.core.column.NumericalColumn", res_col) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index d7d2e1acd85..31a748da856 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -1407,7 +1407,7 @@ def __setitem__(self, arg, value): allow_non_unique=True, ) if is_scalar(value): - self._data[arg] = column.full(len(self), value) + self._data[arg] = as_column(value, length=len(self)) else: value = as_column(value) self._data[arg] = value @@ -1455,8 +1455,8 @@ def __setitem__(self, arg, value): else: for col in arg: if is_scalar(value): - self._data[col] = column.full( - size=len(self), fill_value=value + self._data[col] = as_column( + value, length=len(self) ) else: self._data[col] = column.as_column(value) @@ -3205,10 +3205,16 @@ def _insert(self, loc, name, value, nan_as_null=None, ignore_index=True): ) if _is_scalar_or_zero_d_array(value): - value = column.full( - len(self), + dtype = None + if isinstance(value, (np.ndarray, cupy.ndarray)): + dtype = value.dtype + value = value.item() + if libcudf.scalar._is_null_host_scalar(value): + dtype = "str" + value = as_column( value, - "str" if libcudf.scalar._is_null_host_scalar(value) else None, + length=len(self), + dtype=dtype, ) if len(self) == 0: @@ -5912,7 +5918,7 @@ def isin(self, values): fill_value = cudf.Scalar(False) def make_false_column_like_self(): - return column.full(len(self), fill_value, "bool") + return column.as_column(fill_value, length=len(self), dtype="bool") # Preprocess different input types into a mapping from column names to # a list of values to check. @@ -6031,7 +6037,7 @@ def _prepare_for_rowwise_op(self, method, skipna, numeric_only): { name: filtered._data[name]._get_mask_as_column() if filtered._data[name].nullable - else column.full(len(filtered._data[name]), True) + else as_column(True, length=len(filtered._data[name])) for name in filtered._data.names } ) @@ -7822,8 +7828,8 @@ def func(left, right, output): return output for name in uncommon_columns: - output._data[name] = column.full( - size=len(output), fill_value=value, dtype="bool" + output._data[name] = as_column( + value, length=len(output), dtype="bool" ) return output diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 9d481037ec6..bd9dc1ae3da 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -1231,9 +1231,9 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): ) needle = as_column(target) - result = cudf.core.column.full( - len(needle), - fill_value=-1, + result = as_column( + -1, + length=len(needle), dtype=libcudf.types.size_type_dtype, ) diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 3c6e1e17142..df703370f78 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -50,7 +50,7 @@ from cudf.core._base_index import BaseIndex from cudf.core._compat import PANDAS_LT_300 from cudf.core.buffer import acquire_spill_lock -from cudf.core.column import ColumnBase, as_column, full +from cudf.core.column import ColumnBase, as_column from cudf.core.column_accessor import ColumnAccessor from cudf.core.copy_types import BooleanMask, GatherMap from cudf.core.dtypes import ListDtype @@ -3048,7 +3048,7 @@ def duplicated(self, subset=None, keep="first"): (result,) = libcudf.copying.scatter( [cudf.Scalar(False, dtype=bool)], distinct, - [full(len(self), True, dtype=bool)], + [as_column(True, length=len(self), dtype=bool)], bounds_check=False, ) return cudf.Series(result, index=self.index) @@ -3327,9 +3327,7 @@ def _apply(self, func, kernel_getter, *args, **kwargs): # Mask and data column preallocated ans_col = _return_arr_from_dtype(retty, len(self)) - ans_mask = cudf.core.column.full( - size=len(self), fill_value=True, dtype="bool" - ) + ans_mask = as_column(True, length=len(self), dtype="bool") output_args = [(ans_col, ans_mask), len(self)] input_args = _get_input_args_from_frame(self) launch_args = output_args + input_args + list(args) @@ -6260,10 +6258,10 @@ def _get_replacement_values_for_columns( values_columns = { col: [value] if _is_non_decimal_numeric_dtype(columns_dtype_map[col]) - else full( - len(to_replace), + else as_column( value, - cudf.dtype(type(value)), + length=len(to_replace), + dtype=cudf.dtype(type(value)), ) for col in columns_dtype_map } diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 70112044f75..315a21020a2 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -667,7 +667,7 @@ def isin(self, values, level=None): self_df = self.to_frame(index=False).reset_index() values_df = values_idx.to_frame(index=False) idx = self_df.merge(values_df, how="leftsemi")._data["index"] - res = cudf.core.column.full(size=len(self), fill_value=False) + res = column.as_column(False, length=len(self)) res[idx] = True result = res.values else: @@ -1845,9 +1845,9 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): "index must be monotonic increasing or decreasing" ) - result = cudf.core.column.full( - len(target), - fill_value=-1, + result = column.as_column( + -1, + length=len(target), dtype=libcudf.types.size_type_dtype, ) if not len(self): diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index cb5008af3ad..1b18e11c047 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -55,7 +55,6 @@ IntervalColumn, TimeDeltaColumn, as_column, - full, ) from cudf.core.column.categorical import ( CategoricalAccessor as CategoricalAccessor, @@ -1311,7 +1310,7 @@ def map(self, arg, na_action=None) -> "Series": { "x": arg.keys(), "s": arg.values(), - "bool": full(len(arg), True, dtype=self.dtype), + "bool": as_column(True, length=len(arg), dtype=self.dtype), } ) res = lhs.merge(rhs, on="x", how="left").sort_values( @@ -1333,7 +1332,7 @@ def map(self, arg, na_action=None) -> "Series": { "x": arg.keys(), "s": arg, - "bool": full(len(arg), True, dtype=self.dtype), + "bool": as_column(True, length=len(arg), dtype=self.dtype), } ) res = lhs.merge(rhs, on="x", how="left").sort_values( diff --git a/python/cudf/cudf/core/tools/datetimes.py b/python/cudf/cudf/core/tools/datetimes.py index 0e0df4ecf6e..d182b7b4a7c 100644 --- a/python/cudf/cudf/core/tools/datetimes.py +++ b/python/cudf/cudf/core/tools/datetimes.py @@ -770,7 +770,7 @@ def _isin_datetimelike( was_string = len(rhs) and rhs.dtype.kind == "O" if rhs.dtype.kind in {"f", "i", "u"}: - return cudf.core.column.full(len(lhs), False, dtype="bool") + return column.as_column(False, length=len(lhs), dtype="bool") rhs = rhs.astype(lhs.dtype) if was_string: warnings.warn( @@ -787,7 +787,7 @@ def _isin_datetimelike( except ValueError: # pandas functionally returns all False when cleansing via # typecasting fails - return cudf.core.column.full(len(lhs), False, dtype="bool") + return column.as_column(False, length=len(lhs), dtype="bool") res = lhs._obtain_isin_result(rhs) return res diff --git a/python/cudf/cudf/core/window/rolling.py b/python/cudf/cudf/core/window/rolling.py index 890e4ecc2f0..2037b1682db 100644 --- a/python/cudf/cudf/core/window/rolling.py +++ b/python/cudf/cudf/core/window/rolling.py @@ -9,7 +9,6 @@ import cudf from cudf import _lib as libcudf from cudf.api.types import is_integer, is_number -from cudf.core import column from cudf.core.buffer import acquire_spill_lock from cudf.core.column.column import as_column from cudf.core.mixins import Reducible @@ -236,8 +235,8 @@ def _apply_agg_column(self, source_column, agg_name): window = None else: preceding_window = as_column(self.window) - following_window = column.full( - self.window.size, 0, dtype=self.window.dtype + following_window = as_column( + 0, length=self.window.size, dtype=self.window.dtype ) window = None diff --git a/python/cudf/cudf/io/parquet.py b/python/cudf/cudf/io/parquet.py index 6c70b08384f..bead9c352ef 100644 --- a/python/cudf/cudf/io/parquet.py +++ b/python/cudf/cudf/io/parquet.py @@ -20,7 +20,7 @@ import cudf from cudf._lib import parquet as libparquet from cudf.api.types import is_list_like -from cudf.core.column import build_categorical_column, column_empty, full +from cudf.core.column import as_column, build_categorical_column, column_empty from cudf.utils import ioutils from cudf.utils.nvtx_annotation import _cudf_nvtx_annotate @@ -762,9 +762,9 @@ def _parquet_to_frame( _len = len(dfs[-1]) if partition_categories and name in partition_categories: # Build the categorical column from `codes` - codes = full( - size=_len, - fill_value=partition_categories[name].index(value), + codes = as_column( + partition_categories[name].index(value), + length=_len, ) dfs[-1][name] = build_categorical_column( categories=partition_categories[name], @@ -788,10 +788,10 @@ def _parquet_to_frame( masked=True, ) else: - dfs[-1][name] = full( - size=_len, - fill_value=value, + dfs[-1][name] = as_column( + value, dtype=_dtype, + length=_len, ) if len(dfs) > 1: diff --git a/python/cudf/cudf/tests/test_testing.py b/python/cudf/cudf/tests/test_testing.py index 091cd6b57a4..1994536f395 100644 --- a/python/cudf/cudf/tests/test_testing.py +++ b/python/cudf/cudf/tests/test_testing.py @@ -6,7 +6,7 @@ import pytest import cudf -from cudf.core.column.column import as_column, full +from cudf.core.column.column import as_column from cudf.testing import ( assert_frame_equal, assert_index_equal, @@ -172,8 +172,8 @@ def test_assert_column_equal_dtype_edge_cases(other): assert_column_equal(base.slice(0, 0), other.slice(0, 0), check_dtype=False) assert_column_equal(other.slice(0, 0), base.slice(0, 0), check_dtype=False) - base = full(len(base), fill_value=cudf.NA, dtype=base.dtype) - other = full(len(other), fill_value=cudf.NA, dtype=other.dtype) + base = as_column(cudf.NA, length=len(base), dtype=base.dtype) + other = as_column(cudf.NA, length=len(other), dtype=other.dtype) assert_column_equal(base, other, check_dtype=False) assert_column_equal(other, base, check_dtype=False) diff --git a/python/cudf/cudf/utils/utils.py b/python/cudf/cudf/utils/utils.py index ec5693e14d2..95621cf9519 100644 --- a/python/cudf/cudf/utils/utils.py +++ b/python/cudf/cudf/utils/utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. import decimal import functools @@ -396,8 +396,8 @@ def _all_bools_with_nulls(lhs, rhs, bool_fill_value): else: result_mask = None - result_col = column.full( - size=len(lhs), fill_value=bool_fill_value, dtype=cudf.dtype(np.bool_) + result_col = column.as_column( + bool_fill_value, dtype=cudf.dtype(np.bool_), length=len(lhs) ) if result_mask is not None: result_col = result_col.set_mask(result_mask.as_mask()) diff --git a/python/dask_cudf/dask_cudf/backends.py b/python/dask_cudf/dask_cudf/backends.py index 454cce76ff2..317c45ba582 100644 --- a/python/dask_cudf/dask_cudf/backends.py +++ b/python/dask_cudf/dask_cudf/backends.py @@ -105,8 +105,10 @@ def _get_non_empty_data(s): categories = ( s.categories if len(s.categories) else [UNKNOWN_CATEGORIES] ) - codes = cudf.core.column.full( - size=2, fill_value=0, dtype=cudf._lib.types.size_type_dtype + codes = cudf.core.column.as_column( + 0, + dtype=cudf._lib.types.size_type_dtype, + length=2, ) ordered = s.ordered data = cudf.core.column.build_categorical_column( From 427ce014bbefba17c47fc032c71c3f513f2fce06 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 4 Mar 2024 17:34:20 -1000 Subject: [PATCH 342/384] Add ListColumns.to_pandas(arrow_type=) (#15228) I think there will be a mypy error on main soon as https://github.com/rapidsai/cudf/pull/15182 and https://github.com/rapidsai/cudf/pull/15155 were merge in close succession (my fault for not rebasing first) Also address a review I forgot in https://github.com/rapidsai/cudf/pull/15182/files#r1507154770 cc @galipremsagar Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15228 --- python/cudf/cudf/core/column/interval.py | 2 +- python/cudf/cudf/core/column/lists.py | 18 ++++++++++++------ python/cudf/cudf/tests/test_series.py | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/python/cudf/cudf/core/column/interval.py b/python/cudf/cudf/core/column/interval.py index dcec8957bb2..dc609f732e0 100644 --- a/python/cudf/cudf/core/column/interval.py +++ b/python/cudf/cudf/core/column/interval.py @@ -123,7 +123,7 @@ def to_pandas( if nullable: raise NotImplementedError(f"{nullable=} is not implemented.") elif arrow_type: - raise NotImplementedError(f"{nullable=} is not implemented.") + raise NotImplementedError(f"{arrow_type=} is not implemented.") return pd.Series( self.dtype.to_pandas().__from_arrow__(self.to_arrow()), index=index ) diff --git a/python/cudf/cudf/core/column/lists.py b/python/cudf/cudf/core/column/lists.py index d1bf0b74d3c..1c2bcbef2ec 100644 --- a/python/cudf/cudf/core/column/lists.py +++ b/python/cudf/cudf/core/column/lists.py @@ -294,17 +294,23 @@ def to_pandas( *, index: Optional[pd.Index] = None, nullable: bool = False, + arrow_type: bool = False, ) -> pd.Series: # Can't rely on Column.to_pandas implementation for lists. # Need to perform `to_pylist` to preserve list types. + if arrow_type and nullable: + raise ValueError( + f"{arrow_type=} and {nullable=} cannot both be set." + ) if nullable: raise NotImplementedError(f"{nullable=} is not implemented.") - - pd_series = pd.Series(self.to_arrow().to_pylist(), dtype="object") - - if index is not None: - pd_series.index = index - return pd_series + pa_array = self.to_arrow() + if arrow_type: + return pd.Series( + pd.arrays.ArrowExtensionArray(pa_array), index=index + ) + else: + return pd.Series(pa_array.tolist(), dtype="object", index=index) class ListMethods(ColumnMethods): diff --git a/python/cudf/cudf/tests/test_series.py b/python/cudf/cudf/tests/test_series.py index 6b5c0406deb..e043f358bbe 100644 --- a/python/cudf/cudf/tests/test_series.py +++ b/python/cudf/cudf/tests/test_series.py @@ -2726,7 +2726,7 @@ def test_series_from_large_string(): def test_series_to_pandas_arrow_type_nullable_raises(scalar): pa_array = pa.array([scalar, None]) ser = cudf.Series(pa_array) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match=".* cannot both be set"): ser.to_pandas(nullable=True, arrow_type=True) From cd79fe55d9e4d296f5b865b7b556448fbc50a828 Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Mon, 4 Mar 2024 20:04:19 -0800 Subject: [PATCH 343/384] Implement zero-copy host buffer source instead of using an arrow implementation (#15189) Avoids an arrow dependency with a bit of simple code. No real impact on performance. Authors: - Vukasin Milovanovic (https://github.com/vuule) Approvers: - Bradley Dice (https://github.com/bdice) - Yunsong Wang (https://github.com/PointKernel) URL: https://github.com/rapidsai/cudf/pull/15189 --- cpp/src/io/utilities/datasource.cpp | 33 ++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/cpp/src/io/utilities/datasource.cpp b/cpp/src/io/utilities/datasource.cpp index cf2ba369023..d2026473b6c 100644 --- a/cpp/src/io/utilities/datasource.cpp +++ b/cpp/src/io/utilities/datasource.cpp @@ -18,7 +18,6 @@ #include "io/utilities/config_utils.hpp" #include -#include #include #include #include @@ -27,7 +26,6 @@ #include -#include #include #include #include @@ -338,6 +336,33 @@ class device_buffer_source final : public datasource { cudf::device_span _d_buffer; ///< A non-owning view of the existing device data }; +// zero-copy host buffer source +class host_buffer_source final : public datasource { + public: + explicit host_buffer_source(cudf::host_span h_buffer) : _h_buffer{h_buffer} {} + + size_t host_read(size_t offset, size_t size, uint8_t* dst) override + { + auto const count = std::min(size, this->size() - offset); + std::memcpy(dst, _h_buffer.data() + offset, count); + return count; + } + + std::unique_ptr host_read(size_t offset, size_t size) override + { + auto const count = std::min(size, this->size() - offset); + return std::make_unique( + reinterpret_cast(_h_buffer.data() + offset), count); + } + + [[nodiscard]] bool supports_device_read() const override { return false; } + + [[nodiscard]] size_t size() const override { return _h_buffer.size(); } + + private: + cudf::host_span _h_buffer; ///< A non-owning view of the existing host data +}; + /** * @brief Wrapper class for user implemented data sources * @@ -424,9 +449,7 @@ std::unique_ptr datasource::create(host_buffer const& buffer) std::unique_ptr datasource::create(cudf::host_span buffer) { - // Use Arrow IO buffer class for zero-copy reads of host memory - return std::make_unique(std::make_shared( - reinterpret_cast(buffer.data()), buffer.size())); + return std::make_unique(buffer); } std::unique_ptr datasource::create(cudf::device_span buffer) From f804aa69ca22124f648aba70096df6f1efe27629 Mon Sep 17 00:00:00 2001 From: Alessandro Bellina Date: Mon, 4 Mar 2024 23:26:09 -0600 Subject: [PATCH 344/384] Fix testchunkedPackTwoPasses to copy from the bounce buffer (#15220) This is a follow on from https://github.com/rapidsai/cudf/pull/15210. We bring back the test and fix it so it copies from the right buffer this time. I also set the original column to have some values and nulls, to make sure we are checking something interesting. Authors: - Alessandro Bellina (https://github.com/abellina) Approvers: - Nghia Truong (https://github.com/ttnghia) - Gera Shegalov (https://github.com/gerashegalov) URL: https://github.com/rapidsai/cudf/pull/15220 --- java/src/test/java/ai/rapids/cudf/TableTest.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/java/src/test/java/ai/rapids/cudf/TableTest.java b/java/src/test/java/ai/rapids/cudf/TableTest.java index 3f0470d854a..44dd20561bf 100644 --- a/java/src/test/java/ai/rapids/cudf/TableTest.java +++ b/java/src/test/java/ai/rapids/cudf/TableTest.java @@ -3758,12 +3758,16 @@ void testChunkedPackBasic() { } } } -/* + @Test void testChunkedPackTwoPasses() { // this test packes ~2MB worth of long into a 1MB bounce buffer // this is 3 iterations because of the validity buffer Long[] longs = new Long[256*1024]; + // Initialize elements at odd-numbered indices + for (int i = 1; i < longs.length; i += 2) { + longs[i] = (long)i; + } try (Table t1 = new Table.TestBuilder().column(longs).build(); DeviceMemoryBuffer bounceBuffer = DeviceMemoryBuffer.allocate(1L*1024*1024); ChunkedPack cp = t1.makeChunkedPack(1L*1024*1024); @@ -3776,7 +3780,7 @@ void testChunkedPackTwoPasses() { while (cp.hasNext()) { long copied = cp.next(bounceBuffer); target.copyFromDeviceBufferAsync( - offset, target, 0, copied, Cuda.DEFAULT_STREAM); + offset, bounceBuffer, 0, copied, Cuda.DEFAULT_STREAM); offset += copied; } @@ -3787,7 +3791,6 @@ void testChunkedPackTwoPasses() { } } } -*/ @Test void testContiguousSplitWithStrings() { From 8d073e4ca0a6cb9d9a4d9fe5e4e0147f01d7eb36 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 5 Mar 2024 08:06:46 -0500 Subject: [PATCH 345/384] Change strings_column_view::char_size to return int64 (#15197) Changes the `cudf::strings_column_view::chars_size()` function to return `int64_t` instead of `size_type` Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Nghia Truong (https://github.com/ttnghia) - Mark Harris (https://github.com/harrism) URL: https://github.com/rapidsai/cudf/pull/15197 --- cpp/benchmarks/string/case.cpp | 4 +++- cpp/include/cudf/strings/strings_column_view.hpp | 2 +- cpp/src/strings/strings_column_view.cpp | 8 ++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cpp/benchmarks/string/case.cpp b/cpp/benchmarks/string/case.cpp index 639a3dc1181..a7db972d39f 100644 --- a/cpp/benchmarks/string/case.cpp +++ b/cpp/benchmarks/string/case.cpp @@ -45,7 +45,9 @@ void bench_case(nvbench::state& state) cudf::type_id::INT8, distribution_id::UNIFORM, 32, 126); // nice ASCII range auto input = cudf::strings_column_view(col_view); auto ascii_column = create_random_column( - cudf::type_id::INT8, row_count{input.chars_size(cudf::get_default_stream())}, ascii_profile); + cudf::type_id::INT8, + row_count{static_cast(input.chars_size(cudf::get_default_stream()))}, + ascii_profile); auto ascii_data = ascii_column->view(); col_view = cudf::column_view(col_view.type(), diff --git a/cpp/include/cudf/strings/strings_column_view.hpp b/cpp/include/cudf/strings/strings_column_view.hpp index 840a2dd1165..036589e17fe 100644 --- a/cpp/include/cudf/strings/strings_column_view.hpp +++ b/cpp/include/cudf/strings/strings_column_view.hpp @@ -112,7 +112,7 @@ class strings_column_view : private column_view { * @param stream CUDA stream used for device memory operations and kernel launches * @return Number of bytes in the chars child column */ - [[nodiscard]] size_type chars_size(rmm::cuda_stream_view stream) const noexcept; + [[nodiscard]] int64_t chars_size(rmm::cuda_stream_view stream) const noexcept; /** * @brief Return an iterator for the chars child column. diff --git a/cpp/src/strings/strings_column_view.cpp b/cpp/src/strings/strings_column_view.cpp index 6be22d8e729..83ae916afc3 100644 --- a/cpp/src/strings/strings_column_view.cpp +++ b/cpp/src/strings/strings_column_view.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include #include @@ -45,10 +45,10 @@ strings_column_view::offset_iterator strings_column_view::offsets_end() const return offsets_begin() + size() + 1; } -size_type strings_column_view::chars_size(rmm::cuda_stream_view stream) const noexcept +int64_t strings_column_view::chars_size(rmm::cuda_stream_view stream) const noexcept { - if (size() == 0) return 0; - return detail::get_value(offsets(), offsets().size() - 1, stream); + if (size() == 0) { return 0L; } + return cudf::strings::detail::get_offset_value(offsets(), offsets().size() - 1, stream); } strings_column_view::chars_iterator strings_column_view::chars_begin(rmm::cuda_stream_view) const From 1f5fcf679ee6052ab320220ee7218fcad51d99f2 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 5 Mar 2024 08:17:53 -0800 Subject: [PATCH 346/384] Improvements for `__cuda_array_interface__` tests (#15188) This PR contains a few minor improvements for `__cuda_array_interface__` and its tests. Found while working on #15111. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15188 --- python/cudf/cudf/core/single_column_frame.py | 5 ++++- .../cudf/tests/test_cuda_array_interface.py | 20 ++++++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/python/cudf/cudf/core/single_column_frame.py b/python/cudf/cudf/core/single_column_frame.py index 97779522b8b..19dde2e51b9 100644 --- a/python/cudf/cudf/core/single_column_frame.py +++ b/python/cudf/cudf/core/single_column_frame.py @@ -242,7 +242,10 @@ def __cuda_array_interface__(self): try: return self._column.__cuda_array_interface__ except NotImplementedError: - raise AttributeError + raise AttributeError( + f"'{type(self).__name__}' object has no attribute " + "'__cuda_array_interface__'" + ) @_cudf_nvtx_annotate def factorize(self, sort=False, use_na_sentinel=True): diff --git a/python/cudf/cudf/tests/test_cuda_array_interface.py b/python/cudf/cudf/tests/test_cuda_array_interface.py index 1f20152172b..213c6c2c1f9 100644 --- a/python/cudf/cudf/tests/test_cuda_array_interface.py +++ b/python/cudf/cudf/tests/test_cuda_array_interface.py @@ -4,10 +4,10 @@ from contextlib import ExitStack as does_not_raise import cupy +import numba.cuda import numpy as np import pandas as pd import pytest -from numba import cuda import cudf from cudf.core.buffer.spill_manager import get_global_manager @@ -25,7 +25,7 @@ def test_cuda_array_interface_interop_in(dtype, module): if dtype in DATETIME_TYPES: expectation = pytest.raises(ValueError) elif module == "numba": - module_constructor = cuda.to_device + module_constructor = numba.cuda.to_device with expectation: module_data = module_constructor(np_data) @@ -55,7 +55,7 @@ def to_host_function(x): return cupy.asnumpy(x) elif module == "numba": - module_constructor = cuda.as_cuda_array + module_constructor = numba.cuda.as_cuda_array def to_host_function(x): return x.copy_to_host() @@ -89,7 +89,7 @@ def to_host_function(x): elif module == "numba": expectation = pytest.raises(NotImplementedError) - module_constructor = cuda.as_cuda_array + module_constructor = numba.cuda.as_cuda_array def to_host_function(x): return x.copy_to_host() @@ -135,9 +135,11 @@ def test_cuda_array_interface_as_column(dtype, nulls, mask_type): if mask_type == "bools": if nulls == "some": - obj.__cuda_array_interface__["mask"] = cuda.to_device(mask) + obj.__cuda_array_interface__["mask"] = numba.cuda.to_device(mask) elif nulls == "all": - obj.__cuda_array_interface__["mask"] = cuda.to_device([False] * 10) + obj.__cuda_array_interface__["mask"] = numba.cuda.to_device( + [False] * 10 + ) expect = sr got = cudf.Series(obj) @@ -193,7 +195,11 @@ def test_cuda_array_interface_pytorch(): assert_eq(got, cudf.Series(buffer, dtype=np.bool_)) - # TODO: This test fails with PyTorch 2. Is it still expected to be valid? + # TODO: This test fails with PyTorch 2. It appears that PyTorch + # checks that the pointer is device-accessible even when the + # size is zero. See + # https://github.com/pytorch/pytorch/issues/98133 + # # index = cudf.Index([], dtype="float64") # tensor = torch.tensor(index) # got = cudf.Index(tensor) From d4368e98a4b92ade651a5f5df98035a297658f16 Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Tue, 5 Mar 2024 16:45:18 +0000 Subject: [PATCH 347/384] Fix GroupBy.get_group and GroupBy.indices (#15143) These are supposed to index based on row indices, not row labels. - Closes https://github.com/rapidsai/cudf/issues/14955 Authors: - Lawrence Mitchell (https://github.com/wence-) - Richard (Rick) Zamora (https://github.com/rjzamora) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/15143 --- python/cudf/cudf/core/groupby/groupby.py | 22 +++++++++++++------ .../cudf/tests/groupby/test_groupby_obj.py | 15 +++++++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 python/cudf/cudf/tests/groupby/test_groupby_obj.py diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index e4370be304a..caf5ac5928f 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -363,13 +363,22 @@ def indices(self): >>> df.groupby(by=["a"]).indices {10: array([0, 1]), 40: array([2])} """ - group_names, offsets, _, grouped_values = self._grouped() + offsets, group_keys, (indices,) = self._groupby.groups( + [ + cudf.core.column.as_column( + range(len(self.obj)), dtype=size_type_dtype + ) + ] + ) + group_keys = libcudf.stream_compaction.drop_duplicates(group_keys) + if len(group_keys) > 1: + index = cudf.MultiIndex.from_arrays(group_keys) + else: + (group_keys,) = group_keys + index = cudf.Index(group_keys) return dict( - zip( - group_names.to_pandas(), - np.split(grouped_values.index.values, offsets[1:-1]), - ) + zip(index.to_pandas(), cp.split(indices.values, offsets[1:-1])) ) @_cudf_nvtx_annotate @@ -414,8 +423,7 @@ def get_group(self, name, obj=None): "instead of ``gb.get_group(name, obj=df)``.", FutureWarning, ) - - return obj.loc[self.groups[name].drop_duplicates()] + return obj.iloc[self.indices[name]] @_cudf_nvtx_annotate def size(self): diff --git a/python/cudf/cudf/tests/groupby/test_groupby_obj.py b/python/cudf/cudf/tests/groupby/test_groupby_obj.py new file mode 100644 index 00000000000..04b483e08dc --- /dev/null +++ b/python/cudf/cudf/tests/groupby/test_groupby_obj.py @@ -0,0 +1,15 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +from numpy.testing import assert_array_equal + +import cudf +from cudf.testing._utils import assert_eq + + +def test_groupby_14955(): + # https://github.com/rapidsai/cudf/issues/14955 + df = cudf.DataFrame({"a": [1, 2] * 2}, index=[0] * 4) + agg = df.groupby("a") + pagg = df.to_pandas().groupby("a") + for key in agg.groups: + assert_array_equal(pagg.indices[key], agg.indices[key].get()) + assert_eq(pagg.get_group(key), agg.get_group(key)) From d53df8c88e9c62acb90744bfb1df6580909065d0 Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Tue, 5 Mar 2024 10:39:22 -0800 Subject: [PATCH 348/384] Tune up row size estimation in the data generator (#15202) - Make string offsets a part of the strings column size; - Fix erroneous "last element" inclusion in list columns; - Minimize rounding errors by switching to double for the average row size; - Account for null frequency for columns that don't store null elements (strings, lists); - Account for the null masks size. With these changes, actual table size should be much closer to the requested value. Tested indirectly through Parquet file size in benchmarks. Authors: - Vukasin Milovanovic (https://github.com/vuule) Approvers: - Bradley Dice (https://github.com/bdice) - David Wendt (https://github.com/davidwendt) URL: https://github.com/rapidsai/cudf/pull/15202 --- cpp/benchmarks/common/generate_input.cu | 138 ++++++++++++++--------- cpp/benchmarks/common/generate_input.hpp | 7 +- 2 files changed, 89 insertions(+), 56 deletions(-) diff --git a/cpp/benchmarks/common/generate_input.cu b/cpp/benchmarks/common/generate_input.cu index 8952b86b5a3..71ce45879dd 100644 --- a/cpp/benchmarks/common/generate_input.cu +++ b/cpp/benchmarks/common/generate_input.cu @@ -71,7 +71,7 @@ auto deterministic_engine(unsigned seed) { return thrust::minstd_rand{seed}; } * Computes the mean value for a distribution of given type and value bounds. */ template -T get_distribution_mean(distribution_params const& dist) +double get_distribution_mean(distribution_params const& dist) { switch (dist.id) { case distribution_id::NORMAL: @@ -90,6 +90,39 @@ T get_distribution_mean(distribution_params const& dist) } } +/** + * @brief Calculates the number of direct parents needed to generate a struct column hierarchy with + * lowest maximum number of children in any nested column. + * + * Used to generate an "evenly distributed" struct column hierarchy with the given number of leaf + * columns and nesting levels. The column tree is considered evenly distributed if all columns have + * nearly the same number of child columns (difference not larger than one). + */ +int num_direct_parents(int num_lvls, int num_leaf_columns) +{ + // Estimated average number of children in the hierarchy; + auto const num_children_avg = std::pow(num_leaf_columns, 1. / num_lvls); + // Minimum number of children columns for any column in the hierarchy + int const num_children_min = std::floor(num_children_avg); + // Maximum number of children columns for any column in the hierarchy + int const num_children_max = num_children_min + 1; + + // Minimum number of columns needed so that their number of children does not exceed the maximum + int const min_for_current_nesting = + std::ceil(static_cast(num_leaf_columns) / num_children_max); + // Minimum number of columns needed so that columns at the higher levels have at least the minimum + // number of children + int const min_for_upper_nesting = std::pow(num_children_min, num_lvls - 1); + // Both conditions need to be satisfied + return std::max(min_for_current_nesting, min_for_upper_nesting); +} + +// Size of the null mask for each row, in bytes +[[nodiscard]] double row_null_mask_size(data_profile const& profile) +{ + return profile.get_null_probability().has_value() ? 1. / 8 : 0.; +} + /** * @brief Computes the average element size in a column, given the data profile. * @@ -97,26 +130,27 @@ T get_distribution_mean(distribution_params const& dist) * the element size of non-fixed-width columns. For lists and structs, `avg_element_size` is called * recursively to determine the size of nested columns. */ -size_t avg_element_size(data_profile const& profile, cudf::data_type dtype); +double avg_element_size(data_profile const& profile, cudf::data_type dtype); // Utilities to determine the mean size of an element, given the data profile template ())> -size_t non_fixed_width_size(data_profile const& profile) +double non_fixed_width_size(data_profile const& profile) { CUDF_FAIL("Should not be called, use `size_of` for this type instead"); } template ())> -size_t non_fixed_width_size(data_profile const& profile) +double non_fixed_width_size(data_profile const& profile) { CUDF_FAIL("not implemented!"); } template <> -size_t non_fixed_width_size(data_profile const& profile) +double non_fixed_width_size(data_profile const& profile) { auto const dist = profile.get_distribution_params().length_params; - return get_distribution_mean(dist); + return get_distribution_mean(dist) * profile.get_valid_probability() + sizeof(cudf::size_type) + + row_null_mask_size(profile); } double geometric_sum(size_t n, double p) @@ -126,45 +160,65 @@ double geometric_sum(size_t n, double p) } template <> -size_t non_fixed_width_size(data_profile const& profile) +double non_fixed_width_size(data_profile const& profile) { - auto const dist_params = profile.get_distribution_params(); - auto const single_level_mean = get_distribution_mean(dist_params.length_params); + auto const dist_params = profile.get_distribution_params(); + auto const single_level_mean = + get_distribution_mean(dist_params.length_params) * profile.get_valid_probability(); + // Leaf column size auto const element_size = avg_element_size(profile, cudf::data_type{dist_params.element_type}); auto const element_count = std::pow(single_level_mean, dist_params.max_depth); + auto const offset_size = avg_element_size(profile, cudf::data_type{cudf::type_id::INT32}); // Each nesting level includes offsets, this is the sum of all levels - // Also include an additional offset per level for the size of the last element - auto const total_offset_count = - geometric_sum(dist_params.max_depth, single_level_mean) + dist_params.max_depth; + auto const total_offset_count = geometric_sum(dist_params.max_depth, single_level_mean); - return sizeof(cudf::size_type) * total_offset_count + element_size * element_count; + return element_size * element_count + offset_size * total_offset_count; +} + +[[nodiscard]] cudf::size_type num_struct_columns(data_profile const& profile) +{ + auto const dist_params = profile.get_distribution_params(); + + cudf::size_type children_count = dist_params.leaf_types.size(); + cudf::size_type total_parent_count = 0; + for (cudf::size_type lvl = dist_params.max_depth; lvl > 0; --lvl) { + children_count = num_direct_parents(lvl, children_count); + total_parent_count += children_count; + } + return total_parent_count; } template <> -size_t non_fixed_width_size(data_profile const& profile) +double non_fixed_width_size(data_profile const& profile) { auto const dist_params = profile.get_distribution_params(); - return std::accumulate(dist_params.leaf_types.cbegin(), - dist_params.leaf_types.cend(), - 0ul, - [&](auto& sum, auto type_id) { - return sum + avg_element_size(profile, cudf::data_type{type_id}); - }); + auto const total_children_size = + std::accumulate(dist_params.leaf_types.cbegin(), + dist_params.leaf_types.cend(), + 0ul, + [&](auto& sum, auto type_id) { + return sum + avg_element_size(profile, cudf::data_type{type_id}); + }); + + // struct columns have a null mask for each row + auto const structs_null_mask_size = num_struct_columns(profile) * row_null_mask_size(profile); + + return total_children_size + structs_null_mask_size; } struct non_fixed_width_size_fn { template - size_t operator()(data_profile const& profile) + double operator()(data_profile const& profile) { return non_fixed_width_size(profile); } }; -size_t avg_element_size(data_profile const& profile, cudf::data_type dtype) +double avg_element_size(data_profile const& profile, cudf::data_type dtype) { - if (cudf::is_fixed_width(dtype)) { return cudf::size_of(dtype); } + if (cudf::is_fixed_width(dtype)) { return cudf::size_of(dtype) + row_null_mask_size(profile); } return cudf::type_dispatcher(dtype, non_fixed_width_size_fn{}, profile); } @@ -596,32 +650,6 @@ struct create_rand_col_fn { } }; -/** - * @brief Calculates the number of direct parents needed to generate a struct column hierarchy with - * lowest maximum number of children in any nested column. - * - * Used to generate an "evenly distributed" struct column hierarchy with the given number of leaf - * columns and nesting levels. The column tree is considered evenly distributed if all columns have - * nearly the same number of child columns (difference not larger than one). - */ -int num_direct_parents(int num_lvls, int num_leaf_columns) -{ - // Estimated average number of children in the hierarchy; - auto const num_children_avg = std::pow(num_leaf_columns, 1. / num_lvls); - // Minimum number of children columns for any column in the hierarchy - int const num_children_min = std::floor(num_children_avg); - // Maximum number of children columns for any column in the hierarchy - int const num_children_max = num_children_min + 1; - - // Minimum number of columns needed so that their number of children does not exceed the maximum - int const min_for_current_nesting = std::ceil((double)num_leaf_columns / num_children_max); - // Minimum number of columns needed so that columns at the higher levels have at least the minimum - // number of children - int const min_for_upper_nesting = std::pow(num_children_min, num_lvls - 1); - // Both conditions need to be satisfied - return std::max(min_for_current_nesting, min_for_upper_nesting); -} - template <> std::unique_ptr create_random_column(data_profile const& profile, thrust::minstd_rand& engine, @@ -825,13 +853,17 @@ std::unique_ptr create_random_table(std::vector cons data_profile const& profile, unsigned seed) { - size_t const avg_row_bytes = - std::accumulate(dtype_ids.begin(), dtype_ids.end(), 0ul, [&](size_t sum, auto tid) { + auto const avg_row_bytes = + std::accumulate(dtype_ids.begin(), dtype_ids.end(), 0., [&](size_t sum, auto tid) { return sum + avg_element_size(profile, cudf::data_type(tid)); }); - cudf::size_type const num_rows = table_bytes.size / avg_row_bytes; + std::size_t const num_rows = std::lround(table_bytes.size / avg_row_bytes); + CUDF_EXPECTS(num_rows > 0, "Table size is too small for the given data types"); + CUDF_EXPECTS(num_rows < std::numeric_limits::max(), + "Table size is too large for the given data types"); - return create_random_table(dtype_ids, row_count{num_rows}, profile, seed); + return create_random_table( + dtype_ids, row_count{static_cast(num_rows)}, profile, seed); } std::unique_ptr create_random_table(std::vector const& dtype_ids, diff --git a/cpp/benchmarks/common/generate_input.hpp b/cpp/benchmarks/common/generate_input.hpp index a2efdb819bf..3bc53e1b5c9 100644 --- a/cpp/benchmarks/common/generate_input.hpp +++ b/cpp/benchmarks/common/generate_input.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -313,8 +313,9 @@ class data_profile { } } - auto get_bool_probability_true() const { return bool_probability_true; } - auto get_null_probability() const { return null_probability; }; + [[nodiscard]] auto get_bool_probability_true() const { return bool_probability_true; } + [[nodiscard]] auto get_null_probability() const { return null_probability; }; + [[nodiscard]] auto get_valid_probability() const { return 1. - null_probability.value_or(0.); }; [[nodiscard]] auto get_cardinality() const { return cardinality; }; [[nodiscard]] auto get_avg_run_length() const { return avg_run_length; }; From 176f75b1da0559c024a62a98f13ff15491f18a95 Mon Sep 17 00:00:00 2001 From: Alessandro Bellina Date: Tue, 5 Mar 2024 12:47:40 -0600 Subject: [PATCH 349/384] [JNI] rmm based pinned pool (#15219) Part of https://github.com/rapidsai/cudf/issues/14782. This PR removes our old implementation of the java based pinned memory pool and replaces it with a jni layer on top of `rmm::pool_memory_resource` This PR does NOT set the default cuIO pinned host resource. That is happening after this PR goes in https://github.com/rapidsai/cudf/pull/15079. We'll need a follow on PR to change `PinnedMemoryPool.initialize` method to add an argument to set the cuIO pinned host resource. I have run with this and version of it that are shared with cuIO and I can't find regressions in NDS at SF3K. Note that we don't align anymore on our side. RMM is doing the same alignment we were doing before, using `std::max_align_t`. Note also that the rmm pool doesn't have a quick way to find out what the current size is. So we had some tests that were asserting for this, and I have removed the asserts. If we would like to get that back I am happy to work with RMM to figure out how to do that. Authors: - Alessandro Bellina (https://github.com/abellina) Approvers: - Jim Brennan (https://github.com/jbrennan333) URL: https://github.com/rapidsai/cudf/pull/15219 --- .../java/ai/rapids/cudf/PinnedMemoryPool.java | 281 ++---------------- java/src/main/java/ai/rapids/cudf/Rmm.java | 10 +- java/src/main/native/src/RmmJni.cpp | 45 +++ .../ai/rapids/cudf/HostMemoryBufferTest.java | 4 +- .../ai/rapids/cudf/PinnedMemoryPoolTest.java | 37 ++- 5 files changed, 108 insertions(+), 269 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/PinnedMemoryPool.java b/java/src/main/java/ai/rapids/cudf/PinnedMemoryPool.java index 9ce72ba237e..17f05a9baf6 100644 --- a/java/src/main/java/ai/rapids/cudf/PinnedMemoryPool.java +++ b/java/src/main/java/ai/rapids/cudf/PinnedMemoryPool.java @@ -1,6 +1,6 @@ /* * - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,93 +22,30 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Comparator; -import java.util.Iterator; import java.util.Objects; -import java.util.Optional; -import java.util.SortedSet; -import java.util.TreeSet; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; /** - * This provides a pool of pinned memory similar to what RMM does for device memory. + * This is the JNI interface to a rmm::pool_memory_resource. */ public final class PinnedMemoryPool implements AutoCloseable { private static final Logger log = LoggerFactory.getLogger(PinnedMemoryPool.class); - private static final long ALIGNMENT = ColumnView.hostPaddingSizeInBytes(); // These static fields should only ever be accessed when class-synchronized. // Do NOT use singleton_ directly! Use the getSingleton accessor instead. private static volatile PinnedMemoryPool singleton_ = null; private static Future initFuture = null; - - private final long totalPoolSize; - private final long pinnedPoolBase; - private final SortedSet freeHeap = new TreeSet<>(new SortedByAddress()); - private int numAllocatedSections = 0; - private long availableBytes; - - private static class SortedBySize implements Comparator { - @Override - public int compare(MemorySection s0, MemorySection s1) { - return Long.compare(s0.size, s1.size); - } - } - - private static class SortedByAddress implements Comparator { - @Override - public int compare(MemorySection s0, MemorySection s1) { - return Long.compare(s0.baseAddress, s1.baseAddress); - } - } - - private static class MemorySection { - private long baseAddress; - private long size; - - MemorySection(long baseAddress, long size) { - this.baseAddress = baseAddress; - this.size = size; - } - - boolean canCombine(MemorySection other) { - boolean ret = (other.baseAddress + other.size) == baseAddress || - (baseAddress + size) == other.baseAddress; - log.trace("CAN {} COMBINE WITH {} ? {}", this, other, ret); - return ret; - } - - void combineWith(MemorySection other) { - assert canCombine(other); - log.trace("COMBINING {} AND {}", this, other); - this.baseAddress = Math.min(baseAddress, other.baseAddress); - this.size = other.size + this.size; - log.trace("COMBINED TO {}\n", this); - } - - MemorySection splitOff(long newSize) { - assert this.size > newSize; - MemorySection ret = new MemorySection(baseAddress, newSize); - this.baseAddress += newSize; - this.size -= newSize; - return ret; - } - - @Override - public String toString() { - return "PINNED: " + size + " bytes (0x" + Long.toHexString(baseAddress) - + " to 0x" + Long.toHexString(baseAddress + size) + ")"; - } - } + private long poolHandle; + private long poolSize; private static final class PinnedHostBufferCleaner extends MemoryBuffer.MemoryBufferCleaner { - private MemorySection section; + private long address; private final long origLength; - PinnedHostBufferCleaner(MemorySection section, long length) { - this.section = section; + PinnedHostBufferCleaner(long address, long length) { + this.address = address; origLength = length; } @@ -116,15 +53,15 @@ private static final class PinnedHostBufferCleaner extends MemoryBuffer.MemoryBu protected synchronized boolean cleanImpl(boolean logErrorIfNotClean) { boolean neededCleanup = false; long origAddress = 0; - if (section != null) { - origAddress = section.baseAddress; + if (address != -1) { + origAddress = address; try { - PinnedMemoryPool.freeInternal(section); + PinnedMemoryPool.freeInternal(address, origLength); } finally { // Always mark the resource as freed even if an exception is thrown. // We cannot know how far it progressed before the exception, and // therefore it is unsafe to retry. - section = null; + address = -1; } neededCleanup = true; } @@ -137,7 +74,7 @@ protected synchronized boolean cleanImpl(boolean logErrorIfNotClean) { @Override public boolean isClean() { - return section == null; + return address == -1; } } @@ -161,16 +98,8 @@ private static PinnedMemoryPool getSingleton() { return singleton_; } - private static void freeInternal(MemorySection section) { - Objects.requireNonNull(getSingleton()).free(section); - } - - /** - * Used to indicate that memory was allocated from a reservation. This primarily is for - * keeping track of outstanding allocations. - */ - private static void reserveAllocInternal(MemorySection section) { - Objects.requireNonNull(getSingleton()).reserveAllocHappened(section); + private static void freeInternal(long address, long origLength) { + Objects.requireNonNull(getSingleton()).free(address, origLength); } /** @@ -209,12 +138,14 @@ public static boolean isInitialized() { } /** - * Shut down the pool of memory. If there are outstanding allocations this may fail. + * Shut down the RMM pool_memory_resource, nulling out our reference. Any allocation + * or free that is in flight will fail after this. */ public static synchronized void shutdown() { PinnedMemoryPool pool = getSingleton(); if (pool != null) { pool.close(); + pool = null; } initFuture = null; singleton_ = null; @@ -235,21 +166,6 @@ public static HostMemoryBuffer tryAllocate(long bytes) { return result; } - /** - * Factory method to create a pinned host memory reservation. - * - * @param bytes size in bytes to reserve - * @return newly created reservation or null if insufficient pinned memory to cover it. - */ - public static HostMemoryReservation tryReserve(long bytes) { - HostMemoryReservation result = null; - PinnedMemoryPool pool = getSingleton(); - if (pool != null) { - result = pool.tryReserveInternal(bytes); - } - return result; - } - /** * Factory method to create a host buffer but preferably pointing to pinned memory. * It is not guaranteed that the returned buffer will be pointer to pinned memory. @@ -276,26 +192,13 @@ public static HostMemoryBuffer allocate(long bytes) { return allocate(bytes, DefaultHostMemoryAllocator.get()); } - /** - * Get the number of bytes free in the pinned memory pool. - * - * @return amount of free memory in bytes or 0 if the pool is not initialized - */ - public static long getAvailableBytes() { - PinnedMemoryPool pool = getSingleton(); - if (pool != null) { - return pool.getAvailableBytesInternal(); - } - return 0; - } - /** * Get the number of bytes that the pinned memory pool was allocated with. */ public static long getTotalPoolSizeBytes() { PinnedMemoryPool pool = getSingleton(); if (pool != null) { - return pool.getTotalPoolSizeInternal(); + return pool.poolSize; } return 0; } @@ -306,157 +209,31 @@ private PinnedMemoryPool(long poolSize, int gpuId) { Cuda.setDevice(gpuId); Cuda.freeZero(); } - this.totalPoolSize = poolSize; - this.pinnedPoolBase = Cuda.hostAllocPinned(poolSize); - freeHeap.add(new MemorySection(pinnedPoolBase, poolSize)); - this.availableBytes = poolSize; + this.poolHandle = Rmm.newPinnedPoolMemoryResource(poolSize, poolSize); + this.poolSize = poolSize; } @Override public void close() { - assert numAllocatedSections == 0 : "Leaked " + numAllocatedSections + " pinned allocations"; - Cuda.freePinned(pinnedPoolBase); + Rmm.releasePinnedPoolMemoryResource(this.poolHandle); + this.poolHandle = -1; } /** - * Pads a length of bytes to the alignment the CPU wants in the worst case. This helps to - * calculate the size needed for a reservation if there are multiple buffers. - * @param bytes the size in bytes - * @return the new padded size in bytes. + * This makes an attempt to allocate pinned memory, and if the pinned memory allocation fails + * it will return null, instead of throw. */ - public static long padToCpuAlignment(long bytes) { - return ((bytes + ALIGNMENT - 1) / ALIGNMENT) * ALIGNMENT; - } - - private synchronized MemorySection tryGetInternal(long bytes, String what) { - if (freeHeap.isEmpty()) { - log.debug("No free pinned memory left"); - return null; - } - // Align the allocation - long alignedBytes = padToCpuAlignment(bytes); - Optional firstFit = freeHeap.stream() - .filter(section -> section.size >= alignedBytes) - .findFirst(); - if (!firstFit.isPresent()) { - if (log.isDebugEnabled()) { - MemorySection largest = freeHeap.stream() - .max(new SortedBySize()) - .orElse(new MemorySection(0, 0)); - log.debug("Insufficient pinned memory. {} needed, {} found", alignedBytes, largest.size); - } - return null; - } - MemorySection first = firstFit.get(); - log.debug("{} {}/{} bytes pinned from {} FREE COUNT {} OUTSTANDING COUNT {}", - what, bytes, alignedBytes, first, freeHeap.size(), numAllocatedSections); - freeHeap.remove(first); - MemorySection allocated; - if (first.size == alignedBytes) { - allocated = first; - } else { - allocated = first.splitOff(alignedBytes); - freeHeap.add(first); - } - numAllocatedSections++; - availableBytes -= allocated.size; - log.debug("{} {} free {} outstanding {}", what, allocated, freeHeap, numAllocatedSections); - return allocated; - } - private synchronized HostMemoryBuffer tryAllocateInternal(long bytes) { - MemorySection allocated = tryGetInternal(bytes, "allocate"); - if (allocated == null) { + long allocated = Rmm.allocFromPinnedPool(this.poolHandle, bytes); + if (allocated == -1) { return null; } else { - return new HostMemoryBuffer(allocated.baseAddress, bytes, + return new HostMemoryBuffer(allocated, bytes, new PinnedHostBufferCleaner(allocated, bytes)); } } - private class PinnedReservation implements HostMemoryReservation { - private MemorySection section = null; - - public PinnedReservation(MemorySection section) { - this.section = section; - } - - @Override - public synchronized HostMemoryBuffer allocate(long bytes, boolean preferPinned) { - return this.allocate(bytes); - } - - @Override - public synchronized HostMemoryBuffer allocate(long bytes) { - if (section == null || section.size < bytes) { - throw new OutOfMemoryError("Reservation didn't have enough space " + bytes + " / " + - (section == null ? 0 : section.size)); - } - long alignedSize = padToCpuAlignment(bytes); - MemorySection allocated; - if (section.size >= bytes && section.size <= alignedSize) { - allocated = section; - section = null; - // No need for reserveAllocInternal because the original section is already tracked - } else { - allocated = section.splitOff(alignedSize); - PinnedMemoryPool.reserveAllocInternal(allocated); - } - return new HostMemoryBuffer(allocated.baseAddress, bytes, - new PinnedHostBufferCleaner(allocated, bytes)); - } - - @Override - public synchronized void close() throws Exception { - if (section != null) { - try { - PinnedMemoryPool.freeInternal(section); - } finally { - // Always mark the resource as freed even if an exception is thrown. - // We cannot know how far it progressed before the exception, and - // therefore it is unsafe to retry. - section = null; - } - } - } - } - - private HostMemoryReservation tryReserveInternal(long bytes) { - MemorySection allocated = tryGetInternal(bytes, "allocate"); - if (allocated == null) { - return null; - } else { - return new PinnedReservation(allocated); - } - } - - private synchronized void free(MemorySection section) { - log.debug("Freeing {} with {} outstanding {}", section, freeHeap, numAllocatedSections); - availableBytes += section.size; - Iterator it = freeHeap.iterator(); - while(it.hasNext()) { - MemorySection current = it.next(); - if (section.canCombine(current)) { - it.remove(); - section.combineWith(current); - } - } - freeHeap.add(section); - numAllocatedSections--; - log.debug("After freeing {} outstanding {}", freeHeap, numAllocatedSections); - } - - private synchronized void reserveAllocHappened(MemorySection section) { - if (section != null && section.size > 0) { - numAllocatedSections++; - } - } - - private synchronized long getAvailableBytesInternal() { - return this.availableBytes; - } - - private long getTotalPoolSizeInternal() { - return this.totalPoolSize; + private synchronized void free(long address, long size) { + Rmm.freeFromPinnedPool(this.poolHandle, address, size); } } diff --git a/java/src/main/java/ai/rapids/cudf/Rmm.java b/java/src/main/java/ai/rapids/cudf/Rmm.java index 66c053f15b2..552da62382a 100755 --- a/java/src/main/java/ai/rapids/cudf/Rmm.java +++ b/java/src/main/java/ai/rapids/cudf/Rmm.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -581,4 +581,12 @@ static native long newEventHandlerResourceAdaptor(long handle, long trackerHandl static native long releaseEventHandlerResourceAdaptor(long handle, boolean debug); private static native void setCurrentDeviceResourceInternal(long newHandle); + + public static native long newPinnedPoolMemoryResource(long initSize, long maxSize); + + public static native void releasePinnedPoolMemoryResource(long poolPtr); + + public static native long allocFromPinnedPool(long poolPtr, long size); + + public static native void freeFromPinnedPool(long poolPtr, long ptr, long size); } diff --git a/java/src/main/native/src/RmmJni.cpp b/java/src/main/native/src/RmmJni.cpp index 81b8241bab0..7b81b5ff4de 100644 --- a/java/src/main/native/src/RmmJni.cpp +++ b/java/src/main/native/src/RmmJni.cpp @@ -31,11 +31,13 @@ #include #include #include +#include #include "cudf_jni_apis.hpp" using rmm::mr::device_memory_resource; using rmm::mr::logging_resource_adaptor; +using rmm_pinned_pool_t = rmm::mr::pool_memory_resource; namespace { @@ -746,4 +748,47 @@ JNIEXPORT void JNICALL Java_ai_rapids_cudf_Rmm_setCurrentDeviceResourceInternal( } CATCH_STD(env, ) } + +JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Rmm_newPinnedPoolMemoryResource(JNIEnv *env, + jclass clazz, + jlong init, jlong max) { + try { + cudf::jni::auto_set_device(env); + auto pool = new rmm_pinned_pool_t(new rmm::mr::pinned_host_memory_resource(), init, max); + return reinterpret_cast(pool); + } + CATCH_STD(env, 0) +} + +JNIEXPORT void JNICALL Java_ai_rapids_cudf_Rmm_releasePinnedPoolMemoryResource(JNIEnv *env, + jclass clazz, + jlong pool_ptr) { + try { + cudf::jni::auto_set_device(env); + delete reinterpret_cast(pool_ptr); + } + CATCH_STD(env, ) +} + +JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Rmm_allocFromPinnedPool(JNIEnv *env, jclass clazz, + jlong pool_ptr, jlong size) { + try { + cudf::jni::auto_set_device(env); + auto pool = reinterpret_cast(pool_ptr); + void *ret = pool->allocate(size); + return reinterpret_cast(ret); + } catch (const std::exception &unused) { return -1; } +} + +JNIEXPORT void JNICALL Java_ai_rapids_cudf_Rmm_freeFromPinnedPool(JNIEnv *env, jclass clazz, + jlong pool_ptr, jlong ptr, + jlong size) { + try { + cudf::jni::auto_set_device(env); + auto pool = reinterpret_cast(pool_ptr); + void *cptr = reinterpret_cast(ptr); + pool->deallocate(cptr, size); + } + CATCH_STD(env, ) +} } diff --git a/java/src/test/java/ai/rapids/cudf/HostMemoryBufferTest.java b/java/src/test/java/ai/rapids/cudf/HostMemoryBufferTest.java index e848d4a89bf..b7fde511c38 100644 --- a/java/src/test/java/ai/rapids/cudf/HostMemoryBufferTest.java +++ b/java/src/test/java/ai/rapids/cudf/HostMemoryBufferTest.java @@ -1,6 +1,6 @@ /* * - * Copyright (c) 2019-2020, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -187,7 +187,7 @@ public void testFilemap() throws Exception { } public static void initPinnedPoolIfNeeded(long size) { - long available = PinnedMemoryPool.getAvailableBytes(); + long available = PinnedMemoryPool.getTotalPoolSizeBytes(); if (available < size) { if (PinnedMemoryPool.isInitialized()) { PinnedMemoryPool.shutdown(); diff --git a/java/src/test/java/ai/rapids/cudf/PinnedMemoryPoolTest.java b/java/src/test/java/ai/rapids/cudf/PinnedMemoryPoolTest.java index 16628d7be36..8c6e29dbd0c 100644 --- a/java/src/test/java/ai/rapids/cudf/PinnedMemoryPoolTest.java +++ b/java/src/test/java/ai/rapids/cudf/PinnedMemoryPoolTest.java @@ -1,6 +1,6 @@ /* * - * Copyright (c) 2019, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ package ai.rapids.cudf; +import java.nio.ByteBuffer; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.slf4j.Logger; @@ -80,35 +81,27 @@ void allocate() { void testFragmentationAndExhaustion() { final long poolSize = 15 * 1024L; PinnedMemoryPool.initialize(poolSize); - assertEquals(poolSize, PinnedMemoryPool.getAvailableBytes()); + assertEquals(poolSize, PinnedMemoryPool.getTotalPoolSizeBytes()); HostMemoryBuffer[] buffers = new HostMemoryBuffer[5]; try { buffers[0] = PinnedMemoryPool.tryAllocate(1024); assertNotNull(buffers[0]); - assertEquals(14*1024L, PinnedMemoryPool.getAvailableBytes()); buffers[1] = PinnedMemoryPool.tryAllocate(2048); assertNotNull(buffers[1]); - assertEquals(12*1024L, PinnedMemoryPool.getAvailableBytes()); buffers[2] = PinnedMemoryPool.tryAllocate(4096); assertNotNull(buffers[2]); - assertEquals(8*1024L, PinnedMemoryPool.getAvailableBytes()); buffers[1].close(); - assertEquals(10*1024L, PinnedMemoryPool.getAvailableBytes()); buffers[1] = null; buffers[1] = PinnedMemoryPool.tryAllocate(8192); assertNotNull(buffers[1]); - assertEquals(2*1024L, PinnedMemoryPool.getAvailableBytes()); buffers[3] = PinnedMemoryPool.tryAllocate(2048); assertNotNull(buffers[3]); - assertEquals(0L, PinnedMemoryPool.getAvailableBytes()); buffers[4] = PinnedMemoryPool.tryAllocate(64); assertNull(buffers[4]); buffers[0].close(); - assertEquals(1024L, PinnedMemoryPool.getAvailableBytes()); buffers[0] = null; buffers[4] = PinnedMemoryPool.tryAllocate(64); assertNotNull(buffers[4]); - assertEquals(1024L - 64, PinnedMemoryPool.getAvailableBytes()); } finally { for (HostMemoryBuffer buffer : buffers) { if (buffer != null) { @@ -116,19 +109,35 @@ void testFragmentationAndExhaustion() { } } } - assertEquals(poolSize, PinnedMemoryPool.getAvailableBytes()); + } + + @Test + void testTouchPinnedMemory() { + final long poolSize = 15 * 1024L; + PinnedMemoryPool.initialize(poolSize); + int bufLength = 256; + try(HostMemoryBuffer hmb = PinnedMemoryPool.allocate(bufLength); + HostMemoryBuffer hmb2 = PinnedMemoryPool.allocate(bufLength)) { + ByteBuffer bb = hmb.asByteBuffer(0, bufLength); + for (int i = 0; i < bufLength; i++) { + bb.put(i, (byte)i); + } + hmb2.copyFromHostBuffer(0, hmb, 0, bufLength); + ByteBuffer bb2 = hmb2.asByteBuffer(0, bufLength); + for (int i = 0; i < bufLength; i++) { + assertEquals(bb.get(i), bb2.get(i)); + } + } } @Test void testZeroSizedAllocation() { final long poolSize = 4 * 1024L; PinnedMemoryPool.initialize(poolSize); - assertEquals(poolSize, PinnedMemoryPool.getAvailableBytes()); + assertEquals(poolSize, PinnedMemoryPool.getTotalPoolSizeBytes()); try (HostMemoryBuffer buffer = PinnedMemoryPool.tryAllocate(0)) { assertNotNull(buffer); assertEquals(0, buffer.getLength()); - assertEquals(poolSize, PinnedMemoryPool.getAvailableBytes()); } - assertEquals(poolSize, PinnedMemoryPool.getAvailableBytes()); } } From 3ea947a7b22e76c741cc6b076bd09cd53ea64f3c Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Tue, 5 Mar 2024 12:45:31 -0800 Subject: [PATCH 350/384] Use `hostdevice_vector` in `kernel_error` to avoid the pageable copy (#15140) Issue #15122 The addition of kernel error checking introduced a 5% performance regression in Spark-RAPIDS. It was determined that the pageable copy of the error back to host caused this overhead, presumably because of the CUDA's bounce buffer bottleneck. This PR aims to eliminate most of the error checking overhead by using `hostdevice_vector` in the `kernel_error` class. The `hostdevice_vector` uses pinned memory so the copy is no longer pageable. The PR also removes the redundant sync after we read the error. Authors: - Vukasin Milovanovic (https://github.com/vuule) Approvers: - Mike Wilson (https://github.com/hyperbolic2346) - Paul Mattione (https://github.com/pmattione-nvidia) URL: https://github.com/rapidsai/cudf/pull/15140 --- cpp/src/io/parquet/error.hpp | 32 +++++++++++++------- cpp/src/io/parquet/reader_impl.cpp | 6 ++-- cpp/src/io/parquet/reader_impl_preprocess.cu | 15 ++++----- cpp/src/io/utilities/hostdevice_span.hpp | 1 + 4 files changed, 32 insertions(+), 22 deletions(-) diff --git a/cpp/src/io/parquet/error.hpp b/cpp/src/io/parquet/error.hpp index bff0713a1ef..4e2eb4c66d3 100644 --- a/cpp/src/io/parquet/error.hpp +++ b/cpp/src/io/parquet/error.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,8 @@ #pragma once #include -#include + +#include #include #include @@ -37,7 +38,7 @@ class kernel_error { using pointer = value_type*; private: - rmm::device_scalar _error_code; + mutable cudf::detail::hostdevice_vector _error_code; public: /** @@ -50,30 +51,39 @@ class kernel_error { * * @param CUDA stream to use */ - kernel_error(rmm::cuda_stream_view stream) : _error_code{0, stream} {} + kernel_error(rmm::cuda_stream_view stream) : _error_code(1, stream) + { + _error_code[0] = 0; + _error_code.host_to_device_async(stream); + } /** * @brief Return a pointer to the device memory for the error */ - [[nodiscard]] auto data() { return _error_code.data(); } + [[nodiscard]] auto data() { return _error_code.device_ptr(); } /** * @brief Return the current value of the error * - * This uses the stream used to create this instance. This does a synchronize on the stream - * this object was instantiated with. + * @param stream The CUDA stream to synchronize with */ - [[nodiscard]] auto value() const { return _error_code.value(_error_code.stream()); } + [[nodiscard]] auto value_sync(rmm::cuda_stream_view stream) const + { + _error_code.device_to_host_sync(stream); + return _error_code[0]; + } /** - * @brief Return a hexadecimal string representation of the current error code + * @brief Return a hexadecimal string representation of an error code * * Returned string will have "0x" prepended. + * + * @param value The error code to convert to a string */ - [[nodiscard]] std::string str() const + [[nodiscard]] static std::string to_string(value_type value) { std::stringstream sstream; - sstream << std::hex << value(); + sstream << std::hex << value; return "0x" + sstream.str(); } }; diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 93fc6bd6bb5..207f908febf 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -246,11 +246,9 @@ void reader::impl::decode_page_data(size_t skip_rows, size_t num_rows) page_nesting.device_to_host_async(_stream); page_nesting_decode.device_to_host_async(_stream); - if (error_code.value() != 0) { - CUDF_FAIL("Parquet data decode failed with code(s) " + error_code.str()); + if (auto const error = error_code.value_sync(_stream); error != 0) { + CUDF_FAIL("Parquet data decode failed with code(s) " + kernel_error::to_string(error)); } - // error_code.value() is synchronous; explicitly sync here for better visibility - _stream.synchronize(); // for list columns, add the final offset to every offset buffer. // TODO : make this happen in more efficiently. Maybe use thrust::for_each diff --git a/cpp/src/io/parquet/reader_impl_preprocess.cu b/cpp/src/io/parquet/reader_impl_preprocess.cu index c524547c4d7..aa4f96aa2e0 100644 --- a/cpp/src/io/parquet/reader_impl_preprocess.cu +++ b/cpp/src/io/parquet/reader_impl_preprocess.cu @@ -296,10 +296,10 @@ void generate_depth_remappings(std::map, std::ve // so that we can actually compile a list of all the unsupported encodings found // in the pages. That cannot be done here since we do not have the pages vector here. // see https://github.com/rapidsai/cudf/pull/14453#pullrequestreview-1778346688 - if (error_code.value() != 0 and - error_code.value() != static_cast(decode_error::UNSUPPORTED_ENCODING)) { + if (auto const error = error_code.value_sync(stream); + error != 0 and error != static_cast(decode_error::UNSUPPORTED_ENCODING)) { CUDF_FAIL("Parquet header parsing failed with code(s) while counting page headers " + - error_code.str()); + kernel_error::to_string(error)); } for (size_t c = 0; c < chunks.size(); c++) { @@ -480,13 +480,14 @@ void decode_page_headers(pass_intermediate_data& pass, error_code.data(), stream); - if (error_code.value() != 0) { - if (BitAnd(error_code.value(), decode_error::UNSUPPORTED_ENCODING) != 0) { + if (auto const error = error_code.value_sync(stream); error != 0) { + if (BitAnd(error, decode_error::UNSUPPORTED_ENCODING) != 0) { auto const unsupported_str = ". With unsupported encodings found: " + list_unsupported_encodings(pass.pages, stream); - CUDF_FAIL("Parquet header parsing failed with code(s) " + error_code.str() + unsupported_str); + CUDF_FAIL("Parquet header parsing failed with code(s) " + kernel_error::to_string(error) + + unsupported_str); } else { - CUDF_FAIL("Parquet header parsing failed with code(s) " + error_code.str()); + CUDF_FAIL("Parquet header parsing failed with code(s) " + kernel_error::to_string(error)); } } diff --git a/cpp/src/io/utilities/hostdevice_span.hpp b/cpp/src/io/utilities/hostdevice_span.hpp index ec5e0410bc0..c9a58ab31cf 100644 --- a/cpp/src/io/utilities/hostdevice_span.hpp +++ b/cpp/src/io/utilities/hostdevice_span.hpp @@ -16,6 +16,7 @@ #pragma once +#include #include #include From 2d1e3c7fba0801453e5e93bae6942d1e02da33e9 Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Tue, 5 Mar 2024 13:10:41 -0800 Subject: [PATCH 351/384] Ignore `byte_range` in `read_json` when the size is not smaller than the input data (#15180) Deduce that the entire file will the loaded when byte_range is not smaller than the input size and use the faster "no byte_range" path. Avoids double IO that happens with regular `byte_range` code path. Authors: - Vukasin Milovanovic (https://github.com/vuule) Approvers: - Shruti Shivakumar (https://github.com/shrshi) - Mike Wilson (https://github.com/hyperbolic2346) URL: https://github.com/rapidsai/cudf/pull/15180 --- cpp/src/io/json/read_json.cu | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cpp/src/io/json/read_json.cu b/cpp/src/io/json/read_json.cu index 506d7b6cddc..b03e0dd452b 100644 --- a/cpp/src/io/json/read_json.cu +++ b/cpp/src/io/json/read_json.cu @@ -140,10 +140,11 @@ size_type find_first_delimiter_in_chunk(host_span= source_size); } /** @@ -168,7 +169,7 @@ auto get_record_range_raw_input(host_span> sources, reader_opts.get_byte_range_offset(), reader_opts.get_byte_range_size(), stream); - if (should_load_whole_source(reader_opts)) return buffer; + if (should_load_whole_source(reader_opts, sources[0]->size())) return buffer; auto first_delim_pos = reader_opts.get_byte_range_offset() == 0 ? 0 : find_first_delimiter(buffer, '\n', stream); if (first_delim_pos == -1) { @@ -212,7 +213,7 @@ table_with_metadata read_json(host_span> sources, return legacy::read_json(sources, reader_opts, stream, mr); } - if (not should_load_whole_source(reader_opts)) { + if (reader_opts.get_byte_range_offset() != 0 or reader_opts.get_byte_range_size() != 0) { CUDF_EXPECTS(reader_opts.is_enabled_lines(), "Specifying a byte range is supported only for JSON Lines"); CUDF_EXPECTS(sources.size() == 1, From b60bf182b3b5bd425cbc1ad49a92de72010afc98 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 5 Mar 2024 13:55:38 -0800 Subject: [PATCH 352/384] Clean up usage of __CUDA_ARCH__ and other macros. (#15218) Closes #15030. This PR cleans up references to `__CUDA_ARCH__` and other macros. - We can safely drop Pascal support now that the required minimum is Volta (`__CUDA_ARCH__` of 700). - Removed a leftover reference to CUDA 10. - Removed an instance of `#if 1` that was no longer needed. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Michael Schellenberger Costa (https://github.com/miscco) - Nghia Truong (https://github.com/ttnghia) - Mike Wilson (https://github.com/hyperbolic2346) URL: https://github.com/rapidsai/cudf/pull/15218 --- .../cudf/detail/utilities/device_atomics.cuh | 3 - cpp/src/filling/repeat.cu | 9 +- cpp/src/hash/managed.cuh | 4 - cpp/src/io/comp/snap.cu | 10 -- cpp/src/io/fst/agent_dfa.cuh | 2 +- cpp/src/transform/row_conversion.cu | 100 +----------------- 6 files changed, 8 insertions(+), 120 deletions(-) diff --git a/cpp/include/cudf/detail/utilities/device_atomics.cuh b/cpp/include/cudf/detail/utilities/device_atomics.cuh index 1e3fe3d08dc..6f23abc59a8 100644 --- a/cpp/include/cudf/detail/utilities/device_atomics.cuh +++ b/cpp/include/cudf/detail/utilities/device_atomics.cuh @@ -170,8 +170,6 @@ struct genericAtomicOperationImpl { } }; -#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 600) -// `atomicAdd(double)` is supported after cuda architecture 6.0 template <> struct genericAtomicOperationImpl { using T = double; @@ -180,7 +178,6 @@ struct genericAtomicOperationImpl { return atomicAdd(addr, update_value); } }; -#endif template <> struct genericAtomicOperationImpl { diff --git a/cpp/src/filling/repeat.cu b/cpp/src/filling/repeat.cu index bd53eeddbb5..87cc0f21d0e 100644 --- a/cpp/src/filling/repeat.cu +++ b/cpp/src/filling/repeat.cu @@ -55,13 +55,8 @@ struct count_accessor { std::enable_if_t, cudf::size_type> operator()(rmm::cuda_stream_view stream) { using ScalarType = cudf::scalar_type_t; -#if 1 - // TODO: temporary till cudf::scalar's value() function is marked as const - auto p_count = const_cast(static_cast(this->p_scalar)); -#else - auto p_count = static_cast(this->p_scalar); -#endif - auto count = p_count->value(stream); + auto p_count = static_cast(this->p_scalar); + auto count = p_count->value(stream); // static_cast is necessary due to bool CUDF_EXPECTS(static_cast(count) <= std::numeric_limits::max(), "count should not exceed the column size limit", diff --git a/cpp/src/hash/managed.cuh b/cpp/src/hash/managed.cuh index aa7bff85ea6..9797c83c47c 100644 --- a/cpp/src/hash/managed.cuh +++ b/cpp/src/hash/managed.cuh @@ -37,9 +37,5 @@ struct managed { inline bool isPtrManaged(cudaPointerAttributes attr) { -#if CUDART_VERSION >= 10000 return (attr.type == cudaMemoryTypeManaged); -#else - return attr.isManaged; -#endif } diff --git a/cpp/src/io/comp/snap.cu b/cpp/src/io/comp/snap.cu index 252c96f496a..7d4dcffa713 100644 --- a/cpp/src/io/comp/snap.cu +++ b/cpp/src/io/comp/snap.cu @@ -153,17 +153,7 @@ static __device__ uint8_t* StoreCopy(uint8_t* dst, */ static inline __device__ uint32_t HashMatchAny(uint32_t v, uint32_t t) { -#if (__CUDA_ARCH__ >= 700) return __match_any_sync(~0, v); -#else - uint32_t err_map = 0; - for (uint32_t i = 0; i < hash_bits; i++, v >>= 1) { - uint32_t b = v & 1; - uint32_t match_b = ballot(b); - err_map |= match_b ^ -(int32_t)b; - } - return ~err_map; -#endif } /** diff --git a/cpp/src/io/fst/agent_dfa.cuh b/cpp/src/io/fst/agent_dfa.cuh index 9ba8696370a..2171764decd 100644 --- a/cpp/src/io/fst/agent_dfa.cuh +++ b/cpp/src/io/fst/agent_dfa.cuh @@ -91,7 +91,7 @@ class DFASimulationCallbackWrapper { { uint32_t const count = transducer_table(old_state, symbol_id, read_symbol); if (write) { -#if __CUDA_ARCH__ > 0 +#if defined(__CUDA_ARCH__) #pragma unroll 1 #endif for (uint32_t out_char = 0; out_char < count; out_char++) { diff --git a/cpp/src/transform/row_conversion.cu b/cpp/src/transform/row_conversion.cu index 32faa097d0e..359e1ccb80d 100644 --- a/cpp/src/transform/row_conversion.cu +++ b/cpp/src/transform/row_conversion.cu @@ -39,24 +39,14 @@ #include #include +#include +#include #include #include #include #include #include -#include - -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700 -#define ASYNC_MEMCPY_SUPPORTED -#endif - -#if !defined(__CUDA_ARCH__) || defined(ASYNC_MEMCPY_SUPPORTED) -#include -#endif // #if !defined(__CUDA_ARCH__) || defined(ASYNC_MEMCPY_SUPPORTED) - -#include - #include #include #include @@ -65,6 +55,7 @@ #include #include #include +#include namespace { @@ -90,13 +81,6 @@ using detail::make_device_uvector_async; using detail::make_device_uvector_sync; using rmm::device_uvector; -#ifdef ASYNC_MEMCPY_SUPPORTED -using cuda::aligned_size_t; -#else -template -using aligned_size_t = size_t; // Local stub for cuda::aligned_size_t. -#endif // ASYNC_MEMCPY_SUPPORTED - namespace cudf { namespace detail { @@ -569,12 +553,6 @@ CUDF_KERNEL void copy_to_rows_fixed_width_optimized(const size_type start_row, } } -#ifdef ASYNC_MEMCPY_SUPPORTED -#define MEMCPY(dst, src, size, barrier) cuda::memcpy_async(dst, src, size, barrier) -#else -#define MEMCPY(dst, src, size, barrier) memcpy(dst, src, size) -#endif // ASYNC_MEMCPY_SUPPORTED - /** * @brief copy data from cudf columns into JCUDF format, which is row-based * @@ -615,11 +593,9 @@ CUDF_KERNEL void copy_to_rows(const size_type num_rows, auto const warp = cooperative_groups::tiled_partition(group); extern __shared__ int8_t shared_data[]; -#ifdef ASYNC_MEMCPY_SUPPORTED __shared__ cuda::barrier tile_barrier; if (group.thread_rank() == 0) { init(&tile_barrier, group.size()); } group.sync(); -#endif // ASYNC_MEMCPY_SUPPORTED auto const tile = tile_infos[blockIdx.x]; auto const num_tile_cols = tile.num_cols(); @@ -702,21 +678,11 @@ CUDF_KERNEL void copy_to_rows(const size_type num_rows, auto const src = &shared_data[tile_row_size * copy_row]; auto const dst = tile_output_buffer + row_offsets(copy_row + tile.start_row, row_batch_start) + starting_column_offset; -#ifdef ASYNC_MEMCPY_SUPPORTED cuda::memcpy_async(warp, dst, src, tile_row_size, tile_barrier); -#else - for (int b = warp.thread_rank(); b < tile_row_size; b += warp.size()) { - dst[b] = src[b]; - } -#endif } -#ifdef ASYNC_MEMCPY_SUPPORTED // wait on the last copies to complete tile_barrier.arrive_and_wait(); -#else - group.sync(); -#endif // ASYNC_MEMCPY_SUPPORTED } /** @@ -752,12 +718,10 @@ CUDF_KERNEL void copy_validity_to_rows(const size_type num_rows, auto const group = cooperative_groups::this_thread_block(); auto const warp = cooperative_groups::tiled_partition(group); -#ifdef ASYNC_MEMCPY_SUPPORTED // Initialize cuda barriers for each tile. __shared__ cuda::barrier shared_tile_barrier; if (group.thread_rank() == 0) { init(&shared_tile_barrier, group.size()); } group.sync(); -#endif // ASYNC_MEMCPY_SUPPORTED auto tile = tile_infos[blockIdx.x]; auto const num_tile_cols = tile.num_cols(); @@ -822,21 +786,11 @@ CUDF_KERNEL void copy_validity_to_rows(const size_type num_rows, relative_row += warp.meta_group_size()) { auto const src = &shared_data[validity_data_row_length * relative_row]; auto const dst = output_data_base + row_offsets(relative_row + tile.start_row, row_batch_start); -#ifdef ASYNC_MEMCPY_SUPPORTED cuda::memcpy_async(warp, dst, src, row_bytes, shared_tile_barrier); -#else - for (int b = warp.thread_rank(); b < row_bytes; b += warp.size()) { - dst[b] = src[b]; - } -#endif } -#ifdef ASYNC_MEMCPY_SUPPORTED // wait for tile of data to arrive shared_tile_barrier.arrive_and_wait(); -#else - group.sync(); -#endif // ASYNC_MEMCPY_SUPPORTED } /** @@ -871,9 +825,7 @@ CUDF_KERNEL void copy_strings_to_rows(size_type const num_rows, // memcpy of the string data. auto const my_block = cooperative_groups::this_thread_block(); auto const warp = cooperative_groups::tiled_partition(my_block); -#ifdef ASYNC_MEMCPY_SUPPORTED cuda::barrier block_barrier; -#endif auto const start_row = blockIdx.x * NUM_STRING_ROWS_PER_BLOCK_TO_ROWS + warp.meta_group_rank() + batch_row_offset; @@ -896,13 +848,7 @@ CUDF_KERNEL void copy_strings_to_rows(size_type const num_rows, auto string_output_dest = &output_data[base_row_offset + offset]; auto string_output_src = &variable_input_data[col][string_start_offset]; warp.sync(); -#ifdef ASYNC_MEMCPY_SUPPORTED cuda::memcpy_async(warp, string_output_dest, string_output_src, string_length, block_barrier); -#else - for (int c = warp.thread_rank(); c < string_length; c += warp.size()) { - string_output_dest[c] = string_output_src[c]; - } -#endif offset += string_length; } } @@ -950,12 +896,10 @@ CUDF_KERNEL void copy_from_rows(const size_type num_rows, auto const warp = cooperative_groups::tiled_partition(group); extern __shared__ int8_t shared[]; -#ifdef ASYNC_MEMCPY_SUPPORTED // Initialize cuda barriers for each tile. __shared__ cuda::barrier tile_barrier; if (group.thread_rank() == 0) { init(&tile_barrier, group.size()); } group.sync(); -#endif // ASYNC_MEMCPY_SUPPORTED { auto const fetch_tile = tile_infos[blockIdx.x]; @@ -973,13 +917,7 @@ CUDF_KERNEL void copy_from_rows(const size_type num_rows, auto dst = &shared[shared_offset]; auto src = &input_data[row_offsets(absolute_row, row_batch_start) + starting_col_offset]; // copy the data -#ifdef ASYNC_MEMCPY_SUPPORTED cuda::memcpy_async(warp, dst, src, fetch_tile_row_size, tile_barrier); -#else - for (int b = warp.thread_rank(); b < fetch_tile_row_size; b += warp.size()) { - dst[b] = src[b]; - } -#endif } } @@ -989,12 +927,8 @@ CUDF_KERNEL void copy_from_rows(const size_type num_rows, auto const cols_in_tile = tile.num_cols(); auto const tile_row_size = tile.get_shared_row_size(col_offsets, col_sizes); -#ifdef ASYNC_MEMCPY_SUPPORTED // ensure our data is ready tile_barrier.arrive_and_wait(); -#else - group.sync(); -#endif // ASYNC_MEMCPY_SUPPORTED // Now we copy from shared memory to final destination. The data is laid out in rows in shared // memory, so the reads for a column will be "vertical". Because of this and the different sizes @@ -1017,17 +951,13 @@ CUDF_KERNEL void copy_from_rows(const size_type num_rows, int8_t* shmem_src = &shared[shared_memory_offset]; int8_t* dst = &output_data[absolute_col][absolute_row * column_size]; - MEMCPY(dst, shmem_src, column_size, tile_barrier); + cuda::memcpy_async(dst, shmem_src, column_size, tile_barrier); } } } -#ifdef ASYNC_MEMCPY_SUPPORTED // wait on the last copies to complete tile_barrier.arrive_and_wait(); -#else - group.sync(); -#endif // ASYNC_MEMCPY_SUPPORTED } /** @@ -1077,12 +1007,10 @@ CUDF_KERNEL void copy_validity_from_rows(const size_type num_rows, auto const group = cooperative_groups::this_thread_block(); auto const warp = cooperative_groups::tiled_partition(group); -#ifdef ASYNC_MEMCPY_SUPPORTED // Initialize cuda barriers for each tile. __shared__ cuda::barrier shared_tile_barrier; if (group.thread_rank() == 0) { init(&shared_tile_barrier, group.size()); } group.sync(); -#endif // ASYNC_MEMCPY_SUPPORTED auto const tile = tile_infos[blockIdx.x]; auto const tile_start_col = tile.start_col; @@ -1147,22 +1075,12 @@ CUDF_KERNEL void copy_validity_from_rows(const size_type num_rows, auto const src = reinterpret_cast(&shared[validity_data_col_length * relative_col]); -#ifdef ASYNC_MEMCPY_SUPPORTED cuda::memcpy_async( - warp, dst, src, aligned_size_t<4>(validity_data_col_length), shared_tile_barrier); -#else - for (int b = warp.thread_rank(); b < col_words; b += warp.size()) { - dst[b] = src[b]; - } -#endif + warp, dst, src, cuda::aligned_size_t<4>(validity_data_col_length), shared_tile_barrier); } -#ifdef ASYNC_MEMCPY_SUPPORTED // wait for tile of data to arrive shared_tile_barrier.arrive_and_wait(); -#else - group.sync(); -#endif // ASYNC_MEMCPY_SUPPORTED } /** @@ -1193,9 +1111,7 @@ CUDF_KERNEL void copy_strings_from_rows(RowOffsetFunctor row_offsets, // Traversing in row-major order to coalesce the offsets and size reads. auto my_block = cooperative_groups::this_thread_block(); auto warp = cooperative_groups::tiled_partition(my_block); -#ifdef ASYNC_MEMCPY_SUPPORTED cuda::barrier block_barrier; -#endif // workaround for not being able to take a reference to a constexpr host variable auto const ROWS_PER_BLOCK = NUM_STRING_ROWS_PER_BLOCK_FROM_ROWS; @@ -1216,13 +1132,7 @@ CUDF_KERNEL void copy_strings_from_rows(RowOffsetFunctor row_offsets, auto const src = &row_data[row_offsets(row, 0) + str_row_off[row]]; auto dst = &str_col_data[str_col_off[row]]; -#ifdef ASYNC_MEMCPY_SUPPORTED cuda::memcpy_async(warp, dst, src, str_len[row], block_barrier); -#else - for (int c = warp.thread_rank(); c < str_len[row]; c += warp.size()) { - dst[c] = src[c]; - } -#endif } } } From 13d807edff0fb2356e27da520451fafd8db106f2 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 5 Mar 2024 14:10:51 -0800 Subject: [PATCH 353/384] Generalize GHA selectors for pure Python testing (#15191) To eliminate hard-coding, generalize the GHA workflow logic to select one build for testing. This should simplify future updates. This is a follow-up to #15174. xref: https://github.com/rapidsai/build-planning/issues/25 Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Jake Awe (https://github.com/AyodeAwe) - https://github.com/jakirkham URL: https://github.com/rapidsai/cudf/pull/15191 --- .github/workflows/build.yaml | 3 ++- .github/workflows/pr.yaml | 9 ++++++--- .github/workflows/test.yaml | 3 ++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e60c47fae2b..ef2141ed934 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -92,7 +92,8 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.11" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4a662ed0f43..7599616a0c5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -128,7 +128,8 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.11" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request script: "ci/build_wheel_dask_cudf.sh" wheel-tests-dask-cudf: @@ -136,7 +137,8 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.11" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request script: ci/test_wheel_dask_cudf.sh devcontainer: @@ -154,7 +156,8 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.11" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request script: ci/cudf_pandas_scripts/run_tests.sh # pandas-tests: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e66b2e1f872..bc5eeb2777b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -99,7 +99,8 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.11" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} From e612a8aee5ba54f397b8d5be14201776bac9dd2d Mon Sep 17 00:00:00 2001 From: Nghia Truong <7416935+ttnghia@users.noreply.github.com> Date: Tue, 5 Mar 2024 16:42:41 -0700 Subject: [PATCH 354/384] Remove row conversion code from libcudf (#15234) This is to remove the row conversion code from libcudf. It was move from spark-rapids-jni (by https://github.com/rapidsai/cudf/pull/14664) to temporarily workaround the issue due to conflict of kernel names that causes invalid memory access when calling to `thrust::in(ex)clusive_scan` (https://github.com/NVIDIA/spark-rapids-jni/issues/1567). Now we have fixes for the namespace visibility issue (by marking all libcudf kenels private in https://github.com/rapidsai/rapids-cmake/pull/523 and https://github.com/NVIDIA/cuCollections/pull/422) and need to move back the code. Closes https://github.com/rapidsai/cudf/issues/14853. Authors: - Nghia Truong (https://github.com/ttnghia) Approvers: - David Wendt (https://github.com/davidwendt) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/15234 --- cpp/CMakeLists.txt | 1 - cpp/include/cudf/row_conversion.hpp | 55 - cpp/src/transform/row_conversion.cu | 2514 ------------------------ cpp/tests/CMakeLists.txt | 2 - cpp/tests/transform/row_conversion.cpp | 1011 ---------- 5 files changed, 3583 deletions(-) delete mode 100644 cpp/include/cudf/row_conversion.hpp delete mode 100644 cpp/src/transform/row_conversion.cu delete mode 100644 cpp/tests/transform/row_conversion.cpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index c74963be50d..5e8d13aa32d 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -641,7 +641,6 @@ add_library( src/transform/nans_to_nulls.cu src/transform/one_hot_encode.cu src/transform/row_bit_count.cu - src/transform/row_conversion.cu src/transform/transform.cpp src/transpose/transpose.cu src/unary/cast_ops.cu diff --git a/cpp/include/cudf/row_conversion.hpp b/cpp/include/cudf/row_conversion.hpp deleted file mode 100644 index e2c0577b885..00000000000 --- a/cpp/include/cudf/row_conversion.hpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include - -#include - -#include - -namespace cudf { -//! @cond Doxygen_Suppress - -std::vector> convert_to_rows_fixed_width_optimized( - cudf::table_view const& tbl, - // TODO need something for validity - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()); - -std::vector> convert_to_rows( - cudf::table_view const& tbl, - // TODO need something for validity - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()); - -std::unique_ptr convert_from_rows_fixed_width_optimized( - cudf::lists_column_view const& input, - std::vector const& schema, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()); - -std::unique_ptr convert_from_rows( - cudf::lists_column_view const& input, - std::vector const& schema, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()); - -//! @endcond -} // namespace cudf diff --git a/cpp/src/transform/row_conversion.cu b/cpp/src/transform/row_conversion.cu deleted file mode 100644 index 359e1ccb80d..00000000000 --- a/cpp/src/transform/row_conversion.cu +++ /dev/null @@ -1,2514 +0,0 @@ -/* - * Copyright (c) 2020-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace { - -constexpr auto JCUDF_ROW_ALIGNMENT = 8; - -constexpr auto MAX_BATCH_SIZE = std::numeric_limits::max(); - -// Number of rows each block processes in the two kernels. Tuned via nsight -constexpr auto NUM_STRING_ROWS_PER_BLOCK_TO_ROWS = 1024; -constexpr auto NUM_STRING_ROWS_PER_BLOCK_FROM_ROWS = 64; -constexpr auto MIN_STRING_BLOCKS = 32; -constexpr auto MAX_STRING_BLOCKS = MAX_BATCH_SIZE; - -constexpr auto NUM_WARPS_IN_BLOCK = 32; - -} // anonymous namespace - -// needed to suppress warning about cuda::barrier -#pragma nv_diag_suppress static_var_with_dynamic_init - -using namespace cudf; -using detail::make_device_uvector_async; -using detail::make_device_uvector_sync; -using rmm::device_uvector; - -namespace cudf { -namespace detail { - -/* - * This module converts data from row-major to column-major and from column-major to row-major. It - * is a transpose of the data of sorts, but there are a few complicating factors. They are spelled - * out below: - * - * Row Batches: - * The row data has to fit inside a cuDF column, which limits it to 2 gigs currently. The calling - * code attempts to keep the data size under 2 gigs, but due to padding this isn't always the case, - * so being able to break this up into multiple columns is necessary. Internally, this is referred - * to as the row batch, which is a group of rows that will fit into this 2 gig space requirement. - * There are typically 1 of these batches, but there can be 2. - * - * Async Memcpy: - * The CUDA blocks are using memcpy_async, which allows for the device to schedule memcpy operations - * and then wait on them to complete at a later time with a barrier. On Ampere or later hardware - * there is dedicated hardware to do this copy and on pre-Ampere it should generate the same code - * that a hand-rolled loop would generate, so performance should be the same or better than a - * hand-rolled kernel. - * - * Tile Info: - * Each CUDA block will work on a single tile info before exiting. This single tile consumes all - * available shared memory. The kernel reads data into shared memory and then back out from shared - * memory to device memory via memcpy_async. This kernel is completely memory bound. - * - * Batch Data: - * This structure contains all the row batches and some book-keeping data necessary for the batches - * such as row numbers for the batches. - * - * Tiles: - * The tile info describes a tile of data to process. In a GPU with 48KB this equates to about 221 - * bytes in each direction of a table. The tiles are kept as square as possible to attempt to - * coalesce memory operations. The taller a tile is the better coalescing of columns, but row - * coalescing suffers. The wider a tile is the better the row coalescing, but columns coalescing - * suffers. The code attempts to produce a square tile to balance the coalescing. It starts by - * figuring out the optimal byte length and then adding columns to the data until the tile is too - * large. Since rows are different width with different alignment requirements, this isn't typically - * exact. Once a width is found the tiles are generated vertically with that width and height and - * then the process repeats. This means all the tiles will be the same height, but will have - * different widths based on what columns they encompass. Tiles in a vertical row will all have the - * same dimensions. - * - * -------------------------------- - * | 4 5.0f || True 8 3 1 | - * | 3 6.0f || False 3 1 1 | - * | 2 7.0f || True 7 4 1 | - * | 1 8.0f || False 2 5 1 | - * -------------------------------- - * | 0 9.0f || True 6 7 1 | - * ... - */ - -/** - * @brief The CUDA blocks work on one tile_info struct of data. - * This structure defines the workspaces for the blocks. - * - */ -struct tile_info { - int start_col; - int start_row; - int end_col; - int end_row; - int batch_number; - - __device__ inline size_type get_shared_row_size(size_type const* const col_offsets, - size_type const* const col_sizes) const - { - // this calculation is invalid if there are holes in the data such as a variable-width column. - // It is wrong in a safe way in that it will say this row size is larger than it should be, so - // we are not losing data we are just not as efficient as we could be with shared memory. This - // may be a problem if the tile is computed without regard to variable width offset/length sizes - // in that we overrun shared memory. - return util::round_up_unsafe(col_offsets[end_col] + col_sizes[end_col] - col_offsets[start_col], - JCUDF_ROW_ALIGNMENT); - } - - __device__ inline size_type num_cols() const { return end_col - start_col + 1; } - - __device__ inline size_type num_rows() const { return end_row - start_row + 1; } -}; - -/** - * @brief Returning rows is done in a byte cudf column. This is limited in size by - * `size_type` and so output is broken into batches of rows that fit inside - * this limit. - * - */ -struct row_batch { - size_type num_bytes; // number of bytes in this batch - size_type row_count; // number of rows in the batch - device_uvector row_offsets; // offsets column of output cudf column -}; - -/** - * @brief Holds information about the batches of data to be processed - * - */ -struct batch_data { - device_uvector batch_row_offsets; // offsets to each row in incoming data - device_uvector d_batch_row_boundaries; // row numbers for the start of each batch - std::vector - batch_row_boundaries; // row numbers for the start of each batch: 0, 1500, 2700 - std::vector row_batches; // information about each batch such as byte count -}; - -/** - * @brief builds row size information for tables that contain strings - * - * @param tbl table from which to compute row size information - * @param fixed_width_and_validity_size size of fixed-width and validity data in this table - * @param stream cuda stream on which to operate - * @return pair of device vector of size_types of the row sizes of the table and a device vector of - * offsets into the string column - */ -std::pair, rmm::device_uvector> -build_string_row_offsets(table_view const& tbl, - size_type fixed_width_and_validity_size, - rmm::cuda_stream_view stream) -{ - auto const num_rows = tbl.num_rows(); - rmm::device_uvector d_row_sizes(num_rows, stream); - thrust::uninitialized_fill(rmm::exec_policy(stream), d_row_sizes.begin(), d_row_sizes.end(), 0); - - auto d_offsets_iterators = [&]() { - std::vector offsets_iterators; - auto itr = thrust::make_transform_iterator( - tbl.begin(), [](auto const& col) -> cudf::detail::input_offsetalator { - return cudf::detail::offsetalator_factory::make_input_iterator( - strings_column_view(col).offsets(), col.offset()); - }); - auto stencil = thrust::make_transform_iterator( - tbl.begin(), [](auto const& col) -> bool { return !is_fixed_width(col.type()); }); - thrust::copy_if(thrust::host, - itr, - itr + tbl.num_columns(), - stencil, - std::back_inserter(offsets_iterators), - thrust::identity{}); - return make_device_uvector_sync( - offsets_iterators, stream, rmm::mr::get_current_device_resource()); - }(); - - auto const num_columns = static_cast(d_offsets_iterators.size()); - - thrust::for_each(rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(num_columns * num_rows), - [d_offsets_iterators = d_offsets_iterators.data(), - num_columns, - num_rows, - d_row_sizes = d_row_sizes.data()] __device__(auto element_idx) { - auto const row = element_idx % num_rows; - auto const col = element_idx / num_rows; - auto const val = - d_offsets_iterators[col][row + 1] - d_offsets_iterators[col][row]; - atomicAdd(&d_row_sizes[row], val); - }); - - // transform the row sizes to include fixed width size and alignment - thrust::transform(rmm::exec_policy(stream), - d_row_sizes.begin(), - d_row_sizes.end(), - d_row_sizes.begin(), - cuda::proclaim_return_type( - [fixed_width_and_validity_size] __device__(auto row_size) { - return util::round_up_unsafe(fixed_width_and_validity_size + row_size, - JCUDF_ROW_ALIGNMENT); - })); - - return {std::move(d_row_sizes), std::move(d_offsets_iterators)}; -} - -/** - * @brief functor to return the offset of a row in a table with string columns - * - */ -struct string_row_offset_functor { - string_row_offset_functor(device_span d_row_offsets) - : d_row_offsets(d_row_offsets){}; - - __device__ inline size_type operator()(int row_number, int) const - { - return d_row_offsets[row_number]; - } - - device_span d_row_offsets; -}; - -/** - * @brief functor to return the offset of a row in a table with only fixed-width columns - * - */ -struct fixed_width_row_offset_functor { - fixed_width_row_offset_functor(size_type fixed_width_only_row_size) - : _fixed_width_only_row_size(fixed_width_only_row_size){}; - - __device__ inline size_type operator()(int row_number, int tile_row_start) const - { - return (row_number - tile_row_start) * _fixed_width_only_row_size; - } - - size_type _fixed_width_only_row_size; -}; - -/** - * @brief Copies data from row-based JCUDF format to column-based cudf format. - * - * This optimized version of the conversion is faster for fixed-width tables that do not have more - * than 100 columns. - * - * @param num_rows number of rows in the incoming table - * @param num_columns number of columns in the incoming table - * @param row_size length in bytes of each row - * @param input_offset_in_row offset to each row of data - * @param num_bytes total number of bytes in the incoming data - * @param output_data array of pointers to the output data - * @param output_nm array of pointers to the output null masks - * @param input_data pointing to the incoming row data - */ -CUDF_KERNEL void copy_from_rows_fixed_width_optimized(const size_type num_rows, - const size_type num_columns, - const size_type row_size, - const size_type* input_offset_in_row, - const size_type* num_bytes, - int8_t** output_data, - bitmask_type** output_nm, - const int8_t* input_data) -{ - // We are going to copy the data in two passes. - // The first pass copies a chunk of data into shared memory. - // The second pass copies that chunk from shared memory out to the final location. - - // Because shared memory is limited we copy a subset of the rows at a time. - // For simplicity we will refer to this as a row_group - - // In practice we have found writing more than 4 columns of data per thread - // results in performance loss. As such we are using a 2 dimensional - // kernel in terms of threads, but not in terms of blocks. Columns are - // controlled by the y dimension (there is no y dimension in blocks). Rows - // are controlled by the x dimension (there are multiple blocks in the x - // dimension). - - size_type const rows_per_group = blockDim.x; - size_type const row_group_start = blockIdx.x; - size_type const row_group_stride = gridDim.x; - size_type const row_group_end = (num_rows + rows_per_group - 1) / rows_per_group + 1; - - extern __shared__ int8_t shared_data[]; - - // Because we are copying fixed width only data and we stride the rows - // this thread will always start copying from shared data in the same place - int8_t* row_tmp = &shared_data[row_size * threadIdx.x]; - int8_t* row_vld_tmp = &row_tmp[input_offset_in_row[num_columns - 1] + num_bytes[num_columns - 1]]; - - for (auto row_group_index = row_group_start; row_group_index < row_group_end; - row_group_index += row_group_stride) { - // Step 1: Copy the data into shared memory - // We know row_size is always aligned with and a multiple of int64_t; - int64_t* long_shared = reinterpret_cast(shared_data); - int64_t const* long_input = reinterpret_cast(input_data); - - auto const shared_output_index = threadIdx.x + (threadIdx.y * blockDim.x); - auto const shared_output_stride = blockDim.x * blockDim.y; - auto const row_index_end = std::min(num_rows, ((row_group_index + 1) * rows_per_group)); - auto const num_rows_in_group = row_index_end - (row_group_index * rows_per_group); - auto const shared_length = row_size * num_rows_in_group; - - size_type const shared_output_end = shared_length / sizeof(int64_t); - - auto const start_input_index = (row_size * row_group_index * rows_per_group) / sizeof(int64_t); - - for (size_type shared_index = shared_output_index; shared_index < shared_output_end; - shared_index += shared_output_stride) { - long_shared[shared_index] = long_input[start_input_index + shared_index]; - } - // Wait for all of the data to be in shared memory - __syncthreads(); - - // Step 2 copy the data back out - - // Within the row group there should be 1 thread for each row. This is a - // requirement for launching the kernel - auto const row_index = (row_group_index * rows_per_group) + threadIdx.x; - // But we might not use all of the threads if the number of rows does not go - // evenly into the thread count. We don't want those threads to exit yet - // because we may need them to copy data in for the next row group. - uint32_t active_mask = __ballot_sync(0xffffffff, row_index < num_rows); - if (row_index < num_rows) { - auto const col_index_start = threadIdx.y; - auto const col_index_stride = blockDim.y; - for (auto col_index = col_index_start; col_index < num_columns; - col_index += col_index_stride) { - auto const col_size = num_bytes[col_index]; - int8_t const* col_tmp = &(row_tmp[input_offset_in_row[col_index]]); - int8_t* col_output = output_data[col_index]; - switch (col_size) { - case 1: { - col_output[row_index] = *col_tmp; - break; - } - case 2: { - int16_t* short_col_output = reinterpret_cast(col_output); - short_col_output[row_index] = *reinterpret_cast(col_tmp); - break; - } - case 4: { - int32_t* int_col_output = reinterpret_cast(col_output); - int_col_output[row_index] = *reinterpret_cast(col_tmp); - break; - } - case 8: { - int64_t* long_col_output = reinterpret_cast(col_output); - long_col_output[row_index] = *reinterpret_cast(col_tmp); - break; - } - default: { - auto const output_offset = col_size * row_index; - // TODO this should just not be supported for fixed width columns, but just in case... - for (auto b = 0; b < col_size; b++) { - col_output[b + output_offset] = col_tmp[b]; - } - break; - } - } - - bitmask_type* nm = output_nm[col_index]; - int8_t* valid_byte = &row_vld_tmp[col_index / 8]; - size_type byte_bit_offset = col_index % 8; - int predicate = *valid_byte & (1 << byte_bit_offset); - uint32_t bitmask = __ballot_sync(active_mask, predicate); - if (row_index % 32 == 0) { nm[word_index(row_index)] = bitmask; } - } // end column loop - } // end row copy - // wait for the row_group to be totally copied before starting on the next row group - __syncthreads(); - } -} - -CUDF_KERNEL void copy_to_rows_fixed_width_optimized(const size_type start_row, - const size_type num_rows, - const size_type num_columns, - const size_type row_size, - const size_type* output_offset_in_row, - const size_type* num_bytes, - const int8_t** input_data, - const bitmask_type** input_nm, - int8_t* output_data) -{ - // We are going to copy the data in two passes. - // The first pass copies a chunk of data into shared memory. - // The second pass copies that chunk from shared memory out to the final location. - - // Because shared memory is limited we copy a subset of the rows at a time. - // We do not support copying a subset of the columns in a row yet, so we don't - // currently support a row that is wider than shared memory. - // For simplicity we will refer to this as a row_group - - // In practice we have found reading more than 4 columns of data per thread - // results in performance loss. As such we are using a 2 dimensional - // kernel in terms of threads, but not in terms of blocks. Columns are - // controlled by the y dimension (there is no y dimension in blocks). Rows - // are controlled by the x dimension (there are multiple blocks in the x - // dimension). - - size_type rows_per_group = blockDim.x; - size_type row_group_start = blockIdx.x; - size_type row_group_stride = gridDim.x; - size_type row_group_end = (num_rows + rows_per_group - 1) / rows_per_group + 1; - - extern __shared__ int8_t shared_data[]; - - // Because we are copying fixed width only data and we stride the rows - // this thread will always start copying to shared data in the same place - int8_t* row_tmp = &shared_data[row_size * threadIdx.x]; - int8_t* row_vld_tmp = - &row_tmp[output_offset_in_row[num_columns - 1] + num_bytes[num_columns - 1]]; - - for (size_type row_group_index = row_group_start; row_group_index < row_group_end; - row_group_index += row_group_stride) { - // Within the row group there should be 1 thread for each row. This is a - // requirement for launching the kernel - size_type row_index = start_row + (row_group_index * rows_per_group) + threadIdx.x; - // But we might not use all of the threads if the number of rows does not go - // evenly into the thread count. We don't want those threads to exit yet - // because we may need them to copy data back out. - if (row_index < (start_row + num_rows)) { - size_type col_index_start = threadIdx.y; - size_type col_index_stride = blockDim.y; - for (size_type col_index = col_index_start; col_index < num_columns; - col_index += col_index_stride) { - size_type col_size = num_bytes[col_index]; - int8_t* col_tmp = &(row_tmp[output_offset_in_row[col_index]]); - const int8_t* col_input = input_data[col_index]; - switch (col_size) { - case 1: { - *col_tmp = col_input[row_index]; - break; - } - case 2: { - const int16_t* short_col_input = reinterpret_cast(col_input); - *reinterpret_cast(col_tmp) = short_col_input[row_index]; - break; - } - case 4: { - const int32_t* int_col_input = reinterpret_cast(col_input); - *reinterpret_cast(col_tmp) = int_col_input[row_index]; - break; - } - case 8: { - const int64_t* long_col_input = reinterpret_cast(col_input); - *reinterpret_cast(col_tmp) = long_col_input[row_index]; - break; - } - default: { - size_type input_offset = col_size * row_index; - // TODO this should just not be supported for fixed width columns, but just in case... - for (size_type b = 0; b < col_size; b++) { - col_tmp[b] = col_input[b + input_offset]; - } - break; - } - } - // atomicOr only works on 32 bit or 64 bit aligned values, and not byte aligned - // so we have to rewrite the addresses to make sure that it is 4 byte aligned - int8_t* valid_byte = &row_vld_tmp[col_index / 8]; - size_type byte_bit_offset = col_index % 8; - uint64_t fixup_bytes = reinterpret_cast(valid_byte) % 4; - int32_t* valid_int = reinterpret_cast(valid_byte - fixup_bytes); - size_type int_bit_offset = byte_bit_offset + (fixup_bytes * 8); - // Now copy validity for the column - if (input_nm[col_index]) { - if (bit_is_set(input_nm[col_index], row_index)) { - atomicOr_block(valid_int, 1 << int_bit_offset); - } else { - atomicAnd_block(valid_int, ~(1 << int_bit_offset)); - } - } else { - // It is valid so just set the bit - atomicOr_block(valid_int, 1 << int_bit_offset); - } - } // end column loop - } // end row copy - // wait for the row_group to be totally copied into shared memory - __syncthreads(); - - // Step 2: Copy the data back out - // We know row_size is always aligned with and a multiple of int64_t; - int64_t* long_shared = reinterpret_cast(shared_data); - int64_t* long_output = reinterpret_cast(output_data); - - size_type shared_input_index = threadIdx.x + (threadIdx.y * blockDim.x); - size_type shared_input_stride = blockDim.x * blockDim.y; - size_type row_index_end = ((row_group_index + 1) * rows_per_group); - if (row_index_end > num_rows) { row_index_end = num_rows; } - size_type num_rows_in_group = row_index_end - (row_group_index * rows_per_group); - size_type shared_length = row_size * num_rows_in_group; - - size_type shared_input_end = shared_length / sizeof(int64_t); - - size_type start_output_index = (row_size * row_group_index * rows_per_group) / sizeof(int64_t); - - for (size_type shared_index = shared_input_index; shared_index < shared_input_end; - shared_index += shared_input_stride) { - long_output[start_output_index + shared_index] = long_shared[shared_index]; - } - __syncthreads(); - // Go for the next round - } -} - -/** - * @brief copy data from cudf columns into JCUDF format, which is row-based - * - * @tparam RowOffsetFunctor iterator that gives the size of a specific row of the table. - * @param num_rows total number of rows in the table - * @param num_columns total number of columns in the table - * @param shmem_used_per_tile shared memory amount each `tile_info` is using - * @param tile_infos span of `tile_info` structs the define the work - * @param input_data pointer to raw table data - * @param col_sizes array of sizes for each element in a column - one per column - * @param col_offsets offset into input data row for each column's start - * @param row_offsets offset to a specific row in the output data - * @param batch_row_boundaries row numbers for batch starts - * @param output_data pointer to output data - * - */ -template -CUDF_KERNEL void copy_to_rows(const size_type num_rows, - const size_type num_columns, - const size_type shmem_used_per_tile, - device_span tile_infos, - const int8_t** input_data, - const size_type* col_sizes, - const size_type* col_offsets, - RowOffsetFunctor row_offsets, - size_type const* batch_row_boundaries, - int8_t** output_data) -{ - // We are going to copy the data in two passes. - // The first pass copies a chunk of data into shared memory. - // The second pass copies that chunk from shared memory out to the final location. - - // Because shared memory is limited we copy a subset of the rows at a time. - // This has been broken up for us in the tile_info struct, so we don't have - // any calculation to do here, but it is important to note. - - auto const group = cooperative_groups::this_thread_block(); - auto const warp = cooperative_groups::tiled_partition(group); - extern __shared__ int8_t shared_data[]; - - __shared__ cuda::barrier tile_barrier; - if (group.thread_rank() == 0) { init(&tile_barrier, group.size()); } - group.sync(); - - auto const tile = tile_infos[blockIdx.x]; - auto const num_tile_cols = tile.num_cols(); - auto const num_tile_rows = tile.num_rows(); - auto const tile_row_size = tile.get_shared_row_size(col_offsets, col_sizes); - auto const starting_column_offset = col_offsets[tile.start_col]; - - // to do the copy we need to do n column copies followed by m element copies OR we have to do m - // element copies followed by r row copies. When going from column to row it is much easier to - // copy by elements first otherwise we would need a running total of the column sizes for our - // tile, which isn't readily available. This makes it more appealing to copy element-wise from - // input data into shared matching the end layout and do row-based memcopies out. - - // read each column across the tile - // each warp takes a column with each thread of a warp taking a row this is done with cooperative - // groups where each column is chosen by the tiled partition and each thread in that partition - // works on a row - for (int relative_col = warp.meta_group_rank(); relative_col < num_tile_cols; - relative_col += warp.meta_group_size()) { - auto const absolute_col = relative_col + tile.start_col; - auto const col_size = col_sizes[absolute_col]; - auto const col_offset = col_offsets[absolute_col]; - auto const relative_col_offset = col_offset - starting_column_offset; - auto const col_ptr = input_data[absolute_col]; - - if (col_ptr == nullptr) { - // variable-width data column - continue; - } - - for (int relative_row = warp.thread_rank(); relative_row < num_tile_rows; - relative_row += warp.size()) { - if (relative_row >= num_tile_rows) { - // out of bounds - continue; - } - auto const absolute_row = relative_row + tile.start_row; - - auto const shared_offset = relative_row * tile_row_size + relative_col_offset; - auto const input_src = col_ptr + col_size * absolute_row; - - // copy the element from global memory - switch (col_size) { - case 2: { - const int16_t* short_col_input = reinterpret_cast(input_src); - *reinterpret_cast(&shared_data[shared_offset]) = *short_col_input; - break; - } - case 4: { - const int32_t* int_col_input = reinterpret_cast(input_src); - *reinterpret_cast(&shared_data[shared_offset]) = *int_col_input; - break; - } - case 8: { - const int64_t* long_col_input = reinterpret_cast(input_src); - *reinterpret_cast(&shared_data[shared_offset]) = *long_col_input; - break; - } - case 1: shared_data[shared_offset] = *input_src; break; - default: { - for (int i = 0; i < col_size; ++i) { - shared_data[shared_offset] = *input_src; - } - break; - } - } - } - } - - auto const tile_output_buffer = output_data[tile.batch_number]; - auto const row_batch_start = tile.batch_number == 0 ? 0 : batch_row_boundaries[tile.batch_number]; - - // no async copies above waiting on the barrier, so we sync the group here to ensure all copies to - // shared memory are completed before copying data out - group.sync(); - - // each warp takes a row - for (int copy_row = warp.meta_group_rank(); copy_row < tile.num_rows(); - copy_row += warp.meta_group_size()) { - auto const src = &shared_data[tile_row_size * copy_row]; - auto const dst = tile_output_buffer + row_offsets(copy_row + tile.start_row, row_batch_start) + - starting_column_offset; - cuda::memcpy_async(warp, dst, src, tile_row_size, tile_barrier); - } - - // wait on the last copies to complete - tile_barrier.arrive_and_wait(); -} - -/** - * @brief copy data from row-based format to cudf columns - * - * @tparam RowOffsetFunctor iterator that gives the size of a specific row of the table. - * @param num_rows total number of rows in the table - * @param num_columns total number of columns in the table - * @param shmem_used_per_tile amount of shared memory that is used by a tile - * @param row_offsets offset to a specific row in the output data - * @param batch_row_boundaries row numbers for batch starts - * @param output_data pointer to output data, partitioned by data size - * @param validity_offsets offset into input data row for validity data - * @param tile_infos information about the tiles of work - * @param input_nm pointer to input data - * - */ -template -CUDF_KERNEL void copy_validity_to_rows(const size_type num_rows, - const size_type num_columns, - const size_type shmem_used_per_tile, - RowOffsetFunctor row_offsets, - size_type const* batch_row_boundaries, - int8_t** output_data, - const size_type validity_offset, - device_span tile_infos, - const bitmask_type** input_nm) -{ - extern __shared__ int8_t shared_data[]; - - // each thread of warp reads a single int32 of validity - so we read 128 bytes then ballot_sync - // the bits and write the result to shmem after we fill shared mem memcpy it out in a blob. - auto const group = cooperative_groups::this_thread_block(); - auto const warp = cooperative_groups::tiled_partition(group); - - // Initialize cuda barriers for each tile. - __shared__ cuda::barrier shared_tile_barrier; - if (group.thread_rank() == 0) { init(&shared_tile_barrier, group.size()); } - group.sync(); - - auto tile = tile_infos[blockIdx.x]; - auto const num_tile_cols = tile.num_cols(); - auto const num_tile_rows = tile.num_rows(); - - auto const threads_per_warp = warp.size(); - auto const rows_per_read = cudf::detail::size_in_bits(); - - auto const num_sections_x = util::div_rounding_up_unsafe(num_tile_cols, threads_per_warp); - auto const num_sections_y = util::div_rounding_up_unsafe(num_tile_rows, rows_per_read); - auto const validity_data_row_length = util::round_up_unsafe( - util::div_rounding_up_unsafe(num_tile_cols, CHAR_BIT), JCUDF_ROW_ALIGNMENT); - auto const total_sections = num_sections_x * num_sections_y; - - // the tile is divided into sections. A warp operates on a section at a time. - for (int my_section_idx = warp.meta_group_rank(); my_section_idx < total_sections; - my_section_idx += warp.meta_group_size()) { - // convert to rows and cols - auto const section_x = my_section_idx % num_sections_x; - auto const section_y = my_section_idx / num_sections_x; - auto const relative_col = section_x * threads_per_warp + warp.thread_rank(); - auto const relative_row = section_y * rows_per_read; - auto const absolute_col = relative_col + tile.start_col; - auto const absolute_row = relative_row + tile.start_row; - auto const participating = absolute_col < num_columns && absolute_row < num_rows; - auto const participation_mask = __ballot_sync(0xFFFFFFFF, participating); - - if (participating) { - auto my_data = input_nm[absolute_col] != nullptr - ? input_nm[absolute_col][word_index(absolute_row)] - : std::numeric_limits::max(); - - // every thread that is participating in the warp has 4 bytes, but it's column-based data and - // we need it in row-based. So we shuffle the bits around with ballot_sync to make the bytes - // we actually write. - bitmask_type dw_mask = 0x1; - for (int i = 0; i < threads_per_warp && relative_row + i < num_rows; ++i, dw_mask <<= 1) { - auto validity_data = __ballot_sync(participation_mask, my_data & dw_mask); - // lead thread in each warp writes data - auto const validity_write_offset = - validity_data_row_length * (relative_row + i) + (relative_col / CHAR_BIT); - if (warp.thread_rank() == 0) { - *reinterpret_cast(&shared_data[validity_write_offset]) = validity_data; - } - } - } - } - - auto const output_data_base = - output_data[tile.batch_number] + validity_offset + tile.start_col / CHAR_BIT; - - // each warp copies a row at a time - auto const row_bytes = util::div_rounding_up_unsafe(num_tile_cols, CHAR_BIT); - auto const row_batch_start = tile.batch_number == 0 ? 0 : batch_row_boundaries[tile.batch_number]; - - // make sure entire tile has finished copy - // Note that this was copied from above just under the for loop due to nsight complaints about - // divergent threads - group.sync(); - - for (int relative_row = warp.meta_group_rank(); relative_row < num_tile_rows; - relative_row += warp.meta_group_size()) { - auto const src = &shared_data[validity_data_row_length * relative_row]; - auto const dst = output_data_base + row_offsets(relative_row + tile.start_row, row_batch_start); - cuda::memcpy_async(warp, dst, src, row_bytes, shared_tile_barrier); - } - - // wait for tile of data to arrive - shared_tile_barrier.arrive_and_wait(); -} - -/** - * @brief kernel to copy string data to JCUDF row format - * - * @tparam RowOffsetFunctor iterator for row offsets into the destination data - * @param num_rows number of rows in this portion of the table - * @param num_variable_columns number of columns of variable-width data - * @param variable_input_data variable width data column pointers - * @param variable_col_output_offsets output offset information for variable-width columns - * @param variable_col_offsets input offset information for variable-width columns - * @param fixed_width_row_size offset to variable-width data in a row - * @param row_offsets offsets for each row in output data - * @param batch_row_offset row start for this batch - * @param output_data pointer to output data for this batch - * - */ -template -CUDF_KERNEL void copy_strings_to_rows(size_type const num_rows, - size_type const num_variable_columns, - int8_t const** variable_input_data, - size_type const* variable_col_output_offsets, - cudf::detail::input_offsetalator* variable_col_offsets, - size_type fixed_width_row_size, - RowOffsetFunctor row_offsets, - size_type const batch_row_offset, - int8_t* output_data) -{ - // Each block will take a group of rows controlled by NUM_STRING_ROWS_PER_BLOCK_TO_ROWS. Each warp - // will copy a row at a time. The base thread will first go through column data and fill out - // offset/length information for the column. Then all threads of the warp will participate in the - // memcpy of the string data. - auto const my_block = cooperative_groups::this_thread_block(); - auto const warp = cooperative_groups::tiled_partition(my_block); - cuda::barrier block_barrier; - - auto const start_row = - blockIdx.x * NUM_STRING_ROWS_PER_BLOCK_TO_ROWS + warp.meta_group_rank() + batch_row_offset; - auto const end_row = - std::min(num_rows, static_cast(start_row + NUM_STRING_ROWS_PER_BLOCK_TO_ROWS)); - - for (int row = start_row; row < end_row; row += warp.meta_group_size()) { - auto offset = fixed_width_row_size; // initial offset to variable-width data - auto const base_row_offset = row_offsets(row, 0); - for (int col = 0; col < num_variable_columns; ++col) { - auto const string_start_offset = variable_col_offsets[col][row]; - auto const string_length = variable_col_offsets[col][row + 1] - string_start_offset; - if (warp.thread_rank() == 0) { - // write the offset/length to column - uint32_t* output_dest = reinterpret_cast( - &output_data[base_row_offset + variable_col_output_offsets[col]]); - output_dest[0] = offset; - output_dest[1] = string_length; - } - auto string_output_dest = &output_data[base_row_offset + offset]; - auto string_output_src = &variable_input_data[col][string_start_offset]; - warp.sync(); - cuda::memcpy_async(warp, string_output_dest, string_output_src, string_length, block_barrier); - offset += string_length; - } - } -} -/** - * @brief copy data from row-based format to cudf columns - * - * @tparam RowOffsetFunctor iterator that gives the size of a specific row of the table. - * @param num_rows total number of rows in the table - * @param num_columns total number of columns in the table - * @param shmem_used_per_tile amount of shared memory that is used by a tile - * @param row_offsets offset to a specific row in the input data - * @param batch_row_boundaries row numbers for batch starts - * @param output_data pointers to column data - * @param col_sizes array of sizes for each element in a column - one per column - * @param col_offsets offset into input data row for each column's start - * @param tile_infos information about the tiles of work - * @param input_data pointer to input data - * - */ -template -CUDF_KERNEL void copy_from_rows(const size_type num_rows, - const size_type num_columns, - const size_type shmem_used_per_tile, - RowOffsetFunctor row_offsets, - size_type const* batch_row_boundaries, - int8_t** output_data, - const size_type* col_sizes, - const size_type* col_offsets, - device_span tile_infos, - const int8_t* input_data) -{ - // We are going to copy the data in two passes. - // The first pass copies a chunk of data into shared memory. - // The second pass copies that chunk from shared memory out to the final location. - - // Because shared memory is limited we copy a subset of the rows at a time. This has been broken - // up for us in the tile_info struct, so we don't have any calculation to do here, but it is - // important to note. - - // To speed up some of the random access memory we do, we copy col_sizes and col_offsets to shared - // memory for each of the tiles that we work on - - auto const group = cooperative_groups::this_thread_block(); - auto const warp = cooperative_groups::tiled_partition(group); - extern __shared__ int8_t shared[]; - - // Initialize cuda barriers for each tile. - __shared__ cuda::barrier tile_barrier; - if (group.thread_rank() == 0) { init(&tile_barrier, group.size()); } - group.sync(); - - { - auto const fetch_tile = tile_infos[blockIdx.x]; - auto const fetch_tile_start_row = fetch_tile.start_row; - auto const starting_col_offset = col_offsets[fetch_tile.start_col]; - auto const fetch_tile_row_size = fetch_tile.get_shared_row_size(col_offsets, col_sizes); - auto const row_batch_start = - fetch_tile.batch_number == 0 ? 0 : batch_row_boundaries[fetch_tile.batch_number]; - - for (int absolute_row = warp.meta_group_rank() + fetch_tile.start_row; - absolute_row <= fetch_tile.end_row; - absolute_row += warp.meta_group_size()) { - warp.sync(); - auto shared_offset = (absolute_row - fetch_tile_start_row) * fetch_tile_row_size; - auto dst = &shared[shared_offset]; - auto src = &input_data[row_offsets(absolute_row, row_batch_start) + starting_col_offset]; - // copy the data - cuda::memcpy_async(warp, dst, src, fetch_tile_row_size, tile_barrier); - } - } - - { - auto const tile = tile_infos[blockIdx.x]; - auto const rows_in_tile = tile.num_rows(); - auto const cols_in_tile = tile.num_cols(); - auto const tile_row_size = tile.get_shared_row_size(col_offsets, col_sizes); - - // ensure our data is ready - tile_barrier.arrive_and_wait(); - - // Now we copy from shared memory to final destination. The data is laid out in rows in shared - // memory, so the reads for a column will be "vertical". Because of this and the different sizes - // for each column, this portion is handled on row/column basis. to prevent each thread working - // on a single row and also to ensure that all threads can do work in the case of more threads - // than rows, we do a global index instead of a double for loop with col/row. - for (int relative_row = warp.thread_rank(); relative_row < rows_in_tile; - relative_row += warp.size()) { - auto const absolute_row = relative_row + tile.start_row; - auto const shared_memory_row_offset = tile_row_size * relative_row; - - for (int relative_col = warp.meta_group_rank(); relative_col < cols_in_tile; - relative_col += warp.meta_group_size()) { - auto const absolute_col = relative_col + tile.start_col; - - auto const shared_memory_offset = - col_offsets[absolute_col] - col_offsets[tile.start_col] + shared_memory_row_offset; - auto const column_size = col_sizes[absolute_col]; - - int8_t* shmem_src = &shared[shared_memory_offset]; - int8_t* dst = &output_data[absolute_col][absolute_row * column_size]; - - cuda::memcpy_async(dst, shmem_src, column_size, tile_barrier); - } - } - } - - // wait on the last copies to complete - tile_barrier.arrive_and_wait(); -} - -/** - * @brief copy data from row-based format to cudf columns - * - * @tparam RowOffsetFunctor iterator that gives the size of a specific row of the table. - * @param num_rows total number of rows in the table - * @param num_columns total number of columns in the table - * @param shmem_used_per_tile amount of shared memory that is used by a tile - * @param row_offsets offset to the first column a specific row in the input data - * @param batch_row_boundaries row numbers for batch starts - * @param output_nm pointers to null masks for columns - * @param validity_offsets offset into input data row for validity data - * @param tile_infos information about the tiles of work - * @param input_data pointer to input data - * - */ -template -CUDF_KERNEL void copy_validity_from_rows(const size_type num_rows, - const size_type num_columns, - const size_type shmem_used_per_tile, - RowOffsetFunctor row_offsets, - size_type const* batch_row_boundaries, - bitmask_type** output_nm, - const size_type validity_offset, - device_span tile_infos, - const int8_t* input_data) -{ - extern __shared__ int8_t shared[]; - - using cudf::detail::warp_size; - - // each thread of warp reads a single byte of validity - so we read 32 bytes then ballot_sync the - // bits and write the result to shmem after we fill shared mem memcpy it out in a blob. Probably - // need knobs for number of rows vs columns to balance read/write - - // C0 C1 C2 C3 C4 C5 C6 C7 - // R0 1 0 1 0 0 1 1 0 <-- thread 0 reads byte r0 - // R1 1 1 1 1 1 1 1 0 <-- thread 1 reads byte r1 - // R2 0 0 1 0 0 1 1 0 <-- thread 2 reads byte r2 - // ... - // R31 1 1 1 1 1 1 1 1 <-- thread 31 reads byte r31 - // ^ - // | 1 bit of each input byte, by column, are swizzled into a single 32 bit word via - // __ballot_sync, representing 32 rows of that column. - - auto const group = cooperative_groups::this_thread_block(); - auto const warp = cooperative_groups::tiled_partition(group); - - // Initialize cuda barriers for each tile. - __shared__ cuda::barrier shared_tile_barrier; - if (group.thread_rank() == 0) { init(&shared_tile_barrier, group.size()); } - group.sync(); - - auto const tile = tile_infos[blockIdx.x]; - auto const tile_start_col = tile.start_col; - auto const tile_start_row = tile.start_row; - auto const num_tile_cols = tile.num_cols(); - auto const num_tile_rows = tile.num_rows(); - - auto const threads_per_warp = warp.size(); - auto const cols_per_read = CHAR_BIT; - - auto const rows_per_read = static_cast(threads_per_warp); - auto const num_sections_x = util::div_rounding_up_safe(num_tile_cols, cols_per_read); - auto const num_sections_y = util::div_rounding_up_safe(num_tile_rows, rows_per_read); - auto const validity_data_col_length = num_sections_y * 4; // words to bytes - auto const total_sections = num_sections_x * num_sections_y; - - // the tile is divided into sections. A warp operates on a section at a time. - for (int my_section_idx = warp.meta_group_rank(); my_section_idx < total_sections; - my_section_idx += warp.meta_group_size()) { - // convert section to row and col - auto const section_x = my_section_idx % num_sections_x; - auto const section_y = my_section_idx / num_sections_x; - auto const relative_col = section_x * cols_per_read; - auto const relative_row = section_y * rows_per_read + warp.thread_rank(); - auto const absolute_col = relative_col + tile_start_col; - auto const absolute_row = relative_row + tile_start_row; - auto const row_batch_start = - tile.batch_number == 0 ? 0 : batch_row_boundaries[tile.batch_number]; - - auto const participation_mask = __ballot_sync(0xFFFFFFFF, absolute_row < num_rows); - - if (absolute_row < num_rows) { - auto const my_byte = input_data[row_offsets(absolute_row, row_batch_start) + validity_offset + - (absolute_col / cols_per_read)]; - - // so every thread that is participating in the warp has a byte, but it's row-based data and - // we need it in column-based. So we shuffle the bits around to make the bytes we actually - // write. - for (int i = 0, byte_mask = 0x1; (i < cols_per_read) && ((relative_col + i) < num_columns); - ++i, byte_mask <<= 1) { - auto const validity_data = __ballot_sync(participation_mask, my_byte & byte_mask); - // lead thread in each warp writes data - if (warp.thread_rank() == 0) { - auto const validity_write_offset = - validity_data_col_length * (relative_col + i) + relative_row / cols_per_read; - *reinterpret_cast(&shared[validity_write_offset]) = validity_data; - } - } - } - } - - // now memcpy the shared memory out to the final destination - auto const col_words = util::div_rounding_up_unsafe(num_tile_rows, CHAR_BIT * 4); - - // make sure entire tile has finished copy - group.sync(); - - for (int relative_col = warp.meta_group_rank(); relative_col < num_tile_cols; - relative_col += warp.meta_group_size()) { - auto const absolute_col = relative_col + tile_start_col; - auto dst = output_nm[absolute_col] + word_index(tile_start_row); - auto const src = - reinterpret_cast(&shared[validity_data_col_length * relative_col]); - - cuda::memcpy_async( - warp, dst, src, cuda::aligned_size_t<4>(validity_data_col_length), shared_tile_barrier); - } - - // wait for tile of data to arrive - shared_tile_barrier.arrive_and_wait(); -} - -/** - * @brief copies string data from jcudf row format to cudf columns - * - * @tparam RowOffsetFunctor iterator for row offsets into the destination data - * @param row_offsets offsets for each row in input data - * @param string_row_offsets offset data into jcudf row data for each string - * @param string_lengths length of each incoming string in each column - * @param string_column_offsets offset column data for cudf column - * @param string_col_data output cudf string column data - * @param row_data jcudf row data - * @param num_rows number of rows in data - * @param num_string_columns number of string columns in the table - */ -template -CUDF_KERNEL void copy_strings_from_rows(RowOffsetFunctor row_offsets, - int32_t** string_row_offsets, - int32_t** string_lengths, - size_type** string_column_offsets, - char** string_col_data, - int8_t const* row_data, - size_type const num_rows, - size_type const num_string_columns) -{ - // Each warp takes a tile, which is a single column and up to ROWS_PER_BLOCK rows. A tile will not - // wrap around the bottom of the table. The warp will copy the strings for each row in the tile. - // Traversing in row-major order to coalesce the offsets and size reads. - auto my_block = cooperative_groups::this_thread_block(); - auto warp = cooperative_groups::tiled_partition(my_block); - cuda::barrier block_barrier; - - // workaround for not being able to take a reference to a constexpr host variable - auto const ROWS_PER_BLOCK = NUM_STRING_ROWS_PER_BLOCK_FROM_ROWS; - auto const tiles_per_col = util::div_rounding_up_unsafe(num_rows, ROWS_PER_BLOCK); - auto const starting_tile = blockIdx.x * warp.meta_group_size() + warp.meta_group_rank(); - auto const num_tiles = tiles_per_col * num_string_columns; - auto const tile_stride = warp.meta_group_size() * gridDim.x; - // Each warp will copy strings in its tile. This is handled by all the threads of a warp passing - // the same parameters to async_memcpy and all threads in the warp participating in the copy. - for (auto my_tile = starting_tile; my_tile < num_tiles; my_tile += tile_stride) { - auto const starting_row = (my_tile % tiles_per_col) * ROWS_PER_BLOCK; - auto const col = my_tile / tiles_per_col; - auto const str_len = string_lengths[col]; - auto const str_row_off = string_row_offsets[col]; - auto const str_col_off = string_column_offsets[col]; - auto str_col_data = string_col_data[col]; - for (int row = starting_row; row < starting_row + ROWS_PER_BLOCK && row < num_rows; ++row) { - auto const src = &row_data[row_offsets(row, 0) + str_row_off[row]]; - auto dst = &str_col_data[str_col_off[row]]; - - cuda::memcpy_async(warp, dst, src, str_len[row], block_barrier); - } - } -} - -/** - * @brief Calculate the dimensions of the kernel for fixed width only columns. - * - * @param [in] num_columns the number of columns being copied. - * @param [in] num_rows the number of rows being copied. - * @param [in] size_per_row the size each row takes up when padded. - * @param [out] blocks the size of the blocks for the kernel - * @param [out] threads the size of the threads for the kernel - * @return the size in bytes of shared memory needed for each block. - */ -static int calc_fixed_width_kernel_dims(const size_type num_columns, - const size_type num_rows, - const size_type size_per_row, - dim3& blocks, - dim3& threads) -{ - // We have found speed degrades when a thread handles more than 4 columns. - // Each block is 2 dimensional. The y dimension indicates the columns. - // We limit this to 32 threads in the y dimension so we can still - // have at least 32 threads in the x dimension (1 warp) which should - // result in better coalescing of memory operations. We also - // want to guarantee that we are processing a multiple of 32 threads - // in the x dimension because we use atomic operations at the block - // level when writing validity data out to main memory, and that would - // need to change if we split a word of validity data between blocks. - int const y_block_size = min(util::div_rounding_up_safe(num_columns, 4), 32); - int const x_possible_block_size = 1024 / y_block_size; - // 48KB is the default setting for shared memory per block according to the cuda tutorials - // If someone configures the GPU to only have 16 KB this might not work. - int const max_shared_size = 48 * 1024; - // If we don't have enough shared memory there is no point in having more threads - // per block that will just sit idle - auto const max_block_size = std::min(x_possible_block_size, max_shared_size / size_per_row); - // Make sure that the x dimension is a multiple of 32 this not only helps - // coalesce memory access it also lets us do a ballot sync for validity to write - // the data back out the warp level. If x is a multiple of 32 then each thread in the y - // dimension is associated with one or more warps, that should correspond to the validity - // words directly. - int const block_size = (max_block_size / 32) * 32; - CUDF_EXPECTS(block_size != 0, "Row size is too large to fit in shared memory"); - - // The maximum number of blocks supported in the x dimension is 2 ^ 31 - 1 - // but in practice having too many can cause some overhead that I don't totally - // understand. Playing around with this having as little as 600 blocks appears - // to be able to saturate memory on V100, so this is an order of magnitude higher - // to try and future proof this a bit. - int const num_blocks = std::clamp((num_rows + block_size - 1) / block_size, 1, 10240); - - blocks.x = num_blocks; - blocks.y = 1; - blocks.z = 1; - threads.x = block_size; - threads.y = y_block_size; - threads.z = 1; - return size_per_row * block_size; -} - -/** - * When converting to rows it is possible that the size of the table was too big to fit - * in a single column. This creates an output column for a subset of the rows in a table - * going from start row and containing the next num_rows. Most of the parameters passed - * into this function are common between runs and should be calculated once. - */ -static std::unique_ptr fixed_width_convert_to_rows( - const size_type start_row, - const size_type num_rows, - const size_type num_columns, - const size_type size_per_row, - rmm::device_uvector& column_start, - rmm::device_uvector& column_size, - rmm::device_uvector& input_data, - rmm::device_uvector& input_nm, - const scalar& zero, - const scalar& scalar_size_per_row, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) -{ - int64_t const total_allocation = size_per_row * num_rows; - // We made a mistake in the split somehow - CUDF_EXPECTS(total_allocation < std::numeric_limits::max(), - "Table is too large to fit!"); - - // Allocate and set the offsets row for the byte array - std::unique_ptr offsets = - cudf::detail::sequence(num_rows + 1, zero, scalar_size_per_row, stream, mr); - - std::unique_ptr data = make_numeric_column(data_type(type_id::INT8), - static_cast(total_allocation), - mask_state::UNALLOCATED, - stream, - mr); - - dim3 blocks; - dim3 threads; - int shared_size = - detail::calc_fixed_width_kernel_dims(num_columns, num_rows, size_per_row, blocks, threads); - - copy_to_rows_fixed_width_optimized<<>>( - start_row, - num_rows, - num_columns, - size_per_row, - column_start.data(), - column_size.data(), - input_data.data(), - input_nm.data(), - data->mutable_view().data()); - - return make_lists_column(num_rows, - std::move(offsets), - std::move(data), - 0, - rmm::device_buffer{0, cudf::get_default_stream(), mr}, - stream, - mr); -} - -static inline bool are_all_fixed_width(std::vector const& schema) -{ - return std::all_of( - schema.begin(), schema.end(), [](const data_type& t) { return is_fixed_width(t); }); -} - -/** - * @brief Given a set of fixed width columns, calculate how the data will be laid out in memory. - * - * @param [in] schema the types of columns that need to be laid out. - * @param [out] column_start the byte offset where each column starts in the row. - * @param [out] column_size the size in bytes of the data for each columns in the row. - * @return the size in bytes each row needs. - */ -static inline int32_t compute_fixed_width_layout(std::vector const& schema, - std::vector& column_start, - std::vector& column_size) -{ - // We guarantee that the start of each column is 64-bit aligned so anything can go - // there, but to make the code simple we will still do an alignment for it. - int32_t at_offset = 0; - for (auto col = schema.begin(); col < schema.end(); col++) { - size_type s = size_of(*col); - column_size.emplace_back(s); - std::size_t allocation_needed = s; - std::size_t alignment_needed = allocation_needed; // They are the same for fixed width types - at_offset = util::round_up_unsafe(at_offset, static_cast(alignment_needed)); - column_start.emplace_back(at_offset); - at_offset += allocation_needed; - } - - // Now we need to add in space for validity - // Eventually we can think about nullable vs not nullable, but for now we will just always add - // it in - int32_t const validity_bytes_needed = - util::div_rounding_up_safe(schema.size(), CHAR_BIT); - // validity comes at the end and is byte aligned so we can pack more in. - at_offset += validity_bytes_needed; - // Now we need to pad the end so all rows are 64 bit aligned - return util::round_up_unsafe(at_offset, JCUDF_ROW_ALIGNMENT); -} - -/** - * @brief column sizes and column start offsets for a table - */ -struct column_info_s { - size_type size_per_row; - std::vector column_starts; - std::vector column_sizes; - std::vector variable_width_column_starts; - - column_info_s& operator=(column_info_s const& other) = delete; - column_info_s& operator=(column_info_s&& other) = delete; -}; - -/** - * @brief Compute information about a table such as bytes per row and offsets. - * - * @tparam iterator iterator of column schema data - * @param begin starting iterator of column schema - * @param end ending iterator of column schema - * @param column_starts column start offsets - * @param column_sizes size in bytes of each column - * @return size of the fixed_width data portion of a row. - */ -template -column_info_s compute_column_information(iterator begin, iterator end) -{ - size_type size_per_row = 0; - std::vector column_starts; - std::vector column_sizes; - std::vector variable_width_column_starts; - - column_starts.reserve(std::distance(begin, end) + 1); - column_sizes.reserve(std::distance(begin, end)); - - for (auto col_type = begin; col_type != end; ++col_type) { - bool const compound_type = is_compound(*col_type); - - // a list or string column will write a single uint64 of data here for offset/length - auto const col_size = compound_type ? sizeof(uint32_t) + sizeof(uint32_t) : size_of(*col_type); - - // align size for this type - They are the same for fixed width types and 4 bytes for variable - // width length/offset combos - size_type const alignment_needed = compound_type ? __alignof(uint32_t) : col_size; - size_per_row = util::round_up_unsafe(size_per_row, alignment_needed); - if (compound_type) { variable_width_column_starts.push_back(size_per_row); } - column_starts.push_back(size_per_row); - column_sizes.push_back(col_size); - size_per_row += col_size; - } - - // add validity offset to the end of fixed_width offsets - auto validity_offset = size_per_row; - column_starts.push_back(validity_offset); - - // validity is byte-aligned in the JCUDF format - size_per_row += - util::div_rounding_up_safe(static_cast(std::distance(begin, end)), CHAR_BIT); - - return {size_per_row, - std::move(column_starts), - std::move(column_sizes), - std::move(variable_width_column_starts)}; -} - -/** - * @brief Build `tile_info` for the validity data to break up the work. - * - * @param num_columns number of columns in the table - * @param num_rows number of rows in the table - * @param shmem_limit_per_tile size of shared memory available to a single gpu tile - * @param row_batches batched row information for multiple output locations - * @return vector of `tile_info` structs for validity data - */ -std::vector build_validity_tile_infos(size_type const& num_columns, - size_type const& num_rows, - size_type const& shmem_limit_per_tile, - std::vector const& row_batches) -{ - auto const desired_rows_and_columns = static_cast(sqrt(shmem_limit_per_tile)); - auto const column_stride = util::round_up_unsafe( - [&]() { - if (desired_rows_and_columns > num_columns) { - // not many columns, build a single tile for table width and ship it off - return num_columns; - } else { - return util::round_down_safe(desired_rows_and_columns, CHAR_BIT); - } - }(), - JCUDF_ROW_ALIGNMENT); - - // we fit as much as we can given the column stride note that an element in the table takes just 1 - // bit, but a row with a single element still takes 8 bytes! - auto const bytes_per_row = - util::round_up_safe(util::div_rounding_up_unsafe(column_stride, CHAR_BIT), JCUDF_ROW_ALIGNMENT); - auto const row_stride = - std::min(num_rows, util::round_down_safe(shmem_limit_per_tile / bytes_per_row, 64)); - std::vector validity_tile_infos; - validity_tile_infos.reserve(num_columns / column_stride * num_rows / row_stride); - for (int col = 0; col < num_columns; col += column_stride) { - int current_tile_row_batch = 0; - int rows_left_in_batch = row_batches[current_tile_row_batch].row_count; - int row = 0; - while (row < num_rows) { - if (rows_left_in_batch == 0) { - current_tile_row_batch++; - rows_left_in_batch = row_batches[current_tile_row_batch].row_count; - } - int const tile_height = std::min(row_stride, rows_left_in_batch); - validity_tile_infos.emplace_back( - detail::tile_info{col, - row, - std::min(col + column_stride - 1, num_columns - 1), - row + tile_height - 1, - current_tile_row_batch}); - row += tile_height; - rows_left_in_batch -= tile_height; - } - } - - return validity_tile_infos; -} - -/** - * @brief functor that returns the size of a row or 0 is row is greater than the number of rows in - * the table - * - * @tparam RowSize iterator that returns the size of a specific row - */ -template -struct row_size_functor { - row_size_functor(size_type row_end, RowSize row_sizes, size_type last_row_end) - : _row_end(row_end), _row_sizes(row_sizes), _last_row_end(last_row_end) - { - } - - __device__ inline uint64_t operator()(int i) const - { - return i >= _row_end ? 0 : _row_sizes[i + _last_row_end]; - } - - size_type _row_end; - RowSize _row_sizes; - size_type _last_row_end; -}; - -/** - * @brief Builds batches of rows that will fit in the size limit of a column. - * - * @tparam RowSize iterator that gives the size of a specific row of the table. - * @param num_rows Total number of rows in the table - * @param row_sizes iterator that gives the size of a specific row of the table. - * @param all_fixed_width bool indicating all data in this table is fixed width - * @param stream stream to operate on for this work - * @param mr memory resource used to allocate any returned data - * @returns vector of size_type's that indicate row numbers for batch boundaries and a - * device_uvector of row offsets - */ -template -batch_data build_batches(size_type num_rows, - RowSize row_sizes, - bool all_fixed_width, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) -{ - auto const total_size = thrust::reduce(rmm::exec_policy(stream), row_sizes, row_sizes + num_rows); - auto const num_batches = static_cast( - util::div_rounding_up_safe(total_size, static_cast(MAX_BATCH_SIZE))); - auto const num_offsets = num_batches + 1; - std::vector row_batches; - std::vector batch_row_boundaries; - device_uvector batch_row_offsets(all_fixed_width ? 0 : num_rows, stream); - - // at most max gpu memory / 2GB iterations. - batch_row_boundaries.reserve(num_offsets); - batch_row_boundaries.push_back(0); - size_type last_row_end = 0; - device_uvector cumulative_row_sizes(num_rows, stream); - - thrust::inclusive_scan( - rmm::exec_policy(stream), row_sizes, row_sizes + num_rows, cumulative_row_sizes.begin()); - - // This needs to be split this into 2 gig batches. Care must be taken to avoid a batch larger than - // 2 gigs. Imagine a table with 900 meg rows. The batches should occur every 2 rows, but if a - // lower bound is run at 2 gigs, 4 gigs, 6 gigs. the batches will be 2 rows, 2 rows, 3 rows, which - // will be invalid. The previous batch size must be taken into account when building a new batch. - // One way is to pull the batch size back to the host and add it to MAX_BATCH_SIZE for the lower - // bound search. The other method involves keeping everything on device, but subtracting the - // previous batch from cumulative_row_sizes based on index. This involves no synchronization - // between GPU and CPU, but involves more work on the GPU. These further need to be broken on a - // 32-row boundary to match the fixed_width optimized versions. - - while (last_row_end < num_rows) { - auto offset_row_sizes = thrust::make_transform_iterator( - cumulative_row_sizes.begin(), - cuda::proclaim_return_type( - [last_row_end, cumulative_row_sizes = cumulative_row_sizes.data()] __device__(auto i) { - return i - cumulative_row_sizes[last_row_end]; - })); - auto search_start = offset_row_sizes + last_row_end; - auto search_end = offset_row_sizes + num_rows; - - // find the next MAX_BATCH_SIZE boundary - auto const lb = - thrust::lower_bound(rmm::exec_policy(stream), search_start, search_end, MAX_BATCH_SIZE); - size_type const batch_size = lb - search_start; - - size_type const row_end = lb == search_end - ? batch_size + last_row_end - : last_row_end + util::round_down_safe(batch_size, 32); - - // build offset list for each row in this batch - auto const num_rows_in_batch = row_end - last_row_end; - - // build offset list for each row in this batch - auto const num_entries = row_end - last_row_end + 1; - device_uvector output_batch_row_offsets(num_entries, stream, mr); - - auto row_size_iter_bounded = cudf::detail::make_counting_transform_iterator( - 0, row_size_functor(row_end, row_sizes, last_row_end)); - - thrust::exclusive_scan(rmm::exec_policy(stream), - row_size_iter_bounded, - row_size_iter_bounded + num_entries, - output_batch_row_offsets.begin()); - - auto const batch_bytes = output_batch_row_offsets.element(num_rows_in_batch, stream); - - // The output_batch_row_offsets vector is used as the offset column of the returned data. This - // needs to be individually allocated, but the kernel needs a contiguous array of offsets or - // more global lookups are necessary. - if (!all_fixed_width) { - cudaMemcpy(batch_row_offsets.data() + last_row_end, - output_batch_row_offsets.data(), - num_rows_in_batch * sizeof(size_type), - cudaMemcpyDeviceToDevice); - } - - batch_row_boundaries.push_back(row_end); - row_batches.push_back({batch_bytes, num_rows_in_batch, std::move(output_batch_row_offsets)}); - - last_row_end = row_end; - } - - return { - std::move(batch_row_offsets), - make_device_uvector_async(batch_row_boundaries, stream, rmm::mr::get_current_device_resource()), - std::move(batch_row_boundaries), - std::move(row_batches)}; -} - -/** - * @brief Computes the number of tiles necessary given a tile height and batch offsets - * - * @param batch_row_boundaries row boundaries for each batch - * @param desired_tile_height height of each tile in the table - * @param stream stream to use - * @return number of tiles necessary - */ -int compute_tile_counts(device_span const& batch_row_boundaries, - int desired_tile_height, - rmm::cuda_stream_view stream) -{ - size_type const num_batches = batch_row_boundaries.size() - 1; - device_uvector num_tiles(num_batches, stream); - auto iter = thrust::make_counting_iterator(0); - thrust::transform( - rmm::exec_policy(stream), - iter, - iter + num_batches, - num_tiles.begin(), - cuda::proclaim_return_type( - [desired_tile_height, batch_row_boundaries = batch_row_boundaries.data()] __device__( - auto batch_index) -> size_type { - return util::div_rounding_up_unsafe( - batch_row_boundaries[batch_index + 1] - batch_row_boundaries[batch_index], - desired_tile_height); - })); - return thrust::reduce(rmm::exec_policy(stream), num_tiles.begin(), num_tiles.end()); -} - -/** - * @brief Builds the `tile_info` structs for a given table. - * - * @param tiles span of tiles to populate - * @param batch_row_boundaries boundary to row batches - * @param column_start starting column of the tile - * @param column_end ending column of the tile - * @param desired_tile_height height of the tile - * @param total_number_of_rows total number of rows in the table - * @param stream stream to use - * @return number of tiles created - */ -size_type build_tiles( - device_span tiles, - device_uvector const& batch_row_boundaries, // comes from build_batches - int column_start, - int column_end, - int desired_tile_height, - int total_number_of_rows, - rmm::cuda_stream_view stream) -{ - size_type const num_batches = batch_row_boundaries.size() - 1; - device_uvector num_tiles(num_batches, stream); - auto iter = thrust::make_counting_iterator(0); - thrust::transform( - rmm::exec_policy(stream), - iter, - iter + num_batches, - num_tiles.begin(), - cuda::proclaim_return_type( - [desired_tile_height, batch_row_boundaries = batch_row_boundaries.data()] __device__( - auto batch_index) -> size_type { - return util::div_rounding_up_unsafe( - batch_row_boundaries[batch_index + 1] - batch_row_boundaries[batch_index], - desired_tile_height); - })); - - size_type const total_tiles = - thrust::reduce(rmm::exec_policy(stream), num_tiles.begin(), num_tiles.end()); - - device_uvector tile_starts(num_batches + 1, stream); - auto tile_iter = cudf::detail::make_counting_transform_iterator( - 0, - cuda::proclaim_return_type( - [num_tiles = num_tiles.data(), num_batches] __device__(auto i) { - return (i < num_batches) ? num_tiles[i] : 0; - })); - thrust::exclusive_scan(rmm::exec_policy(stream), - tile_iter, - tile_iter + num_batches + 1, - tile_starts.begin()); // in tiles - - thrust::transform( - rmm::exec_policy(stream), - iter, - iter + total_tiles, - tiles.begin(), - cuda::proclaim_return_type( - [ =, - tile_starts = tile_starts.data(), - batch_row_boundaries = batch_row_boundaries.data()] __device__(size_type tile_index) { - // what batch this tile falls in - auto const batch_index_iter = - thrust::upper_bound(thrust::seq, tile_starts, tile_starts + num_batches, tile_index); - auto const batch_index = std::distance(tile_starts, batch_index_iter) - 1; - // local index within the tile - int const local_tile_index = tile_index - tile_starts[batch_index]; - // the start row for this batch. - int const batch_row_start = batch_row_boundaries[batch_index]; - // the start row for this tile - int const tile_row_start = batch_row_start + (local_tile_index * desired_tile_height); - // the end row for this tile - int const max_row = - std::min(total_number_of_rows - 1, - batch_index + 1 > num_batches - ? std::numeric_limits::max() - : static_cast(batch_row_boundaries[batch_index + 1]) - 1); - int const tile_row_end = - std::min(batch_row_start + ((local_tile_index + 1) * desired_tile_height) - 1, max_row); - - // stuff the tile - return tile_info{ - column_start, tile_row_start, column_end, tile_row_end, static_cast(batch_index)}; - })); - - return total_tiles; -} - -/** - * @brief Determines what data should be operated on by each tile for the incoming table. - * - * @tparam TileCallback Callback that receives the start and end columns of tiles - * @param column_sizes vector of the size of each column - * @param column_starts vector of the offset of each column - * @param first_row_batch_size size of the first row batch to limit max tile size since a tile - * is unable to span batches - * @param total_number_of_rows total number of rows in the table - * @param shmem_limit_per_tile shared memory allowed per tile - * @param f callback function called when building a tile - */ -template -void determine_tiles(std::vector const& column_sizes, - std::vector const& column_starts, - size_type const first_row_batch_size, - size_type const total_number_of_rows, - size_type const& shmem_limit_per_tile, - TileCallback f) -{ - // tile infos are organized with the tile going "down" the columns this provides the most - // coalescing of memory access - int current_tile_width = 0; - int current_tile_start_col = 0; - - // the ideal tile height has lots of 8-byte reads and 8-byte writes. The optimal read/write would - // be memory cache line sized access, but since other tiles will read/write the edges this may not - // turn out to be overly important. For now, we will attempt to build a square tile as far as byte - // sizes. x * y = shared_mem_size. Which translates to x^2 = shared_mem_size since we want them - // equal, so height and width are sqrt(shared_mem_size). The trick is that it's in bytes, not rows - // or columns. - auto const square_bias = 32; // bias towards columns for performance reasons - auto const optimal_square_len = static_cast(sqrt(shmem_limit_per_tile)); - auto const desired_tile_height = util::round_up_safe( - std::min(optimal_square_len / square_bias, total_number_of_rows), cudf::detail::warp_size); - auto const tile_height = std::clamp(desired_tile_height, 1, first_row_batch_size); - - int row_size = 0; - - // march each column and build the tiles of appropriate sizes - for (uint col = 0; col < column_sizes.size(); ++col) { - auto const col_size = column_sizes[col]; - - // align size for this type - auto const alignment_needed = col_size; // They are the same for fixed width types - auto const row_size_aligned = util::round_up_unsafe(row_size, alignment_needed); - auto const row_size_with_this_col = row_size_aligned + col_size; - auto const row_size_with_end_pad = - util::round_up_unsafe(row_size_with_this_col, JCUDF_ROW_ALIGNMENT); - - if (row_size_with_end_pad * tile_height > shmem_limit_per_tile) { - // too large, close this tile, generate vertical tiles and restart - f(current_tile_start_col, col == 0 ? col : col - 1, tile_height); - - row_size = - util::round_up_unsafe((column_starts[col] + column_sizes[col]) & 7, alignment_needed); - row_size += col_size; // alignment required for shared memory tile boundary to match - // alignment of output row - current_tile_start_col = col; - current_tile_width = 0; - } else { - row_size = row_size_with_this_col; - current_tile_width++; - } - } - - // build last set of tiles - if (current_tile_width > 0) { - f(current_tile_start_col, static_cast(column_sizes.size()) - 1, tile_height); - } -} - -/** - * @brief convert cudf table into JCUDF row format - * - * @tparam offsetFunctor functor type for offset functor - * @param tbl table to convert to JCUDF row format - * @param batch_info information about the batches of data - * @param offset_functor functor that returns the starting offset of each row - * @param column_info information about incoming columns - * @param variable_width_offsets optional vector of offsets for variable-with columns - * @param stream stream used - * @param mr selected memory resource for returned data - * @return vector of list columns containing byte columns of the JCUDF row data - */ -template -std::vector> convert_to_rows( - table_view const& tbl, - batch_data& batch_info, - offsetFunctor offset_functor, - column_info_s const& column_info, - std::optional> variable_width_offsets, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) -{ - int device_id; - CUDF_CUDA_TRY(cudaGetDevice(&device_id)); - int total_shmem_in_bytes; - CUDF_CUDA_TRY( - cudaDeviceGetAttribute(&total_shmem_in_bytes, cudaDevAttrMaxSharedMemoryPerBlock, device_id)); - -#ifndef __CUDA_ARCH__ // __host__ code. - // Need to reduce total shmem available by the size of barriers in the kernel's shared memory - total_shmem_in_bytes -= - util::round_up_unsafe(sizeof(cuda::barrier), 16ul); -#endif // __CUDA_ARCH__ - - auto const shmem_limit_per_tile = total_shmem_in_bytes; - - auto const num_rows = tbl.num_rows(); - auto const fixed_width_only = !variable_width_offsets.has_value(); - - auto select_columns = [](auto const& tbl, auto column_predicate) { - std::vector cols; - std::copy_if(tbl.begin(), tbl.end(), std::back_inserter(cols), [&](auto c) { - return column_predicate(c); - }); - return table_view(cols); - }; - - auto dev_col_sizes = make_device_uvector_async( - column_info.column_sizes, stream, rmm::mr::get_current_device_resource()); - auto dev_col_starts = make_device_uvector_async( - column_info.column_starts, stream, rmm::mr::get_current_device_resource()); - - // Get the pointers to the input columnar data ready - auto const data_begin = thrust::make_transform_iterator(tbl.begin(), [](auto const& c) { - return is_compound(c.type()) ? nullptr : c.template data(); - }); - std::vector input_data(data_begin, data_begin + tbl.num_columns()); - - // validity code handles variable and fixed-width data, so give it everything - auto const nm_begin = - thrust::make_transform_iterator(tbl.begin(), [](auto const& c) { return c.null_mask(); }); - std::vector input_nm(nm_begin, nm_begin + tbl.num_columns()); - - auto dev_input_data = - make_device_uvector_async(input_data, stream, rmm::mr::get_current_device_resource()); - auto dev_input_nm = - make_device_uvector_async(input_nm, stream, rmm::mr::get_current_device_resource()); - - // the first batch always exists unless we were sent an empty table - auto const first_batch_size = batch_info.row_batches[0].row_count; - - std::vector output_buffers; - std::vector output_data; - output_data.reserve(batch_info.row_batches.size()); - output_buffers.reserve(batch_info.row_batches.size()); - std::transform( - batch_info.row_batches.begin(), - batch_info.row_batches.end(), - std::back_inserter(output_buffers), - [&](auto const& batch) { return rmm::device_buffer(batch.num_bytes, stream, mr); }); - std::transform( - output_buffers.begin(), output_buffers.end(), std::back_inserter(output_data), [](auto& buf) { - return static_cast(buf.data()); - }); - - auto dev_output_data = make_device_uvector_async(output_data, stream, mr); - - int info_count = 0; - detail::determine_tiles( - column_info.column_sizes, - column_info.column_starts, - first_batch_size, - num_rows, - shmem_limit_per_tile, - [&gpu_batch_row_boundaries = batch_info.d_batch_row_boundaries, &info_count, &stream]( - int const start_col, int const end_col, int const tile_height) { - int i = detail::compute_tile_counts(gpu_batch_row_boundaries, tile_height, stream); - info_count += i; - }); - - // allocate space for tiles - device_uvector gpu_tile_infos(info_count, stream); - int tile_offset = 0; - - detail::determine_tiles( - column_info.column_sizes, - column_info.column_starts, - first_batch_size, - num_rows, - shmem_limit_per_tile, - [&gpu_batch_row_boundaries = batch_info.d_batch_row_boundaries, - &gpu_tile_infos, - num_rows, - &tile_offset, - stream](int const start_col, int const end_col, int const tile_height) { - tile_offset += detail::build_tiles( - {gpu_tile_infos.data() + tile_offset, gpu_tile_infos.size() - tile_offset}, - gpu_batch_row_boundaries, - start_col, - end_col, - tile_height, - num_rows, - stream); - }); - - // build validity tiles for ALL columns, variable and fixed width. - auto validity_tile_infos = detail::build_validity_tile_infos( - tbl.num_columns(), num_rows, shmem_limit_per_tile, batch_info.row_batches); - - auto dev_validity_tile_infos = - make_device_uvector_async(validity_tile_infos, stream, rmm::mr::get_current_device_resource()); - - auto const validity_offset = column_info.column_starts.back(); - - // blast through the entire table and convert it - detail::copy_to_rows<<>>(num_rows, - tbl.num_columns(), - shmem_limit_per_tile, - gpu_tile_infos, - dev_input_data.data(), - dev_col_sizes.data(), - dev_col_starts.data(), - offset_functor, - batch_info.d_batch_row_boundaries.data(), - reinterpret_cast(dev_output_data.data())); - - // note that validity gets the entire table and not the fixed-width portion - detail::copy_validity_to_rows<<>>(num_rows, - tbl.num_columns(), - shmem_limit_per_tile, - offset_functor, - batch_info.d_batch_row_boundaries.data(), - dev_output_data.data(), - validity_offset, - dev_validity_tile_infos, - dev_input_nm.data()); - - if (!fixed_width_only) { - // build table view for variable-width data only - auto const variable_width_table = - select_columns(tbl, [](auto col) { return is_compound(col.type()); }); - - CUDF_EXPECTS(!variable_width_table.is_empty(), "No variable-width columns when expected!"); - CUDF_EXPECTS(variable_width_offsets.has_value(), "No variable width offset data!"); - - auto const variable_data_begin = thrust::make_transform_iterator( - variable_width_table.begin(), - [](auto const& c) { return is_compound(c.type()) ? c.template data() : nullptr; }); - std::vector variable_width_input_data( - variable_data_begin, variable_data_begin + variable_width_table.num_columns()); - - auto dev_variable_input_data = make_device_uvector_async( - variable_width_input_data, stream, rmm::mr::get_current_device_resource()); - auto dev_variable_col_output_offsets = make_device_uvector_async( - column_info.variable_width_column_starts, stream, rmm::mr::get_current_device_resource()); - - for (uint i = 0; i < batch_info.row_batches.size(); i++) { - auto const batch_row_offset = batch_info.batch_row_boundaries[i]; - auto const batch_num_rows = batch_info.row_batches[i].row_count; - - dim3 const string_blocks( - std::min(MAX_STRING_BLOCKS, - util::div_rounding_up_unsafe(batch_num_rows, NUM_STRING_ROWS_PER_BLOCK_TO_ROWS))); - - detail::copy_strings_to_rows<<>>(batch_num_rows, - variable_width_table.num_columns(), - dev_variable_input_data.data(), - dev_variable_col_output_offsets.data(), - variable_width_offsets->data(), - column_info.size_per_row, - offset_functor, - batch_row_offset, - reinterpret_cast(output_data[i])); - } - } - - // split up the output buffer into multiple buffers based on row batch sizes and create list of - // byte columns - std::vector> ret; - ret.reserve(batch_info.row_batches.size()); - auto counting_iter = thrust::make_counting_iterator(0); - std::transform(counting_iter, - counting_iter + batch_info.row_batches.size(), - std::back_inserter(ret), - [&](auto batch) { - auto const offset_count = batch_info.row_batches[batch].row_offsets.size(); - auto offsets = - std::make_unique(data_type{type_id::INT32}, - (size_type)offset_count, - batch_info.row_batches[batch].row_offsets.release(), - rmm::device_buffer{}, - 0); - auto data = std::make_unique(data_type{type_id::INT8}, - batch_info.row_batches[batch].num_bytes, - std::move(output_buffers[batch]), - rmm::device_buffer{}, - 0); - - return make_lists_column(batch_info.row_batches[batch].row_count, - std::move(offsets), - std::move(data), - 0, - rmm::device_buffer{0, cudf::get_default_stream(), mr}, - stream, - mr); - }); - - return ret; -} - -} // namespace detail - -/** - * @brief convert a cudf table to JCUDF row format - * - * @param tbl incoming table to convert - * @param stream stream to use for operations - * @param mr memory resource used for returned data - * @return vector of list columns containing byte columns of the JCUDF row data - */ -std::vector> convert_to_rows(table_view const& tbl, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) -{ - auto const num_columns = tbl.num_columns(); - auto const num_rows = tbl.num_rows(); - - auto const fixed_width_only = std::all_of( - tbl.begin(), tbl.end(), [](column_view const& c) { return is_fixed_width(c.type()); }); - - // Break up the work into tiles, which are a starting and ending row/col #. This tile size is - // calculated based on the shared memory size available we want a single tile to fill up the - // entire shared memory space available for the transpose-like conversion. - - // There are two different processes going on here. The GPU conversion of the data and the writing - // of the data into the list of byte columns that are a maximum of 2 gigs each due to offset - // maximum size. The GPU conversion portion has to understand this limitation because the column - // must own the data inside and as a result it must be a distinct allocation for that column. - // Copying the data into these final buffers would be prohibitively expensive, so care is taken to - // ensure the GPU writes to the proper buffer. The tiles are broken at the boundaries of specific - // rows based on the row sizes up to that point. These are row batches and they are decided first - // before building the tiles so the tiles can be properly cut around them. - - auto schema_column_iter = - thrust::make_transform_iterator(tbl.begin(), [](auto const& i) { return i.type(); }); - - auto column_info = - detail::compute_column_information(schema_column_iter, schema_column_iter + num_columns); - auto const size_per_row = column_info.size_per_row; - if (fixed_width_only) { - // total encoded row size. This includes fixed-width data and validity only. It does not include - // variable-width data since it isn't copied with the fixed-width and validity kernel. - auto row_size_iter = thrust::make_constant_iterator( - util::round_up_unsafe(size_per_row, JCUDF_ROW_ALIGNMENT)); - - auto batch_info = detail::build_batches(num_rows, row_size_iter, fixed_width_only, stream, mr); - - detail::fixed_width_row_offset_functor offset_functor( - util::round_up_unsafe(size_per_row, JCUDF_ROW_ALIGNMENT)); - - return detail::convert_to_rows( - tbl, batch_info, offset_functor, std::move(column_info), std::nullopt, stream, mr); - } else { - auto offset_data = detail::build_string_row_offsets(tbl, size_per_row, stream); - auto& row_sizes = std::get<0>(offset_data); - - auto row_size_iter = cudf::detail::make_counting_transform_iterator( - 0, detail::row_size_functor(num_rows, row_sizes.data(), 0)); - - auto batch_info = detail::build_batches(num_rows, row_size_iter, fixed_width_only, stream, mr); - - detail::string_row_offset_functor offset_functor(batch_info.batch_row_offsets); - - return detail::convert_to_rows(tbl, - batch_info, - offset_functor, - std::move(column_info), - std::make_optional(std::move(std::get<1>(offset_data))), - stream, - mr); - } -} - -std::vector> convert_to_rows_fixed_width_optimized( - table_view const& tbl, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) -{ - auto const num_columns = tbl.num_columns(); - - std::vector schema; - schema.resize(num_columns); - std::transform( - tbl.begin(), tbl.end(), schema.begin(), [](auto i) -> data_type { return i.type(); }); - - if (detail::are_all_fixed_width(schema)) { - std::vector column_start; - std::vector column_size; - - int32_t const size_per_row = - detail::compute_fixed_width_layout(schema, column_start, column_size); - auto dev_column_start = make_device_uvector_async(column_start, stream, mr); - auto dev_column_size = make_device_uvector_async(column_size, stream, mr); - - // Make the number of rows per batch a multiple of 32 so we don't have to worry about splitting - // validity at a specific row offset. This might change in the future. - auto const max_rows_per_batch = - util::round_down_safe(std::numeric_limits::max() / size_per_row, 32); - - auto const num_rows = tbl.num_rows(); - - // Get the pointers to the input columnar data ready - std::vector input_data; - std::vector input_nm; - for (size_type column_number = 0; column_number < num_columns; column_number++) { - column_view cv = tbl.column(column_number); - input_data.emplace_back(cv.data()); - input_nm.emplace_back(cv.null_mask()); - } - auto dev_input_data = make_device_uvector_async(input_data, stream, mr); - auto dev_input_nm = make_device_uvector_async(input_nm, stream, mr); - - using ScalarType = scalar_type_t; - auto zero = make_numeric_scalar(data_type(type_id::INT32), stream.value()); - zero->set_valid_async(true, stream); - static_cast(zero.get())->set_value(0, stream); - - auto step = make_numeric_scalar(data_type(type_id::INT32), stream.value()); - step->set_valid_async(true, stream); - static_cast(step.get())->set_value(static_cast(size_per_row), stream); - - std::vector> ret; - for (size_type row_start = 0; row_start < num_rows; row_start += max_rows_per_batch) { - size_type row_count = num_rows - row_start; - row_count = row_count > max_rows_per_batch ? max_rows_per_batch : row_count; - ret.emplace_back(detail::fixed_width_convert_to_rows(row_start, - row_count, - num_columns, - size_per_row, - dev_column_start, - dev_column_size, - dev_input_data, - dev_input_nm, - *zero, - *step, - stream, - mr)); - } - - return ret; - } else { - CUDF_FAIL("Only fixed width types are currently supported"); - } -} - -namespace { - -/// @brief Calculates and sets null counts for specified columns -void fixup_null_counts(std::vector>& output_columns, - rmm::cuda_stream_view stream) -{ - for (auto& col : output_columns) { - col->set_null_count(cudf::detail::null_count(col->view().null_mask(), 0, col->size(), stream)); - } -} - -} // namespace - -/** - * @brief convert from JCUDF row format to cudf columns - * - * @param input vector of list columns containing byte columns of the JCUDF row data - * @param schema incoming schema of the data - * @param stream stream to use for compute - * @param mr memory resource for returned data - * @return cudf table of the data - */ -std::unique_ptr
convert_from_rows(lists_column_view const& input, - std::vector const& schema, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) -{ - // verify that the types are what we expect - column_view child = input.child(); - auto const list_type = child.type().id(); - CUDF_EXPECTS(list_type == type_id::INT8 || list_type == type_id::UINT8, - "Only a list of bytes is supported as input"); - - // convert any strings in the schema to two int32 columns - // This allows us to leverage the fixed-width copy code to fill in our offset and string length - // data. - std::vector string_schema; - string_schema.reserve(schema.size()); - for (auto i : schema) { - if (i.id() == type_id::STRING) { - string_schema.push_back(data_type(type_id::INT32)); - string_schema.push_back(data_type(type_id::INT32)); - } else { - string_schema.push_back(i); - } - } - - auto const num_columns = string_schema.size(); - auto const num_rows = input.parent().size(); - - int device_id; - CUDF_CUDA_TRY(cudaGetDevice(&device_id)); - int total_shmem_in_bytes; - CUDF_CUDA_TRY( - cudaDeviceGetAttribute(&total_shmem_in_bytes, cudaDevAttrMaxSharedMemoryPerBlock, device_id)); - -#ifndef __CUDA_ARCH__ // __host__ code. - // Need to reduce total shmem available by the size of barriers in the kernel's shared memory - total_shmem_in_bytes -= - util::round_up_unsafe(sizeof(cuda::barrier), 16ul); -#endif // __CUDA_ARCH__ - - auto const shmem_limit_per_tile = total_shmem_in_bytes; - - auto column_info = detail::compute_column_information(string_schema.begin(), string_schema.end()); - auto const size_per_row = util::round_up_unsafe(column_info.size_per_row, JCUDF_ROW_ALIGNMENT); - - // Ideally we would check that the offsets are all the same, etc. but for now this is probably - // fine - CUDF_EXPECTS(size_per_row * num_rows <= child.size(), "The layout of the data appears to be off"); - auto dev_col_starts = make_device_uvector_async( - column_info.column_starts, stream, rmm::mr::get_current_device_resource()); - auto dev_col_sizes = make_device_uvector_async( - column_info.column_sizes, stream, rmm::mr::get_current_device_resource()); - - // Allocate the columns we are going to write into - std::vector> output_columns; - std::vector> string_row_offset_columns; - std::vector> string_length_columns; - std::vector output_data; - std::vector output_nm; - std::vector string_row_offsets; - std::vector string_lengths; - for (auto i : schema) { - auto make_col = [&output_data, &output_nm](data_type type, - size_type num_rows, - bool include_nm, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) { - auto column = - make_fixed_width_column(type, - num_rows, - include_nm ? mask_state::UNINITIALIZED : mask_state::UNALLOCATED, - stream, - mr); - auto mut = column->mutable_view(); - output_data.emplace_back(mut.data()); - if (include_nm) { output_nm.emplace_back(mut.null_mask()); } - return column; - }; - if (i.id() == type_id::STRING) { - auto const int32type = data_type(type_id::INT32); - auto offset_col = - make_col(int32type, num_rows, true, stream, rmm::mr::get_current_device_resource()); - string_row_offsets.push_back(offset_col->mutable_view().data()); - string_row_offset_columns.emplace_back(std::move(offset_col)); - auto length_col = - make_col(int32type, num_rows, false, stream, rmm::mr::get_current_device_resource()); - string_lengths.push_back(length_col->mutable_view().data()); - string_length_columns.emplace_back(std::move(length_col)); - // placeholder - output_columns.emplace_back(make_empty_column(type_id::STRING)); - } else { - output_columns.emplace_back(make_col(i, num_rows, true, stream, mr)); - } - } - - auto dev_string_row_offsets = - make_device_uvector_async(string_row_offsets, stream, rmm::mr::get_current_device_resource()); - auto dev_string_lengths = - make_device_uvector_async(string_lengths, stream, rmm::mr::get_current_device_resource()); - - // build the row_batches from the passed in list column - std::vector row_batches; - row_batches.push_back( - {detail::row_batch{child.size(), num_rows, device_uvector(0, stream)}}); - - auto dev_output_data = - make_device_uvector_async(output_data, stream, rmm::mr::get_current_device_resource()); - auto dev_output_nm = - make_device_uvector_async(output_nm, stream, rmm::mr::get_current_device_resource()); - - // only ever get a single batch when going from rows, so boundaries are 0, num_rows - constexpr auto num_batches = 2; - device_uvector gpu_batch_row_boundaries(num_batches, stream); - - thrust::transform(rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(num_batches), - gpu_batch_row_boundaries.begin(), - cuda::proclaim_return_type( - [num_rows] __device__(auto i) { return i == 0 ? 0 : num_rows; })); - - int info_count = 0; - detail::determine_tiles(column_info.column_sizes, - column_info.column_starts, - num_rows, - num_rows, - shmem_limit_per_tile, - [&gpu_batch_row_boundaries, &info_count, &stream]( - int const start_col, int const end_col, int const tile_height) { - info_count += detail::compute_tile_counts( - gpu_batch_row_boundaries, tile_height, stream); - }); - - // allocate space for tiles - device_uvector gpu_tile_infos(info_count, stream); - - int tile_offset = 0; - detail::determine_tiles( - column_info.column_sizes, - column_info.column_starts, - num_rows, - num_rows, - shmem_limit_per_tile, - [&gpu_batch_row_boundaries, &gpu_tile_infos, num_rows, &tile_offset, stream]( - int const start_col, int const end_col, int const tile_height) { - tile_offset += detail::build_tiles( - {gpu_tile_infos.data() + tile_offset, gpu_tile_infos.size() - tile_offset}, - gpu_batch_row_boundaries, - start_col, - end_col, - tile_height, - num_rows, - stream); - }); - - dim3 const blocks(gpu_tile_infos.size()); - - // validity needs to be calculated based on the actual number of final table columns - auto validity_tile_infos = - detail::build_validity_tile_infos(schema.size(), num_rows, shmem_limit_per_tile, row_batches); - - auto dev_validity_tile_infos = - make_device_uvector_async(validity_tile_infos, stream, rmm::mr::get_current_device_resource()); - - dim3 const validity_blocks(validity_tile_infos.size()); - - if (dev_string_row_offsets.size() == 0) { - detail::fixed_width_row_offset_functor offset_functor(size_per_row); - - detail::copy_from_rows<<>>(num_rows, - num_columns, - shmem_limit_per_tile, - offset_functor, - gpu_batch_row_boundaries.data(), - dev_output_data.data(), - dev_col_sizes.data(), - dev_col_starts.data(), - gpu_tile_infos, - child.data()); - - detail::copy_validity_from_rows<<>>(num_rows, - num_columns, - shmem_limit_per_tile, - offset_functor, - gpu_batch_row_boundaries.data(), - dev_output_nm.data(), - column_info.column_starts.back(), - dev_validity_tile_infos, - child.data()); - - } else { - detail::string_row_offset_functor offset_functor(device_span{input.offsets()}); - detail::copy_from_rows<<>>(num_rows, - num_columns, - shmem_limit_per_tile, - offset_functor, - gpu_batch_row_boundaries.data(), - dev_output_data.data(), - dev_col_sizes.data(), - dev_col_starts.data(), - gpu_tile_infos, - child.data()); - - detail::copy_validity_from_rows<<>>(num_rows, - num_columns, - shmem_limit_per_tile, - offset_functor, - gpu_batch_row_boundaries.data(), - dev_output_nm.data(), - column_info.column_starts.back(), - dev_validity_tile_infos, - child.data()); - - std::vector> string_col_offsets; - std::vector> string_data_cols; - std::vector string_col_offset_ptrs; - std::vector string_data_col_ptrs; - for (auto& col_string_lengths : string_lengths) { - device_uvector output_string_offsets(num_rows + 1, stream, mr); - auto tmp = cuda::proclaim_return_type( - [num_rows, col_string_lengths] __device__(auto const& i) { - return i < num_rows ? col_string_lengths[i] : 0; - }); - auto bounded_iter = cudf::detail::make_counting_transform_iterator(0, tmp); - thrust::exclusive_scan(rmm::exec_policy(stream), - bounded_iter, - bounded_iter + num_rows + 1, - output_string_offsets.begin()); - - // allocate destination string column - rmm::device_uvector string_data( - output_string_offsets.element(num_rows, stream), stream, mr); - - string_col_offset_ptrs.push_back(output_string_offsets.data()); - string_data_col_ptrs.push_back(string_data.data()); - string_col_offsets.push_back(std::move(output_string_offsets)); - string_data_cols.push_back(std::move(string_data)); - } - auto dev_string_col_offsets = make_device_uvector_async( - string_col_offset_ptrs, stream, rmm::mr::get_current_device_resource()); - auto dev_string_data_cols = make_device_uvector_async( - string_data_col_ptrs, stream, rmm::mr::get_current_device_resource()); - - dim3 const string_blocks( - std::min(std::max(MIN_STRING_BLOCKS, num_rows / NUM_STRING_ROWS_PER_BLOCK_FROM_ROWS), - MAX_STRING_BLOCKS)); - - detail::copy_strings_from_rows<<>>( - offset_functor, - dev_string_row_offsets.data(), - dev_string_lengths.data(), - dev_string_col_offsets.data(), - dev_string_data_cols.data(), - child.data(), - num_rows, - static_cast(string_col_offsets.size())); - - // merge strings back into output_columns - int string_idx = 0; - for (int i = 0; i < static_cast(schema.size()); ++i) { - if (schema[i].id() == type_id::STRING) { - // stuff real string column - auto string_data = string_row_offset_columns[string_idx].release()->release(); - output_columns[i] = - make_strings_column(num_rows, - std::make_unique( - std::move(string_col_offsets[string_idx]), rmm::device_buffer{}, 0), - string_data_cols[string_idx].release(), - 0, - std::move(*string_data.null_mask.release())); - // Null count set to 0, temporarily. Will be fixed up before return. - string_idx++; - } - } - } - - // Set null counts, because output_columns are modified via mutable-view, - // in the kernel above. - // TODO(future): Consider setting null count in the kernel itself. - fixup_null_counts(output_columns, stream); - - return std::make_unique
(std::move(output_columns)); -} - -std::unique_ptr
convert_from_rows_fixed_width_optimized(lists_column_view const& input, - std::vector const& schema, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) -{ - // verify that the types are what we expect - column_view child = input.child(); - auto const list_type = child.type().id(); - CUDF_EXPECTS(list_type == type_id::INT8 || list_type == type_id::UINT8, - "Only a list of bytes is supported as input"); - - auto const num_columns = schema.size(); - - if (detail::are_all_fixed_width(schema)) { - std::vector column_start; - std::vector column_size; - - auto const num_rows = input.parent().size(); - auto const size_per_row = detail::compute_fixed_width_layout(schema, column_start, column_size); - - // Ideally we would check that the offsets are all the same, etc. but for now this is probably - // fine - CUDF_EXPECTS(size_per_row * num_rows == child.size(), - "The layout of the data appears to be off"); - auto dev_column_start = - make_device_uvector_async(column_start, stream, rmm::mr::get_current_device_resource()); - auto dev_column_size = - make_device_uvector_async(column_size, stream, rmm::mr::get_current_device_resource()); - - // Allocate the columns we are going to write into - std::vector> output_columns; - std::vector output_data; - std::vector output_nm; - for (int i = 0; i < static_cast(num_columns); i++) { - auto column = - make_fixed_width_column(schema[i], num_rows, mask_state::UNINITIALIZED, stream, mr); - auto mut = column->mutable_view(); - output_data.emplace_back(mut.data()); - output_nm.emplace_back(mut.null_mask()); - output_columns.emplace_back(std::move(column)); - } - - auto dev_output_data = make_device_uvector_async(output_data, stream, mr); - auto dev_output_nm = make_device_uvector_async(output_nm, stream, mr); - - dim3 blocks; - dim3 threads; - int shared_size = - detail::calc_fixed_width_kernel_dims(num_columns, num_rows, size_per_row, blocks, threads); - - detail::copy_from_rows_fixed_width_optimized<<>>( - num_rows, - num_columns, - size_per_row, - dev_column_start.data(), - dev_column_size.data(), - dev_output_data.data(), - dev_output_nm.data(), - child.data()); - - // Set null counts, because output_columns are modified via mutable-view, - // in the kernel above. - // TODO(future): Consider setting null count in the kernel itself. - fixup_null_counts(output_columns, stream); - - return std::make_unique
(std::move(output_columns)); - } else { - CUDF_FAIL("Only fixed width types are currently supported"); - } -} - -} // namespace cudf diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 93443b04bd5..fa9d2ee88ce 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -263,8 +263,6 @@ ConfigureTest( transform/one_hot_encode_tests.cpp ) -ConfigureTest(ROW_CONVERSION_TEST transform/row_conversion.cpp) - # ################################################################################################## # * interop tests ------------------------------------------------------------------------- ConfigureTest( diff --git a/cpp/tests/transform/row_conversion.cpp b/cpp/tests/transform/row_conversion.cpp deleted file mode 100644 index 77cc236a4c4..00000000000 --- a/cpp/tests/transform/row_conversion.cpp +++ /dev/null @@ -1,1011 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include - -struct ColumnToRowTests : public cudf::test::BaseFixture {}; -struct RowToColumnTests : public cudf::test::BaseFixture {}; - -TEST_F(ColumnToRowTests, Single) -{ - cudf::test::fixed_width_column_wrapper a({-1}); - cudf::table_view in(std::vector{a}); - std::vector schema = {cudf::data_type{cudf::type_id::INT32}}; - - auto old_rows = cudf::convert_to_rows_fixed_width_optimized(in); - auto new_rows = cudf::convert_to_rows(in); - - EXPECT_EQ(old_rows.size(), new_rows.size()); - for (uint i = 0; i < old_rows.size(); ++i) { - auto new_tbl = cudf::convert_from_rows(cudf::lists_column_view(*new_rows[i]), schema); - auto old_tbl = - cudf::convert_from_rows_fixed_width_optimized(cudf::lists_column_view(*old_rows[i]), schema); - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*old_tbl, *new_tbl); - } -} - -TEST_F(ColumnToRowTests, SimpleString) -{ - cudf::test::fixed_width_column_wrapper a({-1, 0, 1, 0, -1}); - cudf::test::strings_column_wrapper b( - {"hello", "world", "this is a really long string to generate a longer row", "dlrow", "olleh"}); - cudf::table_view in(std::vector{a, b}); - std::vector schema = {cudf::data_type{cudf::type_id::INT32}}; - - auto new_rows = cudf::convert_to_rows(in); - - EXPECT_EQ(new_rows[0]->size(), 5); -} - -TEST_F(ColumnToRowTests, DoubleString) -{ - cudf::test::strings_column_wrapper a( - {"hello", "world", "this is a really long string to generate a longer row", "dlrow", "olleh"}); - cudf::test::fixed_width_column_wrapper b({0, 1, 2, 3, 4}); - cudf::test::strings_column_wrapper c({"world", - "hello", - "this string isn't as long", - "this one isn't so short though when you think about it", - "dlrow"}); - cudf::table_view in(std::vector{a, b, c}); - - auto new_rows = cudf::convert_to_rows(in); - - EXPECT_EQ(new_rows[0]->size(), 5); -} - -TEST_F(ColumnToRowTests, BigStrings) -{ - char const* TEST_STRINGS[] = { - "These", - "are", - "the", - "test", - "strings", - "that", - "we", - "have", - "some are really long", - "and some are kinda short", - "They are all over on purpose with different sizes for the strings in order to test the code " - "on all different lengths of strings", - "a", - "good test", - "is required to produce reasonable confidence that this is working"}; - auto num_generator = - cudf::detail::make_counting_transform_iterator(0, [](auto i) -> int32_t { return rand(); }); - auto string_generator = - cudf::detail::make_counting_transform_iterator(0, [&](auto i) -> char const* { - return TEST_STRINGS[rand() % (sizeof(TEST_STRINGS) / sizeof(TEST_STRINGS[0]))]; - }); - - auto const num_rows = 50; - auto const num_cols = 50; - std::vector schema; - - std::vector cols; - std::vector views; - - for (auto col = 0; col < num_cols; ++col) { - if (rand() % 2) { - cols.emplace_back( - cudf::test::fixed_width_column_wrapper(num_generator, num_generator + num_rows)); - views.push_back(cols.back()); - schema.emplace_back(cudf::data_type{cudf::type_id::INT32}); - } else { - cols.emplace_back( - cudf::test::strings_column_wrapper(string_generator, string_generator + num_rows)); - views.push_back(cols.back()); - schema.emplace_back(cudf::type_id::STRING); - } - } - - cudf::table_view in(views); - auto new_rows = cudf::convert_to_rows(in); - - EXPECT_EQ(new_rows[0]->size(), num_rows); -} - -TEST_F(ColumnToRowTests, ManyStrings) -{ - char const* TEST_STRINGS[] = { - "These", - "are", - "the", - "test", - "strings", - "that", - "we", - "have", - "some are really long", - "and some are kinda short", - "They are all over on purpose with different sizes for the strings in order to test the code " - "on all different lengths of strings", - "a", - "good test", - "is required to produce reasonable confidence that this is working", - "some strings", - "are split into multiple strings", - "some strings have all their data", - "lots of choices of strings and sizes is sure to test the offset calculation code to ensure " - "that even a really long string ends up in the correct spot for the final destination allowing " - "for even crazy run-on sentences to be inserted into the data"}; - auto num_generator = - cudf::detail::make_counting_transform_iterator(0, [](auto i) -> int32_t { return rand(); }); - auto string_generator = - cudf::detail::make_counting_transform_iterator(0, [&](auto i) -> char const* { - return TEST_STRINGS[rand() % (sizeof(TEST_STRINGS) / sizeof(TEST_STRINGS[0]))]; - }); - - auto const num_rows = 1'000'000; - auto const num_cols = 50; - std::vector schema; - - std::vector cols; - std::vector views; - - for (auto col = 0; col < num_cols; ++col) { - if (rand() % 2) { - cols.emplace_back( - cudf::test::fixed_width_column_wrapper(num_generator, num_generator + num_rows)); - views.push_back(cols.back()); - schema.emplace_back(cudf::data_type{cudf::type_id::INT32}); - } else { - cols.emplace_back( - cudf::test::strings_column_wrapper(string_generator, string_generator + num_rows)); - views.push_back(cols.back()); - schema.emplace_back(cudf::type_id::STRING); - } - } - - cudf::table_view in(views); - auto new_rows = cudf::convert_to_rows(in); - - EXPECT_EQ(new_rows[0]->size(), num_rows); -} - -TEST_F(ColumnToRowTests, Simple) -{ - cudf::test::fixed_width_column_wrapper a({-1, 0, 1}); - cudf::table_view in(std::vector{a}); - std::vector schema = {cudf::data_type{cudf::type_id::INT32}}; - - auto old_rows = cudf::convert_to_rows_fixed_width_optimized(in); - auto new_rows = cudf::convert_to_rows(in); - - EXPECT_EQ(old_rows.size(), new_rows.size()); - for (uint i = 0; i < old_rows.size(); ++i) { - auto old_tbl = - cudf::convert_from_rows_fixed_width_optimized(cudf::lists_column_view(*old_rows[i]), schema); - auto new_tbl = cudf::convert_from_rows(cudf::lists_column_view(*old_rows[i]), schema); - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*old_tbl, *new_tbl); - } -} - -TEST_F(ColumnToRowTests, Tall) -{ - auto r = - cudf::detail::make_counting_transform_iterator(0, [](auto i) -> int32_t { return rand(); }); - cudf::test::fixed_width_column_wrapper a(r, r + (size_t)4096); - cudf::table_view in(std::vector{a}); - std::vector schema = {cudf::data_type{cudf::type_id::INT32}}; - - auto old_rows = cudf::convert_to_rows_fixed_width_optimized(in); - auto new_rows = cudf::convert_to_rows(in); - - EXPECT_EQ(old_rows.size(), new_rows.size()); - - for (uint i = 0; i < old_rows.size(); ++i) { - auto old_tbl = - cudf::convert_from_rows_fixed_width_optimized(cudf::lists_column_view(*old_rows[i]), schema); - auto new_tbl = cudf::convert_from_rows(cudf::lists_column_view(*old_rows[i]), schema); - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*old_tbl, *new_tbl); - } -} - -TEST_F(ColumnToRowTests, Wide) -{ - std::vector> cols; - std::vector views; - std::vector schema; - - for (int i = 0; i < 256; ++i) { - cols.push_back(cudf::test::fixed_width_column_wrapper({rand()})); - views.push_back(cols.back()); - schema.push_back(cudf::data_type{cudf::type_id::INT32}); - } - cudf::table_view in(views); - - auto old_rows = cudf::convert_to_rows_fixed_width_optimized(in); - auto new_rows = cudf::convert_to_rows(in); - - EXPECT_EQ(old_rows.size(), new_rows.size()); - for (uint i = 0; i < old_rows.size(); ++i) { - auto old_tbl = - cudf::convert_from_rows_fixed_width_optimized(cudf::lists_column_view(*old_rows[i]), schema); - auto new_tbl = cudf::convert_from_rows(cudf::lists_column_view(*old_rows[i]), schema); - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*old_tbl, *new_tbl); - } -} - -TEST_F(ColumnToRowTests, SingleByteWide) -{ - std::vector> cols; - std::vector views; - std::vector schema; - - for (int i = 0; i < 256; ++i) { - cols.push_back(cudf::test::fixed_width_column_wrapper({rand()})); - views.push_back(cols.back()); - - schema.push_back(cudf::data_type{cudf::type_id::INT8}); - } - cudf::table_view in(views); - - auto old_rows = cudf::convert_to_rows_fixed_width_optimized(in); - auto new_rows = cudf::convert_to_rows(in); - - EXPECT_EQ(old_rows.size(), new_rows.size()); - - for (uint i = 0; i < old_rows.size(); ++i) { - auto old_tbl = - cudf::convert_from_rows_fixed_width_optimized(cudf::lists_column_view(*old_rows[i]), schema); - auto new_tbl = cudf::convert_from_rows(cudf::lists_column_view(*old_rows[i]), schema); - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*old_tbl, *new_tbl); - } -} - -TEST_F(ColumnToRowTests, Non2Power) -{ - auto r = - cudf::detail::make_counting_transform_iterator(0, [](auto i) -> int32_t { return rand(); }); - std::vector> cols; - std::vector views; - std::vector schema; - - constexpr auto num_rows = 6 * 1024 + 557; - for (int i = 0; i < 131; ++i) { - cols.push_back(cudf::test::fixed_width_column_wrapper(r + num_rows * i, - r + num_rows * i + num_rows)); - views.push_back(cols.back()); - schema.push_back(cudf::data_type{cudf::type_id::INT32}); - } - cudf::table_view in(views); - - auto old_rows = cudf::convert_to_rows_fixed_width_optimized(in); - auto new_rows = cudf::convert_to_rows(in); - - EXPECT_EQ(old_rows.size(), new_rows.size()); - - for (uint i = 0; i < old_rows.size(); ++i) { - auto old_tbl = - cudf::convert_from_rows_fixed_width_optimized(cudf::lists_column_view(*old_rows[i]), schema); - auto new_tbl = cudf::convert_from_rows(cudf::lists_column_view(*old_rows[i]), schema); - - for (int j = 0; j < old_tbl->num_columns(); ++j) { - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(old_tbl->get_column(j), new_tbl->get_column(j)); - } - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*old_tbl, *new_tbl); - } -} - -TEST_F(ColumnToRowTests, Big) -{ - auto r = - cudf::detail::make_counting_transform_iterator(0, [](auto i) -> int32_t { return rand(); }); - std::vector> cols; - std::vector views; - std::vector schema; - - // 28 columns of 1 million rows - constexpr auto num_rows = 1024 * 1024; - for (int i = 0; i < 28; ++i) { - cols.push_back(cudf::test::fixed_width_column_wrapper(r + num_rows * i, - r + num_rows * i + num_rows)); - views.push_back(cols.back()); - schema.push_back(cudf::data_type{cudf::type_id::INT32}); - } - cudf::table_view in(views); - - auto old_rows = cudf::convert_to_rows_fixed_width_optimized(in); - auto new_rows = cudf::convert_to_rows(in); - - EXPECT_EQ(old_rows.size(), new_rows.size()); - - for (uint i = 0; i < old_rows.size(); ++i) { - auto old_tbl = - cudf::convert_from_rows_fixed_width_optimized(cudf::lists_column_view(*old_rows[i]), schema); - auto new_tbl = cudf::convert_from_rows(cudf::lists_column_view(*old_rows[i]), schema); - - for (int j = 0; j < old_tbl->num_columns(); ++j) { - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(old_tbl->get_column(j), new_tbl->get_column(j)); - } - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*old_tbl, *new_tbl); - } -} - -TEST_F(ColumnToRowTests, Bigger) -{ - auto r = - cudf::detail::make_counting_transform_iterator(0, [](auto i) -> int32_t { return rand(); }); - std::vector> cols; - std::vector views; - std::vector schema; - - // 128 columns of 1 million rows - constexpr auto num_rows = 1024 * 1024; - for (int i = 0; i < 128; ++i) { - cols.push_back(cudf::test::fixed_width_column_wrapper(r + num_rows * i, - r + num_rows * i + num_rows)); - views.push_back(cols.back()); - schema.push_back(cudf::data_type{cudf::type_id::INT32}); - } - cudf::table_view in(views); - - auto old_rows = cudf::convert_to_rows_fixed_width_optimized(in); - auto new_rows = cudf::convert_to_rows(in); - - EXPECT_EQ(old_rows.size(), new_rows.size()); - for (uint i = 0; i < old_rows.size(); ++i) { - auto old_tbl = - cudf::convert_from_rows_fixed_width_optimized(cudf::lists_column_view(*old_rows[i]), schema); - auto new_tbl = cudf::convert_from_rows(cudf::lists_column_view(*old_rows[i]), schema); - - for (int j = 0; j < old_tbl->num_columns(); ++j) { - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(old_tbl->get_column(j), new_tbl->get_column(j)); - } - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*old_tbl, *new_tbl); - } -} - -TEST_F(ColumnToRowTests, Biggest) -{ - auto r = - cudf::detail::make_counting_transform_iterator(0, [](auto i) -> int32_t { return rand(); }); - std::vector> cols; - std::vector views; - std::vector schema; - - // 128 columns of 2 million rows - constexpr auto num_rows = 2 * 1024 * 1024; - for (int i = 0; i < 128; ++i) { - cols.push_back(cudf::test::fixed_width_column_wrapper(r + num_rows * i, - r + num_rows * i + num_rows)); - views.push_back(cols.back()); - schema.push_back(cudf::data_type{cudf::type_id::INT32}); - } - cudf::table_view in(views); - - auto old_rows = cudf::convert_to_rows_fixed_width_optimized(in); - auto new_rows = cudf::convert_to_rows(in); - - EXPECT_EQ(old_rows.size(), new_rows.size()); - - for (uint i = 0; i < old_rows.size(); ++i) { - auto old_tbl = - cudf::convert_from_rows_fixed_width_optimized(cudf::lists_column_view(*old_rows[i]), schema); - auto new_tbl = cudf::convert_from_rows(cudf::lists_column_view(*old_rows[i]), schema); - - for (int j = 0; j < old_tbl->num_columns(); ++j) { - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(old_tbl->get_column(j), new_tbl->get_column(j)); - } - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*old_tbl, *new_tbl); - } -} - -TEST_F(RowToColumnTests, Single) -{ - cudf::test::fixed_width_column_wrapper a({-1}); - cudf::table_view in(std::vector{a}); - - auto old_rows = cudf::convert_to_rows(in); - std::vector schema{cudf::data_type{cudf::type_id::INT32}}; - for (uint i = 0; i < old_rows.size(); ++i) { - auto old_tbl = - cudf::convert_from_rows_fixed_width_optimized(cudf::lists_column_view(*old_rows[i]), schema); - auto new_tbl = cudf::convert_from_rows(cudf::lists_column_view(*old_rows[i]), schema); - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*old_tbl, *new_tbl); - } -} - -TEST_F(RowToColumnTests, Simple) -{ - cudf::test::fixed_width_column_wrapper a({-1, 0, 1}); - cudf::table_view in(std::vector{a}); - - auto old_rows = cudf::convert_to_rows_fixed_width_optimized(in); - std::vector schema{cudf::data_type{cudf::type_id::INT32}}; - for (uint i = 0; i < old_rows.size(); ++i) { - auto old_tbl = - cudf::convert_from_rows_fixed_width_optimized(cudf::lists_column_view(*old_rows[i]), schema); - auto new_tbl = cudf::convert_from_rows(cudf::lists_column_view(*old_rows[i]), schema); - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*old_tbl, *new_tbl); - } -} - -TEST_F(RowToColumnTests, Tall) -{ - auto r = - cudf::detail::make_counting_transform_iterator(0, [](auto i) -> int32_t { return rand(); }); - cudf::test::fixed_width_column_wrapper a(r, r + (size_t)4096); - cudf::table_view in(std::vector{a}); - - auto old_rows = cudf::convert_to_rows_fixed_width_optimized(in); - std::vector schema; - schema.reserve(in.num_columns()); - for (auto col = in.begin(); col < in.end(); ++col) { - schema.push_back(col->type()); - } - for (uint i = 0; i < old_rows.size(); ++i) { - auto old_tbl = - cudf::convert_from_rows_fixed_width_optimized(cudf::lists_column_view(*old_rows[i]), schema); - auto new_tbl = cudf::convert_from_rows(cudf::lists_column_view(*old_rows[i]), schema); - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*old_tbl, *new_tbl); - } -} - -TEST_F(RowToColumnTests, Wide) -{ - std::vector> cols; - std::vector views; - - for (int i = 0; i < 256; ++i) { - cols.push_back(cudf::test::fixed_width_column_wrapper({i})); // rand()})); - views.push_back(cols.back()); - } - cudf::table_view in(views); - - auto old_rows = cudf::convert_to_rows_fixed_width_optimized(in); - std::vector schema; - schema.reserve(in.num_columns()); - for (auto col = in.begin(); col < in.end(); ++col) { - schema.push_back(col->type()); - } - - for (uint i = 0; i < old_rows.size(); ++i) { - auto old_tbl = - cudf::convert_from_rows_fixed_width_optimized(cudf::lists_column_view(*old_rows[i]), schema); - auto new_tbl = cudf::convert_from_rows(cudf::lists_column_view(*old_rows[i]), schema); - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*old_tbl, *new_tbl); - } -} - -TEST_F(RowToColumnTests, SingleByteWide) -{ - std::vector> cols; - std::vector views; - - for (int i = 0; i < 256; ++i) { - cols.push_back(cudf::test::fixed_width_column_wrapper({rand()})); - views.push_back(cols.back()); - } - cudf::table_view in(views); - - auto old_rows = cudf::convert_to_rows_fixed_width_optimized(in); - std::vector schema; - schema.reserve(in.num_columns()); - for (auto col = in.begin(); col < in.end(); ++col) { - schema.push_back(col->type()); - } - for (uint i = 0; i < old_rows.size(); ++i) { - auto old_tbl = - cudf::convert_from_rows_fixed_width_optimized(cudf::lists_column_view(*old_rows[i]), schema); - auto new_tbl = cudf::convert_from_rows(cudf::lists_column_view(*old_rows[i]), schema); - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*old_tbl, *new_tbl); - } -} - -TEST_F(RowToColumnTests, AllTypes) -{ - std::vector> cols; - std::vector views; - std::vector schema{cudf::data_type{cudf::type_id::INT64}, - cudf::data_type{cudf::type_id::FLOAT64}, - cudf::data_type{cudf::type_id::INT8}, - cudf::data_type{cudf::type_id::BOOL8}, - cudf::data_type{cudf::type_id::FLOAT32}, - cudf::data_type{cudf::type_id::INT8}, - cudf::data_type{cudf::type_id::INT32}, - cudf::data_type{cudf::type_id::INT64}}; - - cudf::test::fixed_width_column_wrapper c0({3, 9, 4, 2, 20, 0}, {1, 1, 1, 1, 1, 0}); - cudf::test::fixed_width_column_wrapper c1({5.0, 9.5, 0.9, 7.23, 2.8, 0.0}, - {1, 1, 1, 1, 1, 0}); - cudf::test::fixed_width_column_wrapper c2({5, 1, 0, 2, 7, 0}, {1, 1, 1, 1, 1, 0}); - cudf::test::fixed_width_column_wrapper c3({true, false, false, true, false, false}, - {1, 1, 1, 1, 1, 0}); - cudf::test::fixed_width_column_wrapper c4({1.0f, 3.5f, 5.9f, 7.1f, 9.8f, 0.0f}, - {1, 1, 1, 1, 1, 0}); - cudf::test::fixed_width_column_wrapper c5({2, 3, 4, 5, 9, 0}, {1, 1, 1, 1, 1, 0}); - cudf::test::fixed_point_column_wrapper c6( - {-300, 500, 950, 90, 723, 0}, {1, 1, 1, 1, 1, 1, 1, 0}, numeric::scale_type{-2}); - cudf::test::fixed_point_column_wrapper c7( - {-80, 30, 90, 20, 200, 0}, {1, 1, 1, 1, 1, 1, 0}, numeric::scale_type{-1}); - - cudf::table_view in({c0, c1, c2, c3, c4, c5, c6, c7}); - - auto old_rows = cudf::convert_to_rows_fixed_width_optimized(in); - auto new_rows = cudf::convert_to_rows(in); - - for (uint i = 0; i < old_rows.size(); ++i) { - auto old_tbl = - cudf::convert_from_rows_fixed_width_optimized(cudf::lists_column_view(*old_rows[i]), schema); - auto new_tbl = cudf::convert_from_rows(cudf::lists_column_view(*new_rows[i]), schema); - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*old_tbl, *new_tbl); - } -} - -TEST_F(RowToColumnTests, AllTypesLarge) -{ - std::vector cols; - std::vector schema{}; - - // 15 columns of each type with 1 million entries - constexpr int num_rows{1024 * 1024 * 1}; - - std::default_random_engine re; - std::uniform_real_distribution rand_double(std::numeric_limits::min(), - std::numeric_limits::max()); - std::uniform_int_distribution rand_int64(std::numeric_limits::min(), - std::numeric_limits::max()); - auto r = cudf::detail::make_counting_transform_iterator( - 0, [&](auto i) -> int64_t { return rand_int64(re); }); - auto d = cudf::detail::make_counting_transform_iterator( - 0, [&](auto i) -> double { return rand_double(re); }); - - auto all_valid = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return 1; }); - auto none_valid = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return 0; }); - auto most_valid = cudf::detail::make_counting_transform_iterator( - 0, [](auto i) { return rand() % 2 == 0 ? 0 : 1; }); - auto few_valid = cudf::detail::make_counting_transform_iterator( - 0, [](auto i) { return rand() % 13 == 0 ? 1 : 0; }); - - for (int i = 0; i < 15; ++i) { - cols.push_back(*cudf::test::fixed_width_column_wrapper(r, r + num_rows, all_valid) - .release() - .release()); - schema.push_back(cudf::data_type{cudf::type_id::INT8}); - } - - for (int i = 0; i < 15; ++i) { - cols.push_back(*cudf::test::fixed_width_column_wrapper(r, r + num_rows, few_valid) - .release() - .release()); - schema.push_back(cudf::data_type{cudf::type_id::INT16}); - } - - for (int i = 0; i < 15; ++i) { - if (i < 5) { - cols.push_back(*cudf::test::fixed_width_column_wrapper(r, r + num_rows, few_valid) - .release() - .release()); - } else { - cols.push_back(*cudf::test::fixed_width_column_wrapper(r, r + num_rows, none_valid) - .release() - .release()); - } - schema.push_back(cudf::data_type{cudf::type_id::INT32}); - } - - for (int i = 0; i < 15; ++i) { - cols.push_back(*cudf::test::fixed_width_column_wrapper(d, d + num_rows, most_valid) - .release() - .release()); - schema.push_back(cudf::data_type{cudf::type_id::FLOAT32}); - } - - for (int i = 0; i < 15; ++i) { - cols.push_back(*cudf::test::fixed_width_column_wrapper(d, d + num_rows, most_valid) - .release() - .release()); - schema.push_back(cudf::data_type{cudf::type_id::FLOAT64}); - } - - for (int i = 0; i < 15; ++i) { - cols.push_back(*cudf::test::fixed_width_column_wrapper(r, r + num_rows, few_valid) - .release() - .release()); - schema.push_back(cudf::data_type{cudf::type_id::BOOL8}); - } - - for (int i = 0; i < 15; ++i) { - cols.push_back( - *cudf::test::fixed_width_column_wrapper( - r, r + num_rows, all_valid) - .release() - .release()); - schema.push_back(cudf::data_type{cudf::type_id::TIMESTAMP_MILLISECONDS}); - } - - for (int i = 0; i < 15; ++i) { - cols.push_back( - *cudf::test::fixed_width_column_wrapper( - r, r + num_rows, most_valid) - .release() - .release()); - schema.push_back(cudf::data_type{cudf::type_id::TIMESTAMP_DAYS}); - } - - for (int i = 0; i < 15; ++i) { - cols.push_back(*cudf::test::fixed_point_column_wrapper( - r, r + num_rows, all_valid, numeric::scale_type{-2}) - .release() - .release()); - schema.push_back(cudf::data_type{cudf::type_id::DECIMAL32}); - } - - for (int i = 0; i < 15; ++i) { - cols.push_back(*cudf::test::fixed_point_column_wrapper( - r, r + num_rows, most_valid, numeric::scale_type{-1}) - .release() - .release()); - schema.push_back(cudf::data_type{cudf::type_id::DECIMAL64}); - } - - std::vector views(cols.begin(), cols.end()); - cudf::table_view in(views); - - auto old_rows = cudf::convert_to_rows_fixed_width_optimized(in); - auto new_rows = cudf::convert_to_rows(in); - - for (uint i = 0; i < old_rows.size(); ++i) { - auto old_tbl = - cudf::convert_from_rows_fixed_width_optimized(cudf::lists_column_view(*old_rows[i]), schema); - auto new_tbl = cudf::convert_from_rows(cudf::lists_column_view(*new_rows[i]), schema); - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*old_tbl, *new_tbl); - } -} - -TEST_F(RowToColumnTests, Non2Power) -{ - auto r = - cudf::detail::make_counting_transform_iterator(0, [](auto i) -> int32_t { return rand(); }); - std::vector> cols; - std::vector views; - std::vector schema; - - constexpr auto num_rows = 6 * 1024 + 557; - for (int i = 0; i < 131; ++i) { - cols.push_back(cudf::test::fixed_width_column_wrapper(r + num_rows * i, - r + num_rows * i + num_rows)); - views.push_back(cols.back()); - schema.push_back(cudf::data_type{cudf::type_id::INT32}); - } - cudf::table_view in(views); - - auto old_rows = cudf::convert_to_rows_fixed_width_optimized(in); - - for (uint i = 0; i < old_rows.size(); ++i) { - auto old_tbl = - cudf::convert_from_rows_fixed_width_optimized(cudf::lists_column_view(*old_rows[i]), schema); - auto new_tbl = cudf::convert_from_rows(cudf::lists_column_view(*old_rows[i]), schema); - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*old_tbl, *new_tbl); - } -} - -TEST_F(RowToColumnTests, Big) -{ - auto r = - cudf::detail::make_counting_transform_iterator(0, [](auto i) -> int32_t { return rand(); }); - std::vector> cols; - std::vector views; - std::vector schema; - - // 28 columns of 1 million rows - constexpr auto num_rows = 1024 * 1024; - for (int i = 0; i < 28; ++i) { - cols.push_back(cudf::test::fixed_width_column_wrapper(r + num_rows * i, - r + num_rows * i + num_rows)); - views.push_back(cols.back()); - schema.push_back(cudf::data_type{cudf::type_id::INT32}); - } - cudf::table_view in(views); - - auto old_rows = cudf::convert_to_rows_fixed_width_optimized(in); - - for (uint i = 0; i < old_rows.size(); ++i) { - auto old_tbl = - cudf::convert_from_rows_fixed_width_optimized(cudf::lists_column_view(*old_rows[i]), schema); - auto new_tbl = cudf::convert_from_rows(cudf::lists_column_view(*old_rows[i]), schema); - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*old_tbl, *new_tbl); - } -} - -TEST_F(RowToColumnTests, Bigger) -{ - auto r = - cudf::detail::make_counting_transform_iterator(0, [](auto i) -> int32_t { return rand(); }); - std::vector> cols; - std::vector views; - std::vector schema; - - // 128 columns of 1 million rows - constexpr auto num_rows = 1024 * 1024; - for (int i = 0; i < 128; ++i) { - cols.push_back(cudf::test::fixed_width_column_wrapper(r + num_rows * i, - r + num_rows * i + num_rows)); - views.push_back(cols.back()); - schema.push_back(cudf::data_type{cudf::type_id::INT32}); - } - cudf::table_view in(views); - - auto old_rows = cudf::convert_to_rows_fixed_width_optimized(in); - - for (uint i = 0; i < old_rows.size(); ++i) { - auto old_tbl = - cudf::convert_from_rows_fixed_width_optimized(cudf::lists_column_view(*old_rows[i]), schema); - auto new_tbl = cudf::convert_from_rows(cudf::lists_column_view(*old_rows[i]), schema); - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*old_tbl, *new_tbl); - } -} - -TEST_F(RowToColumnTests, Biggest) -{ - auto r = - cudf::detail::make_counting_transform_iterator(0, [](auto i) -> int32_t { return rand(); }); - std::vector> cols; - std::vector views; - std::vector schema; - - // 128 columns of 2 million rows - constexpr auto num_rows = 2 * 1024 * 1024; - for (int i = 0; i < 128; ++i) { - cols.push_back(cudf::test::fixed_width_column_wrapper(r + num_rows * i, - r + num_rows * i + num_rows)); - views.push_back(cols.back()); - schema.push_back(cudf::data_type{cudf::type_id::INT32}); - } - cudf::table_view in(views); - - auto old_rows = cudf::convert_to_rows_fixed_width_optimized(in); - auto new_rows = cudf::convert_to_rows(in); - - for (uint i = 0; i < old_rows.size(); ++i) { - auto old_tbl = - cudf::convert_from_rows_fixed_width_optimized(cudf::lists_column_view(*old_rows[i]), schema); - auto new_tbl = cudf::convert_from_rows(cudf::lists_column_view(*new_rows[i]), schema); - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*old_tbl, *new_tbl); - } -} - -TEST_F(RowToColumnTests, SimpleString) -{ - cudf::test::fixed_width_column_wrapper a({-1, 0, 1, 0, -1}); - cudf::test::strings_column_wrapper b( - {"hello", "world", "this is a really long string to generate a longer row", "dlrow", "olleh"}); - cudf::table_view in(std::vector{a, b}); - std::vector schema = {cudf::data_type{cudf::type_id::INT32}, - cudf::data_type{cudf::type_id::STRING}}; - - auto new_rows = cudf::convert_to_rows(in); - EXPECT_EQ(new_rows.size(), 1); - for (auto& row : new_rows) { - auto new_cols = cudf::convert_from_rows(cudf::lists_column_view(*row), schema); - EXPECT_EQ(row->size(), 5); - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(in, *new_cols); - } -} - -TEST_F(RowToColumnTests, DoubleString) -{ - cudf::test::strings_column_wrapper a( - {"hello", "world", "this is a really long string to generate a longer row", "dlrow", "olleh"}); - cudf::test::fixed_width_column_wrapper b({0, 1, 2, 3, 4}); - cudf::test::strings_column_wrapper c({"world", - "hello", - "this string isn't as long", - "this one isn't so short though when you think about it", - "dlrow"}); - cudf::table_view in(std::vector{a, b, c}); - std::vector schema = {cudf::data_type{cudf::type_id::STRING}, - cudf::data_type{cudf::type_id::INT32}, - cudf::data_type{cudf::type_id::STRING}}; - - auto new_rows = cudf::convert_to_rows(in); - - for (uint i = 0; i < new_rows.size(); ++i) { - auto new_cols = cudf::convert_from_rows(cudf::lists_column_view(*new_rows[i]), schema); - - EXPECT_EQ(new_rows[0]->size(), 5); - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(in, *new_cols); - } -} - -TEST_F(RowToColumnTests, BigStrings) -{ - char const* TEST_STRINGS[] = { - "These", - "are", - "the", - "test", - "strings", - "that", - "we", - "have", - "some are really long", - "and some are kinda short", - "They are all over on purpose with different sizes for the strings in order to test the code " - "on all different lengths of strings", - "a", - "good test", - "is required to produce reasonable confidence that this is working"}; - auto num_generator = - cudf::detail::make_counting_transform_iterator(0, [](auto i) -> int32_t { return rand(); }); - auto string_generator = - cudf::detail::make_counting_transform_iterator(0, [&](auto i) -> char const* { - return TEST_STRINGS[rand() % (sizeof(TEST_STRINGS) / sizeof(TEST_STRINGS[0]))]; - }); - - auto const num_rows = 50; - auto const num_cols = 50; - std::vector schema; - - std::vector cols; - std::vector views; - - for (auto col = 0; col < num_cols; ++col) { - if (rand() % 2) { - cols.emplace_back( - cudf::test::fixed_width_column_wrapper(num_generator, num_generator + num_rows)); - views.push_back(cols.back()); - schema.emplace_back(cudf::data_type{cudf::type_id::INT32}); - } else { - cols.emplace_back( - cudf::test::strings_column_wrapper(string_generator, string_generator + num_rows)); - views.push_back(cols.back()); - schema.emplace_back(cudf::type_id::STRING); - } - } - - cudf::table_view in(views); - auto new_rows = cudf::convert_to_rows(in); - - for (auto& i : new_rows) { - auto new_cols = cudf::convert_from_rows(cudf::lists_column_view(*i), schema); - - auto in_view = cudf::slice(in, {0, new_cols->num_rows()}); - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(in_view[0], *new_cols); - } -} - -TEST_F(RowToColumnTests, ManyStrings) -{ - // The sizing of this test is very sensitive to the state of the random number generator, - // i.e., depending on the order of execution, the number of times the largest string is - // selected will lead to out-of-memory exceptions. Seeding the RNG here helps prevent that. - srand(1); - char const* TEST_STRINGS[] = { - "These", - "are", - "the", - "test", - "strings", - "that", - "we", - "have", - "some are really long", - "and some are kinda short", - "They are all over on purpose with different sizes for the strings in order to test the code " - "on all different lengths of strings", - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine " - "this string is the longest string because it is duplicated more than you can imagine ", - "a", - "good test", - "is required to produce reasonable confidence that this is working", - "some strings", - "are split into multiple strings", - "some strings have all their data", - "lots of choices of strings and sizes is sure to test the offset calculation code to ensure " - "that even a really long string ends up in the correct spot for the final destination allowing " - "for even crazy run-on sentences to be inserted into the data"}; - auto num_generator = - cudf::detail::make_counting_transform_iterator(0, [](auto i) -> int32_t { return rand(); }); - auto string_generator = - cudf::detail::make_counting_transform_iterator(0, [&](auto i) -> char const* { - return TEST_STRINGS[rand() % (sizeof(TEST_STRINGS) / sizeof(TEST_STRINGS[0]))]; - }); - - auto const num_rows = 300'000; - auto const num_cols = 50; - std::vector schema; - - std::vector cols; - std::vector views; - - for (auto col = 0; col < num_cols; ++col) { - if (rand() % 2) { - cols.emplace_back( - cudf::test::fixed_width_column_wrapper(num_generator, num_generator + num_rows)); - views.push_back(cols.back()); - schema.emplace_back(cudf::data_type{cudf::type_id::INT32}); - } else { - cols.emplace_back( - cudf::test::strings_column_wrapper(string_generator, string_generator + num_rows)); - views.push_back(cols.back()); - schema.emplace_back(cudf::type_id::STRING); - } - } - - cudf::table_view in(views); - auto new_rows = cudf::convert_to_rows(in); - - for (auto& i : new_rows) { - auto new_cols = cudf::convert_from_rows(cudf::lists_column_view(*i), schema); - - auto in_view = cudf::slice(in, {0, new_cols->num_rows()}); - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(in_view[0], *new_cols); - } -} - -CUDF_TEST_PROGRAM_MAIN() From b5bc5316fe9b9319514c202d0517146306976452 Mon Sep 17 00:00:00 2001 From: Mark Harris <783069+harrism@users.noreply.github.com> Date: Wed, 6 Mar 2024 17:01:39 +1100 Subject: [PATCH 355/384] Update `developer_guide.md` with new guidance on quoted internal includes (#15238) Follow up to #15063 to add new guidance for quoting includes of internal headers from `src` paths. Also covers clang-format include grouping. Also fixes a single include that was added with `<>` recently that should be `""`. #15063 updated all includes to match the guidance in this PR (changing a lot of `<>` to `""` for includes from `src/...`. Authors: - Mark Harris (https://github.com/harrism) Approvers: - David Wendt (https://github.com/davidwendt) - Vukasin Milovanovic (https://github.com/vuule) URL: https://github.com/rapidsai/cudf/pull/15238 --- cpp/doxygen/developer_guide/DEVELOPER_GUIDE.md | 17 +++++++++++------ cpp/src/io/parquet/error.hpp | 4 ++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/cpp/doxygen/developer_guide/DEVELOPER_GUIDE.md b/cpp/doxygen/developer_guide/DEVELOPER_GUIDE.md index 5c137433dc5..935ca20b6fa 100644 --- a/cpp/doxygen/developer_guide/DEVELOPER_GUIDE.md +++ b/cpp/doxygen/developer_guide/DEVELOPER_GUIDE.md @@ -127,7 +127,7 @@ and we try to follow his rules: "No raw loops. No raw pointers. No raw synchroni does use raw synchronization primitives. So we should revisit Parent's third rule and improve here. -Additional style guidelines for libcudf code include: +Additional style guidelines for libcudf code: * Prefer "east const", placing `const` after the type. This is not automatically enforced by `clang-format` because the option @@ -152,15 +152,20 @@ The following guidelines apply to organizing `#include` lines. from other RAPIDS libraries, then includes from related libraries, like ``, then includes from dependencies installed with cuDF, and then standard headers (for example ``, ``). - * Use `<>` instead of `""` unless the header is in the same directory as the source file. + * We use clang-format for grouping and sorting headers automatically. See the + `cudf/cpp/.clang-format` file for specifics. + * Use `<>` for all includes except for internal headers that are not in the `include` + directory. In other words, if it is a cuDF internal header (e.g. in the `src` or `test` + directory), the path will not start with `cudf` (e.g. `#include `) so it + should use quotes. Example: `#include "io/utilities/hostdevice_vector.hpp"`. + * `cudf_test` and `nvtext` are separate libraries within the `libcudf` repo. As such, they have + public headers in `include` that should be included with `<>`. * Tools like `clangd` often auto-insert includes when they can, but they usually get the grouping - and brackets wrong. + and brackets wrong. Correct the usage of quotes or brackets and then run clang-format to correct + the grouping. * Always check that includes are only necessary for the file in which they are included. Try to avoid excessive including especially in header files. Double check this when you remove code. - * Use quotes `"` to include local headers from the same relative source directory. This should only - occur in source files and non-public header files. Otherwise use angle brackets `<>` around - included header filenames. * Avoid relative paths with `..` when possible. Paths with `..` are necessary when including (internal) headers from source paths not in the same directory as the including file, because source paths are not passed with `-I`. diff --git a/cpp/src/io/parquet/error.hpp b/cpp/src/io/parquet/error.hpp index 4e2eb4c66d3..f0fc9fab3ab 100644 --- a/cpp/src/io/parquet/error.hpp +++ b/cpp/src/io/parquet/error.hpp @@ -16,9 +16,9 @@ #pragma once -#include +#include "io/utilities/hostdevice_vector.hpp" -#include +#include #include #include From aabfd83f76a070d0bfca2c42c01c84252d22cb25 Mon Sep 17 00:00:00 2001 From: Yunsong Wang Date: Tue, 5 Mar 2024 22:58:17 -0800 Subject: [PATCH 356/384] Add distinct left join (#15149) Contributes to #14948 This PR adds distinct left join. It also cleans up the distinct inner code to use the terms "build" and "probe" consistently instead of "left" and "right". Authors: - Yunsong Wang (https://github.com/PointKernel) - Nghia Truong (https://github.com/ttnghia) Approvers: - Bradley Dice (https://github.com/bdice) - Jason Lowe (https://github.com/jlowe) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/15149 --- cpp/benchmarks/join/distinct_join.cu | 58 ++++- .../cudf/detail/distinct_hash_join.cuh | 6 + cpp/include/cudf/join.hpp | 20 +- cpp/src/join/distinct_hash_join.cu | 80 ++++++- cpp/tests/join/distinct_join_tests.cpp | 198 +++++++++++++++++- 5 files changed, 343 insertions(+), 19 deletions(-) diff --git a/cpp/benchmarks/join/distinct_join.cu b/cpp/benchmarks/join/distinct_join.cu index cbdb82275ef..4a68ee3878e 100644 --- a/cpp/benchmarks/join/distinct_join.cu +++ b/cpp/benchmarks/join/distinct_join.cu @@ -22,21 +22,44 @@ void distinct_inner_join(nvbench::state& state, { skip_helper(state); - auto join = [](cudf::table_view const& left_input, - cudf::table_view const& right_input, + auto join = [](cudf::table_view const& build_input, + cudf::table_view const& probe_input, cudf::null_equality compare_nulls, rmm::cuda_stream_view stream) { - auto const has_nulls = cudf::has_nested_nulls(left_input) || cudf::has_nested_nulls(right_input) - ? cudf::nullable_join::YES - : cudf::nullable_join::NO; - auto hj_obj = cudf::distinct_hash_join{ - left_input, right_input, has_nulls, compare_nulls, stream}; + auto const has_nulls = + cudf::has_nested_nulls(build_input) || cudf::has_nested_nulls(probe_input) + ? cudf::nullable_join::YES + : cudf::nullable_join::NO; + auto hj_obj = cudf::distinct_hash_join{ + build_input, probe_input, has_nulls, compare_nulls, stream}; return hj_obj.inner_join(stream); }; BM_join(state, join); } +template +void distinct_left_join(nvbench::state& state, + nvbench::type_list>) +{ + skip_helper(state); + + auto join = [](cudf::table_view const& build_input, + cudf::table_view const& probe_input, + cudf::null_equality compare_nulls, + rmm::cuda_stream_view stream) { + auto const has_nulls = + cudf::has_nested_nulls(build_input) || cudf::has_nested_nulls(probe_input) + ? cudf::nullable_join::YES + : cudf::nullable_join::NO; + auto hj_obj = cudf::distinct_hash_join{ + build_input, probe_input, has_nulls, compare_nulls, stream}; + return hj_obj.left_join(stream); + }; + + BM_join(state, join); +} + // inner join ----------------------------------------------------------------------- NVBENCH_BENCH_TYPES(distinct_inner_join, NVBENCH_TYPE_AXES(nvbench::type_list, @@ -75,3 +98,24 @@ NVBENCH_BENCH_TYPES(distinct_inner_join, .set_type_axes_names({"Key Type", "Payload Type", "Nullable"}) .add_int64_axis("Build Table Size", {40'000'000, 50'000'000}) .add_int64_axis("Probe Table Size", {50'000'000, 120'000'000}); + +// left join ------------------------------------------------------------------------ +NVBENCH_BENCH_TYPES(distinct_left_join, + NVBENCH_TYPE_AXES(nvbench::type_list, + nvbench::type_list, + nvbench::enum_type_list)) + .set_name("distinct_left_join_32bit") + .set_type_axes_names({"Key Type", "Payload Type", "Nullable"}) + .add_int64_axis("Build Table Size", {100'000, 10'000'000, 80'000'000, 100'000'000}) + .add_int64_axis("Probe Table Size", + {100'000, 400'000, 10'000'000, 40'000'000, 100'000'000, 240'000'000}); + +NVBENCH_BENCH_TYPES(distinct_left_join, + NVBENCH_TYPE_AXES(nvbench::type_list, + nvbench::type_list, + nvbench::enum_type_list)) + .set_name("distinct_left_join_32bit_nulls") + .set_type_axes_names({"Key Type", "Payload Type", "Nullable"}) + .add_int64_axis("Build Table Size", {100'000, 10'000'000, 80'000'000, 100'000'000}) + .add_int64_axis("Probe Table Size", + {100'000, 400'000, 10'000'000, 40'000'000, 100'000'000, 240'000'000}); diff --git a/cpp/include/cudf/detail/distinct_hash_join.cuh b/cpp/include/cudf/detail/distinct_hash_join.cuh index 7827f861bd8..e874151ed36 100644 --- a/cpp/include/cudf/detail/distinct_hash_join.cuh +++ b/cpp/include/cudf/detail/distinct_hash_join.cuh @@ -149,5 +149,11 @@ struct distinct_hash_join { std::pair>, std::unique_ptr>> inner_join(rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) const; + + /** + * @copydoc cudf::distinct_hash_join::left_join + */ + std::unique_ptr> left_join( + rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) const; }; } // namespace cudf::detail diff --git a/cpp/include/cudf/join.hpp b/cpp/include/cudf/join.hpp index d97dc64ac39..b7a3129cfec 100644 --- a/cpp/include/cudf/join.hpp +++ b/cpp/include/cudf/join.hpp @@ -485,7 +485,7 @@ class distinct_hash_join { rmm::cuda_stream_view stream = cudf::get_default_stream()); /** - * Returns the row indices that can be used to construct the result of performing + * @brief Returns the row indices that can be used to construct the result of performing * an inner join between two tables. @see cudf::inner_join(). * * @param stream CUDA stream used for device memory operations and kernel launches @@ -500,6 +500,24 @@ class distinct_hash_join { inner_join(rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()) const; + /** + * @brief Returns the build table indices that can be used to construct the result of performing + * a left join between two tables. + * + * @note For a given row index `i` of the probe table, the resulting `build_indices[i]` contains + * the row index of the matched row from the build table if there is a match. Otherwise, contains + * `JoinNoneValue`. + * + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned table and columns' device + * memory. + * @return A `build_indices` column that can be used to construct the result of performing a left + * join between two tables with `build` and `probe` as the join keys. + */ + std::unique_ptr> left_join( + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()) const; + private: using impl_type = typename cudf::detail::distinct_hash_join; ///< Implementation type diff --git a/cpp/src/join/distinct_hash_join.cu b/cpp/src/join/distinct_hash_join.cu index 981a7bf0dea..85b7c26472d 100644 --- a/cpp/src/join/distinct_hash_join.cu +++ b/cpp/src/join/distinct_hash_join.cu @@ -32,6 +32,9 @@ #include #include #include +#include +#include +#include #include #include @@ -76,6 +79,18 @@ class build_keys_fn { Hasher _hash; }; +/** + * @brief Device output transform functor to construct `size_type` with `cuco::pair` + */ +struct output_fn { + __device__ constexpr cudf::size_type operator()( + cuco::pair const& x) const + { + return static_cast(x.second); + } +}; + template __device__ void flush_buffer(Tile const& tile, cudf::size_type tile_count, @@ -306,9 +321,9 @@ distinct_hash_join::inner_join(rmm::cuda_stream_view stream, std::make_unique>(0, stream, mr)); } - auto left_indices = + auto build_indices = std::make_unique>(probe_table_num_rows, stream, mr); - auto right_indices = + auto probe_indices = std::make_unique>(probe_table_num_rows, stream, mr); auto const probe_row_hasher = @@ -325,14 +340,50 @@ distinct_hash_join::inner_join(rmm::cuda_stream_view stream, probe_table_num_rows, this->_hash_table.ref(cuco::find), counter.data(), - left_indices->data(), - right_indices->data()); + build_indices->data(), + probe_indices->data()); auto const actual_size = counter.value(stream); - left_indices->resize(actual_size, stream); - right_indices->resize(actual_size, stream); + build_indices->resize(actual_size, stream); + probe_indices->resize(actual_size, stream); + + return {std::move(build_indices), std::move(probe_indices)}; +} + +template +std::unique_ptr> distinct_hash_join::left_join( + rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) const +{ + cudf::thread_range range{"distinct_hash_join::left_join"}; + + size_type const probe_table_num_rows{this->_probe.num_rows()}; + + // If output size is zero, return empty + if (probe_table_num_rows == 0) { + return std::make_unique>(0, stream, mr); + } + + auto build_indices = + std::make_unique>(probe_table_num_rows, stream, mr); + + // If build table is empty, return probe table + if (this->_build.num_rows() == 0) { + thrust::fill( + rmm::exec_policy_nosync(stream), build_indices->begin(), build_indices->end(), JoinNoneValue); + } else { + auto const probe_row_hasher = + cudf::experimental::row::hash::row_hasher{this->_preprocessed_probe}; + auto const d_probe_hasher = probe_row_hasher.device_hasher(nullate::DYNAMIC{this->_has_nulls}); + auto const iter = cudf::detail::make_counting_transform_iterator( + 0, build_keys_fn{d_probe_hasher}); + + auto const output_begin = + thrust::make_transform_output_iterator(build_indices->begin(), output_fn{}); + // TODO conditional find for nulls once `cuco::static_set::find_if` is added + this->_hash_table.find_async(iter, iter + probe_table_num_rows, output_begin, stream.value()); + } - return {std::move(left_indices), std::move(right_indices)}; + return build_indices; } } // namespace detail @@ -381,4 +432,19 @@ distinct_hash_join::inner_join(rmm::cuda_stream_view strea { return _impl->inner_join(stream, mr); } + +template <> +std::unique_ptr> +distinct_hash_join::left_join(rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) const +{ + return _impl->left_join(stream, mr); +} + +template <> +std::unique_ptr> distinct_hash_join::left_join( + rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) const +{ + return _impl->left_join(stream, mr); +} } // namespace cudf diff --git a/cpp/tests/join/distinct_join_tests.cpp b/cpp/tests/join/distinct_join_tests.cpp index 27f4c4fdf61..698256251ef 100644 --- a/cpp/tests/join/distinct_join_tests.cpp +++ b/cpp/tests/join/distinct_join_tests.cpp @@ -39,13 +39,23 @@ using strcol_wrapper = cudf::test::strings_column_wrapper; using CVector = std::vector>; using Table = cudf::table; +std::unique_ptr> get_left_indices(cudf::size_type size) +{ + auto sequence = std::vector(size); + std::iota(sequence.begin(), sequence.end(), 0); + auto indices = cudf::detail::make_device_uvector_sync( + sequence, cudf::get_default_stream(), rmm::mr::get_current_device_resource()); + return std::make_unique>(std::move(indices)); +} + struct DistinctJoinTest : public cudf::test::BaseFixture { void compare_to_reference( cudf::table_view const& build_table, cudf::table_view const& probe_table, std::pair>, std::unique_ptr>> const& result, - cudf::table_view const& expected_table) + cudf::table_view const& expected_table, + cudf::out_of_bounds_policy oob_policy = cudf::out_of_bounds_policy::DONT_CHECK) { auto const& [build_join_indices, probe_join_indices] = result; @@ -55,9 +65,8 @@ struct DistinctJoinTest : public cudf::test::BaseFixture { auto build_indices_col = cudf::column_view{build_indices_span}; auto probe_indices_col = cudf::column_view{probe_indices_span}; - auto constexpr oob_policy = cudf::out_of_bounds_policy::DONT_CHECK; - auto joined_cols = cudf::gather(build_table, build_indices_col, oob_policy)->release(); - auto right_cols = cudf::gather(probe_table, probe_indices_col, oob_policy)->release(); + auto joined_cols = cudf::gather(probe_table, probe_indices_col, oob_policy)->release(); + auto right_cols = cudf::gather(build_table, build_indices_col, oob_policy)->release(); joined_cols.insert(joined_cols.end(), std::make_move_iterator(right_cols.begin()), @@ -283,6 +292,31 @@ TEST_F(DistinctJoinTest, EmptyBuildTableInnerJoin) this->compare_to_reference(build.view(), probe.view(), result, build.view()); } +TEST_F(DistinctJoinTest, EmptyBuildTableLeftJoin) +{ + column_wrapper col0_0; + column_wrapper col0_1; + + column_wrapper col1_0{{2, 2, 0, 4, 3}}; + column_wrapper col1_1{{1, 0, 1, 2, 1}, {1, 0, 1, 1, 1}}; + + CVector cols0, cols1; + cols0.push_back(col0_0.release()); + cols0.push_back(col0_1.release()); + cols1.push_back(col1_0.release()); + cols1.push_back(col1_1.release()); + + Table build(std::move(cols0)); + Table probe(std::move(cols1)); + + auto distinct_join = cudf::distinct_hash_join{build.view(), probe.view()}; + auto result = distinct_join.left_join(); + auto gather_map = std::pair{std::move(result), std::move(get_left_indices(result->size()))}; + + this->compare_to_reference( + build.view(), probe.view(), gather_map, probe.view(), cudf::out_of_bounds_policy::NULLIFY); +} + TEST_F(DistinctJoinTest, EmptyProbeTableInnerJoin) { column_wrapper col0_0{{2, 2, 0, 4, 3}}; @@ -305,3 +339,159 @@ TEST_F(DistinctJoinTest, EmptyProbeTableInnerJoin) this->compare_to_reference(build.view(), probe.view(), result, probe.view()); } + +TEST_F(DistinctJoinTest, EmptyProbeTableLeftJoin) +{ + column_wrapper col0_0{{2, 2, 0, 4, 3}}; + column_wrapper col0_1{{1, 0, 1, 2, 1}, {1, 0, 1, 1, 1}}; + + column_wrapper col1_0; + column_wrapper col1_1; + + CVector cols0, cols1; + cols0.push_back(col0_0.release()); + cols0.push_back(col0_1.release()); + cols1.push_back(col1_0.release()); + cols1.push_back(col1_1.release()); + + Table build(std::move(cols0)); + Table probe(std::move(cols1)); + + auto distinct_join = cudf::distinct_hash_join{build.view(), probe.view()}; + auto result = distinct_join.left_join(); + auto gather_map = std::pair{std::move(result), std::move(get_left_indices(result->size()))}; + + this->compare_to_reference( + build.view(), probe.view(), gather_map, probe.view(), cudf::out_of_bounds_policy::NULLIFY); +} + +TEST_F(DistinctJoinTest, LeftJoinNoNulls) +{ + column_wrapper col0_0({3, 1, 2, 0, 3}); + strcol_wrapper col0_1({"s0", "s1", "s2", "s4", "s1"}); + + column_wrapper col1_0({2, 2, 0, 4, 3}); + strcol_wrapper col1_1({"s1", "s0", "s1", "s2", "s1"}); + + CVector cols0, cols1; + cols0.push_back(col0_0.release()); + cols0.push_back(col0_1.release()); + cols1.push_back(col1_0.release()); + cols1.push_back(col1_1.release()); + + Table probe(std::move(cols0)); + Table build(std::move(cols1)); + + column_wrapper col_gold_0({3, 1, 2, 0, 3}); + strcol_wrapper col_gold_1({"s0", "s1", "s2", "s4", "s1"}); + column_wrapper col_gold_2{{-1, -1, -1, -1, 3}, {0, 0, 0, 0, 1}}; + strcol_wrapper col_gold_3{{"", "", "", "", "s1"}, {0, 0, 0, 0, 1}}; + CVector cols_gold; + cols_gold.push_back(col_gold_0.release()); + cols_gold.push_back(col_gold_1.release()); + cols_gold.push_back(col_gold_2.release()); + cols_gold.push_back(col_gold_3.release()); + Table gold(std::move(cols_gold)); + + auto distinct_join = cudf::distinct_hash_join{build.view(), probe.view()}; + auto result = distinct_join.left_join(); + auto gather_map = std::pair{std::move(result), std::move(get_left_indices(result->size()))}; + + this->compare_to_reference( + build.view(), probe.view(), gather_map, gold.view(), cudf::out_of_bounds_policy::NULLIFY); +} + +TEST_F(DistinctJoinTest, LeftJoinWithNulls) +{ + column_wrapper col0_0{{3, 1, 2, 0, 2}}; + strcol_wrapper col0_1({"s1", "s1", "", "s4", "s0"}, {1, 1, 0, 1, 1}); + + column_wrapper col1_0{{2, 2, 0, 4, 3}}; + strcol_wrapper col1_1({"s1", "s0", "s1", "s2", "s1"}); + + CVector cols0, cols1; + cols0.push_back(col0_0.release()); + cols0.push_back(col0_1.release()); + cols1.push_back(col1_0.release()); + cols1.push_back(col1_1.release()); + + Table probe(std::move(cols0)); + Table build(std::move(cols1)); + + auto distinct_join = cudf::distinct_hash_join{build.view(), probe.view()}; + auto result = distinct_join.left_join(); + auto gather_map = std::pair{std::move(result), std::move(get_left_indices(result->size()))}; + + column_wrapper col_gold_0{{3, 1, 2, 0, 2}, {1, 1, 1, 1, 1}}; + strcol_wrapper col_gold_1({"s1", "s1", "", "s4", "s0"}, {1, 1, 0, 1, 1}); + column_wrapper col_gold_2{{3, -1, -1, -1, 2}, {1, 0, 0, 0, 1}}; + strcol_wrapper col_gold_3{{"s1", "", "", "", "s0"}, {1, 0, 0, 0, 1}}; + + CVector cols_gold; + cols_gold.push_back(col_gold_0.release()); + cols_gold.push_back(col_gold_1.release()); + cols_gold.push_back(col_gold_2.release()); + cols_gold.push_back(col_gold_3.release()); + Table gold(std::move(cols_gold)); + + this->compare_to_reference( + build.view(), probe.view(), gather_map, gold.view(), cudf::out_of_bounds_policy::NULLIFY); +} + +TEST_F(DistinctJoinTest, LeftJoinWithStructsAndNulls) +{ + auto col0_names_col = strcol_wrapper{ + "Samuel Vimes", "Carrot Ironfoundersson", "Detritus", "Samuel Vimes", "Angua von Überwald"}; + auto col0_ages_col = column_wrapper{{48, 27, 351, 31, 25}}; + auto col0_is_human_col = column_wrapper{{true, true, false, false, false}, {1, 1, 0, 1, 0}}; + auto col0 = + cudf::test::structs_column_wrapper{{col0_names_col, col0_ages_col, col0_is_human_col}}; + + auto col1_names_col = strcol_wrapper{ + "Samuel Vimes", "Detritus", "Detritus", "Carrot Ironfoundersson", "Angua von Überwald"}; + auto col1_ages_col = column_wrapper{{48, 35, 351, 22, 25}}; + auto col1_is_human_col = column_wrapper{{true, true, false, false, true}, {1, 1, 0, 1, 1}}; + auto col1 = + cudf::test::structs_column_wrapper{{col1_names_col, col1_ages_col, col1_is_human_col}}; + + CVector cols0, cols1; + cols0.push_back(col0.release()); + cols1.push_back(col1.release()); + + Table probe(std::move(cols0)); + Table build(std::move(cols1)); + + auto distinct_join = cudf::distinct_hash_join{build.view(), probe.view()}; + auto result = distinct_join.left_join(); + auto gather_map = std::pair{std::move(result), std::move(get_left_indices(result->size()))}; + + auto col0_gold_names_col = strcol_wrapper{ + "Samuel Vimes", "Detritus", "Carrot Ironfoundersson", "Samuel Vimes", "Angua von Überwald"}; + auto col0_gold_ages_col = column_wrapper{{48, 351, 27, 31, 25}}; + auto col0_gold_is_human_col = + column_wrapper{{true, false, true, false, false}, {1, 0, 1, 1, 0}}; + auto col0_gold = cudf::test::structs_column_wrapper{ + {col0_gold_names_col, col0_gold_ages_col, col0_gold_is_human_col}}; + + auto col1_gold_names_col = strcol_wrapper{{ + "Samuel Vimes", + "Detritus", + "", + "", + "", + }, + {1, 1, 0, 0, 0}}; + auto col1_gold_ages_col = column_wrapper{{48, 351, -1, -1, -1}, {1, 1, 0, 0, 0}}; + auto col1_gold_is_human_col = + column_wrapper{{true, false, false, false, false}, {1, 0, 0, 0, 0}}; + auto col1_gold = cudf::test::structs_column_wrapper{ + {col1_gold_names_col, col1_gold_ages_col, col1_gold_is_human_col}, {1, 1, 0, 0, 0}}; + + CVector cols_gold; + cols_gold.push_back(col0_gold.release()); + cols_gold.push_back(col1_gold.release()); + Table gold(std::move(cols_gold)); + + this->compare_to_reference( + build.view(), probe.view(), gather_map, gold.view(), cudf::out_of_bounds_policy::NULLIFY); +} From dbf7236c4b30ee6f87223b728688cddf39453d14 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Tue, 5 Mar 2024 22:59:21 -0800 Subject: [PATCH 357/384] Add ability to request Parquet encodings on a per-column basis (#15081) Allows users to request specific page encodings to use on a column-by-column basis. This is accomplished by adding an `encoding` property to the `column_input_metadata` struct. This is a necessary change before adding `DELTA_BYTE_ARRAY` encoding. Authors: - Ed Seidl (https://github.com/etseidl) - Vukasin Milovanovic (https://github.com/vuule) Approvers: - Vukasin Milovanovic (https://github.com/vuule) - Yunsong Wang (https://github.com/PointKernel) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/15081 --- cpp/include/cudf/io/types.hpp | 44 ++++++++++ cpp/src/io/parquet/page_enc.cu | 34 +++++++- cpp/src/io/parquet/parquet_gpu.hpp | 1 + cpp/src/io/parquet/writer_impl.cu | 85 +++++++++++++++++-- cpp/tests/io/parquet_writer_test.cpp | 122 +++++++++++++++++++++++++++ 5 files changed, 276 insertions(+), 10 deletions(-) diff --git a/cpp/include/cudf/io/types.hpp b/cpp/include/cudf/io/types.hpp index 3208a81cd63..64d627483e6 100644 --- a/cpp/include/cudf/io/types.hpp +++ b/cpp/include/cudf/io/types.hpp @@ -99,6 +99,26 @@ enum statistics_freq { STATISTICS_COLUMN = 3, ///< Full column and offset indices. Implies STATISTICS_ROWGROUP }; +/** + * @brief Valid encodings for use with `column_in_metadata::set_encoding()` + */ +enum class column_encoding { + // Common encodings: + USE_DEFAULT = -1, ///< No encoding has been requested, use default encoding + DICTIONARY, ///< Use dictionary encoding + // Parquet encodings: + PLAIN, ///< Use plain encoding + DELTA_BINARY_PACKED, ///< Use DELTA_BINARY_PACKED encoding (only valid for integer columns) + DELTA_LENGTH_BYTE_ARRAY, ///< Use DELTA_LENGTH_BYTE_ARRAY encoding (only + ///< valid for BYTE_ARRAY columns) + DELTA_BYTE_ARRAY, ///< Use DELTA_BYTE_ARRAY encoding (only valid for + ///< BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY columns) + // ORC encodings: + DIRECT, ///< Use DIRECT encoding + DIRECT_V2, ///< Use DIRECT_V2 encoding + DICTIONARY_V2, ///< Use DICTIONARY_V2 encoding +}; + /** * @brief Statistics about compression performed by a writer. */ @@ -585,6 +605,7 @@ class column_in_metadata { std::optional _decimal_precision; std::optional _parquet_field_id; std::vector children; + column_encoding _encoding = column_encoding::USE_DEFAULT; public: column_in_metadata() = default; @@ -701,6 +722,22 @@ class column_in_metadata { return *this; } + /** + * @brief Sets the encoding to use for this column. + * + * This is just a request, and the encoder may still choose to use a different encoding + * depending on resource constraints. Use the constants defined in the `parquet_encoding` + * struct. + * + * @param encoding The encoding to use + * @return this for chaining + */ + column_in_metadata& set_encoding(column_encoding encoding) noexcept + { + _encoding = encoding; + return *this; + } + /** * @brief Get reference to a child of this column * @@ -806,6 +843,13 @@ class column_in_metadata { * @return Boolean indicating whether to encode this column as binary data */ [[nodiscard]] bool is_enabled_output_as_binary() const noexcept { return _output_as_binary; } + + /** + * @brief Get the encoding that was set for this column. + * + * @return The encoding that was set for this column + */ + [[nodiscard]] column_encoding get_encoding() const { return _encoding; } }; /** diff --git a/cpp/src/io/parquet/page_enc.cu b/cpp/src/io/parquet/page_enc.cu index 5aad31bd057..617cb1d0992 100644 --- a/cpp/src/io/parquet/page_enc.cu +++ b/cpp/src/io/parquet/page_enc.cu @@ -573,9 +573,13 @@ CUDF_KERNEL void __launch_bounds__(128) // at the worst case number of bytes needed to encode. auto const physical_type = col_g.physical_type; auto const type_id = col_g.leaf_column->type().id(); - auto const is_use_delta = - write_v2_headers && !ck_g.use_dictionary && + auto const is_requested_delta = + col_g.requested_encoding == column_encoding::DELTA_BINARY_PACKED || + col_g.requested_encoding == column_encoding::DELTA_LENGTH_BYTE_ARRAY; + auto const is_fallback_to_delta = + !ck_g.use_dictionary && write_v2_headers && (physical_type == INT32 || physical_type == INT64 || physical_type == BYTE_ARRAY); + auto const is_use_delta = is_requested_delta || is_fallback_to_delta; if (t < 32) { uint32_t fragments_in_chunk = 0; @@ -786,7 +790,31 @@ CUDF_KERNEL void __launch_bounds__(128) if (t == 0) { if (not pages.empty()) { // set encoding - if (is_use_delta) { + if (col_g.requested_encoding != column_encoding::USE_DEFAULT) { + switch (col_g.requested_encoding) { + case column_encoding::PLAIN: page_g.kernel_mask = encode_kernel_mask::PLAIN; break; + case column_encoding::DICTIONARY: + // user may have requested dict, but we may not be able to use it + // TODO: when DELTA_BYTE_ARRAY is added, rework the fallback logic so there + // isn't duplicated code here and below. + if (ck_g.use_dictionary) { + page_g.kernel_mask = encode_kernel_mask::DICTIONARY; + } else if (is_fallback_to_delta) { + page_g.kernel_mask = physical_type == BYTE_ARRAY + ? encode_kernel_mask::DELTA_LENGTH_BA + : encode_kernel_mask::DELTA_BINARY; + } else { + page_g.kernel_mask = encode_kernel_mask::PLAIN; + } + break; + case column_encoding::DELTA_BINARY_PACKED: + page_g.kernel_mask = encode_kernel_mask::DELTA_BINARY; + break; + case column_encoding::DELTA_LENGTH_BYTE_ARRAY: + page_g.kernel_mask = encode_kernel_mask::DELTA_LENGTH_BA; + break; + } + } else if (is_use_delta) { // TODO(ets): at some point make a more intelligent decision on this. DELTA_LENGTH_BA // should always be preferred over PLAIN, but DELTA_BINARY is a different matter. // If the delta encoding size is going to be close to 32 bits anyway, then plain diff --git a/cpp/src/io/parquet/parquet_gpu.hpp b/cpp/src/io/parquet/parquet_gpu.hpp index 86d6ec42c04..af9f1f1267e 100644 --- a/cpp/src/io/parquet/parquet_gpu.hpp +++ b/cpp/src/io/parquet/parquet_gpu.hpp @@ -460,6 +460,7 @@ struct parquet_column_device_view : stats_column_desc { //!< nullability of parent_column. May be different from //!< col.nullable() in case of chunked writing. bool output_as_byte_array; //!< Indicates this list column is being written as a byte array + column_encoding requested_encoding; //!< User specified encoding for this column. }; struct EncColumnChunk; diff --git a/cpp/src/io/parquet/writer_impl.cu b/cpp/src/io/parquet/writer_impl.cu index ecdbdd0fd5f..87c8b2f1611 100644 --- a/cpp/src/io/parquet/writer_impl.cu +++ b/cpp/src/io/parquet/writer_impl.cu @@ -267,11 +267,13 @@ bool is_col_fixed_width(column_view const& column) * 2. stats_dtype: datatype for statistics calculation required for the data stream of a leaf node. * 3. ts_scale: scale to multiply or divide timestamp by in order to convert timestamp to parquet * supported types + * 4. requested_encoding: A user provided encoding to use for the column. */ struct schema_tree_node : public SchemaElement { cudf::detail::LinkedColPtr leaf_column; statistics_dtype stats_dtype; int32_t ts_scale; + column_encoding requested_encoding; // TODO(fut): Think about making schema a class that holds a vector of schema_tree_nodes. The // function construct_schema_tree could be its constructor. It can have method to get the per @@ -588,7 +590,7 @@ std::vector construct_schema_tree( std::function add_schema = [&](cudf::detail::LinkedColPtr const& col, column_in_metadata& col_meta, size_t parent_idx) { - bool col_nullable = is_col_nullable(col, col_meta, write_mode); + bool const col_nullable = is_col_nullable(col, col_meta, write_mode); auto set_field_id = [&schema, parent_idx](schema_tree_node& s, column_in_metadata const& col_meta) { @@ -604,6 +606,52 @@ std::vector construct_schema_tree( return child_col_type == type_id::UINT8; }; + // only call this after col_schema.type has been set + auto set_encoding = [&schema, parent_idx](schema_tree_node& s, + column_in_metadata const& col_meta) { + s.requested_encoding = column_encoding::USE_DEFAULT; + + if (schema[parent_idx].name != "list" and + col_meta.get_encoding() != column_encoding::USE_DEFAULT) { + // do some validation + switch (col_meta.get_encoding()) { + case column_encoding::DELTA_BINARY_PACKED: + if (s.type != Type::INT32 && s.type != Type::INT64) { + CUDF_LOG_WARN( + "DELTA_BINARY_PACKED encoding is only supported for INT32 and INT64 columns; the " + "requested encoding will be ignored"); + return; + } + break; + + case column_encoding::DELTA_LENGTH_BYTE_ARRAY: + if (s.type != Type::BYTE_ARRAY) { + CUDF_LOG_WARN( + "DELTA_LENGTH_BYTE_ARRAY encoding is only supported for BYTE_ARRAY columns; the " + "requested encoding will be ignored"); + return; + } + break; + + // supported parquet encodings + case column_encoding::PLAIN: + case column_encoding::DICTIONARY: break; + + // not yet supported for write (soon...) + case column_encoding::DELTA_BYTE_ARRAY: [[fallthrough]]; + // all others + default: + CUDF_LOG_WARN( + "Unsupported page encoding requested: {}; the requested encoding will be ignored", + static_cast(col_meta.get_encoding())); + return; + } + + // requested encoding seems to be ok, set it + s.requested_encoding = col_meta.get_encoding(); + } + }; + // There is a special case for a list column with one byte column child. This column can // have a special flag that indicates we write this out as binary instead of a list. This is a // more efficient storage mechanism for a single-depth list of bytes, but is a departure from @@ -626,6 +674,7 @@ std::vector construct_schema_tree( col_schema.parent_idx = parent_idx; col_schema.leaf_column = col; set_field_id(col_schema, col_meta); + set_encoding(col_schema, col_meta); col_schema.output_as_byte_array = col_meta.is_enabled_output_as_binary(); schema.push_back(col_schema); } else if (col->type().id() == type_id::STRUCT) { @@ -761,6 +810,7 @@ std::vector construct_schema_tree( col_schema.parent_idx = parent_idx; col_schema.leaf_column = col; set_field_id(col_schema, col_meta); + set_encoding(col_schema, col_meta); schema.push_back(col_schema); } }; @@ -947,9 +997,10 @@ parquet_column_device_view parquet_column_view::get_device_view(rmm::cuda_stream desc.level_bits = CompactProtocolReader::NumRequiredBits(max_rep_level()) << 4 | CompactProtocolReader::NumRequiredBits(max_def_level()); - desc.nullability = _d_nullability.data(); - desc.max_def_level = _max_def_level; - desc.max_rep_level = _max_rep_level; + desc.nullability = _d_nullability.data(); + desc.max_def_level = _max_def_level; + desc.max_rep_level = _max_rep_level; + desc.requested_encoding = schema_node.requested_encoding; return desc; } @@ -1169,9 +1220,15 @@ build_chunk_dictionaries(hostdevice_2dvector& chunks, std::vector> hash_maps_storage; hash_maps_storage.reserve(h_chunks.size()); for (auto& chunk : h_chunks) { - if (col_desc[chunk.col_desc_id].physical_type == Type::BOOLEAN || - (col_desc[chunk.col_desc_id].output_as_byte_array && - col_desc[chunk.col_desc_id].physical_type == Type::BYTE_ARRAY)) { + auto const& chunk_col_desc = col_desc[chunk.col_desc_id]; + auto const is_requested_non_dict = + chunk_col_desc.requested_encoding != column_encoding::USE_DEFAULT && + chunk_col_desc.requested_encoding != column_encoding::DICTIONARY; + auto const is_type_non_dict = + chunk_col_desc.physical_type == Type::BOOLEAN || + (chunk_col_desc.output_as_byte_array && chunk_col_desc.physical_type == Type::BYTE_ARRAY); + + if (is_type_non_dict || is_requested_non_dict) { chunk.use_dictionary = false; } else { chunk.use_dictionary = true; @@ -1191,6 +1248,7 @@ build_chunk_dictionaries(hostdevice_2dvector& chunks, chunks.device_to_host_sync(stream); // Make decision about which chunks have dictionary + bool cannot_honor_request = false; for (auto& ck : h_chunks) { if (not ck.use_dictionary) { continue; } std::tie(ck.use_dictionary, ck.dict_rle_bits) = [&]() -> std::pair { @@ -1217,6 +1275,19 @@ build_chunk_dictionaries(hostdevice_2dvector& chunks, return {true, nbits}; }(); + // If dictionary encoding was requested, but it cannot be used, then print a warning. It will + // actually be disabled in gpuInitPages. + if (not ck.use_dictionary) { + auto const& chunk_col_desc = col_desc[ck.col_desc_id]; + if (chunk_col_desc.requested_encoding == column_encoding::DICTIONARY) { + cannot_honor_request = true; + } + } + } + + // warn if we have to ignore requested encoding + if (cannot_honor_request) { + CUDF_LOG_WARN("DICTIONARY encoding was requested, but resource constraints prevent its use"); } // TODO: (enh) Deallocate hash map storage for chunks that don't use dict and clear pointers. diff --git a/cpp/tests/io/parquet_writer_test.cpp b/cpp/tests/io/parquet_writer_test.cpp index 62a24bf0a73..f4da9f59b8c 100644 --- a/cpp/tests/io/parquet_writer_test.cpp +++ b/cpp/tests/io/parquet_writer_test.cpp @@ -1426,6 +1426,128 @@ TEST_F(ParquetWriterTest, RowGroupMetadata) static_cast(num_rows * sizeof(column_type))); } +TEST_F(ParquetWriterTest, UserRequestedDictFallback) +{ + constexpr int num_rows = 100; + constexpr char const* big_string = + "a " + "very very very very very very very very very very very very very very very very very very " + "very very very very very very very very very very very very very very very very very very " + "very very very very very very very very very very very very very very very very very very " + "very very very very very very very very very very very very very very very very very very " + "very very very very very very very very very very very very very very very very very very " + "very very very very very very very very very very very very very very very very very very " + "long string"; + + auto const max_dict_size = strlen(big_string) * num_rows / 2; + + auto elements1 = cudf::detail::make_counting_transform_iterator( + 0, [big_string](auto i) { return big_string + std::to_string(i); }); + auto const col1 = cudf::test::strings_column_wrapper(elements1, elements1 + num_rows); + auto const table = table_view({col1}); + + cudf::io::table_input_metadata table_metadata(table); + table_metadata.column_metadata[0] + .set_name("big_strings") + .set_encoding(cudf::io::column_encoding::DICTIONARY) + .set_nullability(false); + + auto const filepath = temp_env->get_temp_filepath("UserRequestedDictFallback.parquet"); + cudf::io::parquet_writer_options opts = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, table) + .metadata(table_metadata) + .max_dictionary_size(max_dict_size); + cudf::io::write_parquet(opts); + + auto const source = cudf::io::datasource::create(filepath); + cudf::io::parquet::detail::FileMetaData fmd; + read_footer(source, &fmd); + + // encoding should have fallen back to PLAIN + EXPECT_EQ(fmd.row_groups[0].columns[0].meta_data.encodings[0], + cudf::io::parquet::detail::Encoding::PLAIN); +} + +TEST_F(ParquetWriterTest, UserRequestedEncodings) +{ + using cudf::io::column_encoding; + using cudf::io::parquet::detail::Encoding; + constexpr int num_rows = 500; + + auto const ones = thrust::make_constant_iterator(1); + auto const col = + cudf::test::fixed_width_column_wrapper{ones, ones + num_rows, no_nulls()}; + + auto const strings = thrust::make_constant_iterator("string"); + auto const string_col = + cudf::test::strings_column_wrapper(strings, strings + num_rows, no_nulls()); + + auto const table = table_view( + {col, col, col, col, col, string_col, string_col, string_col, string_col, string_col}); + + cudf::io::table_input_metadata table_metadata(table); + + auto const set_meta = [&table_metadata](int idx, std::string const& name, column_encoding enc) { + table_metadata.column_metadata[idx].set_name(name).set_encoding(enc); + }; + + set_meta(0, "int_plain", column_encoding::PLAIN); + set_meta(1, "int_dict", column_encoding::DICTIONARY); + set_meta(2, "int_db", column_encoding::DELTA_BINARY_PACKED); + set_meta(3, "int_dlba", column_encoding::DELTA_LENGTH_BYTE_ARRAY); + table_metadata.column_metadata[4].set_name("int_none"); + + set_meta(5, "string_plain", column_encoding::PLAIN); + set_meta(6, "string_dict", column_encoding::DICTIONARY); + set_meta(7, "string_dlba", column_encoding::DELTA_LENGTH_BYTE_ARRAY); + set_meta(8, "string_db", column_encoding::DELTA_BINARY_PACKED); + table_metadata.column_metadata[9].set_name("string_none"); + + for (auto& col_meta : table_metadata.column_metadata) { + col_meta.set_nullability(false); + } + + auto const filepath = temp_env->get_temp_filepath("UserRequestedEncodings.parquet"); + cudf::io::parquet_writer_options opts = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, table) + .metadata(table_metadata) + .stats_level(cudf::io::statistics_freq::STATISTICS_COLUMN) + .compression(cudf::io::compression_type::ZSTD); + cudf::io::write_parquet(opts); + + // check page headers to make sure each column is encoded with the appropriate encoder + auto const source = cudf::io::datasource::create(filepath); + cudf::io::parquet::detail::FileMetaData fmd; + read_footer(source, &fmd); + + // no nulls and no repetition, so the only encoding used should be for the data. + // since we're writing v1, both dict and data pages should use PLAIN_DICTIONARY. + auto const expect_enc = [&fmd](int idx, cudf::io::parquet::detail::Encoding enc) { + EXPECT_EQ(fmd.row_groups[0].columns[idx].meta_data.encodings[0], enc); + }; + + // requested plain + expect_enc(0, Encoding::PLAIN); + // requested dictionary + expect_enc(1, Encoding::PLAIN_DICTIONARY); + // requested delta_binary_packed + expect_enc(2, Encoding::DELTA_BINARY_PACKED); + // requested delta_length_byte_array, but should fall back to dictionary + expect_enc(3, Encoding::PLAIN_DICTIONARY); + // no request, should fall back to dictionary + expect_enc(4, Encoding::PLAIN_DICTIONARY); + // requested plain + expect_enc(5, Encoding::PLAIN); + // requested dictionary + expect_enc(6, Encoding::PLAIN_DICTIONARY); + // requested delta_length_byte_array + expect_enc(7, Encoding::DELTA_LENGTH_BYTE_ARRAY); + // requested delta_binary_packed, but should fall back to dictionary + expect_enc(8, Encoding::PLAIN_DICTIONARY); + // no request, should fall back to dictionary + expect_enc(9, Encoding::PLAIN_DICTIONARY); +} + TEST_F(ParquetWriterTest, DeltaBinaryStartsWithNulls) { // test that the DELTA_BINARY_PACKED writer can properly encode a column that begins with From ab20f470090e7a6ebc4a3065feddff77b9e24f27 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Wed, 6 Mar 2024 09:11:41 -0500 Subject: [PATCH 358/384] Deprecate strings_column_view::offsets_begin() (#15205) Deprecates the `cudf::strings_column_view::offsets_begin()` and `cudf::strings_column_view::offsets_end()` since they are hardcoded to return `size_type*`. There are very few places that used these functions. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Mark Harris (https://github.com/harrism) - Bradley Dice (https://github.com/bdice) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/15205 --- cpp/include/cudf/strings/strings_column_view.hpp | 8 ++++++-- cpp/src/strings/replace/multi.cu | 13 ++++++------- cpp/src/strings/replace/replace.cu | 8 ++++---- cpp/src/strings/strings_column_view.cpp | 4 ++-- cpp/tests/strings/array_tests.cpp | 15 --------------- 5 files changed, 18 insertions(+), 30 deletions(-) diff --git a/cpp/include/cudf/strings/strings_column_view.hpp b/cpp/include/cudf/strings/strings_column_view.hpp index 036589e17fe..1156f0a5b73 100644 --- a/cpp/include/cudf/strings/strings_column_view.hpp +++ b/cpp/include/cudf/strings/strings_column_view.hpp @@ -88,20 +88,24 @@ class strings_column_view : private column_view { /** * @brief Return an iterator for the offsets child column. * + * @deprecated Since 24.04 + * * This automatically applies the offset of the parent. * * @return Iterator pointing to the first offset value. */ - [[nodiscard]] offset_iterator offsets_begin() const; + [[deprecated]] offset_iterator offsets_begin() const; /** * @brief Return an end iterator for the offsets child column. * + * @deprecated Since 24.04 + * * This automatically applies the offset of the parent. * * @return Iterator pointing 1 past the last offset value. */ - [[nodiscard]] offset_iterator offsets_end() const; + [[deprecated]] offset_iterator offsets_end() const; /** * @brief Returns the number of bytes in the chars child column. diff --git a/cpp/src/strings/replace/multi.cu b/cpp/src/strings/replace/multi.cu index ffa922d5944..8b5a4317b50 100644 --- a/cpp/src/strings/replace/multi.cu +++ b/cpp/src/strings/replace/multi.cu @@ -302,17 +302,16 @@ std::unique_ptr replace_character_parallel(strings_column_view const& in auto string_indices = rmm::device_uvector(target_count, stream); auto const pos_itr = cudf::detail::make_counting_transform_iterator( - 0, cuda::proclaim_return_type([d_positions] __device__(auto idx) -> size_type { + 0, cuda::proclaim_return_type([d_positions] __device__(auto idx) -> int64_t { return d_positions[idx].first; })); auto pos_count = std::distance(d_positions, copy_end); - thrust::upper_bound(rmm::exec_policy(stream), - input.offsets_begin(), - input.offsets_end(), - pos_itr, - pos_itr + pos_count, - string_indices.begin()); + auto begin = + cudf::detail::offsetalator_factory::make_input_iterator(input.offsets(), input.offset()); + auto end = begin + input.offsets().size(); + thrust::upper_bound( + rmm::exec_policy(stream), begin, end, pos_itr, pos_itr + pos_count, string_indices.begin()); // compute offsets per string auto targets_offsets = rmm::device_uvector(strings_count + 1, stream); diff --git a/cpp/src/strings/replace/replace.cu b/cpp/src/strings/replace/replace.cu index c37c64e348c..1f752f543d0 100644 --- a/cpp/src/strings/replace/replace.cu +++ b/cpp/src/strings/replace/replace.cu @@ -413,10 +413,10 @@ std::unique_ptr replace_char_parallel(strings_column_view const& strings { auto const strings_count = strings.size(); auto const offset_count = strings_count + 1; - auto const d_offsets = strings.offsets_begin(); - auto const d_in_chars = strings.chars_begin(stream); - auto const chars_bytes = chars_end - chars_start; - auto const target_size = d_target.size_bytes(); + auto const d_offsets = strings.offsets().begin() + strings.offset(); // TODO: PR 14824 + auto const d_in_chars = strings.chars_begin(stream); + auto const chars_bytes = chars_end - chars_start; + auto const target_size = d_target.size_bytes(); // detect a target match at the specified byte position device_span const d_chars_span(d_in_chars, chars_end); diff --git a/cpp/src/strings/strings_column_view.cpp b/cpp/src/strings/strings_column_view.cpp index 83ae916afc3..3ae97a00bbf 100644 --- a/cpp/src/strings/strings_column_view.cpp +++ b/cpp/src/strings/strings_column_view.cpp @@ -37,12 +37,12 @@ column_view strings_column_view::offsets() const strings_column_view::offset_iterator strings_column_view::offsets_begin() const { - return offsets().begin() + offset(); + return offsets().begin() + offset(); } strings_column_view::offset_iterator strings_column_view::offsets_end() const { - return offsets_begin() + size() + 1; + return offsets().begin() + offset() + size() + 1; } int64_t strings_column_view::chars_size(rmm::cuda_stream_view stream) const noexcept diff --git a/cpp/tests/strings/array_tests.cpp b/cpp/tests/strings/array_tests.cpp index c6cc8e078bb..b22d7257041 100644 --- a/cpp/tests/strings/array_tests.cpp +++ b/cpp/tests/strings/array_tests.cpp @@ -213,19 +213,4 @@ TEST_F(StringsColumnTest, ScatterZeroSizeStringsColumn) cudf::test::expect_column_empty(results->view().column(0)); } -TEST_F(StringsColumnTest, OffsetsBeginEnd) -{ - cudf::test::strings_column_wrapper input({"eee", "bb", "", "", "aa", "bbb", "ééé"}, - {1, 1, 0, 1, 1, 1, 1}); - - cudf::test::fixed_width_column_wrapper expected({0, 5}); - auto scv = cudf::strings_column_view(input); - EXPECT_EQ(std::distance(scv.offsets_begin(), scv.offsets_end()), - static_cast(scv.size() + 1)); - - scv = cudf::strings_column_view(cudf::slice(input, {1, 5}).front()); - EXPECT_EQ(std::distance(scv.offsets_begin(), scv.offsets_end()), - static_cast(scv.size() + 1)); -} - CUDF_TEST_PROGRAM_MAIN() From bb0e4fdd6f4960d1d5125256dc147f28d83db560 Mon Sep 17 00:00:00 2001 From: Michael Schellenberger Costa Date: Wed, 6 Mar 2024 06:17:24 -0800 Subject: [PATCH 359/384] Add `get_upstream_resource` method to `stream_checking_resource_adaptor` (#15203) Also deprecate `get_upstream` as we want to get away from raw upstreams Authors: - Michael Schellenberger Costa (https://github.com/miscco) Approvers: - Mark Harris (https://github.com/harrism) - David Wendt (https://github.com/davidwendt) URL: https://github.com/rapidsai/cudf/pull/15203 --- .../cudf_test/stream_checking_resource_adaptor.hpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/cpp/include/cudf_test/stream_checking_resource_adaptor.hpp b/cpp/include/cudf_test/stream_checking_resource_adaptor.hpp index d1841ff42a1..cafde6ca7d5 100644 --- a/cpp/include/cudf_test/stream_checking_resource_adaptor.hpp +++ b/cpp/include/cudf_test/stream_checking_resource_adaptor.hpp @@ -20,6 +20,7 @@ #include #include +#include #include @@ -58,11 +59,14 @@ class stream_checking_resource_adaptor final : public rmm::mr::device_memory_res default; /** - * @brief Return pointer to the upstream resource. + * @brief Returns the wrapped upstream resource * - * @return Pointer to the upstream resource. + * @return The wrapped upstream resource */ - Upstream* get_upstream() const noexcept { return upstream_; } + [[nodiscard]] rmm::device_async_resource_ref get_upstream_resource() const noexcept + { + return upstream_; + } private: /** @@ -110,8 +114,8 @@ class stream_checking_resource_adaptor final : public rmm::mr::device_memory_res { if (this == &other) { return true; } auto cast = dynamic_cast const*>(&other); - return cast != nullptr ? upstream_->is_equal(*cast->get_upstream()) - : upstream_->is_equal(other); + if (cast == nullptr) { return upstream_->is_equal(other); } + return get_upstream_resource() == cast->get_upstream_resource(); } /** From db9e6a91968e047b6517951f5fd32c97874eb79e Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Wed, 6 Mar 2024 10:48:36 -0600 Subject: [PATCH 360/384] Update dlpack to version 0.8 (#15237) cuVS Python and Rust APIS use `dlpack` 0.8 to call `libcuvs`. To be able to create a RAPIDS environment that has both cuDF and cuVS (and eventually cuML and other libraries where we will use `dlpack` as we do in cuVS) we require to update dlpack to match. PR notes: - There is 1 key relevant differences to RAPIDS between 0.5 and 0.8: support of `DLDeviceType` for managed memory with `kDLCUDAManaged` We don't currently require to update the existing libcudf to work with this, but this could be a useful addition that I would suggest exploring post 24.04. - DLpack 1.0 release candidate was released recently with the key addition being adding versioning support in `DLManagedTensorVersioned`. Given the timing for cuDF burndown, I think we should pin to 0.8 for 24.04 reducing the number of changes (which AFAIK all changes are non breaking so we shouldn't need code updates, pending CI testing), and then update cuDF/cuVS/etc. to use `DLManagedTensorVersioned` once dlpack 1.0 final version is released. cc @divyegala @cjnolet Authors: - Dante Gama Dessavre (https://github.com/dantegd) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) - Robert Maynard (https://github.com/robertmaynard) - Bradley Dice (https://github.com/bdice) - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/cudf/pull/15237 --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-122_arch-x86_64.yaml | 2 +- conda/recipes/cudf/meta.yaml | 2 +- conda/recipes/libcudf/conda_build_config.yaml | 2 +- cpp/cmake/thirdparty/get_dlpack.cmake | 4 ++-- dependencies.yaml | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index c12e88f1c0f..e13357aa78e 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -28,7 +28,7 @@ dependencies: - cxx-compiler - cython>=3.0.3 - dask-cuda==24.4.* -- dlpack>=0.5,<0.6.0a0 +- dlpack>=0.8,<1.0 - doxygen=1.9.1 - fastavro>=0.22.9 - fmt>=10.1.1,<11 diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index e773812967d..c028c3fde3a 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -29,7 +29,7 @@ dependencies: - cxx-compiler - cython>=3.0.3 - dask-cuda==24.4.* -- dlpack>=0.5,<0.6.0a0 +- dlpack>=0.8,<1.0 - doxygen=1.9.1 - fastavro>=0.22.9 - fmt>=10.1.1,<11 diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 6a85fadaa48..7633fbb00a3 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -64,7 +64,7 @@ requirements: - cython >=3.0.3 - scikit-build-core >=0.7.0 - setuptools - - dlpack >=0.5,<0.6.0a0 + - dlpack >=0.8,<1.0 - numpy 1.23 - pyarrow ==14.0.2.* - libcudf ={{ version }} diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml index 3280ddf185a..53770956ebe 100644 --- a/conda/recipes/libcudf/conda_build_config.yaml +++ b/conda/recipes/libcudf/conda_build_config.yaml @@ -26,7 +26,7 @@ libarrow_version: - "==14.0.2" dlpack_version: - - ">=0.5,<0.6.0a0" + - ">=0.8,<1.0" librdkafka_version: - ">=1.9.0,<1.10.0a0" diff --git a/cpp/cmake/thirdparty/get_dlpack.cmake b/cpp/cmake/thirdparty/get_dlpack.cmake index 65b5f4ff2eb..790d6367745 100644 --- a/cpp/cmake/thirdparty/get_dlpack.cmake +++ b/cpp/cmake/thirdparty/get_dlpack.cmake @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -36,6 +36,6 @@ function(find_and_configure_dlpack VERSION) endif() endfunction() -set(CUDF_MIN_VERSION_dlpack 0.5) +set(CUDF_MIN_VERSION_dlpack 0.8) find_and_configure_dlpack(${CUDF_MIN_VERSION_dlpack}) diff --git a/dependencies.yaml b/dependencies.yaml index a83a03b571b..0352d61b0ff 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -199,7 +199,7 @@ dependencies: - &ninja ninja - c-compiler - cxx-compiler - - dlpack>=0.5,<0.6.0a0 + - dlpack>=0.8,<1.0 - zlib>=1.2.13 specific: - output_types: conda From eb8de186720a7edda90760cb189566df18146911 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 6 Mar 2024 11:23:36 -0800 Subject: [PATCH 361/384] Treat dask-cudf CI artifacts as pure wheels (#15223) This marks `dask-cudf` as a pure wheel, meaning that the CI artifacts are not specific to a Python version or CPU architecture. This change depends on https://github.com/rapidsai/gha-tools/pull/96, and makes CI workflows more robust by allowing the test matrix to be separated from the build matrix. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Jake Awe (https://github.com/AyodeAwe) URL: https://github.com/rapidsai/cudf/pull/15223 --- ci/build_wheel_dask_cudf.sh | 2 +- ci/test_wheel_dask_cudf.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_dask_cudf.sh b/ci/build_wheel_dask_cudf.sh index b09c1e51271..150fec4e2d7 100755 --- a/ci/build_wheel_dask_cudf.sh +++ b/ci/build_wheel_dask_cudf.sh @@ -8,4 +8,4 @@ package_dir="python/dask_cudf" ./ci/build_wheel.sh dask-cudf ${package_dir} RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/dist +RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 ${package_dir}/dist diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index 74fcb43ddca..59f6ecd8483 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -4,7 +4,7 @@ set -eou pipefail RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist # Download the cudf built in the previous step # Set the manylinux version used for downloading the wheels so that we test the From d824fa539ad19b8372904b88cd5e3b24aa58b1ce Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Wed, 6 Mar 2024 15:03:53 -0600 Subject: [PATCH 362/384] Java bindings for left outer distinct join (#15154) Adds Java bindings to the distinct left join functionality added in #15149. Authors: - Jason Lowe (https://github.com/jlowe) Approvers: - Robert (Bobby) Evans (https://github.com/revans2) - Jim Brennan (https://github.com/jbrennan333) URL: https://github.com/rapidsai/cudf/pull/15154 --- java/src/main/java/ai/rapids/cudf/Table.java | 52 +++++++-- java/src/main/native/src/TableJni.cpp | 18 ++++ .../test/java/ai/rapids/cudf/TableTest.java | 101 ++++++++++++++++++ 3 files changed, 160 insertions(+), 11 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/Table.java b/java/src/main/java/ai/rapids/cudf/Table.java index a1bdfe9a796..f3b4b9484ef 100644 --- a/java/src/main/java/ai/rapids/cudf/Table.java +++ b/java/src/main/java/ai/rapids/cudf/Table.java @@ -620,6 +620,9 @@ private static native long[] merge(long[] tableHandles, int[] sortKeyIndexes, private static native long[] leftJoinGatherMaps(long leftKeys, long rightKeys, boolean compareNullsEqual) throws CudfException; + private static native long[] leftDistinctJoinGatherMap(long leftKeys, long rightKeys, + boolean compareNullsEqual) throws CudfException; + private static native long leftJoinRowCount(long leftTable, long rightHashJoin) throws CudfException; private static native long[] leftHashJoinGatherMaps(long leftTable, long rightHashJoin) throws CudfException; @@ -2949,6 +2952,33 @@ public GatherMap[] leftJoinGatherMaps(Table rightKeys, boolean compareNullsEqual return buildJoinGatherMaps(gatherMapData); } + /** + * Computes a gather map that can be used to manifest the result of a left equi-join between + * two tables where the right table is guaranteed to not contain any duplicated join keys. + * The left table can be used as-is to produce the left table columns resulting from the join, + * i.e.: left table ordering is preserved in the join result, so no gather map is required for + * the left table. The resulting gather map can be applied to the right table to produce the + * right table columns resulting from the join. It is assumed this table instance holds the + * key columns from the left table, and the table argument represents the key columns from the + * right table. A {@link GatherMap} instance will be returned that can be used to gather the + * right table and that result combined with the left table to produce a left outer join result. + * + * It is the responsibility of the caller to close the resulting gather map instance. + * + * @param rightKeys join key columns from the right table + * @param compareNullsEqual true if null key values should match otherwise false + * @return right table gather map + */ + public GatherMap leftDistinctJoinGatherMap(Table rightKeys, boolean compareNullsEqual) { + if (getNumberOfColumns() != rightKeys.getNumberOfColumns()) { + throw new IllegalArgumentException("Column count mismatch, this: " + getNumberOfColumns() + + "rightKeys: " + rightKeys.getNumberOfColumns()); + } + long[] gatherMapData = + leftDistinctJoinGatherMap(getNativeView(), rightKeys.getNativeView(), compareNullsEqual); + return buildSingleJoinGatherMap(gatherMapData); + } + /** * Computes the number of rows resulting from a left equi-join between two tables. * It is assumed this table instance holds the key columns from the left table, and the @@ -3576,7 +3606,7 @@ public static GatherMap[] mixedFullJoinGatherMaps(Table leftKeys, Table rightKey return buildJoinGatherMaps(gatherMapData); } - private static GatherMap buildSemiJoinGatherMap(long[] gatherMapData) { + private static GatherMap buildSingleJoinGatherMap(long[] gatherMapData) { long bufferSize = gatherMapData[0]; long leftAddr = gatherMapData[1]; long leftHandle = gatherMapData[2]; @@ -3601,7 +3631,7 @@ public GatherMap leftSemiJoinGatherMap(Table rightKeys, boolean compareNullsEqua } long[] gatherMapData = leftSemiJoinGatherMap(getNativeView(), rightKeys.getNativeView(), compareNullsEqual); - return buildSemiJoinGatherMap(gatherMapData); + return buildSingleJoinGatherMap(gatherMapData); } /** @@ -3634,7 +3664,7 @@ public GatherMap conditionalLeftSemiJoinGatherMap(Table rightTable, long[] gatherMapData = conditionalLeftSemiJoinGatherMap(getNativeView(), rightTable.getNativeView(), condition.getNativeHandle()); - return buildSemiJoinGatherMap(gatherMapData); + return buildSingleJoinGatherMap(gatherMapData); } /** @@ -3659,7 +3689,7 @@ public GatherMap conditionalLeftSemiJoinGatherMap(Table rightTable, long[] gatherMapData = conditionalLeftSemiJoinGatherMapWithCount(getNativeView(), rightTable.getNativeView(), condition.getNativeHandle(), outputRowCount); - return buildSemiJoinGatherMap(gatherMapData); + return buildSingleJoinGatherMap(gatherMapData); } /** @@ -3716,7 +3746,7 @@ public static GatherMap mixedLeftSemiJoinGatherMap(Table leftKeys, Table rightKe leftConditional.getNativeView(), rightConditional.getNativeView(), condition.getNativeHandle(), nullEquality == NullEquality.EQUAL); - return buildSemiJoinGatherMap(gatherMapData); + return buildSingleJoinGatherMap(gatherMapData); } /** @@ -3752,7 +3782,7 @@ public static GatherMap mixedLeftSemiJoinGatherMap(Table leftKeys, Table rightKe condition.getNativeHandle(), nullEquality == NullEquality.EQUAL, joinSize.getOutputRowCount(), joinSize.getMatches().getNativeView()); - return buildSemiJoinGatherMap(gatherMapData); + return buildSingleJoinGatherMap(gatherMapData); } /** @@ -3773,7 +3803,7 @@ public GatherMap leftAntiJoinGatherMap(Table rightKeys, boolean compareNullsEqua } long[] gatherMapData = leftAntiJoinGatherMap(getNativeView(), rightKeys.getNativeView(), compareNullsEqual); - return buildSemiJoinGatherMap(gatherMapData); + return buildSingleJoinGatherMap(gatherMapData); } /** @@ -3806,7 +3836,7 @@ public GatherMap conditionalLeftAntiJoinGatherMap(Table rightTable, long[] gatherMapData = conditionalLeftAntiJoinGatherMap(getNativeView(), rightTable.getNativeView(), condition.getNativeHandle()); - return buildSemiJoinGatherMap(gatherMapData); + return buildSingleJoinGatherMap(gatherMapData); } /** @@ -3831,7 +3861,7 @@ public GatherMap conditionalLeftAntiJoinGatherMap(Table rightTable, long[] gatherMapData = conditionalLeftAntiJoinGatherMapWithCount(getNativeView(), rightTable.getNativeView(), condition.getNativeHandle(), outputRowCount); - return buildSemiJoinGatherMap(gatherMapData); + return buildSingleJoinGatherMap(gatherMapData); } /** @@ -3888,7 +3918,7 @@ public static GatherMap mixedLeftAntiJoinGatherMap(Table leftKeys, Table rightKe leftConditional.getNativeView(), rightConditional.getNativeView(), condition.getNativeHandle(), nullEquality == NullEquality.EQUAL); - return buildSemiJoinGatherMap(gatherMapData); + return buildSingleJoinGatherMap(gatherMapData); } /** @@ -3924,7 +3954,7 @@ public static GatherMap mixedLeftAntiJoinGatherMap(Table leftKeys, Table rightKe condition.getNativeHandle(), nullEquality == NullEquality.EQUAL, joinSize.getOutputRowCount(), joinSize.getMatches().getNativeView()); - return buildSemiJoinGatherMap(gatherMapData); + return buildSingleJoinGatherMap(gatherMapData); } /** diff --git a/java/src/main/native/src/TableJni.cpp b/java/src/main/native/src/TableJni.cpp index 357705824d2..51b8eb853de 100644 --- a/java/src/main/native/src/TableJni.cpp +++ b/java/src/main/native/src/TableJni.cpp @@ -2434,6 +2434,24 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_Table_leftJoinGatherMaps( }); } +JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_Table_leftDistinctJoinGatherMap( + JNIEnv *env, jclass, jlong j_left_keys, jlong j_right_keys, jboolean compare_nulls_equal) { + return cudf::jni::join_gather_single_map( + env, j_left_keys, j_right_keys, compare_nulls_equal, + [](cudf::table_view const &left, cudf::table_view const &right, cudf::null_equality nulleq) { + auto has_nulls = cudf::has_nested_nulls(left) || cudf::has_nested_nulls(right) ? + cudf::nullable_join::YES : + cudf::nullable_join::NO; + if (cudf::detail::has_nested_columns(right)) { + cudf::distinct_hash_join hash(right, left, has_nulls, nulleq); + return hash.left_join(); + } else { + cudf::distinct_hash_join hash(right, left, has_nulls, nulleq); + return hash.left_join(); + } + }); +} + JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_leftJoinRowCount(JNIEnv *env, jclass, jlong j_left_table, jlong j_right_hash_join) { diff --git a/java/src/test/java/ai/rapids/cudf/TableTest.java b/java/src/test/java/ai/rapids/cudf/TableTest.java index 44dd20561bf..d06ea05144b 100644 --- a/java/src/test/java/ai/rapids/cudf/TableTest.java +++ b/java/src/test/java/ai/rapids/cudf/TableTest.java @@ -1734,6 +1734,107 @@ void testLeftJoinGatherMapsNulls() { } } + private void checkLeftDistinctJoin(Table leftKeys, Table rightKeys, ColumnView expected, + boolean compareNullsEqual) { + try (GatherMap map = leftKeys.leftDistinctJoinGatherMap(rightKeys, compareNullsEqual)) { + int numRows = (int) expected.getRowCount(); + assertEquals(numRows, map.getRowCount()); + try (ColumnView view = map.toColumnView(0, numRows)) { + assertColumnsAreEqual(expected, view); + } + } + } + + @Test + void testLeftDistinctJoinGatherMaps() { + final int inv = Integer.MIN_VALUE; + try (Table leftKeys = new Table.TestBuilder().column(2, 3, 9, 0, 1, 7, 4, 6, 5, 8, 6).build(); + Table rightKeys = new Table.TestBuilder().column(6, 5, 9, 8, 10, 32).build(); + ColumnVector expected = ColumnVector.fromInts(inv, inv, 2, inv, inv, inv, inv, 0, 1, 3, 0)) { + checkLeftDistinctJoin(leftKeys, rightKeys, expected, false); + } + } + + @Test + void testLeftDistinctJoinGatherMapsWithNested() { + final int inv = Integer.MIN_VALUE; + StructType structType = new StructType(false, + new BasicType(false, DType.STRING), + new BasicType(false, DType.INT32)); + StructData[] leftData = new StructData[]{ + new StructData("abc", 1), + new StructData("xyz", 1), + new StructData("abc", 2), + new StructData("xyz", 2), + new StructData("abc", 1), + new StructData("abc", 3), + new StructData("xyz", 3) + }; + StructData[] rightData = new StructData[]{ + new StructData("abc", 1), + new StructData("xyz", 4), + new StructData("xyz", 2), + new StructData("abc", -1), + }; + try (Table leftKeys = new Table.TestBuilder().column(structType, leftData).build(); + Table rightKeys = new Table.TestBuilder().column(structType, rightData).build(); + ColumnVector expected = ColumnVector.fromInts(0, inv, inv, 2, 0, inv, inv)) { + checkLeftDistinctJoin(leftKeys, rightKeys, expected, false); + } + } + + @Test + void testLeftDistinctJoinGatherMapsNullsEqual() { + final int inv = Integer.MIN_VALUE; + try (Table leftKeys = new Table.TestBuilder() + .column(2, 3, 9, 0, 1, 7, 4, null, null, 8) + .build(); + Table rightKeys = new Table.TestBuilder() + .column(null, 9, 8, 10, 32) + .build(); + ColumnVector expected = ColumnVector.fromInts(inv, inv, 1, inv, inv, inv, inv, 0, 0, 2)) { + checkLeftDistinctJoin(leftKeys, rightKeys, expected, true); + } + } + + @Test + void testLeftDistinctJoinGatherMapsWithNestedNullsEqual() { + final int inv = Integer.MIN_VALUE; + StructType structType = new StructType(true, + new BasicType(true, DType.STRING), + new BasicType(true, DType.INT32)); + StructData[] leftData = new StructData[]{ + new StructData("abc", 1), + null, + new StructData("xyz", 1), + new StructData("abc", 2), + new StructData("xyz", null), + null, + new StructData("abc", 1), + new StructData("abc", 3), + new StructData("xyz", 3), + new StructData(null, null), + new StructData(null, 1) + }; + StructData[] rightData = new StructData[]{ + null, + new StructData("abc", 1), + new StructData("xyz", 4), + new StructData("xyz", 2), + new StructData(null, null), + new StructData(null, 2), + new StructData(null, 1), + new StructData("xyz", null), + new StructData("abc", null), + new StructData("abc", -1) + }; + try (Table leftKeys = new Table.TestBuilder().column(structType, leftData).build(); + Table rightKeys = new Table.TestBuilder().column(structType, rightData).build(); + ColumnVector expected = ColumnVector.fromInts(1, 0, inv, inv, 7, 0, 1, inv, inv, 4, 6)) { + checkLeftDistinctJoin(leftKeys, rightKeys, expected, true); + } + } + @Test void testLeftHashJoinGatherMaps() { final int inv = Integer.MIN_VALUE; From 5838d7b76a0ec7ddd6b32709857bd3c946c3b80d Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 6 Mar 2024 11:26:18 -1000 Subject: [PATCH 363/384] Clean up Columns.astype & cudf.dtype (#15125) - Able to remove `pandas_dtypes_alias_to_cudf_alias` by using `cudf.dtype` in `Column.astype` - Simplified some branches in `Column.astype` Authors: - Matthew Roeschke (https://github.com/mroeschke) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Ashwin Srinath (https://github.com/shwina) - Michael Wang (https://github.com/isVoid) URL: https://github.com/rapidsai/cudf/pull/15125 --- python/cudf/cudf/api/types.py | 2 + python/cudf/cudf/core/column/column.py | 53 +++++++++----------------- python/cudf/cudf/core/dtypes.py | 38 +++++++++--------- python/cudf/cudf/utils/dtypes.py | 14 ------- 4 files changed, 39 insertions(+), 68 deletions(-) diff --git a/python/cudf/cudf/api/types.py b/python/cudf/cudf/api/types.py index a422eb82231..417d8b0922a 100644 --- a/python/cudf/cudf/api/types.py +++ b/python/cudf/cudf/api/types.py @@ -504,6 +504,8 @@ def _is_pandas_nullable_extension_dtype(dtype_to_check) -> bool: ): return True elif isinstance(dtype_to_check, pd.CategoricalDtype): + if dtype_to_check.categories is None: + return False return _is_pandas_nullable_extension_dtype( dtype_to_check.categories.dtype ) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 8941d111d02..ff1204b6178 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -90,8 +90,6 @@ min_scalar_type, min_unsigned_type, np_to_pa_dtype, - pandas_dtypes_alias_to_cudf_alias, - pandas_dtypes_to_np_dtypes, ) from cudf.utils.utils import _array_ufunc, mask_dtype @@ -974,42 +972,20 @@ def astype(self, dtype: Dtype, copy: bool = False) -> ColumnBase: col = self.copy() else: col = self - if self.dtype == dtype: - return col - if _is_categorical_dtype(dtype): + if dtype == "category": + # TODO: Figure out why `cudf.dtype("category")` + # astype's different than just the string return col.as_categorical_column(dtype) - - if ( - isinstance(dtype, str) - and dtype in pandas_dtypes_alias_to_cudf_alias - ): - if cudf.get_option("mode.pandas_compatible"): - raise NotImplementedError("not supported") - else: - dtype = pandas_dtypes_alias_to_cudf_alias[dtype] - elif _is_pandas_nullable_extension_dtype(dtype) and cudf.get_option( - "mode.pandas_compatible" + elif dtype == "interval" and isinstance( + self.dtype, cudf.IntervalDtype ): - raise NotImplementedError("not supported") - else: - dtype = pandas_dtypes_to_np_dtypes.get(dtype, dtype) - if _is_non_decimal_numeric_dtype(dtype): - return col.as_numerical_column(dtype) - elif _is_categorical_dtype(dtype): + return col + was_object = dtype == object or dtype == np.dtype(object) + dtype = cudf.dtype(dtype) + if self.dtype == dtype: + return col + elif isinstance(dtype, CategoricalDtype): return col.as_categorical_column(dtype) - elif cudf.dtype(dtype).type in { - np.str_, - np.object_, - str, - }: - if cudf.get_option("mode.pandas_compatible") and np.dtype( - dtype - ).type in {np.object_}: - raise ValueError( - f"Casting to {dtype} is not supported, use " - "`.astype('str')` instead." - ) - return col.as_string_column(dtype) elif isinstance(dtype, IntervalDtype): return col.as_interval_column(dtype) elif isinstance(dtype, (ListDtype, StructDtype)): @@ -1024,6 +1000,13 @@ def astype(self, dtype: Dtype, copy: bool = False) -> ColumnBase: return col.as_datetime_column(dtype) elif np.issubdtype(cast(Any, dtype), np.timedelta64): return col.as_timedelta_column(dtype) + elif dtype.kind == "O": + if cudf.get_option("mode.pandas_compatible") and was_object: + raise ValueError( + f"Casting to {dtype} is not supported, use " + "`.astype('str')` instead." + ) + return col.as_string_column(dtype) else: return col.as_numerical_column(dtype) diff --git a/python/cudf/cudf/core/dtypes.py b/python/cudf/cudf/core/dtypes.py index 26d2ea3e992..c658701f851 100644 --- a/python/cudf/cudf/core/dtypes.py +++ b/python/cudf/cudf/core/dtypes.py @@ -42,12 +42,12 @@ def dtype(arbitrary): # next, try interpreting arbitrary as a NumPy dtype that we support: try: np_dtype = np.dtype(arbitrary) - if np_dtype.kind in ("OU"): - return np.dtype("object") except TypeError: pass else: - if np_dtype not in cudf._lib.types.SUPPORTED_NUMPY_TO_LIBCUDF_TYPES: + if np_dtype.kind in set("OU"): + return np.dtype("object") + elif np_dtype not in cudf._lib.types.SUPPORTED_NUMPY_TO_LIBCUDF_TYPES: raise TypeError(f"Unsupported type {np_dtype}") return np_dtype @@ -55,25 +55,25 @@ def dtype(arbitrary): # `arbitrary` as a Pandas extension type. # Return the corresponding NumPy/cuDF type. pd_dtype = pd.api.types.pandas_dtype(arbitrary) - if cudf.get_option( - "mode.pandas_compatible" - ) and cudf.api.types._is_pandas_nullable_extension_dtype(pd_dtype): - raise NotImplementedError("not supported") - try: - return dtype(pd_dtype.numpy_dtype) - except AttributeError: - if isinstance(pd_dtype, pd.CategoricalDtype): - return cudf.CategoricalDtype.from_pandas(pd_dtype) + if cudf.api.types._is_pandas_nullable_extension_dtype(pd_dtype): + if cudf.get_option("mode.pandas_compatible"): + raise NotImplementedError( + "Nullable types not supported in pandas compatibility mode" + ) elif isinstance(pd_dtype, pd.StringDtype): return np.dtype("object") - elif isinstance(pd_dtype, pd.IntervalDtype): - return cudf.IntervalDtype.from_pandas(pd_dtype) - elif isinstance(pd_dtype, pd.DatetimeTZDtype): - return pd_dtype else: - raise TypeError( - f"Cannot interpret {arbitrary} as a valid cuDF dtype" - ) + return dtype(pd_dtype.numpy_dtype) + elif isinstance(pd_dtype, pd.core.dtypes.dtypes.NumpyEADtype): + return dtype(pd_dtype.numpy_dtype) + elif isinstance(pd_dtype, pd.CategoricalDtype): + return cudf.CategoricalDtype.from_pandas(pd_dtype) + elif isinstance(pd_dtype, pd.IntervalDtype): + return cudf.IntervalDtype.from_pandas(pd_dtype) + elif isinstance(pd_dtype, pd.DatetimeTZDtype): + return pd_dtype + else: + raise TypeError(f"Cannot interpret {arbitrary} as a valid cuDF dtype") def _decode_type( diff --git a/python/cudf/cudf/utils/dtypes.py b/python/cudf/cudf/utils/dtypes.py index c8aca94ba19..3780fcc627e 100644 --- a/python/cudf/cudf/utils/dtypes.py +++ b/python/cudf/cudf/utils/dtypes.py @@ -74,25 +74,11 @@ pd.StringDtype(): np.dtype("object"), } -pandas_dtypes_alias_to_cudf_alias = { - "UInt8": "uint8", - "UInt16": "uint16", - "UInt32": "uint32", - "UInt64": "uint64", - "Int8": "int8", - "Int16": "int16", - "Int32": "int32", - "Int64": "int64", - "boolean": "bool", -} - np_dtypes_to_pandas_dtypes[np.dtype("float32")] = pd.Float32Dtype() np_dtypes_to_pandas_dtypes[np.dtype("float64")] = pd.Float64Dtype() pandas_dtypes_to_np_dtypes[pd.Float32Dtype()] = np.dtype("float32") pandas_dtypes_to_np_dtypes[pd.Float64Dtype()] = np.dtype("float64") -pandas_dtypes_alias_to_cudf_alias["Float32"] = "float32" -pandas_dtypes_alias_to_cudf_alias["Float64"] = "float64" SIGNED_INTEGER_TYPES = {"int8", "int16", "int32", "int64"} UNSIGNED_TYPES = {"uint8", "uint16", "uint32", "uint64"} From c299a62394379468da5761aa194056ea1f2cfde1 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 6 Mar 2024 11:28:35 -1000 Subject: [PATCH 364/384] DataFrame.columns = ... retains RangeIndex & set dtype (#15129) Also * Renamed `_set_column_names_like` to `_set_columns_like` (we're not just copying over the names) * Set `verify=False` when building the `ColumnAccessor` (columns are not modified so no need to check the columns) Authors: - Matthew Roeschke (https://github.com/mroeschke) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/15129 --- python/cudf/cudf/core/dataframe.py | 96 ++++++++++++++++-------- python/cudf/cudf/core/indexed_frame.py | 4 +- python/cudf/cudf/tests/test_dataframe.py | 53 +++++++++++++ 3 files changed, 120 insertions(+), 33 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 31a748da856..1dc79127f60 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -1777,7 +1777,7 @@ def _concat( # Reassign index and column names if objs[0]._data.multiindex: - out._set_column_names_like(objs[0]) + out._set_columns_like(objs[0]._data) else: out.columns = names if not ignore_index: @@ -2215,7 +2215,11 @@ def from_dict( next(iter(data.values())), (cudf.Series, cupy.ndarray) ): result = cls(data).T - result.columns = columns + result.columns = ( + columns + if columns is not None + else range(len(result._data)) + ) if dtype is not None: result = result.astype(dtype) return result @@ -2619,39 +2623,69 @@ def columns(self): @columns.setter # type: ignore @_cudf_nvtx_annotate def columns(self, columns): - if isinstance(columns, cudf.BaseIndex): - columns = columns.to_pandas() - if columns is None: - columns = pd.Index(range(len(self._data.columns))) - is_multiindex = isinstance(columns, pd.MultiIndex) - - if isinstance(columns, (Series, cudf.Index, ColumnBase)): - columns = pd.Index(columns.to_numpy(), tupleize_cols=is_multiindex) - elif not isinstance(columns, pd.Index): - columns = pd.Index(columns, tupleize_cols=is_multiindex) + multiindex = False + rangeindex = False + label_dtype = None + level_names = None + if isinstance(columns, (pd.MultiIndex, cudf.MultiIndex)): + multiindex = True + if isinstance(columns, cudf.MultiIndex): + pd_columns = columns.to_pandas() + else: + pd_columns = columns + if pd_columns.nunique(dropna=False) != len(pd_columns): + raise ValueError("Duplicate column names are not allowed") + level_names = list(pd_columns.names) + elif isinstance(columns, (cudf.BaseIndex, ColumnBase, Series)): + level_names = (getattr(columns, "name", None),) + rangeindex = isinstance(columns, cudf.RangeIndex) + columns = as_column(columns) + if columns.distinct_count(dropna=False) != len(columns): + raise ValueError("Duplicate column names are not allowed") + pd_columns = pd.Index(columns.to_pandas()) + label_dtype = pd_columns.dtype + else: + pd_columns = pd.Index(columns) + if pd_columns.nunique(dropna=False) != len(pd_columns): + raise ValueError("Duplicate column names are not allowed") + rangeindex = isinstance(pd_columns, pd.RangeIndex) + level_names = (pd_columns.name,) + label_dtype = pd_columns.dtype - if not len(columns) == len(self._data.names): + if len(pd_columns) != len(self._data.names): raise ValueError( f"Length mismatch: expected {len(self._data.names)} elements, " - f"got {len(columns)} elements" + f"got {len(pd_columns)} elements" ) - self._set_column_names(columns, is_multiindex, columns.names) - - def _set_column_names(self, names, multiindex=False, level_names=None): - data = dict(zip(names, self._data.columns)) - if len(names) != len(data): - raise ValueError("Duplicate column names are not allowed") - self._data = ColumnAccessor( - data, + data=dict(zip(pd_columns, self._data.columns)), multiindex=multiindex, level_names=level_names, + label_dtype=label_dtype, + rangeindex=rangeindex, + verify=False, ) - def _set_column_names_like(self, other): - self._set_column_names( - other._data.names, other._data.multiindex, other._data.level_names + def _set_columns_like(self, other: ColumnAccessor) -> None: + """ + Modify self with the column properties of other. + + * Whether .columns is a MultiIndex/RangeIndex + * The possible .columns.dtype + * The .columns.names/name (depending on if it's a MultiIndex) + """ + if len(self._data.names) != len(other.names): + raise ValueError( + f"Length mismatch: expected {len(other)} elements, " + f"got {len(self)} elements" + ) + self._data = ColumnAccessor( + data=dict(zip(other.names, self._data.columns)), + multiindex=other.multiindex, + level_names=other.level_names, + label_dtype=other.label_dtype, + verify=False, ) @_cudf_nvtx_annotate @@ -3023,7 +3057,7 @@ def where(self, cond, other=None, inplace=False): "Array conditional must be same shape as self" ) # Setting `self` column names to `cond` as it has no column names. - cond._set_column_names_like(self) + cond._set_columns_like(self._data) # If other was provided, process that next. if isinstance(other, DataFrame): @@ -6347,7 +6381,7 @@ def mode(self, axis=0, numeric_only=False, dropna=True): if isinstance(df, Series): df = df.to_frame() - df._set_column_names_like(data_df) + df._set_columns_like(data_df._data) return df @@ -6458,7 +6492,7 @@ def _apply_cupy_method_axis_1(self, method, *args, **kwargs): ) else: result_df = DataFrame(result).set_index(self.index) - result_df._set_column_names_like(prepared) + result_df._set_columns_like(prepared._data) return result_df @_cudf_nvtx_annotate @@ -7082,7 +7116,7 @@ def cov(self, **kwargs): cov = cupy.cov(self.values, rowvar=False) cols = self._data.to_pandas_index() df = DataFrame(cupy.asfortranarray(cov)).set_index(cols) - df._set_column_names_like(self) + df._set_columns_like(self._data) return df def corr(self, method="pearson", min_periods=None): @@ -7118,7 +7152,7 @@ def corr(self, method="pearson", min_periods=None): corr = cupy.corrcoef(values, rowvar=False) cols = self._data.to_pandas_index() df = DataFrame(cupy.asfortranarray(corr)).set_index(cols) - df._set_column_names_like(self) + df._set_columns_like(self._data) return df @_cudf_nvtx_annotate @@ -7455,7 +7489,7 @@ def _from_columns_like_self( index_names, override_dtypes=override_dtypes, ) - result._set_column_names_like(self) + result._set_columns_like(self._data) return result @_cudf_nvtx_annotate diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index df703370f78..af52d7b3659 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -2587,7 +2587,7 @@ def sort_index( isinstance(self, cudf.core.dataframe.DataFrame) and self._data.multiindex ): - out._set_column_names_like(self) + out._set_columns_like(self._data) elif (ascending and idx.is_monotonic_increasing) or ( not ascending and idx.is_monotonic_decreasing ): @@ -2607,7 +2607,7 @@ def sort_index( isinstance(self, cudf.core.dataframe.DataFrame) and self._data.multiindex ): - out._set_column_names_like(self) + out._set_columns_like(self._data) if ignore_index: out = out.reset_index(drop=True) else: diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 3143851ddd6..444a4c60055 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -4,6 +4,7 @@ import contextlib import datetime import decimal +import functools import io import operator import random @@ -10727,6 +10728,9 @@ def test_init_from_2_categoricalindex_series_diff_categories(): ) result = cudf.DataFrame([s1, s2]) expected = pd.DataFrame([s1.to_pandas(), s2.to_pandas()]) + # TODO: Remove once https://github.com/pandas-dev/pandas/issues/57592 + # is adressed + expected.columns = result.columns assert_eq(result, expected, check_dtype=False) @@ -10863,6 +10867,55 @@ def test_dataframe_duplicate_index_reindex(): ) +def test_dataframe_columns_set_none_raises(): + df = cudf.DataFrame({"a": [0]}) + with pytest.raises(TypeError): + df.columns = None + + +@pytest.mark.parametrize( + "columns", + [cudf.RangeIndex(1, name="foo"), pd.RangeIndex(1, name="foo"), range(1)], +) +def test_dataframe_columns_set_rangeindex(columns): + df = cudf.DataFrame([1], columns=["a"]) + df.columns = columns + result = df.columns + expected = pd.RangeIndex(1, name=getattr(columns, "name", None)) + pd.testing.assert_index_equal(result, expected, exact=True) + + +@pytest.mark.parametrize("klass", [cudf.MultiIndex, pd.MultiIndex]) +def test_dataframe_columns_set_multiindex(klass): + columns = klass.from_arrays([[10]], names=["foo"]) + df = cudf.DataFrame([1], columns=["a"]) + df.columns = columns + result = df.columns + expected = pd.MultiIndex.from_arrays([[10]], names=["foo"]) + pd.testing.assert_index_equal(result, expected, exact=True) + + +@pytest.mark.parametrize( + "klass", + [ + functools.partial(cudf.Index, name="foo"), + functools.partial(cudf.Series, name="foo"), + functools.partial(pd.Index, name="foo"), + functools.partial(pd.Series, name="foo"), + np.array, + ], +) +def test_dataframe_columns_set_preserve_type(klass): + df = cudf.DataFrame([1], columns=["a"]) + columns = klass([10], dtype="int8") + df.columns = columns + result = df.columns + expected = pd.Index( + [10], dtype="int8", name=getattr(columns, "name", None) + ) + pd.testing.assert_index_equal(result, expected) + + @pytest.mark.parametrize( "scalar", [ From 9678c900a484818b489b723e2568e7b7c0d0b090 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 6 Mar 2024 12:54:09 -1000 Subject: [PATCH 365/384] Avoid factorization in MultiIndex.to_pandas (#15150) This also uncovered a bug in `DataFrame.rename` where the underlying `MultiIndex` `ColumnAccessor` was not being replaced Authors: - Matthew Roeschke (https://github.com/mroeschke) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/15150 --- python/cudf/cudf/core/dataframe.py | 6 +++++- python/cudf/cudf/core/multiindex.py | 15 +++++++++++---- python/cudf/cudf/tests/test_dataframe.py | 16 ++++------------ python/cudf/cudf/tests/test_dropna.py | 11 +---------- 4 files changed, 21 insertions(+), 27 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 1dc79127f60..6a4fe346eb1 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -3583,12 +3583,16 @@ def rename( ) if level is not None and isinstance(self.index, MultiIndex): + level = self.index._get_level_label(level) out_index = self.index.copy(deep=copy) - out_index.get_level_values(level).to_frame().replace( + level_values = out_index.get_level_values(level) + level_values.to_frame().replace( to_replace=list(index.keys()), value=list(index.values()), inplace=True, ) + out_index._data[level] = column.as_column(level_values) + out_index._compute_levels_and_codes() out = DataFrame(index=out_index) else: to_replace = list(index.keys()) diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 315a21020a2..019daacddba 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1577,10 +1577,17 @@ def droplevel(self, level=-1): def to_pandas( self, *, nullable: bool = False, arrow_type: bool = False ) -> pd.MultiIndex: - result = self.to_frame( - index=False, name=list(range(self.nlevels)) - ).to_pandas(nullable=nullable, arrow_type=arrow_type) - return pd.MultiIndex.from_frame(result, names=self.names) + # cudf uses np.iinfo(size_type_dtype).min as missing code + # pandas uses -1 as missing code + pd_codes = self._codes_frame.replace(np.iinfo(size_type_dtype).min, -1) + return pd.MultiIndex( + levels=[ + level.to_pandas(nullable=nullable, arrow_type=arrow_type) + for level in self.levels + ], + codes=[col.values_host for col in pd_codes._columns], + names=self.names, + ) @classmethod @_cudf_nvtx_annotate diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 444a4c60055..e6cf3988d23 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -9332,18 +9332,10 @@ def test_dataframe_setitem_cupy_array(): assert_eq(pdf, gdf) -@pytest.mark.parametrize( - "data", [{"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]}] -) -@pytest.mark.parametrize( - "index", - [{0: 123, 1: 4, 2: 6}], -) -@pytest.mark.parametrize( - "level", - ["x", 0], -) -def test_rename_for_level_MultiIndex_dataframe(data, index, level): +@pytest.mark.parametrize("level", ["x", 0]) +def test_rename_for_level_MultiIndex_dataframe(level): + data = {"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]} + index = {0: 123, 1: 4, 2: 6} pdf = pd.DataFrame( data, index=pd.MultiIndex.from_tuples([(0, 1, 2), (1, 2, 3), (2, 3, 4)]), diff --git a/python/cudf/cudf/tests/test_dropna.py b/python/cudf/cudf/tests/test_dropna.py index f1acd7b4320..c3c8ed922f0 100644 --- a/python/cudf/cudf/tests/test_dropna.py +++ b/python/cudf/cudf/tests/test_dropna.py @@ -252,21 +252,12 @@ def test_dropna_index(data, dtype): @pytest.mark.parametrize("data", [[[1, None, 2], [None, None, 2]]]) @pytest.mark.parametrize("how", ["all", "any"]) -def test_dropna_multiindex(data, how, request): +def test_dropna_multiindex(data, how): pi = pd.MultiIndex.from_arrays(data) gi = cudf.from_pandas(pi) expect = pi.dropna(how) got = gi.dropna(how) - - if how == "all" and "data0" in request.node.callspec.id: - request.applymarker( - pytest.mark.xfail( - reason="pandas NA value np.nan results in float type. " - "cuDF correctly retains int type " - "(https://github.com/pandas-dev/pandas/issues/44792)" - ) - ) assert_eq(expect, got) From 352d686ff1eafd5f06382c04e56558a27eb457c8 Mon Sep 17 00:00:00 2001 From: brandon-b-miller <53796099+brandon-b-miller@users.noreply.github.com> Date: Wed, 6 Mar 2024 21:30:58 -0600 Subject: [PATCH 366/384] Migrate filling operations to pylibcudf (#15225) This PR migrates the filling operations in cuDF Python to pylibcudf. Authors: - https://github.com/brandon-b-miller - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/15225 --- docs/cudf/source/conf.py | 1 + .../user_guide/api_docs/pylibcudf/filling.rst | 6 + .../user_guide/api_docs/pylibcudf/index.rst | 1 + python/cudf/cudf/_lib/filling.pyx | 110 ++++-------- .../cudf/cudf/_lib/pylibcudf/CMakeLists.txt | 1 + python/cudf/cudf/_lib/pylibcudf/__init__.pxd | 2 + python/cudf/cudf/_lib/pylibcudf/__init__.py | 2 + python/cudf/cudf/_lib/pylibcudf/filling.pxd | 35 ++++ python/cudf/cudf/_lib/pylibcudf/filling.pyx | 170 ++++++++++++++++++ 9 files changed, 250 insertions(+), 78 deletions(-) create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/filling.rst create mode 100644 python/cudf/cudf/_lib/pylibcudf/filling.pxd create mode 100644 python/cudf/cudf/_lib/pylibcudf/filling.pyx diff --git a/docs/cudf/source/conf.py b/docs/cudf/source/conf.py index 1b9e3c179cc..3bba50b482c 100644 --- a/docs/cudf/source/conf.py +++ b/docs/cudf/source/conf.py @@ -379,6 +379,7 @@ def _generate_namespaces(namespaces): "type_id", # Unknown base types "int32_t", + "void" } diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/filling.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/filling.rst new file mode 100644 index 00000000000..542a5e12bc4 --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/filling.rst @@ -0,0 +1,6 @@ +======== +filling +======== + +.. automodule:: cudf._lib.pylibcudf.filling + :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst index 2e5b3916c65..8cad95f61ae 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst @@ -13,6 +13,7 @@ This page provides API documentation for pylibcudf. column concatenate copying + filling gpumemoryview groupby join diff --git a/python/cudf/cudf/_lib/filling.pyx b/python/cudf/cudf/_lib/filling.pyx index 63549f08cbd..b7302f3d07a 100644 --- a/python/cudf/cudf/_lib/filling.pyx +++ b/python/cudf/cudf/_lib/filling.pyx @@ -1,103 +1,57 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. from cudf.core.buffer import acquire_spill_lock -from libcpp.memory cimport unique_ptr -from libcpp.utility cimport move -cimport cudf._lib.cpp.filling as cpp_filling from cudf._lib.column cimport Column -from cudf._lib.cpp.column.column cimport column -from cudf._lib.cpp.column.column_view cimport column_view, mutable_column_view -from cudf._lib.cpp.scalar.scalar cimport scalar -from cudf._lib.cpp.table.table cimport table -from cudf._lib.cpp.table.table_view cimport table_view -from cudf._lib.cpp.types cimport size_type from cudf._lib.scalar cimport DeviceScalar -from cudf._lib.utils cimport columns_from_unique_ptr, table_view_from_columns +from cudf._lib.utils cimport columns_from_pylibcudf_table + +from cudf._lib import pylibcudf +from cudf._lib.scalar import as_device_scalar @acquire_spill_lock() def fill_in_place(Column destination, int begin, int end, DeviceScalar value): - cdef mutable_column_view c_destination = destination.mutable_view() - cdef size_type c_begin = begin - cdef size_type c_end = end - cdef const scalar* c_value = value.get_raw_ptr() - - cpp_filling.fill_in_place( - c_destination, - c_begin, - c_end, - c_value[0] + pylibcudf.filling.fill_in_place( + destination.to_pylibcudf(mode='write'), + begin, + end, + ( as_device_scalar(value, dtype=destination.dtype)).c_value ) @acquire_spill_lock() def fill(Column destination, int begin, int end, DeviceScalar value): - cdef column_view c_destination = destination.view() - cdef size_type c_begin = begin - cdef size_type c_end = end - cdef const scalar* c_value = value.get_raw_ptr() - cdef unique_ptr[column] c_result - - with nogil: - c_result = move(cpp_filling.fill( - c_destination, - c_begin, - c_end, - c_value[0] - )) - - return Column.from_unique_ptr(move(c_result)) + return Column.from_pylibcudf( + pylibcudf.filling.fill( + destination.to_pylibcudf(mode='read'), + begin, + end, + ( as_device_scalar(value)).c_value + ) + ) @acquire_spill_lock() def repeat(list inp, object count): + ctbl = pylibcudf.Table([col.to_pylibcudf(mode="read") for col in inp]) if isinstance(count, Column): - return _repeat_via_column(inp, count) - else: - return _repeat_via_size_type(inp, count) - - -def _repeat_via_column(list inp, Column count): - cdef table_view c_inp = table_view_from_columns(inp) - cdef column_view c_count = count.view() - cdef unique_ptr[table] c_result - - with nogil: - c_result = move(cpp_filling.repeat( - c_inp, - c_count, - )) - - return columns_from_unique_ptr(move(c_result)) - - -def _repeat_via_size_type(list inp, size_type count): - cdef table_view c_inp = table_view_from_columns(inp) - cdef unique_ptr[table] c_result - - with nogil: - c_result = move(cpp_filling.repeat( - c_inp, + count = count.to_pylibcudf(mode="read") + return columns_from_pylibcudf_table( + pylibcudf.filling.repeat( + ctbl, count - )) - - return columns_from_unique_ptr(move(c_result)) + ) + ) @acquire_spill_lock() def sequence(int size, DeviceScalar init, DeviceScalar step): - cdef size_type c_size = size - cdef const scalar* c_init = init.get_raw_ptr() - cdef const scalar* c_step = step.get_raw_ptr() - cdef unique_ptr[column] c_result - - with nogil: - c_result = move(cpp_filling.sequence( - c_size, - c_init[0], - c_step[0] - )) - - return Column.from_unique_ptr(move(c_result)) + return Column.from_pylibcudf( + pylibcudf.filling.sequence( + size, + ( as_device_scalar(init)).c_value, + ( as_device_scalar(step)).c_value + ) + ) diff --git a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt index fd749a5edc1..ada47de5cae 100644 --- a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt +++ b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt @@ -18,6 +18,7 @@ set(cython_sources column.pyx concatenate.pyx copying.pyx + filling.pyx gpumemoryview.pyx groupby.pyx interop.pyx diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd index 96aa42cc257..39b29eace10 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd @@ -6,6 +6,7 @@ from . cimport ( binaryop, concatenate, copying, + filling, groupby, interop, join, @@ -37,6 +38,7 @@ __all__ = [ "binaryop", "concatenate", "copying", + "filling", "gpumemoryview", "groupby", "interop", diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.py b/python/cudf/cudf/_lib/pylibcudf/__init__.py index 19cc782dd92..8ccb0ecc341 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.py +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.py @@ -5,6 +5,7 @@ binaryop, concatenate, copying, + filling, groupby, interop, join, @@ -35,6 +36,7 @@ "binaryop", "concatenate", "copying", + "filling", "gpumemoryview", "groupby", "interop", diff --git a/python/cudf/cudf/_lib/pylibcudf/filling.pxd b/python/cudf/cudf/_lib/pylibcudf/filling.pxd new file mode 100644 index 00000000000..55dbd7b075f --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/filling.pxd @@ -0,0 +1,35 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +from cudf._lib.cpp.types cimport size_type + +from .column cimport Column +from .scalar cimport Scalar +from .table cimport Table + +ctypedef fused ColumnOrSize: + Column + size_type + +cpdef Column fill( + Column destination, + size_type begin, + size_type end, + Scalar value, +) + +cpdef void fill_in_place( + Column destination, + size_type c_begin, + size_type c_end, + Scalar value, +) + +cpdef Column sequence( + size_type size, + Scalar init, + Scalar step, +) + +cpdef Table repeat( + Table input_table, + ColumnOrSize count +) diff --git a/python/cudf/cudf/_lib/pylibcudf/filling.pyx b/python/cudf/cudf/_lib/pylibcudf/filling.pyx new file mode 100644 index 00000000000..588ab58a146 --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/filling.pyx @@ -0,0 +1,170 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from cython.operator cimport dereference +from libcpp.memory cimport unique_ptr +from libcpp.utility cimport move + +from cudf._lib.cpp.column.column cimport column +from cudf._lib.cpp.filling cimport ( + fill as cpp_fill, + fill_in_place as cpp_fill_in_place, + repeat as cpp_repeat, + sequence as cpp_sequence, +) +from cudf._lib.cpp.table.table cimport table +from cudf._lib.cpp.types cimport size_type + +from .column cimport Column +from .scalar cimport Scalar +from .table cimport Table + + +cpdef Column fill( + Column destination, + size_type begin, + size_type end, + Scalar value, +): + + """Fill destination column from begin to end with value. + + For details, see :cpp:func:`fill`. + + Parameters + ---------- + destination : Column + The column to be filled + begin : size_type + The index to begin filling from. + end : size_type + The index at which to stop filling. + value : Scalar + The value to fill with. + + Returns + ------- + pylibcudf.Column + The result of the filling operation + """ + + cdef unique_ptr[column] result + with nogil: + result = move( + cpp_fill( + destination.view(), + begin, + end, + dereference(( value).c_obj) + ) + ) + return Column.from_libcudf(move(result)) + +cpdef void fill_in_place( + Column destination, + size_type begin, + size_type end, + Scalar value, +): + + """Fill destination column in place from begin to end with value. + + For details, see :cpp:func:`fill_in_place`. + + Parameters + ---------- + destination : Column + The column to be filled + begin : size_type + The index to begin filling from. + end : size_type + The index at which to stop filling. + value : Scalar + The value to fill with. + """ + + with nogil: + cpp_fill_in_place( + destination.mutable_view(), + begin, + end, + dereference(value.c_obj) + ) + +cpdef Column sequence(size_type size, Scalar init, Scalar step): + """Create a sequence column of size ``size`` with initial value ``init`` and step + ``step``. + + For details, see :cpp:func:`sequence`. + + Parameters + ---------- + size : int + The size of the sequence + init : Scalar + The initial value of the sequence + step : Scalar + The step of the sequence + Returns + ------- + pylibcudf.Column + The result of the sequence operation + """ + + cdef unique_ptr[column] result + cdef size_type c_size = size + with nogil: + result = move( + cpp_sequence( + c_size, + dereference(init.c_obj), + dereference(step.c_obj), + ) + ) + return Column.from_libcudf(move(result)) + + +cpdef Table repeat( + Table input_table, + ColumnOrSize count +): + """Repeat rows of a Table. + + If an integral value is specified for ``count``, every row is repeated ``count`` + times. If ``count`` is a column, the number of repetitions of each row is defined + by the value at the corresponding index of ``count``. + + For details, see :cpp:func:`repeat`. + + Parameters + ---------- + input_table : Table + The table to be repeated + count : Union[Column, size_type] + Integer value to repeat each row by or + non-nullable column of an integral type + + Returns + ------- + pylibcudf.Table + The result of the repeat operation + """ + + cdef unique_ptr[table] result + + if ColumnOrSize is Column: + with nogil: + result = move( + cpp_repeat( + input_table.view(), + count.view() + ) + ) + if ColumnOrSize is size_type: + with nogil: + result = move( + cpp_repeat( + input_table.view(), + count + ) + ) + return Table.from_libcudf(move(result)) From efae666bac226dc50c1c7b5d7f1145ee9a31fc66 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Wed, 6 Mar 2024 19:57:50 -0800 Subject: [PATCH 367/384] Use page statistics in Parquet reader (#14973) #14000 added the ability to write new page statistics to the Parquet writer. This PR uses these new statistics to avoid some string size computations. Benchmarks show an improvement in read times of up to 20%. Authors: - Ed Seidl (https://github.com/etseidl) - Vukasin Milovanovic (https://github.com/vuule) - Yunsong Wang (https://github.com/PointKernel) Approvers: - Vukasin Milovanovic (https://github.com/vuule) - Yunsong Wang (https://github.com/PointKernel) URL: https://github.com/rapidsai/cudf/pull/14973 --- cpp/src/io/parquet/decode_preprocess.cu | 9 +- cpp/src/io/parquet/page_hdr.cu | 21 +- cpp/src/io/parquet/page_string_decode.cu | 34 ++- cpp/src/io/parquet/parquet_gpu.hpp | 15 +- cpp/src/io/parquet/reader_impl.cpp | 29 ++- cpp/src/io/parquet/reader_impl.hpp | 8 +- cpp/src/io/parquet/reader_impl_chunking.cu | 27 +++ cpp/src/io/parquet/reader_impl_helpers.cpp | 207 +++++++++++++++++-- cpp/src/io/parquet/reader_impl_helpers.hpp | 55 ++++- cpp/src/io/parquet/reader_impl_preprocess.cu | 119 ++++++++++- cpp/tests/io/parquet_reader_test.cpp | 85 ++++++++ 11 files changed, 550 insertions(+), 59 deletions(-) diff --git a/cpp/src/io/parquet/decode_preprocess.cu b/cpp/src/io/parquet/decode_preprocess.cu index 19c398c5965..8d8bed8f8bf 100644 --- a/cpp/src/io/parquet/decode_preprocess.cu +++ b/cpp/src/io/parquet/decode_preprocess.cu @@ -375,9 +375,10 @@ CUDF_KERNEL void __launch_bounds__(preprocess_block_size) if (!t) { s->page.skipped_values = -1; s->page.skipped_leaf_values = 0; - s->page.str_bytes = 0; - s->input_row_count = 0; - s->input_value_count = 0; + // str_bytes_from_index will be 0 if no page stats are present + s->page.str_bytes = s->page.str_bytes_from_index; + s->input_row_count = 0; + s->input_value_count = 0; // in the base pass, we're computing the number of rows, make sure we visit absolutely // everything @@ -462,7 +463,7 @@ CUDF_KERNEL void __launch_bounds__(preprocess_block_size) } // retrieve total string size. - if (compute_string_sizes) { + if (compute_string_sizes && !pp->has_page_index) { auto const str_bytes = gpuDecodeTotalPageStringSize(s, t); if (t == 0) { s->page.str_bytes = str_bytes; } } diff --git a/cpp/src/io/parquet/page_hdr.cu b/cpp/src/io/parquet/page_hdr.cu index 0dae0724823..f502fc837d6 100644 --- a/cpp/src/io/parquet/page_hdr.cu +++ b/cpp/src/io/parquet/page_hdr.cu @@ -385,14 +385,19 @@ void __launch_bounds__(128) gpuDecodePageHeaders(ColumnChunkDesc* chunks, // this computation is only valid for flat schemas. for nested schemas, // they will be recomputed in the preprocess step by examining repetition and // definition levels - bs->page.chunk_row = 0; - bs->page.num_rows = 0; - bs->page.skipped_values = -1; - bs->page.skipped_leaf_values = 0; - bs->page.str_bytes = 0; - bs->page.temp_string_size = 0; - bs->page.temp_string_buf = nullptr; - bs->page.kernel_mask = decode_kernel_mask::NONE; + bs->page.chunk_row = 0; + bs->page.num_rows = 0; + bs->page.skipped_values = -1; + bs->page.skipped_leaf_values = 0; + bs->page.str_bytes = 0; + bs->page.str_bytes_from_index = 0; + bs->page.num_valids = 0; + bs->page.start_val = 0; + bs->page.end_val = 0; + bs->page.has_page_index = false; + bs->page.temp_string_size = 0; + bs->page.temp_string_buf = nullptr; + bs->page.kernel_mask = decode_kernel_mask::NONE; } num_values = bs->ck.num_values; page_info = chunk_pages ? chunk_pages[chunk].pages : nullptr; diff --git a/cpp/src/io/parquet/page_string_decode.cu b/cpp/src/io/parquet/page_string_decode.cu index b63f96fda46..a0dfaa2fa58 100644 --- a/cpp/src/io/parquet/page_string_decode.cu +++ b/cpp/src/io/parquet/page_string_decode.cu @@ -599,10 +599,12 @@ CUDF_KERNEL void __launch_bounds__(preprocess_block_size) gpuComputeStringPageBo PageInfo* const pp = &pages[page_idx]; if (t == 0) { - s->page.num_nulls = 0; - s->page.num_valids = 0; + // don't clobber these if they're already computed from the index + if (!pp->has_page_index) { + s->page.num_nulls = 0; + s->page.num_valids = 0; + } // reset str_bytes to 0 in case it's already been calculated (esp needed for chunked reads). - // TODO: need to rethink this once str_bytes is in the statistics pp->str_bytes = 0; } @@ -632,6 +634,9 @@ CUDF_KERNEL void __launch_bounds__(preprocess_block_size) gpuComputeStringPageBo bool const is_bounds_pg = is_bounds_page(s, min_row, num_rows, has_repetition); + // if we have size info, then we only need to do this for bounds pages + if (pp->has_page_index && !is_bounds_pg) { return; } + // find start/end value indices auto const [start_value, end_value] = page_bounds(s, min_row, num_rows, is_bounds_pg, has_repetition, decoders); @@ -698,6 +703,15 @@ CUDF_KERNEL void __launch_bounds__(delta_preproc_block_size) gpuComputeDeltaPage } } } else { + bool const is_bounds_pg = is_bounds_page(s, min_row, num_rows, has_repetition); + + // if we have size info, then we only need to do this for bounds pages + if (pp->has_page_index && !is_bounds_pg) { + // check if we need to store values from the index + if (is_page_contained(s, min_row, num_rows)) { pp->str_bytes = pp->str_bytes_from_index; } + return; + } + // now process string info in the range [start_value, end_value) // set up for decoding strings...can be either plain or dictionary uint8_t const* data = s->data_start; @@ -759,6 +773,13 @@ CUDF_KERNEL void __launch_bounds__(delta_length_block_size) gpuComputeDeltaLengt bool const is_bounds_pg = is_bounds_page(s, min_row, num_rows, has_repetition); + // if we have size info, then we only need to do this for bounds pages + if (pp->has_page_index && !is_bounds_pg) { + // check if we need to store values from the index + if (is_page_contained(s, min_row, num_rows)) { pp->str_bytes = pp->str_bytes_from_index; } + return; + } + // for DELTA_LENGTH_BYTE_ARRAY, string size is page_data_size - size_of_delta_binary_block. // so all we need to do is skip the encoded string size info and then do pointer arithmetic, // if this isn't a bounds page. @@ -850,6 +871,13 @@ CUDF_KERNEL void __launch_bounds__(preprocess_block_size) gpuComputePageStringSi bool const is_bounds_pg = is_bounds_page(s, min_row, num_rows, has_repetition); + // if we have size info, then we only need to do this for bounds pages + if (pp->has_page_index && !is_bounds_pg) { + // check if we need to store values from the index + if (is_page_contained(s, min_row, num_rows)) { pp->str_bytes = pp->str_bytes_from_index; } + return; + } + auto const& col = s->col; size_t str_bytes = 0; // short circuit for FIXED_LEN_BYTE_ARRAY diff --git a/cpp/src/io/parquet/parquet_gpu.hpp b/cpp/src/io/parquet/parquet_gpu.hpp index af9f1f1267e..c66f69b3567 100644 --- a/cpp/src/io/parquet/parquet_gpu.hpp +++ b/cpp/src/io/parquet/parquet_gpu.hpp @@ -316,7 +316,8 @@ struct PageInfo { // for string columns only, the size of all the chars in the string for // this page. only valid/computed during the base preprocess pass int32_t str_bytes; - int32_t str_offset; // offset into string data for this page + int32_t str_offset; // offset into string data for this page + bool has_page_index; // true if str_bytes, num_valids, etc are derivable from page indexes // nesting information (input/output) for each page. this array contains // input column nesting information, output column nesting information and @@ -335,8 +336,15 @@ struct PageInfo { uint8_t* temp_string_buf; decode_kernel_mask kernel_mask; + + // str_bytes from page index. because str_bytes needs to be reset each iteration + // while doing chunked reads, persist the value from the page index here. + int32_t str_bytes_from_index; }; +// forward declaration +struct column_chunk_info; + /** * @brief Return the column schema id as the key for a PageInfo struct. */ @@ -376,6 +384,7 @@ struct ColumnChunkDesc { int32_t ts_clock_rate_, int32_t src_col_index_, int32_t src_col_schema_, + column_chunk_info const* chunk_info_, float list_bytes_per_row_est_) : compressed_data(compressed_data_), compressed_size(compressed_size_), @@ -400,6 +409,7 @@ struct ColumnChunkDesc { ts_clock_rate(ts_clock_rate_), src_col_index(src_col_index_), src_col_schema(src_col_schema_), + h_chunk_info(chunk_info_), list_bytes_per_row_est(list_bytes_per_row_est_) { } @@ -430,6 +440,9 @@ struct ColumnChunkDesc { int32_t src_col_index{}; // my input column index int32_t src_col_schema{}; // my schema index in the file + // pointer to column_chunk_info struct for this chunk (host only) + column_chunk_info const* h_chunk_info{}; + float list_bytes_per_row_est{}; // for LIST columns, an estimate on number of bytes per row }; diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 207f908febf..89562514564 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -28,7 +28,7 @@ namespace cudf::io::parquet::detail { -void reader::impl::decode_page_data(size_t skip_rows, size_t num_rows) +void reader::impl::decode_page_data(bool uses_custom_row_bounds, size_t skip_rows, size_t num_rows) { auto& pass = *_pass_itm_data; auto& subpass = *pass.subpass; @@ -62,14 +62,23 @@ void reader::impl::decode_page_data(size_t skip_rows, size_t num_rows) auto const has_strings = (kernel_mask & STRINGS_MASK) != 0; std::vector col_string_sizes(_input_columns.size(), 0L); if (has_strings) { - ComputePageStringSizes(subpass.pages, - pass.chunks, - delta_temp_buf, - skip_rows, - num_rows, - level_type_size, - kernel_mask, - _stream); + // need to compute pages bounds/sizes if we lack page indexes or are using custom bounds + // TODO: we could probably dummy up size stats for FLBA data since we know the width + auto const has_flba = + std::any_of(pass.chunks.begin(), pass.chunks.end(), [](auto const& chunk) { + return (chunk.data_type & 7) == FIXED_LEN_BYTE_ARRAY && chunk.converted_type != DECIMAL; + }); + + if (!_has_page_index || uses_custom_row_bounds || has_flba) { + ComputePageStringSizes(subpass.pages, + pass.chunks, + delta_temp_buf, + skip_rows, + num_rows, + level_type_size, + kernel_mask, + _stream); + } col_string_sizes = calculate_page_string_offsets(); @@ -426,7 +435,7 @@ table_with_metadata reader::impl::read_chunk_internal( allocate_columns(read_info.skip_rows, read_info.num_rows, uses_custom_row_bounds); // Parse data into the output buffers. - decode_page_data(read_info.skip_rows, read_info.num_rows); + decode_page_data(uses_custom_row_bounds, read_info.skip_rows, read_info.num_rows); // Create the final output cudf columns. for (size_t i = 0; i < _output_buffers.size(); ++i) { diff --git a/cpp/src/io/parquet/reader_impl.hpp b/cpp/src/io/parquet/reader_impl.hpp index 67c56c9c2d7..185419a5b46 100644 --- a/cpp/src/io/parquet/reader_impl.hpp +++ b/cpp/src/io/parquet/reader_impl.hpp @@ -311,10 +311,12 @@ class reader::impl { /** * @brief Converts the page data and outputs to columns. * + * @param uses_custom_row_bounds Whether or not num_rows and skip_rows represents user-specific + * bounds * @param skip_rows Minimum number of rows from start * @param num_rows Number of rows to output */ - void decode_page_data(size_t skip_rows, size_t num_rows); + void decode_page_data(bool uses_custom_row_bounds, size_t skip_rows, size_t num_rows); /** * @brief Creates file-wide parquet chunk information. @@ -365,6 +367,10 @@ class reader::impl { std::unique_ptr _output_metadata; bool _strings_to_categorical = false; + + // are there usable page indexes available + bool _has_page_index = false; + std::optional> _reader_column_schema; data_type _timestamp_type{type_id::EMPTY}; diff --git a/cpp/src/io/parquet/reader_impl_chunking.cu b/cpp/src/io/parquet/reader_impl_chunking.cu index b05318d3a91..9c14902ef2f 100644 --- a/cpp/src/io/parquet/reader_impl_chunking.cu +++ b/cpp/src/io/parquet/reader_impl_chunking.cu @@ -1476,6 +1476,28 @@ void reader::impl::create_global_chunk_info() auto const num_input_columns = _input_columns.size(); auto const num_chunks = row_groups_info.size() * num_input_columns; + // Mapping of input column to page index column + std::vector column_mapping; + + if (_has_page_index and not row_groups_info.empty()) { + // use first row group to define mappings (assumes same schema for each file) + auto const& rg = row_groups_info[0]; + auto const& columns = _metadata->get_row_group(rg.index, rg.source_index).columns; + column_mapping.resize(num_input_columns); + std::transform( + _input_columns.begin(), _input_columns.end(), column_mapping.begin(), [&](auto const& col) { + // translate schema_idx into something we can use for the page indexes + if (auto it = std::find_if( + columns.begin(), + columns.end(), + [&col](auto const& col_chunk) { return col_chunk.schema_idx == col.schema_idx; }); + it != columns.end()) { + return std::distance(columns.begin(), it); + } + CUDF_FAIL("cannot find column mapping"); + }); + } + // Initialize column chunk information auto remaining_rows = num_rows; for (auto const& rg : row_groups_info) { @@ -1505,6 +1527,10 @@ void reader::impl::create_global_chunk_info() static_cast(row_group.num_rows) : 0.0f; + // grab the column_chunk_info for each chunk (if it exists) + column_chunk_info const* const chunk_info = + _has_page_index ? &rg.column_chunks.value()[column_mapping[i]] : nullptr; + chunks.push_back(ColumnChunkDesc(col_meta.total_compressed_size, nullptr, col_meta.num_values, @@ -1524,6 +1550,7 @@ void reader::impl::create_global_chunk_info() clock_rate, i, col.schema_idx, + chunk_info, list_bytes_per_row_est)); } diff --git a/cpp/src/io/parquet/reader_impl_helpers.cpp b/cpp/src/io/parquet/reader_impl_helpers.cpp index 6f11debb8df..776caa99ac9 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.cpp +++ b/cpp/src/io/parquet/reader_impl_helpers.cpp @@ -267,24 +267,45 @@ metadata::metadata(datasource* source) cp.read(this); CUDF_EXPECTS(cp.InitSchema(this), "Cannot initialize schema"); - // loop through the column chunks and read column and offset indexes - for (auto& rg : row_groups) { - for (auto& col : rg.columns) { - if (col.column_index_length > 0 && col.column_index_offset > 0) { - auto const col_idx_buf = - source->host_read(col.column_index_offset, col.column_index_length); - cp.init(col_idx_buf->data(), col_idx_buf->size()); - ColumnIndex ci; - cp.read(&ci); - col.column_index = std::move(ci); - } - if (col.offset_index_length > 0 && col.offset_index_offset > 0) { - auto const off_idx_buf = - source->host_read(col.offset_index_offset, col.offset_index_length); - cp.init(off_idx_buf->data(), off_idx_buf->size()); - OffsetIndex oi; - cp.read(&oi); - col.offset_index = std::move(oi); + // Reading the page indexes is somewhat expensive, so skip if there are no byte array columns. + // Currently the indexes are only used for the string size calculations. + // Could also just read indexes for string columns, but that would require changes elsewhere + // where we're trying to determine if we have the indexes or not. + // Note: This will have to be modified if there are other uses in the future (e.g. calculating + // chunk/pass boundaries). + auto const has_strings = std::any_of( + schema.begin(), schema.end(), [](auto const& elem) { return elem.type == BYTE_ARRAY; }); + + if (has_strings and not row_groups.empty() and not row_groups.front().columns.empty()) { + // column index and offset index are encoded back to back. + // the first column of the first row group will have the first column index, the last + // column of the last row group will have the final offset index. + int64_t const min_offset = row_groups.front().columns.front().column_index_offset; + auto const& last_col = row_groups.back().columns.back(); + int64_t const max_offset = last_col.offset_index_offset + last_col.offset_index_length; + + if (max_offset > 0) { + int64_t const length = max_offset - min_offset; + auto const idx_buf = source->host_read(min_offset, length); + + // now loop over row groups + for (auto& rg : row_groups) { + for (auto& col : rg.columns) { + if (col.column_index_length > 0 && col.column_index_offset > 0) { + int64_t const offset = col.column_index_offset - min_offset; + cp.init(idx_buf->data() + offset, col.column_index_length); + ColumnIndex ci; + cp.read(&ci); + col.column_index = std::move(ci); + } + if (col.offset_index_length > 0 && col.offset_index_offset > 0) { + int64_t const offset = col.offset_index_offset - min_offset; + cp.init(idx_buf->data() + offset, col.offset_index_length); + OffsetIndex oi; + cp.read(&oi); + col.offset_index = std::move(oi); + } + } } } } @@ -346,6 +367,142 @@ size_type aggregate_reader_metadata::calc_num_row_groups() const }); } +// Copies info from the column and offset indexes into the passed in row_group_info. +void aggregate_reader_metadata::column_info_for_row_group(row_group_info& rg_info, + size_type chunk_start_row) const +{ + auto const& fmd = per_file_metadata[rg_info.source_index]; + auto const& rg = fmd.row_groups[rg_info.index]; + + std::vector chunks(rg.columns.size()); + + for (size_t col_idx = 0; col_idx < rg.columns.size(); col_idx++) { + auto const& col_chunk = rg.columns[col_idx]; + auto& schema = get_schema(col_chunk.schema_idx); + auto const max_def_level = schema.max_definition_level; + auto const max_rep_level = schema.max_repetition_level; + + // If any columns lack the page indexes then just return without modifying the + // row_group_info. + if (not col_chunk.offset_index.has_value() or not col_chunk.column_index.has_value()) { + return; + } + + auto const& offset_index = col_chunk.offset_index.value(); + auto const& column_index = col_chunk.column_index.value(); + + auto& chunk_info = chunks[col_idx]; + auto const num_pages = offset_index.page_locations.size(); + + // There is a bug in older versions of parquet-mr where the first data page offset + // really points to the dictionary page. The first possible offset in a file is 4 (after + // the "PAR1" header), so check to see if the dictionary_page_offset is > 0. If it is, then + // we haven't encountered the bug. + if (col_chunk.meta_data.dictionary_page_offset > 0) { + chunk_info.dictionary_offset = col_chunk.meta_data.dictionary_page_offset; + chunk_info.dictionary_size = + col_chunk.meta_data.data_page_offset - chunk_info.dictionary_offset.value(); + } else { + // dictionary_page_offset is 0, so check to see if the data_page_offset does not match + // the first offset in the offset index. If they don't match, then data_page_offset points + // to the dictionary page. + if (num_pages > 0 && + col_chunk.meta_data.data_page_offset < offset_index.page_locations[0].offset) { + chunk_info.dictionary_offset = col_chunk.meta_data.data_page_offset; + chunk_info.dictionary_size = + offset_index.page_locations[0].offset - col_chunk.meta_data.data_page_offset; + } + } + + // Use the definition_level_histogram to get num_valid and num_null. For now, these are + // only ever used for byte array columns. The repetition_level_histogram might be + // necessary to determine the total number of values in the page if the + // definition_level_histogram is absent. + // + // In the future we might want the full histograms saved in the `column_info` struct. + int64_t const* const def_hist = column_index.definition_level_histogram.has_value() + ? column_index.definition_level_histogram.value().data() + : nullptr; + int64_t const* const rep_hist = column_index.repetition_level_histogram.has_value() + ? column_index.repetition_level_histogram.value().data() + : nullptr; + + for (size_t pg_idx = 0; pg_idx < num_pages; pg_idx++) { + auto const& page_loc = offset_index.page_locations[pg_idx]; + // translate chunk-relative row nums to absolute within the file + auto const pg_start_row = chunk_start_row + page_loc.first_row_index; + auto const pg_end_row = + chunk_start_row + (pg_idx == (num_pages - 1) + ? rg.num_rows + : offset_index.page_locations[pg_idx + 1].first_row_index); + + auto const num_rows = pg_end_row - pg_start_row; + page_info pg_info{page_loc, num_rows}; + + // check to see if we already have null counts for each page + if (column_index.null_counts.has_value()) { + pg_info.num_nulls = column_index.null_counts.value()[pg_idx]; + } + + // save variable length byte info if present + if (offset_index.unencoded_byte_array_data_bytes.has_value()) { + pg_info.var_bytes_size = offset_index.unencoded_byte_array_data_bytes.value()[pg_idx]; + } + + // if def histogram is present, then use it to calculate num_valid and num_nulls + if (def_hist != nullptr) { + auto const h = &def_hist[pg_idx * (max_def_level + 1)]; + pg_info.num_valid = h[max_def_level]; + + // calculate num_nulls if not available from column index + if (not pg_info.num_nulls.has_value()) { + pg_info.num_nulls = std::reduce(h, h + max_def_level); + } + } + // there is no def histogram. + // if there is no repetition (no lists), then num_values == num_rows, and num_nulls can be + // obtained from the column index + else if (max_rep_level == 0) { + // if we already have num_nulls from column index + if (pg_info.num_nulls.has_value()) { + pg_info.num_valid = pg_info.num_rows - pg_info.num_nulls.value(); + } + // if max_def is 0, there are no nulls + else if (max_def_level == 0) { + pg_info.num_nulls = 0; + pg_info.num_valid = pg_info.num_rows; + } + } + // if the rep level histogram is present, we can get the total number of values + // from that + else if (rep_hist != nullptr) { + if (pg_info.num_nulls.has_value()) { + auto const h = &rep_hist[pg_idx * (max_rep_level + 1)]; + auto const num_values = std::reduce(h, h + max_rep_level + 1); + pg_info.num_valid = num_values - pg_info.num_nulls.value(); + } + } + + // If none of the ifs above triggered, then we have neither histogram (likely the writer + // doesn't produce them, the r:0 d:1 case should have been handled above). The column index + // doesn't give us value counts, so we'll have to rely on the page headers. If the histogram + // info is missing or insufficient, then just return without modifying the row_group_info. + if (not pg_info.num_nulls.has_value() or not pg_info.num_valid.has_value()) { return; } + + // Like above, if using older page indexes that lack size info, then return without modifying + // the row_group_info. + // TODO: cudf will still set the per-page var_bytes to '0' even for all null pages. Need to + // check the behavior of other implementations (once there are some). Some may not set the + // var bytes for all null pages, so check the `null_pages` field on the column index. + if (schema.type == BYTE_ARRAY and not pg_info.var_bytes_size.has_value()) { return; } + + chunk_info.pages.push_back(std::move(pg_info)); + } + } + + rg_info.column_chunks = std::move(chunks); +} + aggregate_reader_metadata::aggregate_reader_metadata( host_span const> sources) : per_file_metadata(metadatas_from_sources(sources)), @@ -470,23 +627,29 @@ aggregate_reader_metadata::select_row_groups( "Must specify row groups for each source"); for (size_t src_idx = 0; src_idx < row_group_indices.size(); ++src_idx) { + auto const& fmd = per_file_metadata[src_idx]; for (auto const& rowgroup_idx : row_group_indices[src_idx]) { CUDF_EXPECTS( - rowgroup_idx >= 0 && - rowgroup_idx < static_cast(per_file_metadata[src_idx].row_groups.size()), + rowgroup_idx >= 0 && rowgroup_idx < static_cast(fmd.row_groups.size()), "Invalid rowgroup index"); selection.emplace_back(rowgroup_idx, rows_to_read, src_idx); + // if page-level indexes are present, then collect extra chunk and page info. + column_info_for_row_group(selection.back(), 0); rows_to_read += get_row_group(rowgroup_idx, src_idx).num_rows; } } } else { size_type count = 0; for (size_t src_idx = 0; src_idx < per_file_metadata.size(); ++src_idx) { - for (size_t rg_idx = 0; rg_idx < per_file_metadata[src_idx].row_groups.size(); ++rg_idx) { + auto const& fmd = per_file_metadata[src_idx]; + for (size_t rg_idx = 0; rg_idx < fmd.row_groups.size(); ++rg_idx) { + auto const& rg = fmd.row_groups[rg_idx]; auto const chunk_start_row = count; - count += get_row_group(rg_idx, src_idx).num_rows; + count += rg.num_rows; if (count > rows_to_skip || count == 0) { selection.emplace_back(rg_idx, chunk_start_row, src_idx); + // if page-level indexes are present, then collect extra chunk and page info. + column_info_for_row_group(selection.back(), chunk_start_row); } if (count >= rows_to_skip + rows_to_read) { break; } } diff --git a/cpp/src/io/parquet/reader_impl_helpers.hpp b/cpp/src/io/parquet/reader_impl_helpers.hpp index 8d8ab8707be..8295654764e 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.hpp +++ b/cpp/src/io/parquet/reader_impl_helpers.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,6 +34,43 @@ namespace cudf::io::parquet::detail { +/** + * @brief page location and size info + */ +struct page_info { + // page location info from the offset index + PageLocation location; + // number of rows in the page, calculated from offset index + int64_t num_rows; + // number of valid values in page, calculated from definition level histogram if present + std::optional num_valid; + // number of null values in page, calculated from definition level histogram if present + std::optional num_nulls; + // number of bytes of variable-length data from the offset index (byte_array columns only) + std::optional var_bytes_size; +}; + +/** + * @brief column chunk metadata + */ +struct column_chunk_info { + // offset in file of the dictionary (if present) + std::optional dictionary_offset; + // size of dictionary (if present) + std::optional dictionary_size; + std::vector pages; + + /** + * @brief Determine if this column chunk has a dictionary page. + * + * @return `true` if this column chunk has a dictionary page. + */ + [[nodiscard]] constexpr bool has_dictionary() const + { + return dictionary_offset.has_value() && dictionary_size.has_value(); + } +}; + /** * @brief The row_group_info class */ @@ -43,12 +80,20 @@ struct row_group_info { size_t start_row; size_type source_index; // file index. + // Optional metadata pulled from the column and offset indexes, if present. + std::optional> column_chunks; + row_group_info() = default; row_group_info(size_type index, size_t start_row, size_type source_index) : index{index}, start_row{start_row}, source_index{source_index} { } + + /** + * @brief Indicates the presence of page-level indexes. + */ + [[nodiscard]] bool has_page_index() const { return column_chunks.has_value(); } }; /** @@ -104,6 +149,14 @@ class aggregate_reader_metadata { */ [[nodiscard]] size_type calc_num_row_groups() const; + /** + * @brief Calculate column index info for the given `row_group_info` + * + * @param rg_info Struct used to summarize metadata for a single row group + * @param chunk_start_row Global index of first row in the row group + */ + void column_info_for_row_group(row_group_info& rg_info, size_type chunk_start_row) const; + public: aggregate_reader_metadata(host_span const> sources); diff --git a/cpp/src/io/parquet/reader_impl_preprocess.cu b/cpp/src/io/parquet/reader_impl_preprocess.cu index aa4f96aa2e0..51a18de966e 100644 --- a/cpp/src/io/parquet/reader_impl_preprocess.cu +++ b/cpp/src/io/parquet/reader_impl_preprocess.cu @@ -309,6 +309,95 @@ void generate_depth_remappings(std::map, std::ve return total_pages; } +/** + * @brief Count the total number of pages using page index information. + */ +[[nodiscard]] size_t count_page_headers_with_pgidx( + cudf::detail::hostdevice_vector& chunks, rmm::cuda_stream_view stream) +{ + size_t total_pages = 0; + for (auto& chunk : chunks) { + CUDF_EXPECTS(chunk.h_chunk_info != nullptr, "Expected non-null column info struct"); + auto const& chunk_info = *chunk.h_chunk_info; + chunk.num_dict_pages = chunk_info.has_dictionary() ? 1 : 0; + chunk.num_data_pages = chunk_info.pages.size(); + total_pages += chunk.num_data_pages + chunk.num_dict_pages; + } + + // count_page_headers() also pushes chunks to device, so not using thrust here + chunks.host_to_device_async(stream); + + return total_pages; +} + +// struct used to carry info from the page indexes to the device +struct page_index_info { + int32_t num_rows; + int32_t chunk_row; + int32_t num_nulls; + int32_t num_valids; + int32_t str_bytes; +}; + +// functor to copy page_index_info into the PageInfo struct +struct copy_page_info { + device_span page_indexes; + device_span pages; + + __device__ void operator()(size_type idx) + { + auto& pg = pages[idx]; + auto const& pi = page_indexes[idx]; + pg.num_rows = pi.num_rows; + pg.chunk_row = pi.chunk_row; + pg.has_page_index = true; + pg.num_nulls = pi.num_nulls; + pg.num_valids = pi.num_valids; + pg.str_bytes_from_index = pi.str_bytes; + pg.str_bytes = pi.str_bytes; + pg.start_val = 0; + pg.end_val = pg.num_valids; + } +}; + +/** + * @brief Set fields on the pages that can be derived from page indexes. + * + * This replaces some preprocessing steps, such as page string size calculation. + */ +void fill_in_page_info(host_span chunks, + device_span pages, + rmm::cuda_stream_view stream) +{ + auto const num_pages = pages.size(); + std::vector page_indexes(num_pages); + + for (size_t c = 0, page_count = 0; c < chunks.size(); c++) { + auto const& chunk = chunks[c]; + CUDF_EXPECTS(chunk.h_chunk_info != nullptr, "Expected non-null column info struct"); + auto const& chunk_info = *chunk.h_chunk_info; + size_t start_row = 0; + page_count += chunk.num_dict_pages; + for (size_t p = 0; p < chunk_info.pages.size(); p++, page_count++) { + auto& page = page_indexes[page_count]; + page.num_rows = chunk_info.pages[p].num_rows; + page.chunk_row = start_row; + page.num_nulls = chunk_info.pages[p].num_nulls.value_or(0); + page.num_valids = chunk_info.pages[p].num_valid.value_or(0); + page.str_bytes = chunk_info.pages[p].var_bytes_size.value_or(0); + + start_row += page.num_rows; + } + } + + auto d_page_indexes = cudf::detail::make_device_uvector_async( + page_indexes, stream, rmm::mr::get_current_device_resource()); + + auto iter = thrust::make_counting_iterator(0); + thrust::for_each( + rmm::exec_policy_nosync(stream), iter, iter + num_pages, copy_page_info{d_page_indexes, pages}); +} + /** * @brief Returns a string representation of known encodings * @@ -445,6 +534,7 @@ cudf::detail::hostdevice_vector sort_pages(device_span */ void decode_page_headers(pass_intermediate_data& pass, device_span unsorted_pages, + bool has_page_index, rmm::cuda_stream_view stream) { CUDF_FUNC_RANGE(); @@ -491,6 +581,8 @@ void decode_page_headers(pass_intermediate_data& pass, } } + if (has_page_index) { fill_in_page_info(pass.chunks, unsorted_pages, stream); } + // compute max bytes needed for level data auto level_bit_size = cudf::detail::make_counting_transform_iterator( 0, cuda::proclaim_return_type([chunks = pass.chunks.d_begin()] __device__(int i) { @@ -902,12 +994,13 @@ void reader::impl::read_compressed_data() } // Process dataset chunk pages into output columns - auto const total_pages = count_page_headers(chunks, _stream); + auto const total_pages = _has_page_index ? count_page_headers_with_pgidx(chunks, _stream) + : count_page_headers(chunks, _stream); if (total_pages <= 0) { return; } rmm::device_uvector unsorted_pages(total_pages, _stream); // decoding of column/page information - decode_page_headers(pass, unsorted_pages, _stream); + decode_page_headers(pass, unsorted_pages, _has_page_index, _stream); CUDF_EXPECTS(pass.page_offsets.size() - 1 == static_cast(_input_columns.size()), "Encountered page_offsets / num_columns mismatch"); } @@ -1140,6 +1233,11 @@ void reader::impl::preprocess_file( _metadata->select_row_groups( row_group_indices, skip_rows, num_rows, output_types, filter, _stream); + // check for page indexes + _has_page_index = std::all_of(_file_itm_data.row_groups.begin(), + _file_itm_data.row_groups.end(), + [](auto const& row_group) { return row_group.has_page_index(); }); + if (_file_itm_data.global_num_rows > 0 && not _file_itm_data.row_groups.empty() && not _input_columns.empty()) { // fills in chunk information without physically loading or decompressing @@ -1191,13 +1289,16 @@ void reader::impl::generate_list_column_row_count_estimates() // field in ColumnChunkDesc is the absolute row index for the whole file. chunk_row in PageInfo is // relative to the beginning of the chunk. so in the kernels, chunk.start_row + page.chunk_row // gives us the absolute row index - auto key_input = thrust::make_transform_iterator(pass.pages.d_begin(), get_page_chunk_idx{}); - auto page_input = thrust::make_transform_iterator(pass.pages.d_begin(), get_page_num_rows{}); - thrust::exclusive_scan_by_key(rmm::exec_policy_nosync(_stream), - key_input, - key_input + pass.pages.size(), - page_input, - chunk_row_output_iter{pass.pages.device_ptr()}); + // Note: chunk_row is already computed if we have column indexes + if (not _has_page_index) { + auto key_input = thrust::make_transform_iterator(pass.pages.d_begin(), get_page_chunk_idx{}); + auto page_input = thrust::make_transform_iterator(pass.pages.d_begin(), get_page_num_rows{}); + thrust::exclusive_scan_by_key(rmm::exec_policy_nosync(_stream), + key_input, + key_input + pass.pages.size(), + page_input, + chunk_row_output_iter{pass.pages.device_ptr()}); + } // finally, fudge the last page for each column such that it ends on the real known row count // for the pass. this is so that as we march through the subpasses, we will find that every column diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index abbd0c97f07..c13bf488e6a 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -2060,6 +2060,91 @@ TEST_F(ParquetReaderTest, DeltaSkipRowsWithNulls) } } +// test that using page stats is working for full reads and various skip rows +TEST_F(ParquetReaderTest, StringsWithPageStats) +{ + constexpr int num_rows = 10'000; + constexpr auto seed = 21337; + + std::mt19937 engine{seed}; + auto int32_list_nulls = make_parquet_list_col(engine, num_rows, 5, true); + auto int32_list = make_parquet_list_col(engine, num_rows, 5, false); + auto int64_list_nulls = make_parquet_list_col(engine, num_rows, 5, true); + auto int64_list = make_parquet_list_col(engine, num_rows, 5, false); + auto int16_list_nulls = make_parquet_list_col(engine, num_rows, 5, true); + auto int16_list = make_parquet_list_col(engine, num_rows, 5, false); + auto int8_list_nulls = make_parquet_list_col(engine, num_rows, 5, true); + auto int8_list = make_parquet_list_col(engine, num_rows, 5, false); + + auto str_list_nulls = make_parquet_string_list_col(engine, num_rows, 5, 32, true); + auto str_list = make_parquet_string_list_col(engine, num_rows, 5, 32, false); + auto big_str_list_nulls = make_parquet_string_list_col(engine, num_rows, 5, 256, true); + auto big_str_list = make_parquet_string_list_col(engine, num_rows, 5, 256, false); + + auto int32_data = random_values(num_rows); + auto int64_data = random_values(num_rows); + auto int16_data = random_values(num_rows); + auto int8_data = random_values(num_rows); + auto str_data = string_values(engine, num_rows, 32); + auto big_str_data = string_values(engine, num_rows, 256); + + auto const validity = random_validity(engine); + auto const no_nulls = cudf::test::iterators::no_nulls(); + column_wrapper int32_nulls_col{int32_data.begin(), int32_data.end(), validity}; + column_wrapper int32_col{int32_data.begin(), int32_data.end(), no_nulls}; + column_wrapper int64_nulls_col{int64_data.begin(), int64_data.end(), validity}; + column_wrapper int64_col{int64_data.begin(), int64_data.end(), no_nulls}; + + auto str_col = cudf::test::strings_column_wrapper(str_data.begin(), str_data.end(), no_nulls); + auto str_col_nulls = cudf::purge_nonempty_nulls( + cudf::test::strings_column_wrapper(str_data.begin(), str_data.end(), validity)); + auto big_str_col = + cudf::test::strings_column_wrapper(big_str_data.begin(), big_str_data.end(), no_nulls); + auto big_str_col_nulls = cudf::purge_nonempty_nulls( + cudf::test::strings_column_wrapper(big_str_data.begin(), big_str_data.end(), validity)); + + cudf::table_view tbl({int32_col, int32_nulls_col, *int32_list, *int32_list_nulls, + int64_col, int64_nulls_col, *int64_list, *int64_list_nulls, + *int16_list, *int16_list_nulls, *int8_list, *int8_list_nulls, + str_col, *str_col_nulls, *str_list, *str_list_nulls, + big_str_col, *big_str_col_nulls, *big_str_list, *big_str_list_nulls}); + + auto const filepath = temp_env->get_temp_filepath("StringsWithPageStats.parquet"); + auto const out_opts = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, tbl) + .stats_level(cudf::io::statistics_freq::STATISTICS_COLUMN) + .max_page_size_rows(5'000) + .build(); + cudf::io::write_parquet(out_opts); + + // skip_rows / num_rows + // clang-format off + std::vector> params{ + // skip and then read rest of file + {-1, -1}, {1, -1}, {2, -1}, {32, -1}, {33, -1}, {128, -1}, {1'000, -1}, + // no skip but truncate + {0, 1'000}, {0, 6'000}, + // cross page boundaries + {3'000, 5'000} + }; + + // clang-format on + for (auto p : params) { + cudf::io::parquet_reader_options read_args = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}); + if (p.first >= 0) { read_args.set_skip_rows(p.first); } + if (p.second >= 0) { read_args.set_num_rows(p.second); } + auto result = cudf::io::read_parquet(read_args); + + p.first = p.first < 0 ? 0 : p.first; + p.second = p.second < 0 ? num_rows - p.first : p.second; + std::vector slice_indices{p.first, p.first + p.second}; + std::vector expected = cudf::slice(tbl, slice_indices); + + CUDF_TEST_EXPECT_TABLES_EQUAL(result.tbl->view(), expected[0]); + } +} + /////////////////// // metadata tests From 753bf3e525e15c970fc7dc7ce333d96035c4cc55 Mon Sep 17 00:00:00 2001 From: Karthikeyan <6488848+karthikeyann@users.noreply.github.com> Date: Thu, 7 Mar 2024 22:32:14 +0530 Subject: [PATCH 368/384] Fix Null literals to be not parsed as string when mixed types as string is enabled in JSON reader (#14939) Fixes https://github.com/rapidsai/cudf/issues/14864 `null` literal should be ignored (considered as null) during parsing while handling mixed types. Unit tests of complex scenarios are added to test this as well. Authors: - Karthikeyan (https://github.com/karthikeyann) Approvers: - MithunR (https://github.com/mythrocks) - Andy Grove (https://github.com/andygrove) - Shruti Shivakumar (https://github.com/shrshi) - https://github.com/nvdbaranec URL: https://github.com/rapidsai/cudf/pull/14939 --- cpp/src/io/json/json_column.cu | 116 ++++++++++++++++++------- cpp/src/io/json/nested_json.hpp | 14 +++ cpp/src/io/json/nested_json_gpu.cu | 3 +- cpp/tests/io/json_test.cpp | 134 ++++++++++++++++++++++------- 4 files changed, 208 insertions(+), 59 deletions(-) diff --git a/cpp/src/io/json/json_column.cu b/cpp/src/io/json/json_column.cu index 56da1095b81..10646fad354 100644 --- a/cpp/src/io/json/json_column.cu +++ b/cpp/src/io/json/json_column.cu @@ -392,6 +392,54 @@ std::vector copy_strings_to_host(device_span input, return to_host(d_column_names->view()); } +/** + * @brief Checks if all strings in each string column in the tree are nulls. + * For non-string columns, it's set as true. If any of rows in a string column is false, it's set as + * false. + * + * @param input Input JSON string device data + * @param d_column_tree column tree representation of JSON string + * @param tree Node tree representation of the JSON string + * @param col_ids Column ids of the nodes in the tree + * @param options Parsing options specifying the parsing behaviour + * @param stream CUDA stream used for device memory operations and kernel launches + * @return Array of bytes where each byte indicate if it is all nulls string column. + */ +rmm::device_uvector is_all_nulls_each_column(device_span input, + tree_meta_t const& d_column_tree, + tree_meta_t const& tree, + device_span col_ids, + cudf::io::json_reader_options const& options, + rmm::cuda_stream_view stream) +{ + auto const num_nodes = col_ids.size(); + auto const num_cols = d_column_tree.node_categories.size(); + rmm::device_uvector is_all_nulls(num_cols, stream); + thrust::fill(rmm::exec_policy(stream), is_all_nulls.begin(), is_all_nulls.end(), true); + + auto parse_opt = parsing_options(options, stream); + thrust::for_each_n( + rmm::exec_policy(stream), + thrust::counting_iterator(0), + num_nodes, + [options = parse_opt.view(), + data = input.data(), + column_categories = d_column_tree.node_categories.begin(), + col_ids = col_ids.begin(), + range_begin = tree.node_range_begin.begin(), + range_end = tree.node_range_end.begin(), + is_all_nulls = is_all_nulls.begin()] __device__(size_type i) { + auto const node_category = column_categories[col_ids[i]]; + if (node_category == NC_STR or node_category == NC_VAL) { + auto const is_null_literal = serialized_trie_contains( + options.trie_na, + {data + range_begin[i], static_cast(range_end[i] - range_begin[i])}); + if (!is_null_literal) is_all_nulls[col_ids[i]] = false; + } + }); + return is_all_nulls; +} + /** * @brief Holds member data pointers of `d_json_column` * @@ -415,8 +463,10 @@ struct json_column_data { * @param row_offsets Row offsets of the nodes in the tree * @param root Root node of the `d_json_column` tree * @param is_array_of_arrays Whether the tree is an array of arrays - * @param is_enabled_lines Whether the input is a line-delimited JSON - * @param is_enabled_mixed_types_as_string Whether to enable reading mixed types as string + * @param options Parsing options specifying the parsing behaviour + * options affecting behaviour are + * is_enabled_lines: Whether the input is a line-delimited JSON + * is_enabled_mixed_types_as_string: Whether to enable reading mixed types as string * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource used to allocate the device memory * of child_offets and validity members of `d_json_column` @@ -427,13 +477,15 @@ void make_device_json_column(device_span input, device_span row_offsets, device_json_column& root, bool is_array_of_arrays, - bool is_enabled_lines, - bool is_enabled_mixed_types_as_string, + cudf::io::json_reader_options const& options, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { CUDF_FUNC_RANGE(); - auto num_nodes = col_ids.size(); + + bool const is_enabled_lines = options.is_enabled_lines(); + bool const is_enabled_mixed_types_as_string = options.is_enabled_mixed_types_as_string(); + auto const num_nodes = col_ids.size(); rmm::device_uvector sorted_col_ids(col_ids.size(), stream); // make a copy thrust::copy(rmm::exec_policy(stream), col_ids.begin(), col_ids.end(), sorted_col_ids.begin()); @@ -548,6 +600,12 @@ void make_device_json_column(device_span input, return thrust::get<0>(a) < thrust::get<0>(b); }); + std::vector is_str_column_all_nulls{}; + if (is_enabled_mixed_types_as_string) { + is_str_column_all_nulls = cudf::detail::make_std_vector_async( + is_all_nulls_each_column(input, d_column_tree, tree, col_ids, options, stream), stream); + } + // use hash map because we may skip field name's col_ids std::unordered_map> columns; // map{parent_col_id, child_col_name}> = child_col_id, used for null value column tracking @@ -592,29 +650,39 @@ void make_device_json_column(device_span input, auto& parent_col = it->second.get(); bool replaced = false; if (mapped_columns.count({parent_col_id, name}) > 0) { + auto const old_col_id = mapped_columns[{parent_col_id, name}]; // If mixed type as string is enabled, make both of them strings and merge them. // All child columns will be ignored when parsing. if (is_enabled_mixed_types_as_string) { - // VAL/STR or STRUCT or LIST - auto old_col_id = mapped_columns[{parent_col_id, name}]; - - is_mixed_type_column[this_col_id] = 1; - is_mixed_type_column[old_col_id] = 1; - // if old col type (not cat) is list or struct, replace with string. - auto& col = columns.at(old_col_id).get(); - if (col.type == json_col_t::ListColumn or col.type == json_col_t::StructColumn) { - reinitialize_as_string(old_col_id, col); - // all its children (which are already inserted) are ignored later. + bool const is_mixed_type = [&]() { + // If new or old is STR and they are all not null, make it mixed type, else ignore. + if (column_categories[this_col_id] == NC_VAL || + column_categories[this_col_id] == NC_STR) { + if (is_str_column_all_nulls[this_col_id]) return false; + } + if (column_categories[old_col_id] == NC_VAL || column_categories[old_col_id] == NC_STR) { + if (is_str_column_all_nulls[old_col_id]) return false; + } + return true; + }(); + if (is_mixed_type) { + is_mixed_type_column[this_col_id] = 1; + is_mixed_type_column[old_col_id] = 1; + // if old col type (not cat) is list or struct, replace with string. + auto& col = columns.at(old_col_id).get(); + if (col.type == json_col_t::ListColumn or col.type == json_col_t::StructColumn) { + reinitialize_as_string(old_col_id, col); + // all its children (which are already inserted) are ignored later. + } + columns.try_emplace(this_col_id, columns.at(old_col_id)); + continue; } - columns.try_emplace(this_col_id, columns.at(old_col_id)); - continue; } if (column_categories[this_col_id] == NC_VAL || column_categories[this_col_id] == NC_STR) { ignore_vals[this_col_id] = 1; continue; } - auto old_col_id = mapped_columns[{parent_col_id, name}]; if (column_categories[old_col_id] == NC_VAL || column_categories[old_col_id] == NC_STR) { // remap ignore_vals[old_col_id] = 1; @@ -795,15 +863,6 @@ void make_device_json_column(device_span input, } } -/** - * @brief Retrieves the parse_options to be used for type inference and type casting - * - * @param options The reader options to influence the relevant type inference and type casting - * options - */ -cudf::io::parse_options parsing_options(cudf::io::json_reader_options const& options, - rmm::cuda_stream_view stream); - std::pair, std::vector> device_json_column_to_cudf_column( device_json_column& json_col, device_span d_input, @@ -1021,8 +1080,7 @@ table_with_metadata device_parse_nested_json(device_span d_input, gpu_row_offsets, root_column, is_array_of_arrays, - options.is_enabled_lines(), - options.is_enabled_mixed_types_as_string(), + options, stream, mr); diff --git a/cpp/src/io/json/nested_json.hpp b/cpp/src/io/json/nested_json.hpp index c13daf9b9f5..f41b024bb1e 100644 --- a/cpp/src/io/json/nested_json.hpp +++ b/cpp/src/io/json/nested_json.hpp @@ -25,6 +25,10 @@ #include #include +// Forward declaration of parse_options from parsing_utils.cuh +namespace cudf::io { +struct parse_options; +} namespace cudf::io::json { /** @@ -284,6 +288,16 @@ reduce_to_column_tree(tree_meta_t& tree, device_span row_offsets, rmm::cuda_stream_view stream); +/** + * @brief Retrieves the parse_options to be used for type inference and type casting + * + * @param options The reader options to influence the relevant type inference and type casting + * options + * @param stream The CUDA stream to which kernels are dispatched + */ +cudf::io::parse_options parsing_options(cudf::io::json_reader_options const& options, + rmm::cuda_stream_view stream); + /** @copydoc host_parse_nested_json * All processing is done in device memory. * diff --git a/cpp/src/io/json/nested_json_gpu.cu b/cpp/src/io/json/nested_json_gpu.cu index 73af983d108..a6a57c36b08 100644 --- a/cpp/src/io/json/nested_json_gpu.cu +++ b/cpp/src/io/json/nested_json_gpu.cu @@ -2042,7 +2042,8 @@ void make_json_column(json_column& root_column, * options * @param stream The CUDA stream to which kernels are dispatched */ -auto parsing_options(cudf::io::json_reader_options const& options, rmm::cuda_stream_view stream) +cudf::io::parse_options parsing_options(cudf::io::json_reader_options const& options, + rmm::cuda_stream_view stream) { auto parse_opts = cudf::io::parse_options{',', '\n', '\"', '.'}; diff --git a/cpp/tests/io/json_test.cpp b/cpp/tests/io/json_test.cpp index e4ed09d3962..450ea550e99 100644 --- a/cpp/tests/io/json_test.cpp +++ b/cpp/tests/io/json_test.cpp @@ -2052,6 +2052,9 @@ TEST_F(JsonReaderTest, JSONLinesRecoveringIgnoreExcessChars) TEST_F(JsonReaderTest, MixedTypes) { + using LCWS = cudf::test::lists_column_wrapper; + using LCWI = cudf::test::lists_column_wrapper; + using valid_t = std::vector; { // Simple test for mixed types std::string json_string = R"({ "foo": [1,2,3], "bar": 123 } @@ -2084,34 +2087,112 @@ TEST_F(JsonReaderTest, MixedTypes) .lines(true); cudf::io::table_with_metadata result = cudf::io::read_json(in_options); + static int num_case = 0; + num_case++; + std::cout << "case:" << num_case << "\n"; CUDF_TEST_EXPECT_COLUMNS_EQUAL(result.tbl->get_column(0), expected); }; - - // test cases. + // value + string (not mixed type case) test_fn(R"( { "a": "123" } { "a": 123 } )", cudf::test::strings_column_wrapper({"123", "123"})); + // test cases. + // STR + STRUCT, STR + LIST, STR + null + // STRUCT + STR, STRUCT + LIST, STRUCT + null + // LIST + STR, LIST + STRUCT, LIST + null + // LIST + STRUCT + STR, STRUCT + LIST + STR, STR + STRUCT + LIST, STRUCT + LIST + null + // STR + STRUCT + LIST + null + + // STRING mixed: + // STR + STRUCT, STR + LIST, STR + null test_fn(R"( -{ "a": [1,2,3] } +{ "a": "123" } { "a": { "b": 1 } } )", - cudf::test::strings_column_wrapper({"[1,2,3]", "{ \"b\": 1 }"})); + cudf::test::strings_column_wrapper({"123", "{ \"b\": 1 }"})); + test_fn(R"( +{ "a": "123" } +{ "a": [1,2,3] } +)", + cudf::test::strings_column_wrapper({"123", "[1,2,3]"})); + test_fn(R"( +{ "a": "123" } +{ "a": null } +)", + cudf::test::strings_column_wrapper({"123", ""}, std::vector{1, 0}.begin())); + // STRUCT mixed: + // STRUCT + STR, STRUCT + LIST, STRUCT + null test_fn(R"( +{ "a": { "b": 1 } } { "a": "fox" } +)", + cudf::test::strings_column_wrapper({"{ \"b\": 1 }", "fox"})); + test_fn(R"( +{ "a": { "b": 1 } } +{ "a": [1,2,3] } +)", + cudf::test::strings_column_wrapper({"{ \"b\": 1 }", "[1,2,3]"})); + cudf::test::fixed_width_column_wrapper child_int_col_wrapper{1, 2}; + test_fn(R"( { "a": { "b": 1 } } +{ "a": null } )", - cudf::test::strings_column_wrapper({"fox", "{ \"b\": 1 }"})); + cudf::test::structs_column_wrapper{ + {child_int_col_wrapper}, {1, 0} /*Validity*/ + }); + // LIST mixed: + // LIST + STR, LIST + STRUCT, LIST + null test_fn(R"( { "a": [1,2,3] } -{ "a": "fox" } +{ "a": "123" } +)", + cudf::test::strings_column_wrapper({"[1,2,3]", "123"})); + test_fn(R"( +{ "a": [1,2,3] } +{ "a": { "b": 1 } } +)", + cudf::test::strings_column_wrapper({"[1,2,3]", "{ \"b\": 1 }"})); + test_fn( + R"( +{ "a": [1,2,3] } +{ "a": null } +)", + cudf::test::lists_column_wrapper{{LCWI{1L, 2L, 3L}, LCWI{4L, 5L}}, valid_t{1, 0}.begin()}); + + // All mixed: + // LIST + STRUCT + STR, STRUCT + LIST + STR, STR + STRUCT + LIST, STRUCT + LIST + null + test_fn(R"( +{ "a": [1,2,3] } +{ "a": { "b": 1 } } +{ "a": "fox"} +)", + cudf::test::strings_column_wrapper({"[1,2,3]", "{ \"b\": 1 }", "fox"})); + test_fn(R"( +{ "a": { "b": 1 } } +{ "a": [1,2,3] } +{ "a": "fox"} +)", + cudf::test::strings_column_wrapper({"{ \"b\": 1 }", "[1,2,3]", "fox"})); + test_fn(R"( +{ "a": "fox"} +{ "a": { "b": 1 } } +{ "a": [1,2,3] } +)", + cudf::test::strings_column_wrapper({"fox", "{ \"b\": 1 }", "[1,2,3]"})); + test_fn(R"( +{ "a": [1,2,3] } +{ "a": { "b": 1 } } +{ "a": null} )", - cudf::test::strings_column_wrapper({"[1,2,3]", "fox"})); + cudf::test::strings_column_wrapper({"[1,2,3]", "{ \"b\": 1 }", "NA"}, + valid_t{1, 1, 0}.begin())); // RIGHT + // value + string inside list test_fn(R"( { "a": [1,2,3] } { "a": [true,false,true] } @@ -2119,36 +2200,31 @@ TEST_F(JsonReaderTest, MixedTypes) )", cudf::test::lists_column_wrapper{ {"1", "2", "3"}, {"true", "false", "true"}, {"a", "b", "c"}}); - { - std::string json_string = R"( -{ "var1": true } -{ "var1": [{ "var0": true, "var1": "hello", "var2": null }, null, [true, null, null]] } - )"; - - cudf::io::json_reader_options in_options = - cudf::io::json_reader_options::builder( - cudf::io::source_info{json_string.data(), json_string.size()}) - .mixed_types_as_string(true) - .lines(true); - cudf::io::table_with_metadata result = cudf::io::read_json(in_options); - } + // null + list of mixed types and null + test_fn(R"( +{ "var1": null } +{ "var1": [{ "var0": true, "var1": "hello", "var2": null }, null, [true, null, null]] } + )", + cudf::test::lists_column_wrapper( + {{"NA", "NA"}, + {{R"({ "var0": true, "var1": "hello", "var2": null })", "null", "[true, null, null]"}, + valid_t{1, 0, 1}.begin()}}, + valid_t{0, 1}.begin())); // test to confirm if reinitialize a non-string column as string affects max_rowoffsets. // max_rowoffsets is generated based on parent col id, // so, even if mixed types are present, their row offset will be correct. - using LCW = cudf::test::lists_column_wrapper; - using valid_t = std::vector; cudf::test::lists_column_wrapper expected_list{ { - cudf::test::lists_column_wrapper({LCW({"1", "2", "3"}), LCW({"4", "5", "6"})}), - cudf::test::lists_column_wrapper({LCW()}), - cudf::test::lists_column_wrapper({LCW()}), // null - cudf::test::lists_column_wrapper({LCW()}), // null - cudf::test::lists_column_wrapper({LCW({"{\"c\": -1}"}), LCW({"5"})}), - cudf::test::lists_column_wrapper({LCW({"7"}), LCW({"8", "9"})}), - cudf::test::lists_column_wrapper({LCW()}), // null + cudf::test::lists_column_wrapper({LCWS({"1", "2", "3"}), LCWS({"4", "5", "6"})}), + cudf::test::lists_column_wrapper({LCWS()}), + cudf::test::lists_column_wrapper({LCWS()}), // null + cudf::test::lists_column_wrapper({LCWS()}), // null + cudf::test::lists_column_wrapper({LCWS({"{\"c\": -1}"}), LCWS({"5"})}), + cudf::test::lists_column_wrapper({LCWS({"7"}), LCWS({"8", "9"})}), + cudf::test::lists_column_wrapper({LCWS()}), // null }, valid_t{1, 1, 0, 0, 1, 1, 0}.begin()}; test_fn(R"( From 188d7cbf5238c80f3c3b98698db4ec27f28b6b11 Mon Sep 17 00:00:00 2001 From: brandon-b-miller <53796099+brandon-b-miller@users.noreply.github.com> Date: Thu, 7 Mar 2024 13:25:14 -0600 Subject: [PATCH 369/384] Add CUDA 12.4 to supported PTX versions (#15247) This PR updates the mapping from PTX version to toolkit versions to cover CUDA 12.4. Authors: - https://github.com/brandon-b-miller Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/15247 --- python/cudf/cudf/utils/_numba.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/cudf/cudf/utils/_numba.py b/python/cudf/cudf/utils/_numba.py index 6d00fd397df..494b48b3cfd 100644 --- a/python/cudf/cudf/utils/_numba.py +++ b/python/cudf/cudf/utils/_numba.py @@ -181,6 +181,7 @@ def _get_cuda_version_from_ptx_file(path): "8.1": (12, 1), "8.2": (12, 2), "8.3": (12, 3), + "8.4": (12, 4), } cuda_ver = ver_map.get(version) From c2bb860e4b323d1f9efd593938fef3372f36bdef Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 7 Mar 2024 09:35:10 -1000 Subject: [PATCH 370/384] Don't override to_pandas for Datelike columns (#15167) `pandas.Series(pyarrow.array)` is first interpreted as an object data type since pandas doesn't know how to handle pyarrow arrays yet which is bad. Additionally if pyarrow becomes required in pandas this may have different behavior in the future. I think the linked issues might be outdated and we can rely on pyarrow's `to_pandas` Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Ashwin Srinath (https://github.com/shwina) URL: https://github.com/rapidsai/cudf/pull/15167 --- python/cudf/cudf/core/column/datetime.py | 27 ----------------------- python/cudf/cudf/core/column/timedelta.py | 27 ----------------------- 2 files changed, 54 deletions(-) diff --git a/python/cudf/cudf/core/column/datetime.py b/python/cudf/cudf/core/column/datetime.py index 85f07064c97..9a5d9dcd47a 100644 --- a/python/cudf/cudf/core/column/datetime.py +++ b/python/cudf/cudf/core/column/datetime.py @@ -313,33 +313,6 @@ def dayofyear(self) -> ColumnBase: def day_of_year(self) -> ColumnBase: return self.get_dt_field("day_of_year") - def to_pandas( - self, - *, - index: Optional[pd.Index] = None, - nullable: bool = False, - arrow_type: bool = False, - ) -> pd.Series: - if arrow_type and nullable: - raise ValueError( - f"{arrow_type=} and {nullable=} cannot both be set." - ) - elif nullable: - raise NotImplementedError(f"{nullable=} is not implemented.") - elif arrow_type: - return pd.Series( - pd.arrays.ArrowExtensionArray(self.to_arrow()), index=index - ) - else: - # `copy=True` workaround until following issue is fixed: - # https://issues.apache.org/jira/browse/ARROW-9772 - return pd.Series( - self.to_arrow(), - copy=True, - dtype=self.dtype, - index=index, - ) - @property def values(self): """ diff --git a/python/cudf/cudf/core/column/timedelta.py b/python/cudf/cudf/core/column/timedelta.py index ee326b254b9..0d24e8e5120 100644 --- a/python/cudf/cudf/core/column/timedelta.py +++ b/python/cudf/cudf/core/column/timedelta.py @@ -146,33 +146,6 @@ def to_arrow(self) -> pa.Array: null_count=self.null_count, ) - def to_pandas( - self, - *, - index: Optional[pd.Index] = None, - nullable: bool = False, - arrow_type: bool = False, - ) -> pd.Series: - # `copy=True` workaround until following issue is fixed: - # https://issues.apache.org/jira/browse/ARROW-9772 - if arrow_type and nullable: - raise ValueError( - f"{arrow_type=} and {nullable=} cannot both be set." - ) - elif nullable: - raise NotImplementedError(f"{nullable=} is not implemented.") - elif arrow_type: - return pd.Series( - pd.arrays.ArrowExtensionArray(self.to_arrow()), index=index - ) - else: - return pd.Series( - self.to_arrow(), - copy=True, - dtype=self.dtype, - index=index, - ) - def _binaryop(self, other: ColumnBinaryOperand, op: str) -> ColumnBase: reflect, op = self._check_reflected_op(op) other = self._wrap_binop_normalization(other) From abdca82e7f6d1a7386930a2e0d30f987b2f6a633 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 7 Mar 2024 10:30:20 -1000 Subject: [PATCH 371/384] Simplify some to_pandas implementations (#15123) - For `DatetimeTZColumns`, convert via UTC so ambiguous/nonexistent times never become an issue - Dispatch to `super` to reduce duplication Authors: - Matthew Roeschke (https://github.com/mroeschke) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Bradley Dice (https://github.com/bdice) - Ashwin Srinath (https://github.com/shwina) URL: https://github.com/rapidsai/cudf/pull/15123 --- python/cudf/cudf/core/column/numerical.py | 28 ++++++++++++----------- python/cudf/cudf/core/column/string.py | 8 ++----- python/cudf/cudf/core/dataframe.py | 10 ++++---- python/cudf/cudf/core/dtypes.py | 10 +++----- python/cudf/cudf/utils/dtypes.py | 24 +++++-------------- 5 files changed, 31 insertions(+), 49 deletions(-) diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index 8d9da8982ac..b2bd73c9856 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -47,7 +47,6 @@ from cudf.core.dtypes import CategoricalDtype from cudf.core.mixins import BinaryOperand from cudf.utils.dtypes import ( - NUMERIC_TYPES, min_column_type, min_signed_type, np_dtypes_to_pandas_dtypes, @@ -695,23 +694,26 @@ def to_pandas( raise ValueError( f"{arrow_type=} and {nullable=} cannot both be set." ) - if arrow_type: + elif arrow_type: return pd.Series( pd.arrays.ArrowExtensionArray(self.to_arrow()), index=index ) - elif nullable and self.dtype in np_dtypes_to_pandas_dtypes: - pandas_nullable_dtype = np_dtypes_to_pandas_dtypes[self.dtype] + elif ( + nullable + and ( + pandas_nullable_dtype := np_dtypes_to_pandas_dtypes.get( + self.dtype + ) + ) + is not None + ): arrow_array = self.to_arrow() - pandas_array = pandas_nullable_dtype.__from_arrow__(arrow_array) - pd_series = pd.Series(pandas_array, copy=False) - elif str(self.dtype) in NUMERIC_TYPES and not self.has_nulls(): - pd_series = pd.Series(self.values_host, copy=False) + pandas_array = pandas_nullable_dtype.__from_arrow__(arrow_array) # type: ignore[attr-defined] + return pd.Series(pandas_array, copy=False, index=index) + elif self.dtype.kind in set("iuf") and not self.has_nulls(): + return pd.Series(self.values_host, copy=False, index=index) else: - pd_series = self.to_arrow().to_pandas() - - if index is not None: - pd_series.index = index - return pd_series + return super().to_pandas(index=index, nullable=nullable) def _reduction_result_dtype(self, reduction_op: str) -> Dtype: col_dtype = self.dtype diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index e947c9375d7..fb76fcdaf39 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -5805,13 +5805,9 @@ def to_pandas( ) elif nullable: pandas_array = pd.StringDtype().__from_arrow__(self.to_arrow()) - pd_series = pd.Series(pandas_array, copy=False) + return pd.Series(pandas_array, copy=False, index=index) else: - pd_series = self.to_arrow().to_pandas() - - if index is not None: - pd_series.index = index - return pd_series + return super().to_pandas(index=index, nullable=nullable) def can_cast_safely(self, to_dtype: Dtype) -> bool: to_dtype = cudf.api.types.dtype(to_dtype) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 6a4fe346eb1..0440512c467 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -5265,7 +5265,7 @@ def to_pandas( the resulting columns will either convert null values to ``np.nan`` or ``None`` depending on the dtype. arrow_type : bool, Default False - Return the Index with a ``pandas.ArrowDtype`` + Return the columns with a ``pandas.ArrowDtype`` Returns ------- @@ -5324,13 +5324,13 @@ def to_pandas( b bool[pyarrow] dtype: object """ - out_data = {} out_index = self.index.to_pandas() - - for i, col_key in enumerate(self._data): - out_data[i] = self._data[col_key].to_pandas( + out_data = { + i: col.to_pandas( index=out_index, nullable=nullable, arrow_type=arrow_type ) + for i, col in enumerate(self._data.columns) + } out_df = pd.DataFrame(out_data, index=out_index) out_df.columns = self._data.to_pandas_index() diff --git a/python/cudf/cudf/core/dtypes.py b/python/cudf/cudf/core/dtypes.py index c658701f851..3bd342e24c2 100644 --- a/python/cudf/cudf/core/dtypes.py +++ b/python/cudf/cudf/core/dtypes.py @@ -244,14 +244,10 @@ def to_pandas(self) -> pd.CategoricalDtype: """ # noqa: E501 if self._categories is None: categories = None + elif self._categories.dtype.kind == "f": + categories = self._categories.dropna().to_pandas() else: - if self._categories.dtype in { - cudf.dtype("float32"), - cudf.dtype("float64"), - }: - categories = self._categories.dropna().to_pandas() - else: - categories = self._categories.to_pandas() + categories = self._categories.to_pandas() return pd.CategoricalDtype(categories=categories, ordered=self.ordered) def _init_categories(self, categories: Any): diff --git a/python/cudf/cudf/utils/dtypes.py b/python/cudf/cudf/utils/dtypes.py index 3780fcc627e..e9dbc23d767 100644 --- a/python/cudf/cudf/utils/dtypes.py +++ b/python/cudf/cudf/utils/dtypes.py @@ -46,6 +46,12 @@ np.dtype("int64"): pd.Int64Dtype(), np.dtype("bool_"): pd.BooleanDtype(), np.dtype("object"): pd.StringDtype(), + np.dtype("float32"): pd.Float32Dtype(), + np.dtype("float64"): pd.Float64Dtype(), +} +pandas_dtypes_to_np_dtypes = { + pd_dtype: np_dtype + for np_dtype, pd_dtype in np_dtypes_to_pandas_dtypes.items() } pyarrow_dtypes_to_pandas_dtypes = { @@ -61,24 +67,6 @@ pa.string(): pd.StringDtype(), } -pandas_dtypes_to_np_dtypes = { - pd.UInt8Dtype(): np.dtype("uint8"), - pd.UInt16Dtype(): np.dtype("uint16"), - pd.UInt32Dtype(): np.dtype("uint32"), - pd.UInt64Dtype(): np.dtype("uint64"), - pd.Int8Dtype(): np.dtype("int8"), - pd.Int16Dtype(): np.dtype("int16"), - pd.Int32Dtype(): np.dtype("int32"), - pd.Int64Dtype(): np.dtype("int64"), - pd.BooleanDtype(): np.dtype("bool_"), - pd.StringDtype(): np.dtype("object"), -} - - -np_dtypes_to_pandas_dtypes[np.dtype("float32")] = pd.Float32Dtype() -np_dtypes_to_pandas_dtypes[np.dtype("float64")] = pd.Float64Dtype() -pandas_dtypes_to_np_dtypes[pd.Float32Dtype()] = np.dtype("float32") -pandas_dtypes_to_np_dtypes[pd.Float64Dtype()] = np.dtype("float64") SIGNED_INTEGER_TYPES = {"int8", "int16", "int32", "int64"} UNSIGNED_TYPES = {"uint8", "uint16", "uint32", "uint64"} From bd68b1c897741d97684c8555487de759c7576758 Mon Sep 17 00:00:00 2001 From: nvdbaranec <56695930+nvdbaranec@users.noreply.github.com> Date: Thu, 7 Mar 2024 16:50:28 -0600 Subject: [PATCH 372/384] Add general purpose host memory allocator reference to cuIO with a demo of pooled-pinned allocation. (#15079) This PR adds a new interface to cuIO which controls where host memory allocations come from. It adds two core functions: Addresses https://github.com/rapidsai/cudf/issues/14314 ``` rmm::host_async_resource_ref set_host_memory_resource(rmm::host_async_resource_ref mr); rmm::host_async_resource_ref get_host_memory_resource(); ``` `cudf::io::hostdevice_vector` was currently implemented in terms of a `thrust::host_vector<>` that explicitly uses an allocator called `pinned_host_vector`. I copied that and made a new class called `rmm_host_vector` which takes any host_resource_ref. This probably makes `pinned_host_vector` obsolete. Parquet benchmarks have a new commandline option which lets you toggle between 3 modes: ``` --cuio_host_mem pinned (the default, an unpooled, pinned memory source) --cuio_host_mem pinned_pool (the pooled/pinned resource) ``` The ultimate intent here is to reduce the cpu-side overhead of the setup code that comes before the decode kernels in the parquet reader. The wins are pretty significant for our faster kernels (that is, where we are less dominated by gpu time) Edit: Updated to use newly minted resource ref types from rmm itself. I also switched the type to be `host_async_resource_ref` even though in this case the user (`thrust::host_vector`) doesn't explicitly go through the async path. In addition, the pageable memory path (an experimental feature) has been removed. Pinned ``` | data_type | io_type | cardinality | run_length | Samples | CPU Time | Noise | GPU Time | Noise | bytes_per_second | peak_memory_usage | encoded_file_size | |-----------|---------------|-------------|------------|---------|-----------|-------|-----------|-------|------------------|-------------------|-------------------| | INTEGRAL | DEVICE_BUFFER | 0 | 1 | 25x | 20.443 ms | 0.45% | 20.438 ms | 0.45% | 26268890178 | 1.072 GiB | 498.123 MiB | | INTEGRAL | DEVICE_BUFFER | 1000 | 1 | 26x | 19.571 ms | 0.42% | 19.565 ms | 0.42% | 27440146729 | 756.210 MiB | 161.438 MiB | | INTEGRAL | DEVICE_BUFFER | 0 | 32 | 28x | 18.150 ms | 0.18% | 18.145 ms | 0.18% | 29587789525 | 602.424 MiB | 27.720 MiB | | INTEGRAL | DEVICE_BUFFER | 1000 | 32 | 29x | 17.306 ms | 0.37% | 17.300 ms | 0.37% | 31032523423 | 597.181 MiB | 14.403 MiB | ``` Pooled/pinned ``` | data_type | io_type | cardinality | run_length | Samples | CPU Time | Noise | GPU Time | Noise | bytes_per_second | peak_memory_usage | encoded_file_size | |-----------|---------------|-------------|------------|---------|-----------|-------|-----------|-------|------------------|-------------------|-------------------| | INTEGRAL | DEVICE_BUFFER | 0 | 1 | 117x | 17.258 ms | 0.50% | 17.254 ms | 0.50% | 31115706389 | 1.072 GiB | 498.123 MiB | | INTEGRAL | DEVICE_BUFFER | 1000 | 1 | 31x | 16.413 ms | 0.43% | 16.408 ms | 0.43% | 32719609450 | 756.210 MiB | 161.438 MiB | | INTEGRAL | DEVICE_BUFFER | 0 | 32 | 576x | 14.885 ms | 0.58% | 14.881 ms | 0.58% | 36077859564 | 602.519 MiB | 27.720 MiB | | INTEGRAL | DEVICE_BUFFER | 1000 | 32 | 36x | 14.069 ms | 0.48% | 14.065 ms | 0.48% | 38171646940 | 597.243 MiB | 14.403 MiB | ``` Authors: - https://github.com/nvdbaranec Approvers: - Mark Harris (https://github.com/harrism) - Vukasin Milovanovic (https://github.com/vuule) URL: https://github.com/rapidsai/cudf/pull/15079 --- cpp/benchmarks/fixture/nvbench_fixture.hpp | 38 ++++ cpp/benchmarks/fixture/nvbench_main.cpp | 28 +-- .../cudf/detail/utilities/rmm_host_vector.hpp | 208 ++++++++++++++++++ cpp/include/cudf/io/memory_resource.hpp | 44 ++++ cpp/include/cudf/utilities/export.hpp | 26 +++ cpp/src/io/parquet/reader_impl_preprocess.cu | 4 +- cpp/src/io/utilities/config_utils.cpp | 47 +++- cpp/src/io/utilities/hostdevice_vector.hpp | 45 ++-- cpp/tests/CMakeLists.txt | 1 + .../utilities_tests/io_utilities_tests.cpp | 65 ++++++ 10 files changed, 457 insertions(+), 49 deletions(-) create mode 100644 cpp/include/cudf/detail/utilities/rmm_host_vector.hpp create mode 100644 cpp/include/cudf/io/memory_resource.hpp create mode 100644 cpp/include/cudf/utilities/export.hpp create mode 100644 cpp/tests/utilities_tests/io_utilities_tests.cpp diff --git a/cpp/benchmarks/fixture/nvbench_fixture.hpp b/cpp/benchmarks/fixture/nvbench_fixture.hpp index 701ed67e666..4e4eec3547f 100644 --- a/cpp/benchmarks/fixture/nvbench_fixture.hpp +++ b/cpp/benchmarks/fixture/nvbench_fixture.hpp @@ -15,6 +15,7 @@ */ #pragma once +#include #include #include @@ -25,12 +26,17 @@ #include #include #include +#include +#include #include namespace cudf { + namespace detail { static std::string rmm_mode_param{"--rmm_mode"}; ///< RMM mode command-line parameter name +static std::string cuio_host_mem_param{ + "--cuio_host_mem"}; ///< cuio host memory mode parameter name } // namespace detail /** @@ -75,6 +81,30 @@ struct nvbench_base_fixture { "\nExpecting: cuda, pool, async, arena, managed, or managed_pool"); } + inline rmm::host_async_resource_ref make_cuio_host_pinned() + { + static std::shared_ptr mr = + std::make_shared(); + return *mr; + } + + inline rmm::host_async_resource_ref make_cuio_host_pinned_pool() + { + using host_pooled_mr = rmm::mr::pool_memory_resource; + static std::shared_ptr mr = std::make_shared( + std::make_shared().get(), + size_t{1} * 1024 * 1024 * 1024); + + return *mr; + } + + inline rmm::host_async_resource_ref create_cuio_host_memory_resource(std::string const& mode) + { + if (mode == "pinned") return make_cuio_host_pinned(); + if (mode == "pinned_pool") return make_cuio_host_pinned_pool(); + CUDF_FAIL("Unknown cuio_host_mem parameter: " + mode + "\nExpecting: pinned or pinned_pool"); + } + nvbench_base_fixture(int argc, char const* const* argv) { for (int i = 1; i < argc - 1; ++i) { @@ -82,16 +112,24 @@ struct nvbench_base_fixture { if (arg == detail::rmm_mode_param) { i++; rmm_mode = argv[i]; + } else if (arg == detail::cuio_host_mem_param) { + i++; + cuio_host_mode = argv[i]; } } mr = create_memory_resource(rmm_mode); rmm::mr::set_current_device_resource(mr.get()); std::cout << "RMM memory resource = " << rmm_mode << "\n"; + + cudf::io::set_host_memory_resource(create_cuio_host_memory_resource(cuio_host_mode)); + std::cout << "CUIO host memory resource = " << cuio_host_mode << "\n"; } std::shared_ptr mr; std::string rmm_mode{"pool"}; + + std::string cuio_host_mode{"pinned"}; }; } // namespace cudf diff --git a/cpp/benchmarks/fixture/nvbench_main.cpp b/cpp/benchmarks/fixture/nvbench_main.cpp index 64c4d83ac17..f46cb11a6c3 100644 --- a/cpp/benchmarks/fixture/nvbench_main.cpp +++ b/cpp/benchmarks/fixture/nvbench_main.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,20 +21,22 @@ #include -// strip off the rmm_mode parameter before passing the +// strip off the rmm_mode and cuio_host_mem parameters before passing the // remaining arguments to nvbench::option_parser #undef NVBENCH_MAIN_PARSE -#define NVBENCH_MAIN_PARSE(argc, argv) \ - nvbench::option_parser parser; \ - std::vector m_args; \ - for (int i = 0; i < argc; ++i) { \ - std::string arg = argv[i]; \ - if (arg == cudf::detail::rmm_mode_param) { \ - i += 2; \ - } else { \ - m_args.push_back(arg); \ - } \ - } \ +#define NVBENCH_MAIN_PARSE(argc, argv) \ + nvbench::option_parser parser; \ + std::vector m_args; \ + for (int i = 0; i < argc; ++i) { \ + std::string arg = argv[i]; \ + if (arg == cudf::detail::rmm_mode_param) { \ + i += 2; \ + } else if (arg == cudf::detail::cuio_host_mem_param) { \ + i += 2; \ + } else { \ + m_args.push_back(arg); \ + } \ + } \ parser.parse(m_args) // this declares/defines the main() function using the definitions above diff --git a/cpp/include/cudf/detail/utilities/rmm_host_vector.hpp b/cpp/include/cudf/detail/utilities/rmm_host_vector.hpp new file mode 100644 index 00000000000..858501877b0 --- /dev/null +++ b/cpp/include/cudf/detail/utilities/rmm_host_vector.hpp @@ -0,0 +1,208 @@ +/* + * Copyright 2024 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include + +#include + +#include +#include +#include // for bad_alloc + +namespace cudf::detail { + +/*! \p rmm_host_allocator is a CUDA-specific host memory allocator + * that employs \c a `rmm::host_async_resource_ref` for allocation. + * + * This implementation is ported from pinned_host_vector in cudf. + * + * \see https://en.cppreference.com/w/cpp/memory/allocator + */ +template +class rmm_host_allocator; + +/*! \p rmm_host_allocator is a CUDA-specific host memory allocator + * that employs \c an `cudf::host_async_resource_ref` for allocation. + * + * This implementation is ported from pinned_host_vector in cudf. + * + * \see https://en.cppreference.com/w/cpp/memory/allocator + */ +template <> +class rmm_host_allocator { + public: + using value_type = void; ///< The type of the elements in the allocator + using pointer = void*; ///< The type returned by address() / allocate() + using const_pointer = void const*; ///< The type returned by address() + using size_type = std::size_t; ///< The type used for the size of the allocation + using difference_type = std::ptrdiff_t; ///< The type of the distance between two pointers + + /** + * @brief converts a `rmm_host_allocator` to `rmm_host_allocator` + */ + template + struct rebind { + using other = rmm_host_allocator; ///< The rebound type + }; +}; + +/*! \p rmm_host_allocator is a CUDA-specific host memory allocator + * that employs \c `rmm::host_async_resource_ref` for allocation. + * + * The \p rmm_host_allocator provides an interface for host memory allocation through the user + * provided \c `rmm::host_async_resource_ref`. The \p rmm_host_allocator does not take ownership of + * this reference and therefore it is the user's responsibility to ensure its lifetime for the + * duration of the lifetime of the \p rmm_host_allocator. This implementation is ported from + * pinned_host_vector in cudf. + * + * \see https://en.cppreference.com/w/cpp/memory/allocator + */ +template +class rmm_host_allocator { + public: + using value_type = T; ///< The type of the elements in the allocator + using pointer = T*; ///< The type returned by address() / allocate() + using const_pointer = T const*; ///< The type returned by address() + using reference = T&; ///< The parameter type for address() + using const_reference = T const&; ///< The parameter type for address() + using size_type = std::size_t; ///< The type used for the size of the allocation + using difference_type = std::ptrdiff_t; ///< The type of the distance between two pointers + + typedef cuda::std::true_type propagate_on_container_move_assignment; + + /** + * @brief converts a `rmm_host_allocator` to `rmm_host_allocator` + */ + template + struct rebind { + using other = rmm_host_allocator; ///< The rebound type + }; + + /** + * @brief Cannot declare an empty host allocator. + */ + rmm_host_allocator() = delete; + + /** + * @brief Construct from a `cudf::host_async_resource_ref` + */ + rmm_host_allocator(rmm::host_async_resource_ref _mr, rmm::cuda_stream_view _stream) + : mr(_mr), stream(_stream) + { + } + + /** + * @brief Copy constructor + */ + rmm_host_allocator(rmm_host_allocator const& other) = default; + + /** + * @brief Move constructor + */ + rmm_host_allocator(rmm_host_allocator&& other) = default; + + /** + * @brief Assignment operator + */ + rmm_host_allocator& operator=(rmm_host_allocator const& other) + { + mr = other.mr; + return *this; + } + + /** + * @brief rmm_host_allocator's null destructor does nothing. + */ + inline ~rmm_host_allocator() {} + + /** + * @brief This method allocates storage for objects in host memory. + * + * @param cnt The number of objects to allocate. + * @return a \c pointer to the newly allocated objects. + * @note This method does not invoke \p value_type's constructor. + * It is the responsibility of the caller to initialize the + * objects at the returned \c pointer. + */ + inline pointer allocate(size_type cnt) + { + if (cnt > this->max_size()) { throw std::bad_alloc(); } // end if + return static_cast( + mr.allocate_async(cnt * sizeof(value_type), rmm::RMM_DEFAULT_HOST_ALIGNMENT, stream)); + } + + /** + * @brief This method deallocates host memory previously allocated + * with this \c rmm_host_allocator. + * + * @param p A \c pointer to the previously allocated memory. + * @note The second parameter is the number of objects previously allocated. + * @note This method does not invoke \p value_type's destructor. + * It is the responsibility of the caller to destroy + * the objects stored at \p p. + */ + inline void deallocate(pointer p, size_type cnt) + { + mr.deallocate_async(p, cnt * sizeof(value_type), rmm::RMM_DEFAULT_HOST_ALIGNMENT, stream); + } + + /** + * @brief This method returns the maximum size of the \c cnt parameter + * accepted by the \p allocate() method. + * + * @return The maximum number of objects that may be allocated + * by a single call to \p allocate(). + */ + constexpr inline size_type max_size() const + { + return (std::numeric_limits::max)() / sizeof(T); + } + + /** + * @brief This method tests this \p rmm_host_allocator for equality to + * another. + * + * @param x The other \p rmm_host_allocator of interest. + * @return This method always returns \c true. + */ + inline bool operator==(rmm_host_allocator const& x) const { return x.mr == mr; } + + /** + * @brief This method tests this \p rmm_host_allocator for inequality + * to another. + * + * @param x The other \p rmm_host_allocator of interest. + * @return This method always returns \c false. + */ + inline bool operator!=(rmm_host_allocator const& x) const { return !operator==(x); } + + private: + rmm::host_async_resource_ref mr; + rmm::cuda_stream_view stream; +}; + +/** + * @brief A vector class with rmm host memory allocator + */ +template +using rmm_host_vector = thrust::host_vector>; + +} // namespace cudf::detail diff --git a/cpp/include/cudf/io/memory_resource.hpp b/cpp/include/cudf/io/memory_resource.hpp new file mode 100644 index 00000000000..ea79d6a3029 --- /dev/null +++ b/cpp/include/cudf/io/memory_resource.hpp @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace cudf::io { + +/** + * @brief Set the rmm resource to be used for host memory allocations by + * cudf::detail::hostdevice_vector + * + * hostdevice_vector is a utility class that uses a pair of host and device-side buffers for + * bouncing state between the cpu and the gpu. The resource set with this function (typically a + * pinned memory allocator) is what it uses to allocate space for it's host-side buffer. + * + * @param mr The rmm resource to be used for host-side allocations + * @return The previous resource that was in use + */ +rmm::host_async_resource_ref set_host_memory_resource(rmm::host_async_resource_ref mr); + +/** + * @brief Get the rmm resource being used for host memory allocations by + * cudf::detail::hostdevice_vector + * + * @return The rmm resource used for host-side allocations + */ +rmm::host_async_resource_ref get_host_memory_resource(); + +} // namespace cudf::io diff --git a/cpp/include/cudf/utilities/export.hpp b/cpp/include/cudf/utilities/export.hpp new file mode 100644 index 00000000000..dcc72d3e1f6 --- /dev/null +++ b/cpp/include/cudf/utilities/export.hpp @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// Macros used for defining symbol visibility, only GLIBC is supported +#if (defined(__GNUC__) && !defined(__MINGW32__) && !defined(__MINGW64__)) +#define CUDF_EXPORT __attribute__((visibility("default"))) +#define CUDF_HIDDEN __attribute__((visibility("hidden"))) +#else +#define CUDF_EXPORT +#define CUDF_HIDDEN +#endif diff --git a/cpp/src/io/parquet/reader_impl_preprocess.cu b/cpp/src/io/parquet/reader_impl_preprocess.cu index 51a18de966e..1b0a10be811 100644 --- a/cpp/src/io/parquet/reader_impl_preprocess.cu +++ b/cpp/src/io/parquet/reader_impl_preprocess.cu @@ -726,8 +726,8 @@ void reader::impl::build_string_dict_indices() thrust::fill( rmm::exec_policy_nosync(_stream), str_dict_index_count.begin(), str_dict_index_count.end(), 0); thrust::for_each(rmm::exec_policy_nosync(_stream), - pass.pages.begin(), - pass.pages.end(), + pass.pages.d_begin(), + pass.pages.d_end(), set_str_dict_index_count{str_dict_index_count, pass.chunks}); size_t const total_str_dict_indexes = thrust::reduce( diff --git a/cpp/src/io/utilities/config_utils.cpp b/cpp/src/io/utilities/config_utils.cpp index 0f8961334cf..2f7a6131e3d 100644 --- a/cpp/src/io/utilities/config_utils.cpp +++ b/cpp/src/io/utilities/config_utils.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,11 +17,18 @@ #include "config_utils.hpp" #include +#include + +#include +#include +#include #include #include -namespace cudf::io::detail { +namespace cudf::io { + +namespace detail { namespace cufile_integration { @@ -80,4 +87,38 @@ bool is_stable_enabled() { return is_all_enabled() or get_env_policy() == usage_ } // namespace nvcomp_integration -} // namespace cudf::io::detail +inline std::mutex& host_mr_lock() +{ + static std::mutex map_lock; + return map_lock; +} + +inline rmm::host_async_resource_ref default_pinned_mr() +{ + static rmm::mr::pinned_host_memory_resource default_mr{}; + return default_mr; +} + +CUDF_EXPORT inline auto& host_mr() +{ + static rmm::host_async_resource_ref host_mr = default_pinned_mr(); + return host_mr; +} + +} // namespace detail + +rmm::host_async_resource_ref set_host_memory_resource(rmm::host_async_resource_ref mr) +{ + std::lock_guard lock{detail::host_mr_lock()}; + auto last_mr = detail::host_mr(); + detail::host_mr() = mr; + return last_mr; +} + +rmm::host_async_resource_ref get_host_memory_resource() +{ + std::lock_guard lock{detail::host_mr_lock()}; + return detail::host_mr(); +} + +} // namespace cudf::io diff --git a/cpp/src/io/utilities/hostdevice_vector.hpp b/cpp/src/io/utilities/hostdevice_vector.hpp index 3cd70801cdf..a1e8af51858 100644 --- a/cpp/src/io/utilities/hostdevice_vector.hpp +++ b/cpp/src/io/utilities/hostdevice_vector.hpp @@ -19,13 +19,15 @@ #include "config_utils.hpp" #include "hostdevice_span.hpp" -#include +#include +#include #include #include #include #include #include +#include #include @@ -33,13 +35,6 @@ namespace cudf::detail { -inline bool hostdevice_vector_uses_pageable_buffer() -{ - static bool const use_pageable = - cudf::io::detail::getenv_or("LIBCUDF_IO_PREFER_PAGEABLE_TMP_MEMORY", 0); - return use_pageable; -} - /** * @brief A helper class that wraps fixed-length device memory for the GPU, and * a mirror host pinned memory for the CPU. @@ -62,23 +57,12 @@ class hostdevice_vector { } explicit hostdevice_vector(size_t initial_size, size_t max_size, rmm::cuda_stream_view stream) - : d_data(0, stream) + : h_data({cudf::io::get_host_memory_resource(), stream}), d_data(0, stream) { CUDF_EXPECTS(initial_size <= max_size, "initial_size cannot be larger than max_size"); - if (hostdevice_vector_uses_pageable_buffer()) { - h_data_owner = thrust::host_vector(); - } else { - h_data_owner = cudf::detail::pinned_host_vector(); - } - - std::visit( - [&](auto&& v) { - v.reserve(max_size); - v.resize(initial_size); - host_data = v.data(); - }, - h_data_owner); + h_data.reserve(max_size); + h_data.resize(initial_size); current_size = initial_size; d_data.resize(max_size, stream); @@ -88,7 +72,7 @@ class hostdevice_vector { { CUDF_EXPECTS(size() < capacity(), "Cannot insert data into hostdevice_vector because capacity has been exceeded."); - host_data[current_size++] = data; + h_data[current_size++] = data; } [[nodiscard]] size_t capacity() const noexcept { return d_data.size(); } @@ -96,11 +80,11 @@ class hostdevice_vector { [[nodiscard]] size_t size_bytes() const noexcept { return sizeof(T) * size(); } [[nodiscard]] bool empty() const noexcept { return size() == 0; } - [[nodiscard]] T& operator[](size_t i) { return host_data[i]; } - [[nodiscard]] T const& operator[](size_t i) const { return host_data[i]; } + [[nodiscard]] T& operator[](size_t i) { return h_data[i]; } + [[nodiscard]] T const& operator[](size_t i) const { return h_data[i]; } - [[nodiscard]] T* host_ptr(size_t offset = 0) { return host_data + offset; } - [[nodiscard]] T const* host_ptr(size_t offset = 0) const { return host_data + offset; } + [[nodiscard]] T* host_ptr(size_t offset = 0) { return h_data.data() + offset; } + [[nodiscard]] T const* host_ptr(size_t offset = 0) const { return h_data.data() + offset; } [[nodiscard]] T* begin() { return host_ptr(); } [[nodiscard]] T const* begin() const { return host_ptr(); } @@ -171,7 +155,7 @@ class hostdevice_vector { */ [[nodiscard]] operator hostdevice_span() { - return hostdevice_span{host_data, d_data.data(), size()}; + return hostdevice_span{h_data.data(), d_data.data(), size()}; } /** @@ -186,12 +170,11 @@ class hostdevice_vector { CUDF_EXPECTS(offset < d_data.size(), "Offset is out of bounds."); CUDF_EXPECTS(count <= d_data.size() - offset, "The span with given offset and count is out of bounds."); - return hostdevice_span{host_data + offset, d_data.data() + offset, count}; + return hostdevice_span{h_data.data() + offset, d_data.data() + offset, count}; } private: - std::variant, cudf::detail::pinned_host_vector> h_data_owner; - T* host_data = nullptr; + cudf::detail::rmm_host_vector h_data; size_t current_size = 0; rmm::device_uvector d_data; }; diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index fa9d2ee88ce..135a40b076a 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -376,6 +376,7 @@ ConfigureTest( utilities_tests/column_debug_tests.cpp utilities_tests/column_utilities_tests.cpp utilities_tests/column_wrapper_tests.cpp + utilities_tests/io_utilities_tests.cpp utilities_tests/lists_column_wrapper_tests.cpp utilities_tests/logger_tests.cpp utilities_tests/default_stream_tests.cpp diff --git a/cpp/tests/utilities_tests/io_utilities_tests.cpp b/cpp/tests/utilities_tests/io_utilities_tests.cpp new file mode 100644 index 00000000000..6981ad71f1e --- /dev/null +++ b/cpp/tests/utilities_tests/io_utilities_tests.cpp @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include +#include + +#include +#include +#include + +class IoUtilitiesTest : public cudf::test::BaseFixture {}; + +TEST(IoUtilitiesTest, HostMemoryGetAndSet) +{ + // Global environment for temporary files + auto const temp_env = static_cast( + ::testing::AddGlobalTestEnvironment(new cudf::test::TempDirTestEnvironment)); + + // pinned/pooled host memory resource + using host_pooled_mr = rmm::mr::pool_memory_resource; + host_pooled_mr mr(std::make_shared().get(), + size_t{128} * 1024 * 1024); + + // set new resource + auto last_mr = cudf::io::get_host_memory_resource(); + cudf::io::set_host_memory_resource(mr); + + constexpr int num_rows = 32 * 1024; + auto valids = + cudf::detail::make_counting_transform_iterator(0, [&](int index) { return index % 2; }); + auto values = thrust::make_counting_iterator(0); + + cudf::test::fixed_width_column_wrapper col(values, values + num_rows, valids); + + cudf::table_view expected({col}); + auto filepath = temp_env->get_temp_filepath("IoUtilsMemTest.parquet"); + cudf::io::parquet_writer_options out_args = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, expected); + cudf::io::write_parquet(out_args); + + cudf::io::parquet_reader_options const read_opts = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}); + auto const result = cudf::io::read_parquet(read_opts); + CUDF_TEST_EXPECT_TABLES_EQUAL(*result.tbl, expected); + + // reset memory resource back + cudf::io::set_host_memory_resource(last_mr); +} From b909732cd2916b7adca82f4f90a6580e6a7dbd92 Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Thu, 7 Mar 2024 18:20:59 -0800 Subject: [PATCH 373/384] Fix number of rows in randomly generated lists columns (#15248) Changing `single_level_mean` to double introduced a rounding error in the iterative process of generating random lists columns. This PR addressed the issue by enforcing the correct row count in the root lists column. Authors: - Vukasin Milovanovic (https://github.com/vuule) Approvers: - Shruti Shivakumar (https://github.com/shrshi) - Nghia Truong (https://github.com/ttnghia) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/15248 --- cpp/benchmarks/common/generate_input.cu | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/cpp/benchmarks/common/generate_input.cu b/cpp/benchmarks/common/generate_input.cu index 71ce45879dd..ccc7bdef527 100644 --- a/cpp/benchmarks/common/generate_input.cu +++ b/cpp/benchmarks/common/generate_input.cu @@ -740,7 +740,8 @@ std::unique_ptr create_random_column(data_profile { auto const dist_params = profile.get_distribution_params(); auto const single_level_mean = get_distribution_mean(dist_params.length_params); - auto const num_elements = num_rows * pow(single_level_mean, dist_params.max_depth); + cudf::size_type const num_elements = + std::lround(num_rows * std::pow(single_level_mean, dist_params.max_depth)); auto leaf_column = cudf::type_dispatcher( cudf::data_type(dist_params.element_type), create_rand_col_fn{}, profile, engine, num_elements); @@ -751,13 +752,16 @@ std::unique_ptr create_random_column(data_profile // Generate the list column bottom-up auto list_column = std::move(leaf_column); - for (int lvl = 0; lvl < dist_params.max_depth; ++lvl) { + for (int lvl = dist_params.max_depth; lvl > 0; --lvl) { // Generating the next level - offsets point into the current list column - auto current_child_column = std::move(list_column); - cudf::size_type const num_rows = current_child_column->size() / single_level_mean; - - auto offsets = len_dist(engine, num_rows + 1); - auto valids = valid_dist(engine, num_rows); + auto current_child_column = std::move(list_column); + // Because single_level_mean is not a whole number, rounding errors can lead to slightly + // different row count; top-level column needs to have exactly num_rows rows, so enforce it here + cudf::size_type const current_num_rows = + (lvl == 1) ? num_rows : std::lround(current_child_column->size() / single_level_mean); + + auto offsets = len_dist(engine, current_num_rows + 1); + auto valids = valid_dist(engine, current_num_rows); // to ensure these values <= current_child_column->size() auto output_offsets = thrust::make_transform_output_iterator( offsets.begin(), clamp_down{current_child_column->size()}); @@ -767,7 +771,7 @@ std::unique_ptr create_random_column(data_profile current_child_column->size(); // Always include all elements auto offsets_column = std::make_unique(cudf::data_type{cudf::type_id::INT32}, - num_rows + 1, + current_num_rows + 1, offsets.release(), rmm::device_buffer{}, 0); @@ -778,7 +782,7 @@ std::unique_ptr create_random_column(data_profile cudf::get_default_stream(), rmm::mr::get_current_device_resource()); list_column = cudf::make_lists_column( - num_rows, + current_num_rows, std::move(offsets_column), std::move(current_child_column), profile.get_null_probability().has_value() ? null_count : 0, From 65fb21803bd39ddc5e57426d365d1c2d0fa5f357 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Thu, 7 Mar 2024 22:41:15 -0800 Subject: [PATCH 374/384] Add DELTA_BYTE_ARRAY encoder for Parquet (#15239) Re-submission of #14938. Final (delta) piece of #13501. Adds the ability to encode Parquet pages as DELTA_BYTE_ARRAY. Python testing wlll be added as a follow-on when per-column encoding selection is added to the python API (ref this [comment](https://github.com/rapidsai/cudf/pull/15081#issuecomment-1979731930)). Authors: - Ed Seidl (https://github.com/etseidl) - Vukasin Milovanovic (https://github.com/vuule) - Yunsong Wang (https://github.com/PointKernel) Approvers: - Vukasin Milovanovic (https://github.com/vuule) - Yunsong Wang (https://github.com/PointKernel) URL: https://github.com/rapidsai/cudf/pull/15239 --- cpp/src/io/parquet/page_delta_decode.cu | 16 + cpp/src/io/parquet/page_enc.cu | 403 +++++++++++++++++++++--- cpp/src/io/parquet/parquet_gpu.hpp | 9 +- cpp/src/io/parquet/writer_impl.cu | 26 +- cpp/tests/io/parquet_reader_test.cpp | 42 +++ cpp/tests/io/parquet_writer_test.cpp | 79 ++++- 6 files changed, 502 insertions(+), 73 deletions(-) diff --git a/cpp/src/io/parquet/page_delta_decode.cu b/cpp/src/io/parquet/page_delta_decode.cu index c68b6a32c8b..7c0092c6185 100644 --- a/cpp/src/io/parquet/page_delta_decode.cu +++ b/cpp/src/io/parquet/page_delta_decode.cu @@ -462,6 +462,14 @@ CUDF_KERNEL void __launch_bounds__(decode_block_size) return; } + if (s->col.logical_type.has_value() && s->col.logical_type->type == LogicalType::DECIMAL) { + // we cannot read decimal encoded with DELTA_BYTE_ARRAY yet + if (t == 0) { + set_error(static_cast(decode_error::INVALID_DATA_TYPE), error_code); + } + return; + } + bool const has_repetition = s->col.max_level[level_type::REPETITION] > 0; // choose a character parallel string copy when the average string is longer than a warp @@ -620,6 +628,14 @@ CUDF_KERNEL void __launch_bounds__(decode_block_size) return; } + if (s->col.logical_type.has_value() && s->col.logical_type->type == LogicalType::DECIMAL) { + // we cannot read decimal encoded with DELTA_LENGTH_BYTE_ARRAY yet + if (t == 0) { + set_error(static_cast(decode_error::INVALID_DATA_TYPE), error_code); + } + return; + } + bool const has_repetition = s->col.max_level[level_type::REPETITION] > 0; // copying logic from gpuDecodePageData. diff --git a/cpp/src/io/parquet/page_enc.cu b/cpp/src/io/parquet/page_enc.cu index 617cb1d0992..fb17545875a 100644 --- a/cpp/src/io/parquet/page_enc.cu +++ b/cpp/src/io/parquet/page_enc.cu @@ -16,6 +16,7 @@ #include "delta_enc.cuh" #include "io/utilities/block_utils.cuh" +#include "page_string_utils.cuh" #include "parquet_gpu.cuh" #include @@ -30,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -493,10 +495,47 @@ CUDF_KERNEL void __launch_bounds__(128) } } +// given a column chunk, determine which data encoding to use +__device__ encode_kernel_mask data_encoding_for_col(EncColumnChunk const* chunk, + parquet_column_device_view const* col_desc, + bool write_v2_headers) +{ + // first check for dictionary (boolean always uses dict encoder) + if (chunk->use_dictionary or col_desc->physical_type == BOOLEAN) { + return encode_kernel_mask::DICTIONARY; + } + + // next check for user requested encoding, but skip if user requested dictionary encoding + // (if we could use the requested dict encoding, we'd have returned above) + if (col_desc->requested_encoding != column_encoding::USE_DEFAULT and + col_desc->requested_encoding != column_encoding::DICTIONARY) { + switch (col_desc->requested_encoding) { + case column_encoding::PLAIN: return encode_kernel_mask::PLAIN; + case column_encoding::DELTA_BINARY_PACKED: return encode_kernel_mask::DELTA_BINARY; + case column_encoding::DELTA_LENGTH_BYTE_ARRAY: return encode_kernel_mask::DELTA_LENGTH_BA; + case column_encoding::DELTA_BYTE_ARRAY: return encode_kernel_mask::DELTA_BYTE_ARRAY; + } + } + + // Select a fallback encoding. For V1, we always choose PLAIN. For V2 we'll use + // DELTA_BINARY_PACKED for INT32 and INT64, and DELTA_LENGTH_BYTE_ARRAY for + // BYTE_ARRAY. Everything else will still fall back to PLAIN. + if (write_v2_headers) { + switch (col_desc->physical_type) { + case INT32: + case INT64: return encode_kernel_mask::DELTA_BINARY; + case BYTE_ARRAY: return encode_kernel_mask::DELTA_LENGTH_BA; + } + } + + return encode_kernel_mask::PLAIN; +} + __device__ size_t delta_data_len(Type physical_type, cudf::type_id type_id, uint32_t num_values, - size_t page_size) + size_t page_size, + encode_kernel_mask encoding) { auto const dtype_len_out = physical_type_len(physical_type, type_id); auto const dtype_len = [&]() -> uint32_t { @@ -516,6 +555,8 @@ __device__ size_t delta_data_len(Type physical_type, // divisible by 128 (via static assert on delta::block_size), but do safe division anyway. auto const bytes_per_block = cudf::util::div_rounding_up_unsafe(max_bits * vals_per_block, 8); auto const block_size = mini_block_header_size + bytes_per_block; + // the number of DELTA_BINARY_PACKED blocks to encode + auto const num_dbp_blocks = encoding == encode_kernel_mask::DELTA_BYTE_ARRAY ? 2 : 1; // delta header is 2 bytes for the block_size, 1 byte for number of mini-blocks, // max 5 bytes for number of values, and max dtype_len + 1 for first value. @@ -526,12 +567,17 @@ __device__ size_t delta_data_len(Type physical_type, // The above is just a size estimate for a DELTA_BINARY_PACKED data page. For BYTE_ARRAY // data we also need to add size of the char data. `page_size` that is passed in is the // plain encoded size (i.e. num_values * sizeof(size_type) + char_data_len), so the char - // data len is `page_size` minus the first term. - // TODO: this will need to change for DELTA_BYTE_ARRAY encoding - auto const char_data_len = - physical_type == BYTE_ARRAY ? page_size - num_values * sizeof(size_type) : 0; + // data len is `page_size` minus the first term. For FIXED_LEN_BYTE_ARRAY there are no + // lengths, so just use `page_size`. + // `num_dbp_blocks` takes into account the two delta binary blocks for DELTA_BYTE_ARRAY. + size_t char_data_len = 0; + if (physical_type == BYTE_ARRAY) { + char_data_len = page_size - num_values * sizeof(size_type); + } else if (physical_type == FIXED_LEN_BYTE_ARRAY) { + char_data_len = page_size; + } - return header_size + num_blocks * block_size + char_data_len; + return header_size + num_blocks * num_dbp_blocks * block_size + char_data_len; } // blockDim {128,1,1} @@ -573,13 +619,12 @@ CUDF_KERNEL void __launch_bounds__(128) // at the worst case number of bytes needed to encode. auto const physical_type = col_g.physical_type; auto const type_id = col_g.leaf_column->type().id(); - auto const is_requested_delta = - col_g.requested_encoding == column_encoding::DELTA_BINARY_PACKED || - col_g.requested_encoding == column_encoding::DELTA_LENGTH_BYTE_ARRAY; - auto const is_fallback_to_delta = - !ck_g.use_dictionary && write_v2_headers && - (physical_type == INT32 || physical_type == INT64 || physical_type == BYTE_ARRAY); - auto const is_use_delta = is_requested_delta || is_fallback_to_delta; + + // figure out kernel encoding to use for data pages + auto const column_data_encoding = data_encoding_for_col(&ck_g, &col_g, write_v2_headers); + auto const is_use_delta = column_data_encoding == encode_kernel_mask::DELTA_BINARY or + column_data_encoding == encode_kernel_mask::DELTA_LENGTH_BA or + column_data_encoding == encode_kernel_mask::DELTA_BYTE_ARRAY; if (t < 32) { uint32_t fragments_in_chunk = 0; @@ -754,8 +799,8 @@ CUDF_KERNEL void __launch_bounds__(128) } // get a different bound if using delta encoding if (is_use_delta) { - auto const delta_len = - delta_data_len(physical_type, type_id, page_g.num_leaf_values, page_size); + auto const delta_len = delta_data_len( + physical_type, type_id, page_g.num_leaf_values, page_size, column_data_encoding); page_size = max(page_size, delta_len); } auto const max_data_size = @@ -771,11 +816,28 @@ CUDF_KERNEL void __launch_bounds__(128) // 4-byte length indicator, so subtract that. page_g.var_bytes_size = var_bytes_size; } + + page_g.kernel_mask = column_data_encoding; page_g.max_data_size = static_cast(max_data_size); pagestats_g.start_chunk = ck_g.first_fragment + page_start; pagestats_g.num_chunks = page_g.num_fragments; page_offset += util::round_up_unsafe(page_g.max_hdr_size + page_g.max_data_size, page_align); + // if encoding delta_byte_array, need to allocate some space for scratch data. + // if there are leaf nulls, we need space for a mapping array: + // sizeof(size_type) * num_leaf_values + // we always need prefix lengths: sizeof(size_type) * num_valid + if (page_g.kernel_mask == encode_kernel_mask::DELTA_BYTE_ARRAY) { + // scratch needs to be aligned to a size_type boundary + auto const pg_end = reinterpret_cast(ck_g.uncompressed_bfr + page_offset); + auto scratch = util::round_up_unsafe(pg_end, sizeof(size_type)); + if (page_g.num_valid != page_g.num_leaf_values) { + scratch += sizeof(size_type) * page_g.num_leaf_values; + } + scratch += sizeof(size_type) * page_g.num_valid; + page_offset = + thrust::distance(ck_g.uncompressed_bfr, reinterpret_cast(scratch)); + } if (not comp_page_sizes.empty()) { // V2 does not include level data in compressed size estimate comp_page_offset += page_g.max_hdr_size + page_g.max_lvl_size + @@ -789,43 +851,6 @@ CUDF_KERNEL void __launch_bounds__(128) __syncwarp(); if (t == 0) { if (not pages.empty()) { - // set encoding - if (col_g.requested_encoding != column_encoding::USE_DEFAULT) { - switch (col_g.requested_encoding) { - case column_encoding::PLAIN: page_g.kernel_mask = encode_kernel_mask::PLAIN; break; - case column_encoding::DICTIONARY: - // user may have requested dict, but we may not be able to use it - // TODO: when DELTA_BYTE_ARRAY is added, rework the fallback logic so there - // isn't duplicated code here and below. - if (ck_g.use_dictionary) { - page_g.kernel_mask = encode_kernel_mask::DICTIONARY; - } else if (is_fallback_to_delta) { - page_g.kernel_mask = physical_type == BYTE_ARRAY - ? encode_kernel_mask::DELTA_LENGTH_BA - : encode_kernel_mask::DELTA_BINARY; - } else { - page_g.kernel_mask = encode_kernel_mask::PLAIN; - } - break; - case column_encoding::DELTA_BINARY_PACKED: - page_g.kernel_mask = encode_kernel_mask::DELTA_BINARY; - break; - case column_encoding::DELTA_LENGTH_BYTE_ARRAY: - page_g.kernel_mask = encode_kernel_mask::DELTA_LENGTH_BA; - break; - } - } else if (is_use_delta) { - // TODO(ets): at some point make a more intelligent decision on this. DELTA_LENGTH_BA - // should always be preferred over PLAIN, but DELTA_BINARY is a different matter. - // If the delta encoding size is going to be close to 32 bits anyway, then plain - // is a better choice. - page_g.kernel_mask = physical_type == BYTE_ARRAY ? encode_kernel_mask::DELTA_LENGTH_BA - : encode_kernel_mask::DELTA_BINARY; - } else if (ck_g.use_dictionary || physical_type == BOOLEAN) { - page_g.kernel_mask = encode_kernel_mask::DICTIONARY; - } else { - page_g.kernel_mask = encode_kernel_mask::PLAIN; - } // need space for the chunk histograms plus data page histograms auto const num_histograms = num_pages - ck_g.num_dict_pages(); if (ck_g.def_histogram_data != nullptr && col_g.max_def_level > 0) { @@ -2166,6 +2191,273 @@ CUDF_KERNEL void __launch_bounds__(block_size, 8) s, output_ptr + string_data_len, pages, comp_in, comp_out, comp_results, true); } +struct byte_array { + uint8_t const* data; + size_type length; + + // calculate the amount of overlap with a preceding array + __device__ size_type common_prefix_length(byte_array const& preceding) const + { + auto const max_pref_len = min(length, preceding.length); + size_type idx = 0; + while (idx < max_pref_len and data[idx] == preceding.data[idx]) { + idx++; + } + return idx; + } +}; + +// DELTA_BYTE_ARRAY page data encoder +// blockDim(128, 1, 1) +template +CUDF_KERNEL void __launch_bounds__(block_size, 8) + gpuEncodeDeltaByteArrayPages(device_span pages, + device_span> comp_in, + device_span> comp_out, + device_span comp_results) +{ + using cudf::detail::warp_size; + // block of shared memory for value storage and bit packing + __shared__ uleb128_t delta_shared[delta::buffer_size + delta::block_size]; + __shared__ __align__(8) page_enc_state_s<0> state_g; + __shared__ delta_binary_packer packer; + __shared__ uint8_t* scratch_data; + __shared__ size_t avg_suffix_len; + using block_scan = cub::BlockScan; + using block_reduce = cub::BlockReduce; + __shared__ union { + typename block_scan::TempStorage scan_storage; + typename block_reduce::TempStorage reduce_storage; + typename delta_binary_packer::index_scan::TempStorage delta_index_tmp; + typename delta_binary_packer::block_reduce::TempStorage delta_reduce_tmp; + typename delta_binary_packer::warp_reduce::TempStorage + delta_warp_red_tmp[delta::num_mini_blocks]; + } temp_storage; + + auto* const s = &state_g; + uint32_t t = threadIdx.x; + + if (t == 0) { + state_g = page_enc_state_s<0>{}; + s->page = pages[blockIdx.x]; + s->ck = *s->page.chunk; + s->col = *s->ck.col_desc; + s->rle_len_pos = nullptr; + // get s->cur back to where it was at the end of encoding the rep and def level data + set_page_data_start(s); + } + __syncthreads(); + + if (BitAnd(s->page.kernel_mask, encode_kernel_mask::DELTA_BYTE_ARRAY) == 0) { return; } + + // Encode data values + if (t == 0) { + uint8_t* dst = s->cur; + s->rle_run = 0; + s->rle_pos = 0; + s->rle_numvals = 0; + s->rle_out = dst; + s->page.encoding = Encoding::DELTA_BYTE_ARRAY; + s->page_start_val = row_to_value_idx(s->page.start_row, s->col); + s->chunk_start_val = row_to_value_idx(s->ck.start_row, s->col); + + // set pointer to beginning of scratch space (aligned to size_type boundary) + auto scratch_start = + reinterpret_cast(s->page.page_data + s->page.max_hdr_size + s->page.max_data_size); + scratch_start = util::round_up_unsafe(scratch_start, sizeof(size_type)); + scratch_data = reinterpret_cast(scratch_start); + } + __syncthreads(); + + // create offsets map (if needed) + // We only encode valid values, and we need to know adjacent valid strings. So first we'll + // create a mapping of leaf indexes to valid indexes: + // + // validity array is_valid: + // 1 1 0 1 0 1 1 0 + // + // exclusive scan on is_valid yields mapping of leaf index -> valid index: + // 0 1 2 2 3 3 4 5 + // + // Last value should equal page.num_valid. Now we need to transform that into a reverse + // lookup that maps valid index -> leaf index (of length num_valid): + // 0 1 3 5 6 + // + auto const has_leaf_nulls = s->page.num_valid != s->page.num_leaf_values; + + size_type* const offsets_map = + has_leaf_nulls ? reinterpret_cast(scratch_data) : nullptr; + + if (offsets_map != nullptr) { + size_type* const forward_map = offsets_map + s->page.num_valid; + + // create the validity array + for (int idx = t; idx < s->page.num_leaf_values; idx += block_size) { + size_type const idx_in_col = s->page_start_val + idx; + bool const is_valid = + idx_in_col < s->col.leaf_column->size() and s->col.leaf_column->is_valid(idx_in_col); + forward_map[idx] = is_valid ? 1 : 0; + } + __syncthreads(); + + // exclusive scan to get leaf_idx -> valid_idx + block_excl_sum(forward_map, s->page.num_leaf_values, 0); + + // now reverse map to get valid_idx -> leaf_idx mapping + for (int idx = t; idx < s->page.num_leaf_values; idx += block_size) { + size_type const idx_in_col = s->page_start_val + idx; + bool const is_valid = + idx_in_col < s->col.leaf_column->size() and s->col.leaf_column->is_valid(idx_in_col); + if (is_valid) { offsets_map[forward_map[idx]] = idx; } + } + __syncthreads(); + } + + size_type* const prefix_lengths = + has_leaf_nulls ? offsets_map + s->page.num_valid : reinterpret_cast(scratch_data); + + auto const type_id = s->col.leaf_column->type().id(); + + auto const byte_array_at = [type_id, s](size_type idx) -> byte_array { + if (type_id == type_id::STRING) { + auto const str = s->col.leaf_column->element(idx); + return {reinterpret_cast(str.data()), str.size_bytes()}; + } else if (s->col.output_as_byte_array && type_id == type_id::LIST) { + auto const str = get_element(*s->col.leaf_column, idx); + return {reinterpret_cast(str.data()), + static_cast(str.size_bytes())}; + } + return {nullptr, 0}; + }; + + // Calculate prefix lengths. The first prefix length is always 0. loop over num_valid since we + // only encode valid values. + // Note: calculating this on a string-per-thread basis seems bad for large strings with lots + // of overlap. But in testing, it was found that the string copy at the end had a much larger + // impact on performance, and doing this step on a string-per-warp basis was always slower. + if (t == 0) { prefix_lengths[0] = 0; } + for (int idx = t + 1; idx < s->page.num_valid; idx += block_size) { + size_type const leaf_idx = has_leaf_nulls ? offsets_map[idx] : idx; + size_type const pleaf_idx = has_leaf_nulls ? offsets_map[idx - 1] : idx - 1; + + // get this string and the preceding string + auto const current = byte_array_at(leaf_idx + s->page_start_val); + auto const preceding = byte_array_at(pleaf_idx + s->page_start_val); + + // calculate the amount of overlap + prefix_lengths[idx] = current.common_prefix_length(preceding); + } + + // encode prefix lengths + if (t == 0) { + packer.init(s->cur, s->page.num_valid, reinterpret_cast(delta_shared), &temp_storage); + } + __syncthreads(); + + // don't start at `t` because all threads must participate in each iteration + for (int idx = 0; idx < s->page.num_valid; idx += block_size) { + size_type const t_idx = idx + t; + auto const in_range = t_idx < s->page.num_valid; + auto const val = in_range ? prefix_lengths[t_idx] : 0; + packer.add_value(val, in_range); + } + + auto const suffix_ptr = packer.flush(); + __syncthreads(); + + // encode suffix lengths + if (t == 0) { + packer.init( + suffix_ptr, s->page.num_valid, reinterpret_cast(delta_shared), &temp_storage); + } + __syncthreads(); + + size_t non_zero = 0; + size_t suffix_bytes = 0; + + for (int idx = 0; idx < s->page.num_valid; idx += block_size) { + size_type const t_idx = idx + t; + auto const in_range = t_idx < s->page.num_valid; + int32_t val = 0; + if (in_range) { + size_type const leaf_idx = has_leaf_nulls ? offsets_map[t_idx] : t_idx; + auto const byte_arr = byte_array_at(leaf_idx + s->page_start_val); + val = byte_arr.length - prefix_lengths[t_idx]; + if (val > 0) { + non_zero++; + suffix_bytes += val; + } + } + packer.add_value(val, in_range); + } + + auto const strings_ptr = packer.flush(); + + non_zero = block_reduce(temp_storage.reduce_storage).Sum(non_zero); + __syncthreads(); + suffix_bytes = block_reduce(temp_storage.reduce_storage).Sum(suffix_bytes); + if (t == 0) { avg_suffix_len = util::div_rounding_up_unsafe(suffix_bytes, non_zero); } + __syncthreads(); + + // Now copy the byte array data. For shorter suffixes (<= 64 bytes), it is faster to use + // memcpy on a string-per-thread basis. For longer suffixes, it's better to use a parallel + // approach. 64 was a good cutoff in testing. + constexpr size_t suffix_cutoff = 64; + + size_t str_data_len = 0; + if (avg_suffix_len <= suffix_cutoff) { + for (int idx = 0; idx < s->page.num_valid; idx += block_size) { + size_type const t_idx = idx + t; + size_type s_len = 0, pref_len = 0, suff_len = 0; + uint8_t const* s_ptr = nullptr; + if (t_idx < s->page.num_valid) { + size_type const leaf_idx = has_leaf_nulls ? offsets_map[t_idx] : t_idx; + auto const byte_arr = byte_array_at(leaf_idx + s->page_start_val); + s_len = byte_arr.length; + s_ptr = byte_arr.data; + pref_len = prefix_lengths[t_idx]; + suff_len = byte_arr.length - pref_len; + } + + // calculate offsets into output + size_type s_off, total; + block_scan(temp_storage.scan_storage) + .ExclusiveScan(suff_len, s_off, str_data_len, cub::Sum(), total); + + if (t_idx < s->page.num_valid) { + auto const dst = strings_ptr + s_off; + memcpy(dst, s_ptr + pref_len, suff_len); + } + str_data_len += total; + __syncthreads(); + } + } else { + int t0 = 0; // thread 0 for each string + for (int idx = 0; idx < s->page.num_valid; idx++) { + // calculate ids for this string + int const tid = (t - t0 + block_size) % block_size; + + // fetch string for this iter + size_type const leaf_idx = has_leaf_nulls ? offsets_map[idx] : idx; + auto const byte_arr = byte_array_at(leaf_idx + s->page_start_val); + size_type const pref_len = prefix_lengths[idx]; + size_type const suff_len = byte_arr.length - pref_len; + + // now copy the data + auto const dst = strings_ptr + str_data_len; + for (int src_idx = tid; src_idx < suff_len; src_idx += block_size) { + dst[src_idx] = byte_arr.data[pref_len + src_idx]; + } + + str_data_len += suff_len; + t0 = (t0 + suff_len) % block_size; + } + } + + finish_page_encode( + s, strings_ptr + str_data_len, pages, comp_in, comp_out, comp_results, true); +} + constexpr int decide_compression_warps_in_block = 4; constexpr int decide_compression_block_size = decide_compression_warps_in_block * cudf::detail::warp_size; @@ -3137,6 +3429,13 @@ void EncodePages(device_span pages, gpuEncodeDeltaLengthByteArrayPages <<>>(pages, comp_in, comp_out, comp_results); } + if (BitAnd(kernel_mask, encode_kernel_mask::DELTA_BYTE_ARRAY) != 0) { + auto const strm = streams[s_idx++]; + gpuEncodePageLevels<<>>( + pages, write_v2_headers, encode_kernel_mask::DELTA_BYTE_ARRAY); + gpuEncodeDeltaByteArrayPages + <<>>(pages, comp_in, comp_out, comp_results); + } if (BitAnd(kernel_mask, encode_kernel_mask::DICTIONARY) != 0) { auto const strm = streams[s_idx++]; gpuEncodePageLevels<<>>( diff --git a/cpp/src/io/parquet/parquet_gpu.hpp b/cpp/src/io/parquet/parquet_gpu.hpp index c66f69b3567..ca7334be216 100644 --- a/cpp/src/io/parquet/parquet_gpu.hpp +++ b/cpp/src/io/parquet/parquet_gpu.hpp @@ -514,10 +514,11 @@ constexpr uint32_t encoding_to_mask(Encoding encoding) * Used to control which encode kernels to run. */ enum class encode_kernel_mask { - PLAIN = (1 << 0), // Run plain encoding kernel - DICTIONARY = (1 << 1), // Run dictionary encoding kernel - DELTA_BINARY = (1 << 2), // Run DELTA_BINARY_PACKED encoding kernel - DELTA_LENGTH_BA = (1 << 3), // Run DELTA_LENGTH_BYTE_ARRAY encoding kernel + PLAIN = (1 << 0), // Run plain encoding kernel + DICTIONARY = (1 << 1), // Run dictionary encoding kernel + DELTA_BINARY = (1 << 2), // Run DELTA_BINARY_PACKED encoding kernel + DELTA_LENGTH_BA = (1 << 3), // Run DELTA_LENGTH_BYTE_ARRAY encoding kernel + DELTA_BYTE_ARRAY = (1 << 4), // Run DELTA_BYtE_ARRAY encoding kernel }; /** diff --git a/cpp/src/io/parquet/writer_impl.cu b/cpp/src/io/parquet/writer_impl.cu index 87c8b2f1611..5a8d96975ce 100644 --- a/cpp/src/io/parquet/writer_impl.cu +++ b/cpp/src/io/parquet/writer_impl.cu @@ -631,14 +631,36 @@ std::vector construct_schema_tree( "requested encoding will be ignored"); return; } + // we don't yet allow encoding decimal128 with DELTA_LENGTH_BYTE_ARRAY (nor with + // the BYTE_ARRAY physical type, but check anyway) + if (s.converted_type.value_or(ConvertedType::UNKNOWN) == ConvertedType::DECIMAL) { + CUDF_LOG_WARN( + "Decimal types cannot yet be encoded as DELTA_LENGTH_BYTE_ARRAY; the " + "requested encoding will be ignored"); + return; + } + break; + + case column_encoding::DELTA_BYTE_ARRAY: + if (s.type != Type::BYTE_ARRAY && s.type != Type::FIXED_LEN_BYTE_ARRAY) { + CUDF_LOG_WARN( + "DELTA_BYTE_ARRAY encoding is only supported for BYTE_ARRAY and " + "FIXED_LEN_BYTE_ARRAY columns; the requested encoding will be ignored"); + return; + } + // we don't yet allow encoding decimal128 with DELTA_BYTE_ARRAY + if (s.converted_type.value_or(ConvertedType::UNKNOWN) == ConvertedType::DECIMAL) { + CUDF_LOG_WARN( + "Decimal types cannot yet be encoded as DELTA_BYTE_ARRAY; the " + "requested encoding will be ignored"); + return; + } break; // supported parquet encodings case column_encoding::PLAIN: case column_encoding::DICTIONARY: break; - // not yet supported for write (soon...) - case column_encoding::DELTA_BYTE_ARRAY: [[fallthrough]]; // all others default: CUDF_LOG_WARN( diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index c13bf488e6a..85ada9b38fc 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -1955,6 +1955,7 @@ TEST_F(ParquetReaderTest, RepeatedNoAnnotations) TEST_F(ParquetReaderTest, DeltaSkipRowsWithNulls) { + using cudf::io::column_encoding; constexpr int num_rows = 10'000; constexpr auto seed = 21337; @@ -1999,9 +2000,17 @@ TEST_F(ParquetReaderTest, DeltaSkipRowsWithNulls) int64_col, int64_nulls_col, *int64_list, *int64_list_nulls, *int16_list, *int16_list_nulls, *int8_list, *int8_list_nulls, str_col, *str_col_nulls, *str_list, *str_list_nulls, + big_str_col, *big_str_col_nulls, *big_str_list, *big_str_list_nulls, + str_col, *str_col_nulls, *str_list, *str_list_nulls, big_str_col, *big_str_col_nulls, *big_str_list, *big_str_list_nulls}); auto const filepath = temp_env->get_temp_filepath("DeltaSkipRowsWithNulls.parquet"); + auto input_metadata = cudf::io::table_input_metadata{tbl}; + for (int i = 12; i <= 27; ++i) { + input_metadata.column_metadata[i].set_encoding( + i <= 19 ? column_encoding::DELTA_LENGTH_BYTE_ARRAY : column_encoding::DELTA_BYTE_ARRAY); + } + auto const out_opts = cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, tbl) .stats_level(cudf::io::statistics_freq::STATISTICS_COLUMN) @@ -2060,6 +2069,39 @@ TEST_F(ParquetReaderTest, DeltaSkipRowsWithNulls) } } +TEST_F(ParquetReaderTest, DeltaByteArraySkipAllValid) +{ + // test that the DELTA_BYTE_ARRAY decoder can handle the case where skip rows skips all valid + // values in a page. see #15075 + constexpr int num_rows = 500; + constexpr int num_valid = 150; + + auto const ones = thrust::make_constant_iterator("one"); + + auto valids = cudf::detail::make_counting_transform_iterator( + 0, [num_valid](auto i) { return i < num_valid; }); + auto const col = cudf::test::strings_column_wrapper{ones, ones + num_rows, valids}; + auto const expected = table_view({col}); + + auto input_metadata = cudf::io::table_input_metadata{expected}; + input_metadata.column_metadata[0].set_encoding(cudf::io::column_encoding::DELTA_BYTE_ARRAY); + + auto const filepath = temp_env->get_temp_filepath("DeltaByteArraySkipAllValid.parquet"); + cudf::io::parquet_writer_options out_opts = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, expected) + .write_v2_headers(true) + .metadata(input_metadata) + .dictionary_policy(cudf::io::dictionary_policy::NEVER); + cudf::io::write_parquet(out_opts); + + cudf::io::parquet_reader_options in_opts = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) + .skip_rows(num_valid + 1); + auto result = cudf::io::read_parquet(in_opts); + CUDF_TEST_EXPECT_TABLES_EQUAL(cudf::slice(expected, {num_valid + 1, num_rows}), + result.tbl->view()); +} + // test that using page stats is working for full reads and various skip rows TEST_F(ParquetReaderTest, StringsWithPageStats) { diff --git a/cpp/tests/io/parquet_writer_test.cpp b/cpp/tests/io/parquet_writer_test.cpp index f4da9f59b8c..200c58bb9aa 100644 --- a/cpp/tests/io/parquet_writer_test.cpp +++ b/cpp/tests/io/parquet_writer_test.cpp @@ -1482,8 +1482,18 @@ TEST_F(ParquetWriterTest, UserRequestedEncodings) auto const string_col = cudf::test::strings_column_wrapper(strings, strings + num_rows, no_nulls()); - auto const table = table_view( - {col, col, col, col, col, string_col, string_col, string_col, string_col, string_col}); + auto const table = table_view({col, + col, + col, + col, + col, + col, + string_col, + string_col, + string_col, + string_col, + string_col, + string_col}); cudf::io::table_input_metadata table_metadata(table); @@ -1495,13 +1505,15 @@ TEST_F(ParquetWriterTest, UserRequestedEncodings) set_meta(1, "int_dict", column_encoding::DICTIONARY); set_meta(2, "int_db", column_encoding::DELTA_BINARY_PACKED); set_meta(3, "int_dlba", column_encoding::DELTA_LENGTH_BYTE_ARRAY); - table_metadata.column_metadata[4].set_name("int_none"); + set_meta(4, "int_dba", column_encoding::DELTA_BYTE_ARRAY); + table_metadata.column_metadata[5].set_name("int_none"); - set_meta(5, "string_plain", column_encoding::PLAIN); - set_meta(6, "string_dict", column_encoding::DICTIONARY); - set_meta(7, "string_dlba", column_encoding::DELTA_LENGTH_BYTE_ARRAY); - set_meta(8, "string_db", column_encoding::DELTA_BINARY_PACKED); - table_metadata.column_metadata[9].set_name("string_none"); + set_meta(6, "string_plain", column_encoding::PLAIN); + set_meta(7, "string_dict", column_encoding::DICTIONARY); + set_meta(8, "string_dlba", column_encoding::DELTA_LENGTH_BYTE_ARRAY); + set_meta(9, "string_dba", column_encoding::DELTA_BYTE_ARRAY); + set_meta(10, "string_db", column_encoding::DELTA_BINARY_PACKED); + table_metadata.column_metadata[11].set_name("string_none"); for (auto& col_meta : table_metadata.column_metadata) { col_meta.set_nullability(false); @@ -1534,18 +1546,55 @@ TEST_F(ParquetWriterTest, UserRequestedEncodings) expect_enc(2, Encoding::DELTA_BINARY_PACKED); // requested delta_length_byte_array, but should fall back to dictionary expect_enc(3, Encoding::PLAIN_DICTIONARY); - // no request, should fall back to dictionary + // requested delta_byte_array, but should fall back to dictionary expect_enc(4, Encoding::PLAIN_DICTIONARY); + // no request, should use dictionary + expect_enc(5, Encoding::PLAIN_DICTIONARY); + // requested plain - expect_enc(5, Encoding::PLAIN); + expect_enc(6, Encoding::PLAIN); // requested dictionary - expect_enc(6, Encoding::PLAIN_DICTIONARY); + expect_enc(7, Encoding::PLAIN_DICTIONARY); // requested delta_length_byte_array - expect_enc(7, Encoding::DELTA_LENGTH_BYTE_ARRAY); + expect_enc(8, Encoding::DELTA_LENGTH_BYTE_ARRAY); + // requested delta_byte_array + expect_enc(9, Encoding::DELTA_BYTE_ARRAY); // requested delta_binary_packed, but should fall back to dictionary - expect_enc(8, Encoding::PLAIN_DICTIONARY); - // no request, should fall back to dictionary - expect_enc(9, Encoding::PLAIN_DICTIONARY); + expect_enc(10, Encoding::PLAIN_DICTIONARY); + // no request, should use dictionary + expect_enc(11, Encoding::PLAIN_DICTIONARY); +} + +TEST_F(ParquetWriterTest, Decimal128DeltaByteArray) +{ + // decimal128 in cuDF maps to FIXED_LEN_BYTE_ARRAY, which is allowed by the spec to use + // DELTA_BYTE_ARRAY encoding. But this use is not implemented in cuDF. + __int128_t val0 = 0xa1b2'c3d4'e5f6ULL; + __int128_t val1 = val0 << 80; + column_wrapper col0{{numeric::decimal128(val0, numeric::scale_type{0}), + numeric::decimal128(val1, numeric::scale_type{0})}}; + + auto expected = table_view{{col0, col0}}; + cudf::io::table_input_metadata table_metadata(expected); + table_metadata.column_metadata[0] + .set_name("decimal128") + .set_encoding(cudf::io::column_encoding::DELTA_BYTE_ARRAY) + .set_nullability(false); + + auto const filepath = temp_env->get_temp_filepath("Decimal128DeltaByteArray.parquet"); + const cudf::io::parquet_writer_options out_opts = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, expected) + .compression(cudf::io::compression_type::NONE) + .metadata(table_metadata); + cudf::io::write_parquet(out_opts); + + auto const source = cudf::io::datasource::create(filepath); + cudf::io::parquet::detail::FileMetaData fmd; + read_footer(source, &fmd); + + // make sure DELTA_BYTE_ARRAY was not used + EXPECT_NE(fmd.row_groups[0].columns[0].meta_data.encodings[0], + cudf::io::parquet::detail::Encoding::DELTA_BYTE_ARRAY); } TEST_F(ParquetWriterTest, DeltaBinaryStartsWithNulls) From 2ebfc808a46bcabb893a1b8345749fc3dd954a96 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Fri, 8 Mar 2024 07:28:29 -0500 Subject: [PATCH 375/384] Remove create_chars_child_column utility (#15241) Removes the `cudf::strings::detail::create_chars_child_column` utility. This is not longer needed or used. Removing it helps prevent inadvertently using it to wrap chars data with a cudf column. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Muhammad Haseeb (https://github.com/mhaseeb123) - Yunsong Wang (https://github.com/PointKernel) URL: https://github.com/rapidsai/cudf/pull/15241 --- cpp/include/cudf/strings/detail/utilities.hpp | 13 ------------- cpp/src/strings/utilities.cu | 8 -------- 2 files changed, 21 deletions(-) diff --git a/cpp/include/cudf/strings/detail/utilities.hpp b/cpp/include/cudf/strings/detail/utilities.hpp index 3cf2850548d..8d8065dbcaf 100644 --- a/cpp/include/cudf/strings/detail/utilities.hpp +++ b/cpp/include/cudf/strings/detail/utilities.hpp @@ -26,19 +26,6 @@ namespace cudf { namespace strings { namespace detail { -/** - * @brief Create a chars column to be a child of a strings column. - * - * This will return the properly sized column to be filled in by the caller. - * - * @param bytes Number of bytes for the chars column. - * @param stream CUDA stream used for device memory operations and kernel launches. - * @param mr Device memory resource used to allocate the returned column's device memory. - * @return The chars child column for a strings column. - */ -std::unique_ptr create_chars_child_column(size_type bytes, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr); /** * @brief Creates a string_view vector from a strings column. diff --git a/cpp/src/strings/utilities.cu b/cpp/src/strings/utilities.cu index 72c3ccf4ac5..0a7353821b0 100644 --- a/cpp/src/strings/utilities.cu +++ b/cpp/src/strings/utilities.cu @@ -65,14 +65,6 @@ rmm::device_uvector create_string_vector_from_column( return strings_vector; } -std::unique_ptr create_chars_child_column(cudf::size_type total_bytes, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) -{ - return make_numeric_column( - data_type{type_id::INT8}, total_bytes, mask_state::UNALLOCATED, stream, mr); -} - namespace { // The device variables are created here to avoid using a singleton that may cause issues // with RMM initialize/finalize. See PR #3159 for details on this approach. From 7b0eee1d181293929ce9f6ad7b8a3a10fff2e360 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 8 Mar 2024 10:00:05 -0500 Subject: [PATCH 376/384] Use variable substitution for RAPIDS version in Doxyfile (#15231) Doxyfiles support environment variable substitution, so read the version from `VERSION` and put it in an environment variable. Also remove a hard-coded version from `ci/check_style.sh`. Issue: https://github.com/rapidsai/build-planning/issues/15 Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - Bradley Dice (https://github.com/bdice) - Jake Awe (https://github.com/AyodeAwe) URL: https://github.com/rapidsai/cudf/pull/15231 --- ci/build_docs.sh | 3 +++ ci/check_style.sh | 4 +++- ci/checks/doxygen.sh | 6 +++++- ci/release/update-version.sh | 9 --------- cpp/CMakeLists.txt | 3 ++- cpp/doxygen/Doxyfile | 4 ++-- 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 529eaeae696..b94c61cc184 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -5,6 +5,9 @@ set -euo pipefail export RAPIDS_VERSION_NUMBER="$(rapids-generate-version)" +export RAPIDS_VERSION="$(rapids-version)" +export RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" + rapids-logger "Create test conda environment" . /opt/conda/etc/profile.d/conda.sh diff --git a/ci/check_style.sh b/ci/check_style.sh index 8d882743fcc..b3890607f64 100755 --- a/ci/check_style.sh +++ b/ci/check_style.sh @@ -16,7 +16,9 @@ rapids-dependency-file-generator \ rapids-mamba-retry env create --force -f "${ENV_YAML_DIR}/env.yaml" -n checks conda activate checks -FORMAT_FILE_URL=https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-24.04/cmake-format-rapids-cmake.json +RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" + +FORMAT_FILE_URL="https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION_MAJOR_MINOR}/cmake-format-rapids-cmake.json" export RAPIDS_CMAKE_FORMAT_FILE=/tmp/rapids_cmake_ci/cmake-formats-rapids-cmake.json mkdir -p $(dirname ${RAPIDS_CMAKE_FORMAT_FILE}) wget -O ${RAPIDS_CMAKE_FORMAT_FILE} ${FORMAT_FILE_URL} diff --git a/ci/checks/doxygen.sh b/ci/checks/doxygen.sh index d932fa097e9..faf662aa593 100755 --- a/ci/checks/doxygen.sh +++ b/ci/checks/doxygen.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. ############################### # cuDF doxygen warnings check # ############################### @@ -21,6 +21,10 @@ if [ ! $(version "$DOXYGEN_VERSION") -eq $(version "1.9.1") ] ; then exit 0 fi +# Set variables for doxygen +export RAPIDS_VERSION="$(rapids-version)" +export RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" + # Run doxygen, ignore missing tag files error TAG_ERROR1="error: Tag file '.*.tag' does not exist or is not a file. Skipping it..." TAG_ERROR2="error: cannot open tag file .*.tag for writing" diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 811e7825363..7cacdfd39c3 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -44,12 +44,6 @@ echo "${NEXT_FULL_TAG}" > VERSION # Wheel testing script sed_runner "s/branch-.*/branch-${NEXT_SHORT_TAG}/g" ci/test_wheel_dask_cudf.sh -# cmake-format rapids-cmake definitions -sed_runner 's/'"branch-.*\/cmake-format-rapids-cmake.json"'/'"branch-${NEXT_SHORT_TAG}\/cmake-format-rapids-cmake.json"'/g' ci/check_style.sh - -# doxyfile update -sed_runner 's/PROJECT_NUMBER = .*/PROJECT_NUMBER = '${NEXT_FULL_TAG}'/g' cpp/doxygen/Doxyfile - DEPENDENCIES=( cudf cudf_kafka @@ -71,9 +65,6 @@ for DEP in "${DEPENDENCIES[@]}"; do done done -# Doxyfile update -sed_runner "s|\(TAGFILES.*librmm/\).*|\1${NEXT_SHORT_TAG}|" cpp/doxygen/Doxyfile - # README.md update sed_runner "s/version == ${CURRENT_SHORT_TAG}/version == ${NEXT_SHORT_TAG}/g" README.md sed_runner "s/cudf=${CURRENT_SHORT_TAG}/cudf=${NEXT_SHORT_TAG}/g" README.md diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 5e8d13aa32d..36fef2201f1 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1081,7 +1081,8 @@ rapids_export( add_custom_command( OUTPUT CUDF_DOXYGEN WORKING_DIRECTORY ${CUDF_SOURCE_DIR}/doxygen - COMMAND doxygen Doxyfile + COMMAND ${CMAKE_COMMAND} -E env "RAPIDS_VERSION=${RAPIDS_VERSION}" + "RAPIDS_VERSION_MAJOR_MINOR=${RAPIDS_VERSION_MAJOR_MINOR}" doxygen Doxyfile VERBATIM COMMENT "Custom command for building cudf doxygen docs." ) diff --git a/cpp/doxygen/Doxyfile b/cpp/doxygen/Doxyfile index e45f856b870..81d8793d98b 100644 --- a/cpp/doxygen/Doxyfile +++ b/cpp/doxygen/Doxyfile @@ -38,7 +38,7 @@ PROJECT_NAME = libcudf # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 24.04.00 +PROJECT_NUMBER = $(RAPIDS_VERSION) # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -2226,7 +2226,7 @@ SKIP_FUNCTION_MACROS = YES # the path). If a tag file is not located in the directory in which doxygen is # run, you must also specify the path to the tagfile here. -TAGFILES = rmm.tag=https://docs.rapids.ai/api/librmm/24.04 +TAGFILES = rmm.tag=https://docs.rapids.ai/api/librmm/$(RAPIDS_VERSION_MAJOR_MINOR) # When a file name is specified after GENERATE_TAGFILE, doxygen will create a # tag file that is based on the input files it reads. See section "Linking to From ec24c02c1d1f83fe5e407a61dd77d0024d5ebc77 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 8 Mar 2024 09:17:00 -0800 Subject: [PATCH 377/384] Rewrite conversion in terms of column (#15213) It looks like soon after I started investigating scalar conversions for https://github.com/rapidsai/cudf/pull/14121 (but well before I made the PR) a major underlying hole was plugged in pyarrow via https://github.com/apache/arrow/pull/36162. Most of #14121 was created to give us a way to handle scalars from pyarrow generically in libcudf. Now that pyarrow scalars can be easily tossed into arrays, we no longer really need separate scalar functions in libcudf; we can simply create an array from the scalar, put it into a table, and then call the table function. Additionally, arrow also has a function for creating an array from a scalar. This function is not new but [was previously undocumented](https://github.com/apache/arrow/pull/40373). The builder code added to libcudf in #14121 can be removed and replaced with that factory. The scalar conversion is as simple as calling that arrow function and then using our preexisting `from_arrow` function on the resulting array. For now this PR is just a simplification of internals. Future PRs will remove the scalar API once we have a more standard path for the conversion of arrays via the C Data Interface. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Yunsong Wang (https://github.com/PointKernel) - David Wendt (https://github.com/davidwendt) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/15213 --- cpp/include/cudf/detail/interop.hpp | 55 ------------------------- cpp/src/interop/from_arrow.cu | 63 +---------------------------- python/cudf/cudf/_lib/scalar.pyx | 25 ++++++++---- 3 files changed, 20 insertions(+), 123 deletions(-) diff --git a/cpp/include/cudf/detail/interop.hpp b/cpp/include/cudf/detail/interop.hpp index 683b49e1813..296b68d22a9 100644 --- a/cpp/include/cudf/detail/interop.hpp +++ b/cpp/include/cudf/detail/interop.hpp @@ -105,61 +105,6 @@ std::shared_ptr to_arrow_array(cudf::type_id id, Ts&&... args) } } -/** - * @brief Invokes an `operator()` template with the type instantiation based on - * the specified `arrow::DataType`'s `id()`. - * - * This function is analogous to libcudf's type_dispatcher, but instead applies - * to Arrow functions. Its primary use case is to leverage Arrow's - * metaprogramming facilities like arrow::TypeTraits that require translating - * the runtime dtype information into compile-time types. - */ -template -constexpr decltype(auto) arrow_type_dispatcher(arrow::DataType const& dtype, - Functor f, - Ts&&... args) -{ - switch (dtype.id()) { - case arrow::Type::INT8: - return f.template operator()(std::forward(args)...); - case arrow::Type::INT16: - return f.template operator()(std::forward(args)...); - case arrow::Type::INT32: - return f.template operator()(std::forward(args)...); - case arrow::Type::INT64: - return f.template operator()(std::forward(args)...); - case arrow::Type::UINT8: - return f.template operator()(std::forward(args)...); - case arrow::Type::UINT16: - return f.template operator()(std::forward(args)...); - case arrow::Type::UINT32: - return f.template operator()(std::forward(args)...); - case arrow::Type::UINT64: - return f.template operator()(std::forward(args)...); - case arrow::Type::FLOAT: - return f.template operator()(std::forward(args)...); - case arrow::Type::DOUBLE: - return f.template operator()(std::forward(args)...); - case arrow::Type::BOOL: - return f.template operator()(std::forward(args)...); - case arrow::Type::TIMESTAMP: - return f.template operator()(std::forward(args)...); - case arrow::Type::DURATION: - return f.template operator()(std::forward(args)...); - case arrow::Type::STRING: - return f.template operator()(std::forward(args)...); - case arrow::Type::LIST: - return f.template operator()(std::forward(args)...); - case arrow::Type::DECIMAL128: - return f.template operator()(std::forward(args)...); - case arrow::Type::STRUCT: - return f.template operator()(std::forward(args)...); - default: { - CUDF_FAIL("Invalid type."); - } - } -} - // Converting arrow type to cudf type data_type arrow_to_cudf_type(arrow::DataType const& arrow_type); diff --git a/cpp/src/interop/from_arrow.cu b/cpp/src/interop/from_arrow.cu index 7b44fb41288..2a524c773c0 100644 --- a/cpp/src/interop/from_arrow.cu +++ b/cpp/src/interop/from_arrow.cu @@ -419,52 +419,6 @@ std::unique_ptr get_column(arrow::Array const& array, : get_empty_type_column(array.length()); } -struct BuilderGenerator { - template && - !std::is_same_v)> - std::shared_ptr operator()(std::shared_ptr const& type) - { - return std::make_shared::BuilderType>( - type, arrow::default_memory_pool()); - } - - template || - std::is_same_v)> - std::shared_ptr operator()(std::shared_ptr const& type) - { - CUDF_FAIL("Type not supported by BuilderGenerator"); - } -}; - -std::shared_ptr make_builder(std::shared_ptr const& type) -{ - switch (type->id()) { - case arrow::Type::STRUCT: { - std::vector> field_builders; - - for (auto field : type->fields()) { - auto const vt = field->type(); - if (vt->id() == arrow::Type::STRUCT || vt->id() == arrow::Type::LIST) { - field_builders.push_back(make_builder(vt)); - } else { - field_builders.push_back(arrow_type_dispatcher(*vt, BuilderGenerator{}, vt)); - } - } - return std::make_shared( - type, arrow::default_memory_pool(), field_builders); - } - case arrow::Type::LIST: { - return std::make_shared(arrow::default_memory_pool(), - make_builder(type->field(0)->type())); - } - default: { - return arrow_type_dispatcher(*type, BuilderGenerator{}, type); - } - } -} - } // namespace std::unique_ptr
from_arrow(arrow::Table const& input_table, @@ -512,21 +466,8 @@ std::unique_ptr from_arrow(arrow::Scalar const& input, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { - // Get a builder for the scalar type - auto builder = detail::make_builder(input.type); - - auto status = builder->AppendScalar(input); - if (status != arrow::Status::OK()) { - if (status.IsNotImplemented()) { - // The only known failure case here is for nulls - CUDF_FAIL("Cannot create untyped null scalars or nested types with untyped null leaf nodes", - std::invalid_argument); - } - CUDF_FAIL("Arrow ArrayBuilder::AppendScalar failed"); - } - - auto maybe_array = builder->Finish(); - if (!maybe_array.ok()) { CUDF_FAIL("Arrow ArrayBuilder::Finish failed"); } + auto maybe_array = arrow::MakeArrayFromScalar(input, 1); + if (!maybe_array.ok()) { CUDF_FAIL("Failed to create array"); } auto array = *maybe_array; auto field = arrow::field("", input.type); diff --git a/python/cudf/cudf/_lib/scalar.pyx b/python/cudf/cudf/_lib/scalar.pyx index 37708a4e3ba..cd9793270e2 100644 --- a/python/cudf/cudf/_lib/scalar.pyx +++ b/python/cudf/cudf/_lib/scalar.pyx @@ -129,18 +129,29 @@ cdef class DeviceScalar: else: pa_type = pa.from_numpy_dtype(dtype) - pa_scalar = pa.scalar(value, type=pa_type) + if isinstance(pa_type, pa.ListType) and value is None: + # pyarrow doesn't correctly handle None values for list types, so + # we have to create this one manually. + # https://github.com/apache/arrow/issues/40319 + pa_array = pa.array([None], type=pa_type) + else: + pa_array = pa.array([pa.scalar(value, type=pa_type)]) + + pa_table = pa.Table.from_arrays([pa_array], names=[""]) + table = pylibcudf.Table.from_arrow(pa_table) - data_type = None + column = table.columns()[0] if isinstance(dtype, cudf.core.dtypes.DecimalDtype): - tid = pylibcudf.TypeId.DECIMAL128 if isinstance(dtype, cudf.core.dtypes.Decimal32Dtype): - tid = pylibcudf.TypeId.DECIMAL32 + column = pylibcudf.unary.cast( + column, pylibcudf.DataType(pylibcudf.TypeId.DECIMAL32, -dtype.scale) + ) elif isinstance(dtype, cudf.core.dtypes.Decimal64Dtype): - tid = pylibcudf.TypeId.DECIMAL64 - data_type = pylibcudf.DataType(tid, -dtype.scale) + column = pylibcudf.unary.cast( + column, pylibcudf.DataType(pylibcudf.TypeId.DECIMAL64, -dtype.scale) + ) - self.c_value = pylibcudf.Scalar.from_arrow(pa_scalar, data_type) + self.c_value = pylibcudf.copying.get_element(column, 0) self._dtype = dtype def _to_host_scalar(self): From 6c1872921450ad3d76986900a60c8aa7421732b9 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 8 Mar 2024 07:38:07 -1000 Subject: [PATCH 378/384] Respect IntervalDtype and CategoricalDtype objects passed by users (#14961) Broken off of https://github.com/rapidsai/cudf/pull/14636, these cases are strict about a `dtype` being set so no need to be in a try except Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/14961 --- python/cudf/cudf/core/column/column.py | 167 ++++++------------ python/cudf/cudf/core/column/interval.py | 8 +- .../cudf/cudf/tests/indexes/test_interval.py | 16 +- python/cudf/cudf/tests/test_series.py | 16 ++ 4 files changed, 86 insertions(+), 121 deletions(-) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index ff1204b6178..b7080ff7a7c 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -60,7 +60,6 @@ is_datetime64_dtype, is_dtype_equal, is_integer_dtype, - is_list_dtype, is_scalar, is_string_dtype, ) @@ -2144,59 +2143,57 @@ def as_column( return as_column( np.asarray(view), dtype=dtype, nan_as_null=nan_as_null ) + # Start of arbitrary that's not handed above but dtype provided + elif isinstance(dtype, pd.DatetimeTZDtype): + raise NotImplementedError( + "Use `tz_localize()` to construct timezone aware data." + ) + elif isinstance(dtype, cudf.core.dtypes.DecimalDtype): + # Arrow throws a type error if the input is of + # mixed-precision and cannot fit into the provided + # decimal type properly, see: + # https://github.com/apache/arrow/pull/9948 + # Hence we should let the exception propagate to + # the user. + data = pa.array( + arbitrary, + type=pa.decimal128(precision=dtype.precision, scale=dtype.scale), + ) + if isinstance(dtype, cudf.core.dtypes.Decimal128Dtype): + return cudf.core.column.Decimal128Column.from_arrow(data) + elif isinstance(dtype, cudf.core.dtypes.Decimal64Dtype): + return cudf.core.column.Decimal64Column.from_arrow(data) + elif isinstance(dtype, cudf.core.dtypes.Decimal32Dtype): + return cudf.core.column.Decimal32Column.from_arrow(data) + else: + raise NotImplementedError(f"{dtype} not implemented") + elif isinstance( + dtype, + ( + pd.CategoricalDtype, + cudf.CategoricalDtype, + pd.IntervalDtype, + cudf.IntervalDtype, + ), + ) or dtype in {"category", "interval", "str", str, np.str_}: + if isinstance(dtype, (cudf.CategoricalDtype, cudf.IntervalDtype)): + dtype = dtype.to_pandas() + ser = pd.Series(arbitrary, dtype=dtype) + return as_column(ser, nan_as_null=nan_as_null) + elif isinstance(dtype, (cudf.StructDtype, cudf.ListDtype)): + try: + data = pa.array(arbitrary, type=dtype.to_arrow()) + except (pa.ArrowInvalid, pa.ArrowTypeError): + if isinstance(dtype, cudf.ListDtype): + # e.g. test_cudf_list_struct_write + return cudf.core.column.ListColumn.from_sequences(arbitrary) + raise + return as_column(data, nan_as_null=nan_as_null) else: - if dtype is not None: - # Arrow throws a type error if the input is of - # mixed-precision and cannot fit into the provided - # decimal type properly, see: - # https://github.com/apache/arrow/pull/9948 - # Hence we should let the exception propagate to - # the user. - if isinstance(dtype, cudf.core.dtypes.Decimal128Dtype): - data = pa.array( - arbitrary, - type=pa.decimal128( - precision=dtype.precision, scale=dtype.scale - ), - ) - return cudf.core.column.Decimal128Column.from_arrow(data) - elif isinstance(dtype, cudf.core.dtypes.Decimal64Dtype): - data = pa.array( - arbitrary, - type=pa.decimal128( - precision=dtype.precision, scale=dtype.scale - ), - ) - return cudf.core.column.Decimal64Column.from_arrow(data) - elif isinstance(dtype, cudf.core.dtypes.Decimal32Dtype): - data = pa.array( - arbitrary, - type=pa.decimal128( - precision=dtype.precision, scale=dtype.scale - ), - ) - return cudf.core.column.Decimal32Column.from_arrow(data) - pa_type = None - np_type = None try: if dtype is not None: - if dtype in {"category", "interval"} or isinstance( - dtype, - ( - cudf.CategoricalDtype, - cudf.IntervalDtype, - pd.IntervalDtype, - pd.CategoricalDtype, - ), - ): - raise TypeError - if isinstance(dtype, pd.DatetimeTZDtype): - raise NotImplementedError( - "Use `tz_localize()` to construct " - "timezone aware data." - ) - elif is_datetime64_dtype(dtype): + if is_datetime64_dtype(dtype): # Error checking only, actual construction happens # below. pa_array = pa.array(arbitrary) @@ -2208,42 +2205,6 @@ def as_column( "cuDF does not yet support timezone-aware " "datetimes" ) - if is_list_dtype(dtype): - data = pa.array(arbitrary) - if type(data) not in (pa.ListArray, pa.NullArray): - raise ValueError( - "Cannot create list column from given data" - ) - return as_column(data, nan_as_null=nan_as_null) - elif isinstance(dtype, cudf.StructDtype) and not isinstance( - dtype, cudf.IntervalDtype - ): - data = pa.array(arbitrary, type=dtype.to_arrow()) - return as_column(data, nan_as_null=nan_as_null) - elif isinstance(dtype, cudf.core.dtypes.Decimal128Dtype): - data = pa.array( - arbitrary, - type=pa.decimal128( - precision=dtype.precision, scale=dtype.scale - ), - ) - return cudf.core.column.Decimal128Column.from_arrow(data) - elif isinstance(dtype, cudf.core.dtypes.Decimal64Dtype): - data = pa.array( - arbitrary, - type=pa.decimal128( - precision=dtype.precision, scale=dtype.scale - ), - ) - return cudf.core.column.Decimal64Column.from_arrow(data) - elif isinstance(dtype, cudf.core.dtypes.Decimal32Dtype): - data = pa.array( - arbitrary, - type=pa.decimal128( - precision=dtype.precision, scale=dtype.scale - ), - ) - return cudf.core.column.Decimal32Column.from_arrow(data) if is_bool_dtype(dtype): # Need this special case handling for bool dtypes, # since 'boolean' & 'pd.BooleanDtype' are not @@ -2256,7 +2217,6 @@ def as_column( raise NotImplementedError( f"{dtype=} is not supported." ) - np_type = np_dtype.type pa_type = np_to_pa_dtype(np_dtype) else: # By default cudf constructs a 64-bit column. Setting @@ -2279,15 +2239,6 @@ def as_column( _maybe_convert_to_default_type("float") ) - if ( - cudf.get_option("mode.pandas_compatible") - and isinstance( - arbitrary, (pd.Index, pd.api.extensions.ExtensionArray) - ) - and _is_pandas_nullable_extension_dtype(arbitrary.dtype) - ): - raise NotImplementedError("not supported") - pyarrow_array = pa.array( arbitrary, type=pa_type, @@ -2308,16 +2259,6 @@ def as_column( dtype = cudf.dtype("str") pyarrow_array = pyarrow_array.cast(np_to_pa_dtype(dtype)) - if ( - isinstance(arbitrary, pd.Index) - and arbitrary.dtype == cudf.dtype("object") - and ( - cudf.dtype(pyarrow_array.type.to_pandas_dtype()) - != cudf.dtype(arbitrary.dtype) - ) - ): - raise MixedTypeError("Cannot create column with mixed types") - if ( cudf.get_option("mode.pandas_compatible") and pa.types.is_integer(pyarrow_array.type) @@ -2333,17 +2274,6 @@ def as_column( except (pa.ArrowInvalid, pa.ArrowTypeError, TypeError) as e: if isinstance(e, MixedTypeError): raise TypeError(str(e)) - if _is_categorical_dtype(dtype): - sr = pd.Series(arbitrary, dtype="category") - data = as_column(sr, nan_as_null=nan_as_null, dtype=dtype) - elif np_type == np.str_: - sr = pd.Series(arbitrary, dtype="str") - data = as_column(sr, nan_as_null=nan_as_null) - elif dtype == "interval" or isinstance( - dtype, (pd.IntervalDtype, cudf.IntervalDtype) - ): - sr = pd.Series(arbitrary, dtype="interval") - data = as_column(sr, nan_as_null=nan_as_null, dtype=dtype) elif ( isinstance(arbitrary, Sequence) and len(arbitrary) > 0 @@ -2351,6 +2281,9 @@ def as_column( cudf.utils.dtypes.is_column_like(arb) for arb in arbitrary ) ): + # TODO: I think can be removed; covered by + # elif isinstance(dtype, (cudf.StructDtype, cudf.ListDtype)): + # above return cudf.core.column.ListColumn.from_sequences(arbitrary) elif isinstance(arbitrary, abc.Iterable) or isinstance( arbitrary, abc.Sequence diff --git a/python/cudf/cudf/core/column/interval.py b/python/cudf/cudf/core/column/interval.py index dc609f732e0..7bd693966dc 100644 --- a/python/cudf/cudf/core/column/interval.py +++ b/python/cudf/cudf/core/column/interval.py @@ -99,7 +99,9 @@ def as_interval_column(self, dtype): mask=self.mask, offset=self.offset, null_count=self.null_count, - children=self.children, + children=tuple( + child.astype(dtype.subtype) for child in self.children + ), ) else: raise ValueError("dtype must be IntervalDtype") @@ -124,8 +126,10 @@ def to_pandas( raise NotImplementedError(f"{nullable=} is not implemented.") elif arrow_type: raise NotImplementedError(f"{arrow_type=} is not implemented.") + + pd_type = self.dtype.to_pandas() return pd.Series( - self.dtype.to_pandas().__from_arrow__(self.to_arrow()), index=index + pd_type.__from_arrow__(self.to_arrow()), index=index, dtype=pd_type ) def element_indexing(self, index: int): diff --git a/python/cudf/cudf/tests/indexes/test_interval.py b/python/cudf/cudf/tests/indexes/test_interval.py index 36be7c5674d..365465db1e1 100644 --- a/python/cudf/cudf/tests/indexes/test_interval.py +++ b/python/cudf/cudf/tests/indexes/test_interval.py @@ -100,8 +100,18 @@ def test_interval_range_freq_basic_dtype(start_t, end_t, freq_t): gindex = cudf.interval_range( start=start, end=end, freq=freq, closed="left" ) + if gindex.dtype.subtype.kind == "f": + gindex = gindex.astype( + cudf.IntervalDtype(subtype="float64", closed=gindex.dtype.closed) + ) + elif gindex.dtype.subtype.kind == "i": + gindex = gindex.astype( + cudf.IntervalDtype(subtype="int64", closed=gindex.dtype.closed) + ) - assert_eq(pindex, gindex) + # pandas upcasts to 64 bit https://github.com/pandas-dev/pandas/issues/57268 + # using Series to use check_dtype + assert_eq(pd.Series(pindex), cudf.Series(gindex), check_dtype=False) @pytest.mark.parametrize("closed", ["left", "right", "both", "neither"]) @@ -221,7 +231,9 @@ def test_interval_range_periods_freq_start_dtype(periods_t, freq_t, start_t): start=start, freq=freq, periods=periods, closed="left" ) - assert_eq(pindex, gindex) + # pandas upcasts to 64 bit https://github.com/pandas-dev/pandas/issues/57268 + # using Series to use check_dtype + assert_eq(pd.Series(pindex), cudf.Series(gindex), check_dtype=False) @pytest.mark.parametrize("closed", ["right", "left", "both", "neither"]) diff --git a/python/cudf/cudf/tests/test_series.py b/python/cudf/cudf/tests/test_series.py index e043f358bbe..fdf9357cb5d 100644 --- a/python/cudf/cudf/tests/test_series.py +++ b/python/cudf/cudf/tests/test_series.py @@ -2663,6 +2663,22 @@ def test_series_duplicate_index_reindex(): ) +def test_list_category_like_maintains_dtype(): + dtype = cudf.CategoricalDtype(categories=[1, 2, 3, 4], ordered=True) + data = [1, 2, 3] + result = cudf.Series(cudf.core.column.as_column(data, dtype=dtype)) + expected = pd.Series(data, dtype=dtype.to_pandas()) + assert_eq(result, expected) + + +def test_list_interval_like_maintains_dtype(): + dtype = cudf.IntervalDtype(subtype=np.int8) + data = [pd.Interval(1, 2)] + result = cudf.Series(cudf.core.column.as_column(data, dtype=dtype)) + expected = pd.Series(data, dtype=dtype.to_pandas()) + assert_eq(result, expected) + + @pytest.mark.parametrize( "klass", [cudf.Series, cudf.Index, pd.Series, pd.Index] ) From c9e54cfe20c030a3772d4179c750b4a3358c9ee1 Mon Sep 17 00:00:00 2001 From: Shruti Shivakumar Date: Fri, 8 Mar 2024 13:47:22 -0500 Subject: [PATCH 379/384] Improve performance in JSON reader when `mixed_types_as_string` option is enabled (#15236) Addresses #15196 by applying a patch from @karthikeyann to skip the `infer_column_type_kernel` by forcing the mixed types column to be a string. With this optimization, we see a significant improvement in performance. Please refer to the [comment](https://github.com/rapidsai/cudf/pull/15236#issuecomment-1979772672) for a visualization of the results before and after applying this optimization as obtained from the [JSON lines benchmarking exercise](https://github.com/rapidsai/cudf/pull/15124). Authors: - Shruti Shivakumar (https://github.com/shrshi) - Karthikeyan (https://github.com/karthikeyann) Approvers: - Karthikeyan (https://github.com/karthikeyann) - Vukasin Milovanovic (https://github.com/vuule) URL: https://github.com/rapidsai/cudf/pull/15236 --- cpp/src/io/json/json_column.cu | 3 +++ cpp/src/io/json/nested_json.hpp | 2 ++ 2 files changed, 5 insertions(+) diff --git a/cpp/src/io/json/json_column.cu b/cpp/src/io/json/json_column.cu index 10646fad354..6576d41dd72 100644 --- a/cpp/src/io/json/json_column.cu +++ b/cpp/src/io/json/json_column.cu @@ -674,6 +674,7 @@ void make_device_json_column(device_span input, reinitialize_as_string(old_col_id, col); // all its children (which are already inserted) are ignored later. } + col.forced_as_string_column = true; columns.try_emplace(this_col_id, columns.at(old_col_id)); continue; } @@ -915,6 +916,8 @@ std::pair, std::vector> device_json_co : "n/a"); #endif target_type = schema.value().type; + } else if (json_col.forced_as_string_column) { + target_type = data_type{type_id::STRING}; } // Infer column type, if we don't have an explicit type for it else { diff --git a/cpp/src/io/json/nested_json.hpp b/cpp/src/io/json/nested_json.hpp index f41b024bb1e..64fffdb27fc 100644 --- a/cpp/src/io/json/nested_json.hpp +++ b/cpp/src/io/json/nested_json.hpp @@ -160,6 +160,8 @@ struct device_json_column { std::vector column_order; // Counting the current number of items in this column row_offset_t num_rows = 0; + // Force as string column + bool forced_as_string_column{false}; /** * @brief Construct a new d json column object From dc42182c92eea713538799a5d7ea7486d89d65b3 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 8 Mar 2024 15:33:58 -0600 Subject: [PATCH 380/384] Use NVTX from GitHub. (#15178) This PR removes the vendored copy of NVTX and instead fetches it from GitHub. Note: Consumers of libcudf internal `detail` headers will need to provide their own NVTX. This can be done by using the CMake code in this PR (or the sample CMake code in the [NVTX README](https://github.com/NVIDIA/NVTX?tab=readme-ov-file#cmake)), and calling `target_link_libraries(your_target PRIVATE nvtx3-cpp)`. Closes #6476. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Yunsong Wang (https://github.com/PointKernel) - Nghia Truong (https://github.com/ttnghia) - Robert Maynard (https://github.com/robertmaynard) URL: https://github.com/rapidsai/cudf/pull/15178 --- cpp/CMakeLists.txt | 4 +- cpp/benchmarks/CMakeLists.txt | 2 +- cpp/cmake/thirdparty/get_nvtx.cmake | 27 + .../developer_guide/DEVELOPER_GUIDE.md | 4 +- cpp/include/cudf/detail/nvtx/nvtx3.hpp | 1909 ----------------- cpp/include/cudf/detail/nvtx/ranges.hpp | 6 +- cpp/src/join/distinct_hash_join.cu | 4 +- cpp/tests/CMakeLists.txt | 2 +- java/src/main/native/CMakeLists.txt | 6 +- java/src/main/native/src/NvtxRangeJni.cpp | 4 +- .../main/native/src/NvtxUniqueRangeJni.cpp | 4 +- .../native/src/check_nvcomp_output_sizes.cu | 4 +- 12 files changed, 50 insertions(+), 1926 deletions(-) create mode 100644 cpp/cmake/thirdparty/get_nvtx.cmake delete mode 100644 cpp/include/cudf/detail/nvtx/nvtx3.hpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 36fef2201f1..ca8505fdb5e 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -182,6 +182,8 @@ endif() rapids_cpm_init() # find jitify include(cmake/thirdparty/get_jitify.cmake) +# find NVTX +include(cmake/thirdparty/get_nvtx.cmake) # find nvCOMP include(cmake/thirdparty/get_nvcomp.cmake) # find CCCL before rmm so that we get cudf's patched version of CCCL @@ -776,7 +778,7 @@ add_dependencies(cudf jitify_preprocess_run) target_link_libraries( cudf PUBLIC ${ARROW_LIBRARIES} CCCL::CCCL rmm::rmm - PRIVATE cuco::cuco ZLIB::ZLIB nvcomp::nvcomp kvikio::kvikio + PRIVATE nvtx3-cpp cuco::cuco ZLIB::ZLIB nvcomp::nvcomp kvikio::kvikio $ ) diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index ef25278877e..c82e475dece 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -25,7 +25,7 @@ target_compile_options( target_link_libraries( cudf_datagen PUBLIC GTest::gmock GTest::gtest benchmark::benchmark nvbench::nvbench Threads::Threads cudf - cudftestutil + cudftestutil nvtx3-cpp PRIVATE $ ) diff --git a/cpp/cmake/thirdparty/get_nvtx.cmake b/cpp/cmake/thirdparty/get_nvtx.cmake new file mode 100644 index 00000000000..c722c4f70f1 --- /dev/null +++ b/cpp/cmake/thirdparty/get_nvtx.cmake @@ -0,0 +1,27 @@ +# ============================================================================= +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +# This function finds NVTX and sets any additional necessary environment variables. +function(find_and_configure_nvtx) + rapids_cpm_find( + NVTX3 3.1.0 + GLOBAL_TARGETS nvtx3-c nvtx3-cpp + CPM_ARGS + GIT_REPOSITORY https://github.com/NVIDIA/NVTX.git + GIT_TAG v3.1.0 + GIT_SHALLOW TRUE SOURCE_SUBDIR c + ) +endfunction() + +find_and_configure_nvtx() diff --git a/cpp/doxygen/developer_guide/DEVELOPER_GUIDE.md b/cpp/doxygen/developer_guide/DEVELOPER_GUIDE.md index 935ca20b6fa..8188c466312 100644 --- a/cpp/doxygen/developer_guide/DEVELOPER_GUIDE.md +++ b/cpp/doxygen/developer_guide/DEVELOPER_GUIDE.md @@ -664,11 +664,11 @@ defaults. ## NVTX Ranges In order to aid in performance optimization and debugging, all compute intensive libcudf functions -should have a corresponding NVTX range. Choose between `CUDF_FUNC_RANGE` or `cudf::thread_range` +should have a corresponding NVTX range. Choose between `CUDF_FUNC_RANGE` or `cudf::scoped_range` for declaring NVTX ranges in the current scope: - Use the `CUDF_FUNC_RANGE()` macro if you want to use the name of the function as the name of the NVTX range -- Use `cudf::thread_range rng{"custom_name"};` to provide a custom name for the current scope's +- Use `cudf::scoped_range rng{"custom_name"};` to provide a custom name for the current scope's NVTX range For more information about NVTX, see [here](https://github.com/NVIDIA/NVTX/tree/dev/c). diff --git a/cpp/include/cudf/detail/nvtx/nvtx3.hpp b/cpp/include/cudf/detail/nvtx/nvtx3.hpp deleted file mode 100644 index 5d44c565077..00000000000 --- a/cpp/include/cudf/detail/nvtx/nvtx3.hpp +++ /dev/null @@ -1,1909 +0,0 @@ -/* - * Copyright (c) 2020-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#if defined(NVTX3_MINOR_VERSION) and NVTX3_MINOR_VERSION < 0 -#error \ - "Trying to #include NVTX version 3 in a source file where an older NVTX version has already been included. If you are not directly using NVTX (the NVIDIA Tools Extension library), you are getting this error because libraries you are using have included different versions of NVTX. Suggested solutions are: (1) reorder #includes so the newest NVTX version is included first, (2) avoid using the conflicting libraries in the same .c/.cpp file, or (3) update the library using the older NVTX version to use the newer version instead." -#endif - -/** - * @brief Semantic minor version number. - * - * Major version number is hardcoded into the "nvtx3" namespace/prefix. - * - * If this value is incremented, the above version include guard needs to be - * updated. - */ -#define NVTX3_MINOR_VERSION 0 - -#include - -#include - -/** - * @file nvtx3.hpp - * - * @brief Provides C++ constructs making the NVTX library safer and easier to - * use with zero overhead. - */ - -/** - * \mainpage - * \tableofcontents - * - * \section QUICK_START Quick Start - * - * To add NVTX ranges to your code, use the `nvtx3::thread_range` RAII object. A - * range begins when the object is created, and ends when the object is - * destroyed. - * - * \code{.cpp} - * #include "nvtx3.hpp" - * void some_function(){ - * // Begins a NVTX range with the message "some_function" - * // The range ends when some_function() returns and `r` is destroyed - * nvtx3::thread_range r{"some_function"}; - * - * for(int i = 0; i < 6; ++i){ - * nvtx3::thread_range loop{"loop range"}; - * std::this_thread::sleep_for(std::chrono::seconds{1}); - * } - * } // Range ends when `r` is destroyed - * \endcode - * - * The example code above generates the following timeline view in Nsight - * Systems: - * - * \image html - * https://raw.githubusercontent.com/jrhemstad/nvtx_wrappers/master/docs/example_range.png - * - * Alternatively, use the \ref MACROS like `NVTX3_FUNC_RANGE()` to add - * ranges to your code that automatically use the name of the enclosing function - * as the range's message. - * - * \code{.cpp} - * #include "nvtx3.hpp" - * void some_function(){ - * // Creates a range with a message "some_function" that ends when the - * enclosing - * // function returns - * NVTX3_FUNC_RANGE(); - * ... - * } - * \endcode - * - * - * \section Overview - * - * The NVTX library provides a set of functions for users to annotate their code - * to aid in performance profiling and optimization. These annotations provide - * information to tools like Nsight Systems to improve visualization of - * application timelines. - * - * \ref RANGES are one of the most commonly used NVTX constructs for annotating - * a span of time. For example, imagine a user wanted to see every time a - * function, `my_function`, is called and how long it takes to execute. This can - * be accomplished with an NVTX range created on the entry to the function and - * terminated on return from `my_function` using the push/pop C APIs: - * - * ``` - * void my_function(...){ - * nvtxRangePushA("my_function"); // Begins NVTX range - * // do work - * nvtxRangePop(); // Ends NVTX range - * } - * ``` - * - * One of the challenges with using the NVTX C API is that it requires manually - * terminating the end of the range with `nvtxRangePop`. This can be challenging - * if `my_function()` has multiple returns or can throw exceptions as it - * requires calling `nvtxRangePop()` before all possible return points. - * - * NVTX++ solves this inconvenience through the "RAII" technique by providing a - * `nvtx3::thread_range` class that begins a range at construction and ends the - * range on destruction. The above example then becomes: - * - * ``` - * void my_function(...){ - * nvtx3::thread_range r{"my_function"}; // Begins NVTX range - * // do work - * } // Range ends on exit from `my_function` when `r` is destroyed - * ``` - * - * The range object `r` is deterministically destroyed whenever `my_function` - * returns---ending the NVTX range without manual intervention. For more - * information, see \ref RANGES and `nvtx3::domain_thread_range`. - * - * Another inconvenience of the NVTX C APIs are the several constructs where the - * user is expected to initialize an object at the beginning of an application - * and reuse that object throughout the lifetime of the application. For example - * Domains, Categories, and Registered messages. - * - * Example: - * ``` - * nvtxDomainHandle_t D = nvtxDomainCreateA("my domain"); - * // Reuse `D` throughout the rest of the application - * ``` - * - * This can be problematic if the user application or library does not have an - * explicit initialization function called before all other functions to - * ensure that these long-lived objects are initialized before being used. - * - * NVTX++ makes use of the "construct on first use" technique to alleviate this - * inconvenience. In short, a function local static object is constructed upon - * the first invocation of a function and returns a reference to that object on - * all future invocations. See the documentation for - * `nvtx3::registered_message`, `nvtx3::domain`, `nvtx3::named_category`, and - * https://isocpp.org/wiki/faq/ctors#static-init-order-on-first-use for more - * information. - * - * Using construct on first use, the above example becomes: - * ``` - * struct my_domain{ static constexpr char const* name{"my domain"}; }; - * - * // The first invocation of `domain::get` for the type `my_domain` will - * // construct a `nvtx3::domain` object and return a reference to it. Future - * // invocations simply return a reference. - * nvtx3::domain const& D = nvtx3::domain::get(); - * ``` - * For more information about NVTX and how it can be used, see - * https://docs.nvidia.com/cuda/profiler-users-guide/index.html#nvtx and - * https://devblogs.nvidia.com/cuda-pro-tip-generate-custom-application-profile-timelines-nvtx/ - * for more information. - * - * \section RANGES Ranges - * - * Ranges are used to describe a span of time during the execution of an - * application. Common examples are using ranges to annotate the time it takes - * to execute a function or an iteration of a loop. - * - * NVTX++ uses RAII to automate the generation of ranges that are tied to the - * lifetime of objects. Similar to `std::lock_guard` in the C++ Standard - * Template Library. - * - * \subsection THREAD_RANGE Thread Range - * - * `nvtx3::domain_thread_range` is a class that begins a range upon construction - * and ends the range at destruction. This is one of the most commonly used - * constructs in NVTX++ and is useful for annotating spans of time on a - * particular thread. These ranges can be nested to arbitrary depths. - * - * `nvtx3::thread_range` is an alias for a `nvtx3::domain_thread_range` in the - * global NVTX domain. For more information about Domains, see \ref DOMAINS. - * - * Various attributes of a range can be configured constructing a - * `nvtx3::domain_thread_range` with a `nvtx3::event_attributes` object. For - * more information, see \ref ATTRIBUTES. - * - * Example: - * - * \code{.cpp} - * void some_function(){ - * // Creates a range for the duration of `some_function` - * nvtx3::thread_range r{}; - * - * while(true){ - * // Creates a range for every loop iteration - * // `loop_range` is nested inside `r` - * nvtx3::thread_range loop_range{}; - * } - * } - * \endcode - * - * \subsection PROCESS_RANGE Process Range - * - * `nvtx3::domain_process_range` is identical to `nvtx3::domain_thread_range` - * with the exception that a `domain_process_range` can be created and destroyed - * on different threads. This is useful to annotate spans of time that can - * bridge multiple threads. - * - * `nvtx3::domain_thread_range`s should be preferred unless one needs the - * ability to begin and end a range on different threads. - * - * \section MARKS Marks - * - * `nvtx3::mark` allows annotating an instantaneous event in an application's - * timeline. For example, indicating when a mutex is locked or unlocked. - * - * \code{.cpp} - * std::mutex global_lock; - * void lock_mutex(){ - * global_lock.lock(); - * // Marks an event immediately after the mutex is locked - * nvtx3::mark("lock_mutex"); - * } - * \endcode - * - * \section DOMAINS Domains - * - * Similar to C++ namespaces, Domains allow for scoping NVTX events. By default, - * all NVTX events belong to the "global" domain. Libraries and applications - * should scope their events to use a custom domain to differentiate where the - * events originate from. - * - * It is common for a library or application to have only a single domain and - * for the name of that domain to be known at compile time. Therefore, Domains - * in NVTX++ are represented by _tag types_. - * - * For example, to define a custom domain, simply define a new concrete type - * (a `class` or `struct`) with a `static` member called `name` that contains - * the desired name of the domain. - * - * ``` - * struct my_domain{ static constexpr char const* name{"my domain"}; }; - * ``` - * - * For any NVTX++ construct that can be scoped to a domain, the type `my_domain` - * can be passed as an explicit template argument to scope it to the custom - * domain. - * - * The tag type `nvtx3::domain::global` represents the global NVTX domain. - * - * \code{.cpp} - * // By default, `domain_thread_range` belongs to the global domain - * nvtx3::domain_thread_range<> r0{}; - * - * // Alias for a `domain_thread_range` in the global domain - * nvtx3::thread_range r1{}; - * - * // `r` belongs to the custom domain - * nvtx3::domain_thread_range r{}; - * \endcode - * - * When using a custom domain, it is recommended to define type aliases for NVTX - * constructs in the custom domain. - * ``` - * using my_thread_range = nvtx3::domain_thread_range; - * using my_registered_message = nvtx3::registered_message; - * using my_named_category = nvtx3::named_category; - * ``` - * - * See `nvtx3::domain` for more information. - * - * \section ATTRIBUTES Event Attributes - * - * NVTX events can be customized with various attributes to provide additional - * information (such as a custom message) or to control visualization of the - * event (such as the color used). These attributes can be specified per-event - * via arguments to a `nvtx3::event_attributes` object. - * - * NVTX events can be customized via four "attributes": - * - \ref COLOR : color used to visualize the event in tools. - * - \ref MESSAGES : Custom message string. - * - \ref PAYLOAD : User-defined numerical value. - * - \ref CATEGORY : Intra-domain grouping. - * - * It is possible to construct a `nvtx3::event_attributes` from any number of - * attribute objects (nvtx3::color, nvtx3::message, nvtx3::payload, - * nvtx3::category) in any order. If an attribute is not specified, a tool - * specific default value is used. See `nvtx3::event_attributes` for more - * information. - * - * \code{.cpp} - * // Custom color, message - * event_attributes attr{nvtx3::rgb{127, 255, 0}, - * "message"}; - * - * // Custom color, message, payload, category - * event_attributes attr{nvtx3::rgb{127, 255, 0}, - * nvtx3::payload{42}, - * "message", - * nvtx3::category{1}}; - * - * // Arguments can be in any order - * event_attributes attr{nvtx3::payload{42}, - * nvtx3::category{1}, - * "message", - * nvtx3::rgb{127, 255, 0}}; - * - * // "First wins" with multiple arguments of the same type - * event_attributes attr{ nvtx3::payload{42}, nvtx3::payload{7} }; // payload is - * 42 \endcode - * - * \subsection MESSAGES message - * - * A `nvtx3::message` allows associating a custom message string with an NVTX - * event. - * - * Example: - * \code{.cpp} - * // Create an `event_attributes` with the custom message "my message" - * nvtx3::event_attributes attr{nvtx3::message{"my message"}}; - * - * // strings and string literals implicitly assumed to be a `nvtx3::message` - * nvtx3::event_attributes attr{"my message"}; - * \endcode - * - * \subsubsection REGISTERED_MESSAGE Registered Messages - * - * Associating a `nvtx3::message` with an event requires copying the contents of - * the message every time the message is used, i.e., copying the entire message - * string. This may cause non-trivial overhead in performance sensitive code. - * - * To eliminate this overhead, NVTX allows registering a message string, - * yielding a "handle" that is inexpensive to copy that may be used in place of - * a message string. When visualizing the events, tools such as Nsight Systems - * will take care of mapping the message handle to its string. - * - * A message should be registered once and the handle reused throughout the rest - * of the application. This can be done by either explicitly creating static - * `nvtx3::registered_message` objects, or using the - * `nvtx3::registered_message::get` construct on first use helper (recommended). - * - * Similar to \ref DOMAINS, `nvtx3::registered_message::get` requires defining a - * custom tag type with a static `message` member whose value will be the - * contents of the registered string. - * - * Example: - * \code{.cpp} - * // Explicitly constructed, static `registered_message` - * static registered_message static_message{"my message"}; - * - * // Or use construct on first use: - * // Define a tag type with a `message` member string to register - * struct my_message{ static constexpr char const* message{ "my message" }; }; - * - * // Uses construct on first use to register the contents of - * // `my_message::message` - * nvtx3::registered_message const& msg = - * nvtx3::registered_message::get(); \endcode - * - * \subsection COLOR color - * - * Associating a `nvtx3::color` with an event allows controlling how the event - * is visualized in a tool such as Nsight Systems. This is a convenient way to - * visually differentiate among different events. - * - * \code{.cpp} - * // Define a color via rgb color values - * nvtx3::color c{nvtx3::rgb{127, 255, 0}}; - * nvtx3::event_attributes attr{c}; - * - * // rgb color values can be passed directly to an `event_attributes` - * nvtx3::event_attributes attr1{nvtx3::rgb{127,255,0}}; - * \endcode - * - * \subsection CATEGORY category - * - * A `nvtx3::category` is simply an integer id that allows for fine-grain - * grouping of NVTX events. For example, one might use separate categories for - * IO, memory allocation, compute, etc. - * - * \code{.cpp} - * nvtx3::event_attributes{nvtx3::category{1}}; - * \endcode - * - * \subsubsection NAMED_CATEGORIES Named Categories - * - * Associates a `name` string with a category `id` to help differentiate among - * categories. - * - * For any given category id `Id`, a `named_category{Id, "name"}` should only - * be constructed once and reused throughout an application. This can be done by - * either explicitly creating static `nvtx3::named_category` objects, or using - * the `nvtx3::named_category::get` construct on first use helper (recommended). - * - * Similar to \ref DOMAINS, `nvtx3::named_category::get` requires defining a - * custom tag type with static `name` and `id` members. - * - * \code{.cpp} - * // Explicitly constructed, static `named_category` - * static nvtx3::named_category static_category{42, "my category"}; - * - * // OR use construct on first use: - * // Define a tag type with `name` and `id` members - * struct my_category{ - * static constexpr char const* name{"my category"}; // category name - * static constexpr category::id_type id{42}; // category id - * }; - * - * // Use construct on first use to name the category id `42` - * // with name "my category" - * nvtx3::named_category const& my_category = - * named_category::get(); - * - * // Range `r` associated with category id `42` - * nvtx3::event_attributes attr{my_category}; - * \endcode - * - * \subsection PAYLOAD payload - * - * Allows associating a user-defined numerical value with an event. - * - * ``` - * nvtx3:: event_attributes attr{nvtx3::payload{42}}; // Constructs a payload - * from - * // the `int32_t` value 42 - * ``` - * - * - * \section EXAMPLE Example - * - * Putting it all together: - * \code{.cpp} - * // Define a custom domain tag type - * struct my_domain{ static constexpr char const* name{"my domain"}; }; - * - * // Define a named category tag type - * struct my_category{ - * static constexpr char const* name{"my category"}; - * static constexpr uint32_t id{42}; - * }; - * - * // Define a registered message tag type - * struct my_message{ static constexpr char const* message{"my message"}; }; - * - * // For convenience, use aliases for domain scoped objects - * using my_thread_range = nvtx3::domain_thread_range; - * using my_registered_message = nvtx3::registered_message; - * using my_named_category = nvtx3::named_category; - * - * // Default values for all attributes - * nvtx3::event_attributes attr{}; - * my_thread_range r0{attr}; - * - * // Custom (unregistered) message, and unnamed category - * nvtx3::event_attributes attr1{"message", nvtx3::category{2}}; - * my_thread_range r1{attr1}; - * - * // Alternatively, pass arguments of `event_attributes` ctor directly to - * // `my_thread_range` - * my_thread_range r2{"message", nvtx3::category{2}}; - * - * // construct on first use a registered message - * auto msg = my_registered_message::get(); - * - * // construct on first use a named category - * auto category = my_named_category::get(); - * - * // Use registered message and named category - * my_thread_range r3{msg, category, nvtx3::rgb{127, 255, 0}, - * nvtx3::payload{42}}; - * - * // Any number of arguments in any order - * my_thread_range r{nvtx3::rgb{127, 255,0}, msg}; - * - * \endcode - * \section MACROS Convenience Macros - * - * Oftentimes users want to quickly and easily add NVTX ranges to their library - * or application to aid in profiling and optimization. - * - * A convenient way to do this is to use the \ref NVTX3_FUNC_RANGE and - * \ref NVTX3_FUNC_RANGE_IN macros. These macros take care of constructing an - * `nvtx3::domain_thread_range` with the name of the enclosing function as the - * range's message. - * - * \code{.cpp} - * void some_function(){ - * // Automatically generates an NVTX range for the duration of the function - * // using "some_function" as the event's message. - * NVTX3_FUNC_RANGE(); - * } - * \endcode - */ - -/** - * @brief Enables the use of constexpr when support for C++14 relaxed constexpr - * is present. - * - * Initializing a legacy-C (i.e., no constructor) union member requires - * initializing in the constructor body. Non-empty constexpr constructors - * require C++14 relaxed constexpr. - */ -#if __cpp_constexpr >= 201304L -#define NVTX3_RELAXED_CONSTEXPR constexpr -#else -#define NVTX3_RELAXED_CONSTEXPR -#endif - -namespace nvtx3 { -namespace detail { -/** - * @brief Verifies if a type `T` contains a member `T::name` of type `const - * char*` or `const wchar_t*`. - * - * @tparam T The type to verify - * @return True if `T` contains a member `T::name` of type `const char*` or - * `const wchar_t*`. - */ -template -constexpr auto has_name_member() noexcept -> decltype(T::name, bool()) -{ - return (std::is_same_v::type> or - std::is_same_v::type>); -} -} // namespace detail - -/** - * @brief `domain`s allow for grouping NVTX events into a single scope to - * differentiate them from events in other `domain`s. - * - * By default, all NVTX constructs are placed in the "global" NVTX domain. - * - * A custom `domain` may be used in order to differentiate a library's or - * application's NVTX events from other events. - * - * `domain`s are expected to be long-lived and unique to a library or - * application. As such, it is assumed a domain's name is known at compile - * time. Therefore, all NVTX constructs that can be associated with a domain - * require the domain to be specified via a *type* `DomainName` passed as an - * explicit template parameter. - * - * The type `domain::global` may be used to indicate that the global NVTX - * domain should be used. - * - * None of the C++ NVTX constructs require the user to manually construct a - * `domain` object. Instead, if a custom domain is desired, the user is - * expected to define a type `DomainName` that contains a member - * `DomainName::name` which resolves to either a `char const*` or `wchar_t - * const*`. The value of `DomainName::name` is used to name and uniquely - * identify the custom domain. - * - * Upon the first use of an NVTX construct associated with the type - * `DomainName`, the "construct on first use" pattern is used to construct a - * function local static `domain` object. All future NVTX constructs - * associated with `DomainType` will use a reference to the previously - * constructed `domain` object. See `domain::get`. - * - * Example: - * ``` - * // The type `my_domain` defines a `name` member used to name and identify - * the - * // `domain` object identified by `my_domain`. - * struct my_domain{ static constexpr char const* name{"my_domain"}; }; - * - * // The NVTX range `r` will be grouped with all other NVTX constructs - * // associated with `my_domain`. - * nvtx3::domain_thread_range r{}; - * - * // An alias can be created for a `domain_thread_range` in the custom domain - * using my_thread_range = nvtx3::domain_thread_range; - * my_thread_range my_range{}; - * - * // `domain::global` indicates that the global NVTX domain is used - * nvtx3::domain_thread_range r2{}; - * - * // For convenience, `nvtx3::thread_range` is an alias for a range in the - * // global domain - * nvtx3::thread_range r3{}; - * ``` - */ -class domain { - public: - domain(domain const&) = delete; - domain& operator=(domain const&) = delete; - domain(domain&&) = delete; - domain& operator=(domain&&) = delete; - - /** - * @brief Returns reference to an instance of a function local static - * `domain` object. - * - * Uses the "construct on first use" idiom to safely ensure the `domain` - * object is initialized exactly once upon first invocation of - * `domain::get()`. All following invocations will return a - * reference to the previously constructed `domain` object. See - * https://isocpp.org/wiki/faq/ctors#static-init-order-on-first-use - * - * None of the constructs in this header require the user to directly invoke - * `domain::get`. It is automatically invoked when constructing objects like - * a `domain_thread_range` or `category`. Advanced users may wish to use - * `domain::get` for the convenience of the "construct on first use" idiom - * when using domains with their own use of the NVTX C API. - * - * This function is threadsafe as of C++11. If two or more threads call - * `domain::get` concurrently, exactly one of them is guaranteed - * to construct the `domain` object and the other(s) will receive a - * reference to the object after it is fully constructed. - * - * The domain's name is specified via the type `DomainName` pass as an - * explicit template parameter. `DomainName` is required to contain a - * member `DomainName::name` that resolves to either a `char const*` or - * `wchar_t const*`. The value of `DomainName::name` is used to name and - * uniquely identify the `domain`. - * - * Example: - * ``` - * // The type `my_domain` defines a `name` member used to name and identify - * // the `domain` object identified by `my_domain`. - * struct my_domain{ static constexpr char const* name{"my domain"}; }; - * - * auto D = domain::get(); // First invocation constructs a - * // `domain` with the name "my domain" - * - * auto D1 = domain::get(); // Simply returns reference to - * // previously constructed `domain`. - * ``` - * - * @tparam DomainName Type that contains a `DomainName::name` member used to - * name the `domain` object. - * @return Reference to the `domain` corresponding to the type `DomainName`. - */ - template - static domain const& get() - { - static_assert(detail::has_name_member(), - "Type used to identify a domain must contain a name member of" - "type const char* or const wchar_t*"); - static domain const d{DomainName::name}; - return d; - } - - /** - * @brief Conversion operator to `nvtxDomainHandle_t`. - * - * Allows transparently passing a domain object into an API expecting a - * native `nvtxDomainHandle_t` object. - */ - operator nvtxDomainHandle_t() const noexcept { return _domain; } - - /** - * @brief Tag type for the "global" NVTX domain. - * - * This type may be passed as a template argument to any function/class - * expecting a type to identify a domain to indicate that the global domain - * should be used. - * - * All NVTX events in the global domain across all libraries and - * applications will be grouped together. - * - */ - struct global {}; - - private: - /** - * @brief Construct a new domain with the specified `name`. - * - * This constructor is private as it is intended that `domain` objects only - * be created through the `domain::get` function. - * - * @param name A unique name identifying the domain - */ - explicit domain(char const* name) noexcept : _domain{nvtxDomainCreateA(name)} {} - - /** - * @brief Construct a new domain with the specified `name`. - * - * This constructor is private as it is intended that `domain` objects only - * be created through the `domain::get` function. - * - * @param name A unique name identifying the domain - */ - explicit domain(wchar_t const* name) noexcept : _domain{nvtxDomainCreateW(name)} {} - - /** - * @brief Construct a new domain with the specified `name`. - * - * This constructor is private as it is intended that `domain` objects only - * be created through the `domain::get` function. - * - * @param name A unique name identifying the domain - */ - explicit domain(std::string const& name) noexcept : domain{name.c_str()} {} - - /** - * @brief Construct a new domain with the specified `name`. - * - * This constructor is private as it is intended that `domain` objects only - * be created through the `domain::get` function. - * - * @param name A unique name identifying the domain - */ - explicit domain(std::wstring const& name) noexcept : domain{name.c_str()} {} - - /** - * @brief Default constructor creates a `domain` representing the - * "global" NVTX domain. - * - * All events not associated with a custom `domain` are grouped in the - * "global" NVTX domain. - * - */ - domain() = default; - - /** - * @brief Destroy the domain object, unregistering and freeing all domain - * specific resources. - */ - ~domain() noexcept { nvtxDomainDestroy(_domain); } - - private: - nvtxDomainHandle_t const _domain{}; ///< The `domain`s NVTX handle -}; - -/** - * @brief Returns reference to the `domain` object that represents the global - * NVTX domain. - * - * This specialization for `domain::global` returns a default constructed, - * `domain` object for use when the "global" domain is desired. - * - * All NVTX events in the global domain across all libraries and applications - * will be grouped together. - * - * @return Reference to the `domain` corresponding to the global NVTX domain. - */ -template <> -inline domain const& domain::get() -{ - static domain const d{}; - return d; -} - -/** - * @brief Indicates the values of the red, green, blue color channels for - * a rgb color code. - */ -struct rgb { - /// Type used for component values - using component_type = uint8_t; - - /** - * @brief Construct a rgb with red, green, and blue channels - * specified by `red_`, `green_`, and `blue_`, respectively. - * - * Valid values are in the range `[0,255]`. - * - * @param red_ Value of the red channel - * @param green_ Value of the green channel - * @param blue_ Value of the blue channel - */ - constexpr rgb(component_type red_, component_type green_, component_type blue_) noexcept - : red{red_}, green{green_}, blue{blue_} - { - } - - component_type const red{}; ///< Red channel value - component_type const green{}; ///< Green channel value - component_type const blue{}; ///< Blue channel value -}; - -/** - * @brief Indicates the value of the alpha, red, green, and blue color - * channels for an argb color code. - */ -struct argb final : rgb { - /** - * @brief Construct an argb with alpha, red, green, and blue channels - * specified by `alpha_`, `red_`, `green_`, and `blue_`, respectively. - * - * Valid values are in the range `[0,255]`. - * - * @param alpha_ Value of the alpha channel (opacity) - * @param red_ Value of the red channel - * @param green_ Value of the green channel - * @param blue_ Value of the blue channel - * - */ - constexpr argb(component_type alpha_, - component_type red_, - component_type green_, - component_type blue_) noexcept - : rgb{red_, green_, blue_}, alpha{alpha_} - { - } - - component_type const alpha{}; ///< Alpha channel value -}; - -/** - * @brief Represents a custom color that can be associated with an NVTX event - * via it's `event_attributes`. - * - * Specifying colors for NVTX events is a convenient way to visually - * differentiate among different events in a visualization tool such as Nsight - * Systems. - */ -class color { - public: - /// Type used for the color's value - using value_type = uint32_t; - - /** - * @brief Constructs a `color` using the value provided by `hex_code`. - * - * `hex_code` is expected to be a 4 byte argb hex code. - * - * The most significant byte indicates the value of the alpha channel - * (opacity) (0-255) - * - * The next byte indicates the value of the red channel (0-255) - * - * The next byte indicates the value of the green channel (0-255) - * - * The least significant byte indicates the value of the blue channel - * (0-255) - * - * @param hex_code The hex code used to construct the `color` - */ - constexpr explicit color(value_type hex_code) noexcept : _value{hex_code} {} - - /** - * @brief Construct a `color` using the alpha, red, green, blue components - * in `argb`. - * - * @param argb The alpha, red, green, blue components of the desired `color` - */ - constexpr color(argb argb) noexcept - : color{from_bytes_msb_to_lsb(argb.alpha, argb.red, argb.green, argb.blue)} - { - } - - /** - * @brief Construct a `color` using the red, green, blue components in - * `rgb`. - * - * Uses maximum value for the alpha channel (opacity) of the `color`. - * - * @param rgb The red, green, blue components of the desired `color` - */ - constexpr color(rgb rgb) noexcept - : color{from_bytes_msb_to_lsb(0xFF, rgb.red, rgb.green, rgb.blue)} - { - } - - /** - * @brief Returns the `color`s argb hex code - * - */ - constexpr value_type get_value() const noexcept { return _value; } - - /** - * @brief Return the NVTX color type of the color. - * - */ - constexpr nvtxColorType_t get_type() const noexcept { return _type; } - - color() = delete; - ~color() = default; - color(color const&) = default; - color& operator=(color const&) = default; - color(color&&) = default; - color& operator=(color&&) = default; - - private: - /** - * @brief Constructs an unsigned, 4B integer from the component bytes in - * most to least significant byte order. - * - */ - constexpr static value_type from_bytes_msb_to_lsb(uint8_t byte3, - uint8_t byte2, - uint8_t byte1, - uint8_t byte0) noexcept - { - return uint32_t{byte3} << 24 | uint32_t{byte2} << 16 | uint32_t{byte1} << 8 | uint32_t{byte0}; - } - - value_type const _value{}; ///< color's argb color code - nvtxColorType_t const _type{NVTX_COLOR_ARGB}; ///< NVTX color type code -}; - -/** - * @brief Object for intra-domain grouping of NVTX events. - * - * A `category` is simply an integer id that allows for fine-grain grouping of - * NVTX events. For example, one might use separate categories for IO, memory - * allocation, compute, etc. - * - * Example: - * \code{.cpp} - * nvtx3::category cat1{1}; - * - * // Range `r1` belongs to the category identified by the value `1`. - * nvtx3::thread_range r1{cat1}; - * - * // Range `r2` belongs to the same category as `r1` - * nvtx3::thread_range r2{nvtx3::category{1}}; - * \endcode - * - * To associate a name string with a category id, see `named_category`. - */ -class category { - public: - /// Type used for `category`s integer id. - using id_type = uint32_t; - - /** - * @brief Construct a `category` with the specified `id`. - * - * The `category` will be unnamed and identified only by its `id` value. - * - * All `category` objects sharing the same `id` are equivalent. - * - * @param[in] id The `category`'s identifying value - */ - constexpr explicit category(id_type id) noexcept : id_{id} {} - - /** - * @brief Returns the id of the category. - * - */ - constexpr id_type get_id() const noexcept { return id_; } - - category() = delete; - ~category() = default; - category(category const&) = default; - category& operator=(category const&) = default; - category(category&&) = default; - category& operator=(category&&) = default; - - private: - id_type const id_{}; ///< category's unique identifier -}; - -/** - * @brief A `category` with an associated name string. - * - * Associates a `name` string with a category `id` to help differentiate among - * categories. - * - * For any given category id `Id`, a `named_category(Id, "name")` should only - * be constructed once and reused throughout an application. This can be done - * by either explicitly creating static `named_category` objects, or using the - * `named_category::get` construct on first use helper (recommended). - * - * Creating two or more `named_category` objects with the same value for `id` - * in the same domain results in undefined behavior. - * - * Similarly, behavior is undefined when a `named_category` and `category` - * share the same value of `id`. - * - * Example: - * \code{.cpp} - * // Explicitly constructed, static `named_category` - * static nvtx3::named_category static_category{42, "my category"}; - * - * // Range `r` associated with category id `42` - * nvtx3::thread_range r{static_category}; - * - * // OR use construct on first use: - * - * // Define a type with `name` and `id` members - * struct my_category{ - * static constexpr char const* name{"my category"}; // category name - * static constexpr category::id_type id{42}; // category id - * }; - * - * // Use construct on first use to name the category id `42` - * // with name "my category" - * auto my_category = named_category::get(); - * - * // Range `r` associated with category id `42` - * nvtx3::thread_range r{my_category}; - * \endcode - * - * `named_category`'s association of a name to a category id is local to the - * domain specified by the type `D`. An id may have a different name in - * another domain. - * - * @tparam D Type containing `name` member used to identify the `domain` to - * which the `named_category` belongs. Else, `domain::global` to indicate - * that the global NVTX domain should be used. - */ -template -class named_category final : public category { - public: - /** - * @brief Returns a global instance of a `named_category` as a - * function-local static. - * - * Creates a `named_category` with name and id specified by the contents of - * a type `C`. `C::name` determines the name and `C::id` determines the - * category id. - * - * This function is useful for constructing a named `category` exactly once - * and reusing the same instance throughout an application. - * - * Example: - * \code{.cpp} - * // Define a type with `name` and `id` members - * struct my_category{ - * static constexpr char const* name{"my category"}; // category name - * static constexpr uint32_t id{42}; // category id - * }; - * - * // Use construct on first use to name the category id `42` - * // with name "my category" - * auto cat = named_category::get(); - * - * // Range `r` associated with category id `42` - * nvtx3::thread_range r{cat}; - * \endcode - * - * Uses the "construct on first use" idiom to safely ensure the `category` - * object is initialized exactly once. See - * https://isocpp.org/wiki/faq/ctors#static-init-order-on-first-use - * - * @tparam C Type containing a member `C::name` that resolves to either a - * `char const*` or `wchar_t const*` and `C::id`. - */ - template - static named_category const& get() noexcept - { - static_assert(detail::has_name_member(), - "Type used to name a category must contain a name member."); - static named_category const category{C::id, C::name}; - return category; - } - /** - * @brief Construct a `category` with the specified `id` and `name`. - * - * The name `name` will be registered with `id`. - * - * Every unique value of `id` should only be named once. - * - * @param[in] id The category id to name - * @param[in] name The name to associated with `id` - */ - named_category(id_type id, char const* name) noexcept : category{id} - { - nvtxDomainNameCategoryA(domain::get(), get_id(), name); - }; - - /** - * @brief Construct a `category` with the specified `id` and `name`. - * - * The name `name` will be registered with `id`. - * - * Every unique value of `id` should only be named once. - * - * @param[in] id The category id to name - * @param[in] name The name to associated with `id` - */ - named_category(id_type id, wchar_t const* name) noexcept : category{id} - { - nvtxDomainNameCategoryW(domain::get(), get_id(), name); - }; -}; - -/** - * @brief A message registered with NVTX. - * - * Normally, associating a `message` with an NVTX event requires copying the - * contents of the message string. This may cause non-trivial overhead in - * highly performance sensitive regions of code. - * - * message registration is an optimization to lower the overhead of - * associating a message with an NVTX event. Registering a message yields a - * handle that is inexpensive to copy that may be used in place of a message - * string. - * - * A particular message should only be registered once and the handle - * reused throughout the rest of the application. This can be done by either - * explicitly creating static `registered_message` objects, or using the - * `registered_message::get` construct on first use helper (recommended). - * - * Example: - * \code{.cpp} - * // Explicitly constructed, static `registered_message` - * static registered_message static_message{"message"}; - * - * // "message" is associated with the range `r` - * nvtx3::thread_range r{static_message}; - * - * // Or use construct on first use: - * - * // Define a type with a `message` member that defines the contents of the - * // registered message - * struct my_message{ static constexpr char const* message{ "my message" }; }; - * - * // Uses construct on first use to register the contents of - * // `my_message::message` - * auto msg = registered_message::get(); - * - * // "my message" is associated with the range `r` - * nvtx3::thread_range r{msg}; - * \endcode - * - * `registered_message`s are local to a particular domain specified via - * the type `D`. - * - * @tparam D Type containing `name` member used to identify the `domain` to - * which the `registered_message` belongs. Else, `domain::global` to indicate - * that the global NVTX domain should be used. - */ -template -class registered_message { - public: - /** - * @brief Returns a global instance of a `registered_message` as a function - * local static. - * - * Provides a convenient way to register a message with NVTX without having - * to explicitly register the message. - * - * Upon first invocation, constructs a `registered_message` whose contents - * are specified by `message::message`. - * - * All future invocations will return a reference to the object constructed - * in the first invocation. - * - * Example: - * \code{.cpp} - * // Define a type with a `message` member that defines the contents of the - * // registered message - * struct my_message{ static constexpr char const* message{ "my message" }; - * }; - * - * // Uses construct on first use to register the contents of - * // `my_message::message` - * auto msg = registered_message::get(); - * - * // "my message" is associated with the range `r` - * nvtx3::thread_range r{msg}; - * \endcode - * - * @tparam M Type required to contain a member `M::message` that - * resolves to either a `char const*` or `wchar_t const*` used as the - * registered message's contents. - * @return Reference to a `registered_message` associated with the type `M`. - */ - template - static registered_message const& get() noexcept - { - static registered_message const registered_message{M::message}; - return registered_message; - } - - /** - * @brief Constructs a `registered_message` from the specified `msg` string. - * - * Registers `msg` with NVTX and associates a handle with the registered - * message. - * - * A particular message should only be registered once and the handle - * reused throughout the rest of the application. - * - * @param msg The contents of the message - */ - explicit registered_message(char const* msg) noexcept - : handle_{nvtxDomainRegisterStringA(domain::get(), msg)} - { - } - - /** - * @brief Constructs a `registered_message` from the specified `msg` string. - * - * Registers `msg` with NVTX and associates a handle with the registered - * message. - * - * A particular message should only be registered once and the handle - * reused throughout the rest of the application. - * - * @param msg The contents of the message - */ - explicit registered_message(std::string const& msg) noexcept : registered_message{msg.c_str()} {} - - /** - * @brief Constructs a `registered_message` from the specified `msg` string. - * - * Registers `msg` with NVTX and associates a handle with the registered - * message. - * - * A particular message should only be registered once and the handle - * reused throughout the rest of the application. - * - * @param msg The contents of the message - */ - explicit registered_message(wchar_t const* msg) noexcept - : handle_{nvtxDomainRegisterStringW(domain::get(), msg)} - { - } - - /** - * @brief Constructs a `registered_message` from the specified `msg` string. - * - * Registers `msg` with NVTX and associates a handle with the registered - * message. - * - * A particular message should only be registered once and the handle - * reused throughout the rest of the application. - * - * @param msg The contents of the message - */ - explicit registered_message(std::wstring const& msg) noexcept : registered_message{msg.c_str()} {} - - /** - * @brief Returns the registered message's handle - * - */ - nvtxStringHandle_t get_handle() const noexcept { return handle_; } - - registered_message() = delete; - ~registered_message() = default; - registered_message(registered_message const&) = default; - registered_message& operator=(registered_message const&) = default; - registered_message(registered_message&&) = default; - registered_message& operator=(registered_message&&) = default; - - private: - nvtxStringHandle_t const handle_{}; ///< The handle returned from - ///< registering the message with NVTX -}; - -/** - * @brief Allows associating a message string with an NVTX event via - * its `EventAttribute`s. - * - * Associating a `message` with an NVTX event through its `event_attributes` - * allows for naming events to easily differentiate them from other events. - * - * Every time an NVTX event is created with an associated `message`, the - * contents of the message string must be copied. This may cause non-trivial - * overhead in highly performance sensitive sections of code. Use of a - * `nvtx3::registered_message` is recommended in these situations. - * - * Example: - * \code{.cpp} - * // Creates an `event_attributes` with message "message 0" - * nvtx3::event_attributes attr0{nvtx3::message{"message 0"}}; - * - * // `range0` contains message "message 0" - * nvtx3::thread_range range0{attr0}; - * - * // `std::string` and string literals are implicitly assumed to be - * // the contents of an `nvtx3::message` - * // Creates an `event_attributes` with message "message 1" - * nvtx3::event_attributes attr1{"message 1"}; - * - * // `range1` contains message "message 1" - * nvtx3::thread_range range1{attr1}; - * - * // `range2` contains message "message 2" - * nvtx3::thread_range range2{nvtx3::message{"message 2"}}; - * - * // `std::string` and string literals are implicitly assumed to be - * // the contents of an `nvtx3::message` - * // `range3` contains message "message 3" - * nvtx3::thread_range range3{"message 3"}; - * \endcode - */ -class message { - public: - using value_type = nvtxMessageValue_t; - - /** - * @brief Construct a `message` whose contents are specified by `msg`. - * - * @param msg The contents of the message - */ - NVTX3_RELAXED_CONSTEXPR message(char const* msg) noexcept : type_{NVTX_MESSAGE_TYPE_ASCII} - { - value_.ascii = msg; - } - - /** - * @brief Construct a `message` whose contents are specified by `msg`. - * - * @param msg The contents of the message - */ - message(std::string const& msg) noexcept : message{msg.c_str()} {} - - /** - * @brief Disallow construction for `std::string` r-value - * - * `message` is a non-owning type and therefore cannot take ownership of an - * r-value. Therefore, constructing from an r-value is disallowed to prevent - * a dangling pointer. - * - */ - message(std::string&&) = delete; - - /** - * @brief Construct a `message` whose contents are specified by `msg`. - * - * @param msg The contents of the message - */ - NVTX3_RELAXED_CONSTEXPR message(wchar_t const* msg) noexcept : type_{NVTX_MESSAGE_TYPE_UNICODE} - { - value_.unicode = msg; - } - - /** - * @brief Construct a `message` whose contents are specified by `msg`. - * - * @param msg The contents of the message - */ - message(std::wstring const& msg) noexcept : message{msg.c_str()} {} - - /** - * @brief Disallow construction for `std::wstring` r-value - * - * `message` is a non-owning type and therefore cannot take ownership of an - * r-value. Therefore, constructing from an r-value is disallowed to prevent - * a dangling pointer. - * - */ - message(std::wstring&&) = delete; - - /** - * @brief Construct a `message` from a `registered_message`. - * - * @tparam D Type containing `name` member used to identify the `domain` - * to which the `registered_message` belongs. Else, `domain::global` to - * indicate that the global NVTX domain should be used. - * @param msg The message that has already been registered with NVTX. - */ - template - message(registered_message const& msg) noexcept : type_{NVTX_MESSAGE_TYPE_REGISTERED} - { - value_.registered = msg.get_handle(); - } - - /** - * @brief Return the union holding the value of the message. - * - */ - NVTX3_RELAXED_CONSTEXPR value_type get_value() const noexcept { return value_; } - - /** - * @brief Return the type information about the value the union holds. - * - */ - NVTX3_RELAXED_CONSTEXPR nvtxMessageType_t get_type() const noexcept { return type_; } - - private: - nvtxMessageType_t const type_{}; ///< message type - nvtxMessageValue_t value_{}; ///< message contents -}; - -/** - * @brief A numerical value that can be associated with an NVTX event via - * its `event_attributes`. - * - * Example: - * ``` - * nvtx3:: event_attributes attr{nvtx3::payload{42}}; // Constructs a payload - * from - * // the `int32_t` value 42 - * - * // `range0` will have an int32_t payload of 42 - * nvtx3::thread_range range0{attr}; - * - * // range1 has double payload of 3.14 - * nvtx3::thread_range range1{ nvtx3::payload{3.14} }; - * ``` - */ -class payload { - public: - using value_type = typename nvtxEventAttributes_v2::payload_t; - - /** - * @brief Construct a `payload` from a signed, 8 byte integer. - * - * @param value Value to use as contents of the payload - */ - NVTX3_RELAXED_CONSTEXPR explicit payload(int64_t value) noexcept - : type_{NVTX_PAYLOAD_TYPE_INT64}, value_{} - { - value_.llValue = value; - } - - /** - * @brief Construct a `payload` from a signed, 4 byte integer. - * - * @param value Value to use as contents of the payload - */ - NVTX3_RELAXED_CONSTEXPR explicit payload(int32_t value) noexcept - : type_{NVTX_PAYLOAD_TYPE_INT32}, value_{} - { - value_.iValue = value; - } - - /** - * @brief Construct a `payload` from an unsigned, 8 byte integer. - * - * @param value Value to use as contents of the payload - */ - NVTX3_RELAXED_CONSTEXPR explicit payload(uint64_t value) noexcept - : type_{NVTX_PAYLOAD_TYPE_UNSIGNED_INT64}, value_{} - { - value_.ullValue = value; - } - - /** - * @brief Construct a `payload` from an unsigned, 4 byte integer. - * - * @param value Value to use as contents of the payload - */ - NVTX3_RELAXED_CONSTEXPR explicit payload(uint32_t value) noexcept - : type_{NVTX_PAYLOAD_TYPE_UNSIGNED_INT32}, value_{} - { - value_.uiValue = value; - } - - /** - * @brief Construct a `payload` from a single-precision floating point - * value. - * - * @param value Value to use as contents of the payload - */ - NVTX3_RELAXED_CONSTEXPR explicit payload(float value) noexcept - : type_{NVTX_PAYLOAD_TYPE_FLOAT}, value_{} - { - value_.fValue = value; - } - - /** - * @brief Construct a `payload` from a double-precision floating point - * value. - * - * @param value Value to use as contents of the payload - */ - NVTX3_RELAXED_CONSTEXPR explicit payload(double value) noexcept - : type_{NVTX_PAYLOAD_TYPE_DOUBLE}, value_{} - { - value_.dValue = value; - } - - /** - * @brief Return the union holding the value of the payload - * - */ - NVTX3_RELAXED_CONSTEXPR value_type get_value() const noexcept { return value_; } - - /** - * @brief Return the information about the type the union holds. - * - */ - NVTX3_RELAXED_CONSTEXPR nvtxPayloadType_t get_type() const noexcept { return type_; } - - private: - nvtxPayloadType_t const type_; ///< Type of the payload value - value_type value_; ///< Union holding the payload value -}; - -/** - * @brief Describes the attributes of a NVTX event. - * - * NVTX events can be customized via four "attributes": - * - * - color: color used to visualize the event in tools such as Nsight - * Systems. See `color`. - * - message: Custom message string. See `message`. - * - payload: User-defined numerical value. See `payload`. - * - category: Intra-domain grouping. See `category`. - * - * These component attributes are specified via an `event_attributes` object. - * See `nvtx3::color`, `nvtx3::message`, `nvtx3::payload`, and - * `nvtx3::category` for how these individual attributes are constructed. - * - * While it is possible to specify all four attributes, it is common to want - * to only specify a subset of attributes and use default values for the - * others. For convenience, `event_attributes` can be constructed from any - * number of attribute components in any order. - * - * Example: - * \code{.cpp} - * event_attributes attr{}; // No arguments, use defaults for all attributes - * - * event_attributes attr{"message"}; // Custom message, rest defaulted - * - * // Custom color & message - * event_attributes attr{"message", nvtx3::rgb{127, 255, 0}}; - * - * /// Custom color & message, can use any order of arguments - * event_attributes attr{nvtx3::rgb{127, 255, 0}, "message"}; - * - * - * // Custom color, message, payload, category - * event_attributes attr{nvtx3::rgb{127, 255, 0}, - * "message", - * nvtx3::payload{42}, - * nvtx3::category{1}}; - * - * // Custom color, message, payload, category, can use any order of arguments - * event_attributes attr{nvtx3::payload{42}, - * nvtx3::category{1}, - * "message", - * nvtx3::rgb{127, 255, 0}}; - * - * // Multiple arguments of the same type are allowed, but only the first is - * // used. All others are ignored - * event_attributes attr{ nvtx3::payload{42}, nvtx3::payload{7} }; // payload - * is 42 - * - * // Range `r` will be customized according the attributes in `attr` - * nvtx3::thread_range r{attr}; - * - * // For convenience, the arguments that can be passed to the - * `event_attributes` - * // constructor may be passed to the `domain_thread_range` constructor where - * // they will be forwarded to the `EventAttribute`s constructor - * nvtx3::thread_range r{nvtx3::payload{42}, nvtx3::category{1}, "message"}; - * \endcode - */ -class event_attributes { - public: - using value_type = nvtxEventAttributes_t; - - /** - * @brief Default constructor creates an `event_attributes` with no - * category, color, payload, nor message. - */ - constexpr event_attributes() noexcept - : attributes_{ - NVTX_VERSION, // version - sizeof(nvtxEventAttributes_t), // size - 0, // category - NVTX_COLOR_UNKNOWN, // color type - 0, // color value - NVTX_PAYLOAD_UNKNOWN, // payload type - {}, // payload value (union) - NVTX_MESSAGE_UNKNOWN, // message type - {} // message value (union) - } - { - } - - /** - * @brief Variadic constructor where the first argument is a `category`. - * - * Sets the value of the `EventAttribute`s category based on `c` and - * forwards the remaining variadic parameter pack to the next constructor. - * - */ - template - NVTX3_RELAXED_CONSTEXPR explicit event_attributes(category const& c, Args const&... args) noexcept - : event_attributes(args...) - { - attributes_.category = c.get_id(); - } - - /** - * @brief Variadic constructor where the first argument is a `color`. - * - * Sets the value of the `EventAttribute`s color based on `c` and forwards - * the remaining variadic parameter pack to the next constructor. - * - */ - template - NVTX3_RELAXED_CONSTEXPR explicit event_attributes(color const& c, Args const&... args) noexcept - : event_attributes(args...) - { - attributes_.color = c.get_value(); - attributes_.colorType = c.get_type(); - } - - /** - * @brief Variadic constructor where the first argument is a `payload`. - * - * Sets the value of the `EventAttribute`s payload based on `p` and forwards - * the remaining variadic parameter pack to the next constructor. - * - */ - template - NVTX3_RELAXED_CONSTEXPR explicit event_attributes(payload const& p, Args const&... args) noexcept - : event_attributes(args...) - { - attributes_.payload = p.get_value(); - attributes_.payloadType = p.get_type(); - } - - /** - * @brief Variadic constructor where the first argument is a `message`. - * - * Sets the value of the `EventAttribute`s message based on `m` and forwards - * the remaining variadic parameter pack to the next constructor. - * - */ - template - explicit event_attributes(message const& m, Args const&... args) noexcept - : event_attributes(args...) - { - attributes_.message = m.get_value(); - attributes_.messageType = m.get_type(); - } - - ~event_attributes() = default; - event_attributes(event_attributes const&) = default; - event_attributes& operator=(event_attributes const&) = default; - event_attributes(event_attributes&&) = default; - event_attributes& operator=(event_attributes&&) = default; - - /** - * @brief Get raw pointer to underlying NVTX attributes object. - * - */ - constexpr value_type const* get() const noexcept { return &attributes_; } - - private: - value_type attributes_{}; ///< The NVTX attributes structure -}; - -/** - * @brief A RAII object for creating a NVTX range local to a thread within a - * domain. - * - * When constructed, begins a nested NVTX range on the calling thread in the - * specified domain. Upon destruction, ends the NVTX range. - * - * Behavior is undefined if a `domain_thread_range` object is - * created/destroyed on different threads. - * - * `domain_thread_range` is neither moveable nor copyable. - * - * `domain_thread_range`s may be nested within other ranges. - * - * The domain of the range is specified by the template type parameter `D`. - * By default, the `domain::global` is used, which scopes the range to the - * global NVTX domain. The convenience alias `thread_range` is provided for - * ranges scoped to the global domain. - * - * A custom domain can be defined by creating a type, `D`, with a static - * member `D::name` whose value is used to name the domain associated with - * `D`. `D::name` must resolve to either `char const*` or `wchar_t const*` - * - * Example: - * ``` - * // Define a type `my_domain` with a member `name` used to name the domain - * // associated with the type `my_domain`. - * struct my_domain{ - * static constexpr const char * name{"my domain"}; - * }; - * ``` - * - * Usage: - * ``` - * nvtx3::domain_thread_range<> r0{"range 0"}; // Range in global domain - * - * nvtx3::thread_range r1{"range 1"}; // Alias for range in global domain - * - * nvtx3::domain_thread_range r2{"range 2"}; // Range in custom - * domain - * - * // specify an alias to a range that uses a custom domain - * using my_thread_range = nvtx3::domain_thread_range; - * - * my_thread_range r3{"range 3"}; // Alias for range in custom domain - * ``` - */ -template -class domain_thread_range { - public: - /** - * @brief Construct a `domain_thread_range` with the specified - * `event_attributes` - * - * Example: - * ``` - * nvtx3::event_attributes attr{"msg", nvtx3::rgb{127,255,0}}; - * nvtx3::domain_thread_range<> range{attr}; // Creates a range with message - * contents - * // "msg" and green color - * ``` - * - * @param[in] attr `event_attributes` that describes the desired attributes - * of the range. - */ - explicit domain_thread_range(event_attributes const& attr) noexcept - { - nvtxDomainRangePushEx(domain::get(), attr.get()); - } - - /** - * @brief Constructs a `domain_thread_range` from the constructor arguments - * of an `event_attributes`. - * - * Forwards the arguments `first, args...` to construct an - * `event_attributes` object. The `event_attributes` object is then - * associated with the `domain_thread_range`. - * - * For more detail, see `event_attributes` documentation. - * - * Example: - * ``` - * // Creates a range with message "message" and green color - * nvtx3::domain_thread_range<> r{"message", nvtx3::rgb{127,255,0}}; - * ``` - * - * @note To prevent making needless copies of `event_attributes` objects, - * this constructor is disabled when the first argument is an - * `event_attributes` object, instead preferring the explicit - * `domain_thread_range(event_attributes const&)` constructor. - * - * @param[in] first First argument to forward to the `event_attributes` - * constructor. - * @param[in] args Variadic parameter pack of additional arguments to - * forward. - * - */ - template >>> - explicit domain_thread_range(First const& first, Args const&... args) noexcept - : domain_thread_range{event_attributes{first, args...}} - { - } - - /** - * @brief Default constructor creates a `domain_thread_range` with no - * message, color, payload, nor category. - * - */ - domain_thread_range() : domain_thread_range{event_attributes{}} {} - - domain_thread_range(domain_thread_range const&) = delete; - domain_thread_range& operator=(domain_thread_range const&) = delete; - domain_thread_range(domain_thread_range&&) = delete; - domain_thread_range& operator=(domain_thread_range&&) = delete; - - /** - * @brief Destroy the domain_thread_range, ending the NVTX range event. - */ - ~domain_thread_range() noexcept { nvtxDomainRangePop(domain::get()); } -}; - -/** - * @brief Alias for a `domain_thread_range` in the global NVTX domain. - */ -using thread_range = domain_thread_range<>; - -/** - * @brief A RAII object for creating a NVTX range within a domain that can be - * created and destroyed on different threads. - * - * When constructed, begins a NVTX range in the specified domain. Upon - * destruction, ends the NVTX range. - * - * Similar to `nvtx3::domain_thread_range`, the only difference being that - * `domain_process_range` can start and end on different threads. - * - * Use of `nvtx3::domain_thread_range` should be preferred unless one needs - * the ability to start and end a range on different threads. - * - * `domain_process_range` is moveable, but not copyable. - * - * @tparam D Type containing `name` member used to identify the `domain` - * to which the `domain_process_range` belongs. Else, `domain::global` to - * indicate that the global NVTX domain should be used. - */ -template -class domain_process_range { - public: - /** - * @brief Construct a new domain process range object - * - * @param attr - */ - explicit domain_process_range(event_attributes const& attr) noexcept - : range_id_{nvtxDomainRangeStartEx(domain::get(), attr.get())} - { - } - - /** - * @brief Construct a new domain process range object - * - * @param first - * @param args - */ - template >>> - explicit domain_process_range(First const& first, Args const&... args) noexcept - : domain_process_range{event_attributes{first, args...}} - { - } - - /** - * @brief Construct a new domain process range object - * - */ - constexpr domain_process_range() noexcept : domain_process_range{event_attributes{}} {} - - /** - * @brief Destroy the `domain_process_range` ending the range. - * - */ - ~domain_process_range() noexcept - { - if (not moved_from_) { nvtxRangeEnd(range_id_); } - } - - domain_process_range(domain_process_range const&) = delete; - domain_process_range& operator=(domain_process_range const&) = delete; - - domain_process_range(domain_process_range&& other) noexcept : range_id_{other.range_id_} - { - other.moved_from_ = true; - } - - domain_process_range& operator=(domain_process_range&& other) noexcept - { - range_id_ = other.range_id_; - other.moved_from_ = true; - } - - private: - nvtxRangeId_t range_id_; ///< Range id used to correlate - ///< the start/end of the range - bool moved_from_{false}; ///< Indicates if the object has had - ///< it's contents moved from it, - ///< indicating it should not attempt - ///< to end the NVTX range. -}; - -/** - * @brief Alias for a `domain_process_range` in the global NVTX domain. - */ -using process_range = domain_process_range<>; - -/** - * @brief Annotates an instantaneous point in time with the attributes specified - * by `attr`. - * - * Unlike a "range", a mark is an instantaneous event in an application, e.g., - * locking/unlocking a mutex. - * - * \code{.cpp} - * std::mutex global_lock; - * void lock_mutex(){ - * global_lock.lock(); - * nvtx3::mark("lock_mutex"); - * } - * \endcode - * - * @tparam D Type containing `name` member used to identify the `domain` - * to which the `domain_process_range` belongs. Else, `domain::global` to - * indicate that the global NVTX domain should be used. - * @param[in] attr `event_attributes` that describes the desired attributes - * of the mark. - */ -template -inline void mark(event_attributes const& attr) noexcept -{ - nvtxDomainMarkEx(domain::get(), attr.get()); -} - -} // namespace nvtx3 - -/** - * @brief Convenience macro for generating a range in the specified `domain` - * from the lifetime of a function - * - * This macro is useful for generating an NVTX range in `domain` from - * the entry point of a function to its exit. It is intended to be the first - * line of the function. - * - * Constructs a static `registered_message` using the name of the immediately - * enclosing function returned by `__func__` and constructs a - * `nvtx3::thread_range` using the registered function name as the range's - * message. - * - * Example: - * ``` - * struct my_domain{static constexpr char const* name{"my_domain"};}; - * - * void foo(...){ - * NVTX3_FUNC_RANGE_IN(my_domain); // Range begins on entry to foo() - * // do stuff - * ... - * } // Range ends on return from foo() - * ``` - * - * @param[in] D Type containing `name` member used to identify the - * `domain` to which the `registered_message` belongs. Else, - * `domain::global` to indicate that the global NVTX domain should be used. - */ -#define NVTX3_FUNC_RANGE_IN(D) \ - static ::nvtx3::registered_message const nvtx3_func_name__{__func__}; \ - static ::nvtx3::event_attributes const nvtx3_func_attr__{nvtx3_func_name__}; \ - [[maybe_unused]] ::nvtx3::domain_thread_range const nvtx3_range__{nvtx3_func_attr__}; diff --git a/cpp/include/cudf/detail/nvtx/ranges.hpp b/cpp/include/cudf/detail/nvtx/ranges.hpp index 6ed30e871fa..8ad7467a7ba 100644 --- a/cpp/include/cudf/detail/nvtx/ranges.hpp +++ b/cpp/include/cudf/detail/nvtx/ranges.hpp @@ -16,7 +16,7 @@ #pragma once -#include "nvtx3.hpp" +#include namespace cudf { /** @@ -34,12 +34,12 @@ struct libcudf_domain { * Example: * ``` * void some_function(){ - * cudf::thread_range rng{"custom_name"}; // Customizes range name + * cudf::scoped_range rng{"custom_name"}; // Customizes range name * ... * } * ``` */ -using thread_range = ::nvtx3::domain_thread_range; +using scoped_range = ::nvtx3::scoped_range_in; } // namespace cudf diff --git a/cpp/src/join/distinct_hash_join.cu b/cpp/src/join/distinct_hash_join.cu index 85b7c26472d..8bd42d867a3 100644 --- a/cpp/src/join/distinct_hash_join.cu +++ b/cpp/src/join/distinct_hash_join.cu @@ -311,7 +311,7 @@ std::pair>, distinct_hash_join::inner_join(rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) const { - cudf::thread_range range{"distinct_hash_join::inner_join"}; + cudf::scoped_range range{"distinct_hash_join::inner_join"}; size_type const probe_table_num_rows{this->_probe.num_rows()}; @@ -354,7 +354,7 @@ template std::unique_ptr> distinct_hash_join::left_join( rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) const { - cudf::thread_range range{"distinct_hash_join::left_join"}; + cudf::scoped_range range{"distinct_hash_join::left_join"}; size_type const probe_table_num_rows{this->_probe.num_rows()}; diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 135a40b076a..0eaa87f0ece 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -55,7 +55,7 @@ function(ConfigureTest CMAKE_TEST_NAME) ) target_link_libraries( - ${CMAKE_TEST_NAME} PRIVATE cudftestutil GTest::gmock_main GTest::gtest_main + ${CMAKE_TEST_NAME} PRIVATE cudftestutil GTest::gmock_main GTest::gtest_main nvtx3-cpp $ ) rapids_cuda_set_runtime(${CMAKE_TEST_NAME} USE_STATIC ${CUDA_STATIC_RUNTIME}) diff --git a/java/src/main/native/CMakeLists.txt b/java/src/main/native/CMakeLists.txt index 1e7ac1a68ea..0d5339a1402 100644 --- a/java/src/main/native/CMakeLists.txt +++ b/java/src/main/native/CMakeLists.txt @@ -94,6 +94,10 @@ rapids_cmake_build_type("Release") set(cudf_ROOT "${CUDF_CPP_BUILD_DIR}") rapids_find_package(cudf REQUIRED) +# ################################################################################################## +# * nvtx3------------------------------------------------------------------------------------------- +include(${CUDF_SOURCE_DIR}/cmake/thirdparty/get_nvtx.cmake) + # ################################################################################################## # * nvcomp------------------------------------------------------------------------------------------ @@ -235,7 +239,7 @@ endif() # When nvcomp is installed we need to use nvcomp::nvcomp but from the cudf build directory it will # just be nvcomp. target_link_libraries( - cudfjni ${CUDF_LINK} PRIVATE $ + cudfjni ${CUDF_LINK} PRIVATE nvtx3-cpp $ $ ) diff --git a/java/src/main/native/src/NvtxRangeJni.cpp b/java/src/main/native/src/NvtxRangeJni.cpp index 1f12b2ea8cc..2529acfb91d 100644 --- a/java/src/main/native/src/NvtxRangeJni.cpp +++ b/java/src/main/native/src/NvtxRangeJni.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include "jni_utils.hpp" #include "nvtx_common.hpp" diff --git a/java/src/main/native/src/NvtxUniqueRangeJni.cpp b/java/src/main/native/src/NvtxUniqueRangeJni.cpp index d6c321b5fd2..924b5a564e6 100644 --- a/java/src/main/native/src/NvtxUniqueRangeJni.cpp +++ b/java/src/main/native/src/NvtxUniqueRangeJni.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include "jni_utils.hpp" #include "nvtx_common.hpp" diff --git a/java/src/main/native/src/check_nvcomp_output_sizes.cu b/java/src/main/native/src/check_nvcomp_output_sizes.cu index 944399882b8..9d29e66ec59 100644 --- a/java/src/main/native/src/check_nvcomp_output_sizes.cu +++ b/java/src/main/native/src/check_nvcomp_output_sizes.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include #include +#include #include #include From 69952b03852a346f86665f5b60afaa4152870e0f Mon Sep 17 00:00:00 2001 From: Alessandro Bellina Date: Fri, 8 Mar 2024 16:51:55 -0600 Subject: [PATCH 381/384] Use JNI pinned pool resource with cuIO (#15255) ## Description Following https://github.com/rapidsai/cudf/pull/15079, we add a way to share the pinned pool in JNI with cuIO via the new method added by @nvdbaranec `set_host_memory_resource`. ## Checklist - [x] I am familiar with the [Contributing Guidelines](https://github.com/rapidsai/cudf/blob/HEAD/CONTRIBUTING.md). - [x] New or existing tests cover these changes. - [ ] The documentation is up to date with these changes. --------- Signed-off-by: Alessandro Bellina --- .../java/ai/rapids/cudf/PinnedMemoryPool.java | 22 +- java/src/main/java/ai/rapids/cudf/Rmm.java | 8 + java/src/main/native/src/RmmJni.cpp | 221 ++++++++++++++++++ .../ai/rapids/cudf/PinnedMemoryPoolTest.java | 20 ++ 4 files changed, 268 insertions(+), 3 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/PinnedMemoryPool.java b/java/src/main/java/ai/rapids/cudf/PinnedMemoryPool.java index 17f05a9baf6..6cb34683e5a 100644 --- a/java/src/main/java/ai/rapids/cudf/PinnedMemoryPool.java +++ b/java/src/main/java/ai/rapids/cudf/PinnedMemoryPool.java @@ -106,9 +106,10 @@ private static void freeInternal(long address, long origLength) { * Initialize the pool. * * @param poolSize size of the pool to initialize. + * @note when using this method, the pinned pool will be shared with cuIO */ public static synchronized void initialize(long poolSize) { - initialize(poolSize, -1); + initialize(poolSize, -1, true); } /** @@ -116,8 +117,20 @@ public static synchronized void initialize(long poolSize) { * * @param poolSize size of the pool to initialize. * @param gpuId gpu id to set to get memory pool from, -1 means to use default + * @note when using this method, the pinned pool will be shared with cuIO */ public static synchronized void initialize(long poolSize, int gpuId) { + initialize(poolSize, gpuId, true); + } + + /** + * Initialize the pool. + * + * @param poolSize size of the pool to initialize. + * @param gpuId gpu id to set to get memory pool from, -1 means to use default + * @param setCuioHostMemoryResource true if this pinned pool should be used by cuIO for host memory + */ + public static synchronized void initialize(long poolSize, int gpuId, boolean setCuioHostMemoryResource) { if (isInitialized()) { throw new IllegalStateException("Can only initialize the pool once."); } @@ -126,7 +139,7 @@ public static synchronized void initialize(long poolSize, int gpuId) { t.setDaemon(true); return t; }); - initFuture = initService.submit(() -> new PinnedMemoryPool(poolSize, gpuId)); + initFuture = initService.submit(() -> new PinnedMemoryPool(poolSize, gpuId, setCuioHostMemoryResource)); initService.shutdown(); } @@ -203,13 +216,16 @@ public static long getTotalPoolSizeBytes() { return 0; } - private PinnedMemoryPool(long poolSize, int gpuId) { + private PinnedMemoryPool(long poolSize, int gpuId, boolean setCuioHostMemoryResource) { if (gpuId > -1) { // set the gpu device to use Cuda.setDevice(gpuId); Cuda.freeZero(); } this.poolHandle = Rmm.newPinnedPoolMemoryResource(poolSize, poolSize); + if (setCuioHostMemoryResource) { + Rmm.setCuioPinnedPoolMemoryResource(this.poolHandle); + } this.poolSize = poolSize; } diff --git a/java/src/main/java/ai/rapids/cudf/Rmm.java b/java/src/main/java/ai/rapids/cudf/Rmm.java index 552da62382a..6e9f90e477f 100755 --- a/java/src/main/java/ai/rapids/cudf/Rmm.java +++ b/java/src/main/java/ai/rapids/cudf/Rmm.java @@ -584,9 +584,17 @@ static native long newEventHandlerResourceAdaptor(long handle, long trackerHandl public static native long newPinnedPoolMemoryResource(long initSize, long maxSize); + public static native long setCuioPinnedPoolMemoryResource(long poolPtr); + public static native void releasePinnedPoolMemoryResource(long poolPtr); public static native long allocFromPinnedPool(long poolPtr, long size); public static native void freeFromPinnedPool(long poolPtr, long ptr, long size); + + // only for tests + public static native long allocFromFallbackPinnedPool(long size); + + // only for tests + public static native void freeFromFallbackPinnedPool(long ptr, long size); } diff --git a/java/src/main/native/src/RmmJni.cpp b/java/src/main/native/src/RmmJni.cpp index 7b81b5ff4de..68af350d5fe 100644 --- a/java/src/main/native/src/RmmJni.cpp +++ b/java/src/main/native/src/RmmJni.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -366,6 +367,187 @@ class java_debug_event_handler_memory_resource final : public java_event_handler } }; +inline auto &prior_cuio_host_mr() { + static rmm::host_async_resource_ref _prior_cuio_host_mr = cudf::io::get_host_memory_resource(); + return _prior_cuio_host_mr; +} + +/** + * This is a pinned fallback memory resource that will try to allocate `pool` + * and if that fails, attempt to allocate from the prior resource used by cuIO `prior_cuio_host_mr`. + * + * We detect whether a pointer to free is inside of the pool by checking its address (see + * constructor) + * + * Most of this comes directly from `pinned_host_memory_resource` in RMM. + */ +class pinned_fallback_host_memory_resource { +private: + rmm_pinned_pool_t *_pool; + void *pool_begin_; + void *pool_end_; + +public: + pinned_fallback_host_memory_resource(rmm_pinned_pool_t *pool) : _pool(pool) { + // allocate from the pinned pool the full size to figure out + // our beginning and end address. + auto pool_size = pool->pool_size(); + pool_begin_ = pool->allocate(pool_size); + pool_end_ = static_cast(static_cast(pool_begin_) + pool_size); + pool->deallocate(pool_begin_, pool_size); + } + + // Disable clang-tidy complaining about the easily swappable size and alignment parameters + // of allocate and deallocate + // NOLINTBEGIN(bugprone-easily-swappable-parameters) + + /** + * @brief Allocates pinned host memory of size at least \p bytes bytes from either the + * _pool argument provided, or prior_cuio_host_mr. + * + * @throws rmm::bad_alloc if the requested allocation could not be fulfilled due to any other + * reason. + * + * @param bytes The size, in bytes, of the allocation. + * @param alignment Alignment in bytes. Default alignment is used if unspecified. + * + * @return Pointer to the newly allocated memory. + */ + void *allocate(std::size_t bytes, + [[maybe_unused]] std::size_t alignment = rmm::RMM_DEFAULT_HOST_ALIGNMENT) { + try { + return _pool->allocate(bytes, alignment); + } catch (const std::exception &unused) { + // try to allocate using the underlying pinned resource + return prior_cuio_host_mr().allocate(bytes, alignment); + } + // we should not reached here + return nullptr; + } + + /** + * @brief Deallocate memory pointed to by \p ptr of size \p bytes bytes. We attempt + * to deallocate from _pool, if ptr is detected to be in the pool address range, + * otherwise we deallocate from `prior_cuio_host_mr`. + * + * @param ptr Pointer to be deallocated. + * @param bytes Size of the allocation. + * @param alignment Alignment in bytes. Default alignment is used if unspecified. + */ + void deallocate(void *ptr, std::size_t bytes, + std::size_t alignment = rmm::RMM_DEFAULT_HOST_ALIGNMENT) noexcept { + if (ptr >= pool_begin_ && ptr <= pool_end_) { + _pool->deallocate(ptr, bytes, alignment); + } else { + prior_cuio_host_mr().deallocate(ptr, bytes, alignment); + } + } + + /** + * @brief Allocates pinned host memory of size at least \p bytes bytes. + * + * @note Stream argument is ignored and behavior is identical to allocate. + * + * @throws rmm::out_of_memory if the requested allocation could not be fulfilled due to to a + * CUDA out of memory error. + * @throws rmm::bad_alloc if the requested allocation could not be fulfilled due to any other + * error. + * + * @param bytes The size, in bytes, of the allocation. + * @param stream CUDA stream on which to perform the allocation (ignored). + * @return Pointer to the newly allocated memory. + */ + void *allocate_async(std::size_t bytes, [[maybe_unused]] cuda::stream_ref stream) { + return allocate(bytes); + } + + /** + * @brief Allocates pinned host memory of size at least \p bytes bytes and alignment \p alignment. + * + * @note Stream argument is ignored and behavior is identical to allocate. + * + * @throws rmm::out_of_memory if the requested allocation could not be fulfilled due to to a + * CUDA out of memory error. + * @throws rmm::bad_alloc if the requested allocation could not be fulfilled due to any other + * error. + * + * @param bytes The size, in bytes, of the allocation. + * @param alignment Alignment in bytes. + * @param stream CUDA stream on which to perform the allocation (ignored). + * @return Pointer to the newly allocated memory. + */ + void *allocate_async(std::size_t bytes, std::size_t alignment, + [[maybe_unused]] cuda::stream_ref stream) { + return allocate(bytes, alignment); + } + + /** + * @brief Deallocate memory pointed to by \p ptr of size \p bytes bytes. + * + * @note Stream argument is ignored and behavior is identical to deallocate. + * + * @param ptr Pointer to be deallocated. + * @param bytes Size of the allocation. + * @param stream CUDA stream on which to perform the deallocation (ignored). + */ + void deallocate_async(void *ptr, std::size_t bytes, + [[maybe_unused]] cuda::stream_ref stream) noexcept { + return deallocate(ptr, bytes); + } + + /** + * @brief Deallocate memory pointed to by \p ptr of size \p bytes bytes and alignment \p + * alignment bytes. + * + * @note Stream argument is ignored and behavior is identical to deallocate. + * + * @param ptr Pointer to be deallocated. + * @param bytes Size of the allocation. + * @param alignment Alignment in bytes. + * @param stream CUDA stream on which to perform the deallocation (ignored). + */ + void deallocate_async(void *ptr, std::size_t bytes, std::size_t alignment, + [[maybe_unused]] cuda::stream_ref stream) noexcept { + return deallocate(ptr, bytes, alignment); + } + // NOLINTEND(bugprone-easily-swappable-parameters) + + /** + * @briefreturn{true if the specified resource is the same type as this resource.} + */ + bool operator==(const pinned_fallback_host_memory_resource &) const { return true; } + + /** + * @briefreturn{true if the specified resource is not the same type as this resource, otherwise + * false.} + */ + bool operator!=(const pinned_fallback_host_memory_resource &) const { return false; } + + /** + * @brief Enables the `cuda::mr::device_accessible` property + * + * This property declares that a `pinned_host_memory_resource` provides device accessible memory + */ + friend void get_property(pinned_fallback_host_memory_resource const &, + cuda::mr::device_accessible) noexcept {} + + /** + * @brief Enables the `cuda::mr::host_accessible` property + * + * This property declares that a `pinned_host_memory_resource` provides host accessible memory + */ + friend void get_property(pinned_fallback_host_memory_resource const &, + cuda::mr::host_accessible) noexcept {} +}; + +// carryover from RMM pinned_host_memory_resource +static_assert( + cuda::mr::async_resource_with); + +// we set this to our fallback resource if we have set it. +std::unique_ptr pinned_fallback_mr; + } // anonymous namespace extern "C" { @@ -760,11 +942,30 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Rmm_newPinnedPoolMemoryResource(JNIE CATCH_STD(env, 0) } +JNIEXPORT void JNICALL Java_ai_rapids_cudf_Rmm_setCuioPinnedPoolMemoryResource(JNIEnv *env, + jclass clazz, + jlong pool_ptr) { + try { + cudf::jni::auto_set_device(env); + auto pool = reinterpret_cast(pool_ptr); + // create a pinned fallback pool that will allocate pinned memory + // if the regular pinned pool is exhausted + pinned_fallback_mr.reset(new pinned_fallback_host_memory_resource(pool)); + // set the cuio host mr and store the prior resource in our static variable + prior_cuio_host_mr() = cudf::io::set_host_memory_resource(*pinned_fallback_mr); + } + CATCH_STD(env, ) +} + JNIEXPORT void JNICALL Java_ai_rapids_cudf_Rmm_releasePinnedPoolMemoryResource(JNIEnv *env, jclass clazz, jlong pool_ptr) { try { cudf::jni::auto_set_device(env); + // set the cuio host memory resource to what it was before, or the same + // if we didn't overwrite it with setCuioPinnedPoolMemoryResource + cudf::io::set_host_memory_resource(prior_cuio_host_mr()); + pinned_fallback_mr.reset(); delete reinterpret_cast(pool_ptr); } CATCH_STD(env, ) @@ -791,4 +992,24 @@ JNIEXPORT void JNICALL Java_ai_rapids_cudf_Rmm_freeFromPinnedPool(JNIEnv *env, j } CATCH_STD(env, ) } + +// only for tests +JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Rmm_allocFromFallbackPinnedPool(JNIEnv *env, + jclass clazz, + jlong size) { + cudf::jni::auto_set_device(env); + void *ret = cudf::io::get_host_memory_resource().allocate(size); + return reinterpret_cast(ret); +} + +// only for tests +JNIEXPORT void JNICALL Java_ai_rapids_cudf_Rmm_freeFromFallbackPinnedPool(JNIEnv *env, jclass clazz, + jlong ptr, jlong size) { + try { + cudf::jni::auto_set_device(env); + void *cptr = reinterpret_cast(ptr); + cudf::io::get_host_memory_resource().deallocate(cptr, size); + } + CATCH_STD(env, ) +} } diff --git a/java/src/test/java/ai/rapids/cudf/PinnedMemoryPoolTest.java b/java/src/test/java/ai/rapids/cudf/PinnedMemoryPoolTest.java index 8c6e29dbd0c..82182adbb70 100644 --- a/java/src/test/java/ai/rapids/cudf/PinnedMemoryPoolTest.java +++ b/java/src/test/java/ai/rapids/cudf/PinnedMemoryPoolTest.java @@ -140,4 +140,24 @@ void testZeroSizedAllocation() { assertEquals(0, buffer.getLength()); } } + + // This test simulates cuIO using our fallback pinned pool wrapper + // we should be able to either go to the pool, in this case 15KB in size + // or we should be falling back to pinned cudaMallocHost/cudaFreeHost. + @Test + void testFallbackPinnedPool() { + final long poolSize = 15 * 1024L; + PinnedMemoryPool.initialize(poolSize); + assertEquals(poolSize, PinnedMemoryPool.getTotalPoolSizeBytes()); + + long ptr = Rmm.allocFromFallbackPinnedPool(1347); // this doesn't fallback + long ptr2 = Rmm.allocFromFallbackPinnedPool(15 * 1024L); // this does + Rmm.freeFromFallbackPinnedPool(ptr, 1347); // free from pool + Rmm.freeFromFallbackPinnedPool(ptr2, 15*1024); // free from fallback + + ptr = Rmm.allocFromFallbackPinnedPool(15*1024L); // this doesn't fallback + ptr2 = Rmm.allocFromFallbackPinnedPool(15*1024L); // this does + Rmm.freeFromFallbackPinnedPool(ptr, 15*1024L); // free from pool + Rmm.freeFromFallbackPinnedPool(ptr2, 15*1024L); // free from fallback + } } From b08dd9bed15e60d86a561fc0cb47cdbc2428a09b Mon Sep 17 00:00:00 2001 From: Yunsong Wang Date: Fri, 8 Mar 2024 15:36:49 -0800 Subject: [PATCH 382/384] Add cardinality control for groupby benchs with flat types (#15134) Contributes to #15114 This PR adds cardinality control to `group_max`, `group_nunique` and `group_rank` benchmarks. Authors: - Yunsong Wang (https://github.com/PointKernel) Approvers: - Nghia Truong (https://github.com/ttnghia) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/15134 --- cpp/benchmarks/groupby/group_max.cpp | 53 +++++++++++++++++++----- cpp/benchmarks/groupby/group_nunique.cpp | 19 ++++++--- cpp/benchmarks/groupby/group_rank.cpp | 12 +++--- 3 files changed, 62 insertions(+), 22 deletions(-) diff --git a/cpp/benchmarks/groupby/group_max.cpp b/cpp/benchmarks/groupby/group_max.cpp index b7b330f02e5..01ca23ebbf8 100644 --- a/cpp/benchmarks/groupby/group_max.cpp +++ b/cpp/benchmarks/groupby/group_max.cpp @@ -22,25 +22,30 @@ #include template -void bench_groupby_max(nvbench::state& state, nvbench::type_list) +void groupby_max_helper(nvbench::state& state, + cudf::size_type num_rows, + cudf::size_type cardinality, + double null_probability) { - auto const size = static_cast(state.get_int64("num_rows")); - auto const keys = [&] { - data_profile const profile = data_profile_builder().cardinality(0).no_validity().distribution( - cudf::type_to_id(), distribution_id::UNIFORM, 0, 100); - return create_random_column(cudf::type_to_id(), row_count{size}, profile); + data_profile const profile = + data_profile_builder() + .cardinality(cardinality) + .no_validity() + .distribution(cudf::type_to_id(), distribution_id::UNIFORM, 0, num_rows); + return create_random_column(cudf::type_to_id(), row_count{num_rows}, profile); }(); auto const vals = [&] { auto builder = data_profile_builder().cardinality(0).distribution( - cudf::type_to_id(), distribution_id::UNIFORM, 0, 1000); - if (const auto null_freq = state.get_float64("null_probability"); null_freq > 0) { - builder.null_probability(null_freq); + cudf::type_to_id(), distribution_id::UNIFORM, 0, num_rows); + if (null_probability > 0) { + builder.null_probability(null_probability); } else { builder.no_validity(); } - return create_random_column(cudf::type_to_id(), row_count{size}, data_profile{builder}); + return create_random_column( + cudf::type_to_id(), row_count{num_rows}, data_profile{builder}); }(); auto keys_view = keys->view(); @@ -55,13 +60,39 @@ void bench_groupby_max(nvbench::state& state, nvbench::type_list) state.set_cuda_stream(nvbench::make_cuda_stream_view(cudf::get_default_stream().value())); state.exec(nvbench::exec_tag::sync, [&](nvbench::launch& launch) { auto const result = gb_obj.aggregate(requests); }); - + auto const elapsed_time = state.get_summary("nv/cold/time/gpu/mean").get_float64("value"); + state.add_element_count(static_cast(num_rows) / elapsed_time / 1'000'000., "Mrows/s"); state.add_buffer_size( mem_stats_logger.peak_memory_usage(), "peak_memory_usage", "peak_memory_usage"); } +template +void bench_groupby_max(nvbench::state& state, nvbench::type_list) +{ + auto const cardinality = static_cast(state.get_int64("cardinality")); + auto const num_rows = static_cast(state.get_int64("num_rows")); + auto const null_probability = state.get_float64("null_probability"); + + groupby_max_helper(state, num_rows, cardinality, null_probability); +} + +template +void bench_groupby_max_cardinality(nvbench::state& state, nvbench::type_list) +{ + auto constexpr num_rows = 20'000'000; + auto constexpr null_probability = 0.; + auto const cardinality = static_cast(state.get_int64("cardinality")); + + groupby_max_helper(state, num_rows, cardinality, null_probability); +} + NVBENCH_BENCH_TYPES(bench_groupby_max, NVBENCH_TYPE_AXES(nvbench::type_list)) .set_name("groupby_max") + .add_int64_axis("cardinality", {0}) .add_int64_power_of_two_axis("num_rows", {12, 18, 24}) .add_float64_axis("null_probability", {0, 0.1, 0.9}); + +NVBENCH_BENCH_TYPES(bench_groupby_max_cardinality, NVBENCH_TYPE_AXES(nvbench::type_list)) + .set_name("groupby_max_cardinality") + .add_int64_axis("cardinality", {10, 20, 50, 100, 1'000, 10'000, 100'000, 1'000'000, 10'000'000}); diff --git a/cpp/benchmarks/groupby/group_nunique.cpp b/cpp/benchmarks/groupby/group_nunique.cpp index 63d738b2951..c97deeaff92 100644 --- a/cpp/benchmarks/groupby/group_nunique.cpp +++ b/cpp/benchmarks/groupby/group_nunique.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -39,17 +39,23 @@ auto make_aggregation_request_vector(cudf::column_view const& values, Args&&... template void bench_groupby_nunique(nvbench::state& state, nvbench::type_list) { - auto const size = static_cast(state.get_int64("num_rows")); + auto const size = static_cast(state.get_int64("num_rows")); + auto const cardinality = static_cast(state.get_int64("cardinality")); auto const keys = [&] { - data_profile profile = data_profile_builder().cardinality(0).no_validity().distribution( - cudf::type_to_id(), distribution_id::UNIFORM, 0, 100); + data_profile profile = + data_profile_builder() + .cardinality(cardinality) + .no_validity() + .distribution(cudf::type_to_id(), distribution_id::UNIFORM, 0, size); return create_random_column(cudf::type_to_id(), row_count{size}, profile); }(); auto const vals = [&] { - data_profile profile = data_profile_builder().cardinality(0).distribution( - cudf::type_to_id(), distribution_id::UNIFORM, 0, 1000); + data_profile profile = + data_profile_builder() + .cardinality(cardinality) + .distribution(cudf::type_to_id(), distribution_id::UNIFORM, 0, size); if (const auto null_freq = state.get_float64("null_probability"); null_freq > 0) { profile.set_null_probability(null_freq); } else { @@ -71,4 +77,5 @@ void bench_groupby_nunique(nvbench::state& state, nvbench::type_list) NVBENCH_BENCH_TYPES(bench_groupby_nunique, NVBENCH_TYPE_AXES(nvbench::type_list)) .set_name("groupby_nunique") .add_int64_power_of_two_axis("num_rows", {12, 16, 20, 24}) + .add_int64_axis("cardinality", {0}) .add_float64_axis("null_probability", {0, 0.5}); diff --git a/cpp/benchmarks/groupby/group_rank.cpp b/cpp/benchmarks/groupby/group_rank.cpp index 2122720a421..a02494dc769 100644 --- a/cpp/benchmarks/groupby/group_rank.cpp +++ b/cpp/benchmarks/groupby/group_rank.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,10 +31,12 @@ static void nvbench_groupby_rank(nvbench::state& state, bool const is_sorted = state.get_int64("is_sorted"); cudf::size_type const column_size = state.get_int64("data_size"); - constexpr int num_groups = 100; + auto const cardinality = static_cast(state.get_int64("cardinality")); - data_profile const profile = data_profile_builder().cardinality(0).no_validity().distribution( - dtype, distribution_id::UNIFORM, 0, num_groups); + data_profile const profile = data_profile_builder() + .cardinality(cardinality) + .no_validity() + .distribution(dtype, distribution_id::UNIFORM, 0, column_size); auto source_table = create_random_table({dtype, dtype}, row_count{column_size}, profile); @@ -100,5 +102,5 @@ NVBENCH_BENCH_TYPES(nvbench_groupby_rank, NVBENCH_TYPE_AXES(methods)) 10000000, // 10M 100000000, // 100M }) - + .add_int64_axis("cardinality", {0}) .add_int64_axis("is_sorted", {0, 1}); From 6a03827a74aa820e4e9ad241b0bc0450ceb8c018 Mon Sep 17 00:00:00 2001 From: Karthikeyan <6488848+karthikeyann@users.noreply.github.com> Date: Mon, 11 Mar 2024 09:14:47 +0530 Subject: [PATCH 383/384] Support casting of Map type to string in JSON reader (#14936) Addresses part of https://github.com/rapidsai/cudf/issues/14288 Depends on #14939 (mixed type ignore nulls fix) In the input schema, if a struct column is given as STRING type, it's forced to be a STRING column. This could be used to support map type in spark JSON reader. (Force a map type to be a STRING, and use different parser to extract this string column as key, value columns) To enable this forcing, mixed type as string should be enabled in json_reader_options. Authors: - Karthikeyan (https://github.com/karthikeyann) - Nghia Truong (https://github.com/ttnghia) Approvers: - Andy Grove (https://github.com/andygrove) - Mike Wilson (https://github.com/hyperbolic2346) - Shruti Shivakumar (https://github.com/shrshi) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/14936 --- cpp/CMakeLists.txt | 1 + cpp/include/cudf/io/json.hpp | 2 + cpp/src/io/json/json_column.cu | 37 ++++++-- cpp/src/io/json/nested_json.hpp | 26 ++++++ cpp/src/io/json/parser_features.cpp | 126 ++++++++++++++++++++++++++++ cpp/tests/io/json_test.cpp | 52 ++++++++++++ 6 files changed, 235 insertions(+), 9 deletions(-) create mode 100644 cpp/src/io/json/parser_features.cpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index ca8505fdb5e..47e9eb99733 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -384,6 +384,7 @@ add_library( src/io/json/read_json.cu src/io/json/legacy/json_gpu.cu src/io/json/legacy/reader_impl.cu + src/io/json/parser_features.cpp src/io/json/write_json.cu src/io/orc/aggregate_orc_metadata.cpp src/io/orc/dict_enc.cu diff --git a/cpp/include/cudf/io/json.hpp b/cpp/include/cudf/io/json.hpp index 593dd044d51..1f2628deea7 100644 --- a/cpp/include/cudf/io/json.hpp +++ b/cpp/include/cudf/io/json.hpp @@ -333,6 +333,7 @@ class json_reader_options { /** * @brief Set whether to parse mixed types as a string column. + * Also enables forcing to read a struct as string column using schema. * * @param val Boolean value to enable/disable parsing mixed types as a string column */ @@ -491,6 +492,7 @@ class json_reader_options_builder { /** * @brief Set whether to parse mixed types as a string column. + * Also enables forcing to read a struct as string column using schema. * * @param val Boolean value to enable/disable parsing mixed types as a string column * @return this for chaining diff --git a/cpp/src/io/json/json_column.cu b/cpp/src/io/json/json_column.cu index 6576d41dd72..bc5c45d8980 100644 --- a/cpp/src/io/json/json_column.cu +++ b/cpp/src/io/json/json_column.cu @@ -496,15 +496,16 @@ void make_device_json_column(device_span input, rmm::exec_policy(stream), sorted_col_ids.begin(), sorted_col_ids.end(), node_ids.begin()); NodeIndexT const row_array_parent_col_id = [&]() { - if (!is_array_of_arrays) return parent_node_sentinel; - auto const list_node_index = is_enabled_lines ? 0 : 1; - NodeIndexT value; - CUDF_CUDA_TRY(cudaMemcpyAsync(&value, - col_ids.data() + list_node_index, - sizeof(NodeIndexT), - cudaMemcpyDefault, - stream.value())); - stream.synchronize(); + NodeIndexT value = parent_node_sentinel; + if (!col_ids.empty()) { + auto const list_node_index = is_enabled_lines ? 0 : 1; + CUDF_CUDA_TRY(cudaMemcpyAsync(&value, + col_ids.data() + list_node_index, + sizeof(NodeIndexT), + cudaMemcpyDefault, + stream.value())); + stream.synchronize(); + } return value; }(); @@ -592,6 +593,12 @@ void make_device_json_column(device_span input, col.column_order.clear(); }; + path_from_tree tree_path{column_categories, + column_parent_ids, + column_names, + is_array_of_arrays, + row_array_parent_col_id}; + // 2. generate nested columns tree and its device_memory // reorder unique_col_ids w.r.t. column_range_begin for order of column to be in field order. auto h_range_col_id_it = @@ -642,6 +649,7 @@ void make_device_json_column(device_span input, ignore_vals[this_col_id] = 1; continue; } + // If the child is already found, // replace if this column is a nested column and the existing was a value column // ignore this column if this column is a value column and the existing was a nested column @@ -701,6 +709,17 @@ void make_device_json_column(device_span input, "A mix of lists and structs within the same column is not supported"); } } + if (is_enabled_mixed_types_as_string) { + // get path of this column, check if it is a struct forced as string, and enforce it + auto nt = tree_path.get_path(this_col_id); + std::optional user_dt = get_path_data_type(nt, options); + if (column_categories[this_col_id] == NC_STRUCT and user_dt.has_value() and + user_dt.value().id() == type_id::STRING) { + is_mixed_type_column[this_col_id] = 1; + column_categories[this_col_id] = NC_STR; + } + } + CUDF_EXPECTS(parent_col.child_columns.count(name) == 0, "duplicate column name: " + name); // move into parent device_json_column col(stream, mr); diff --git a/cpp/src/io/json/nested_json.hpp b/cpp/src/io/json/nested_json.hpp index 64fffdb27fc..5d54e340e2b 100644 --- a/cpp/src/io/json/nested_json.hpp +++ b/cpp/src/io/json/nested_json.hpp @@ -309,6 +309,32 @@ table_with_metadata device_parse_nested_json(device_span input, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr); +/** + * @brief Get the path data type of a column by path if present in input schema + * + * @param path path of the column + * @param options json reader options which holds schema + * @return data type of the column if present + */ +std::optional get_path_data_type( + host_span> path, + cudf::io::json_reader_options const& options); + +/** + * @brief Helper class to get path of a column by column id from reduced column tree + * + */ +struct path_from_tree { + host_span column_categories; + host_span column_parent_ids; + host_span column_names; + bool is_array_of_arrays; + NodeIndexT const row_array_parent_col_id; + + using path_rep = std::pair; + std::vector get_path(NodeIndexT this_col_id); +}; + /** * @brief Parses the given JSON string and generates table from the given input. * diff --git a/cpp/src/io/json/parser_features.cpp b/cpp/src/io/json/parser_features.cpp new file mode 100644 index 00000000000..740b7523cc1 --- /dev/null +++ b/cpp/src/io/json/parser_features.cpp @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "nested_json.hpp" + +#include + +#include +#include +#include + +namespace cudf::io::json::detail { + +std::optional child_schema_element(std::string const& col_name, + cudf::io::json_reader_options const& options) +{ + return std::visit( + cudf::detail::visitor_overload{ + [col_name](std::vector const& user_dtypes) -> std::optional { + auto column_index = atol(col_name.data()); + return (static_cast(column_index) < user_dtypes.size()) + ? std::optional{{user_dtypes[column_index]}} + : std::optional{}; + }, + [col_name]( + std::map const& user_dtypes) -> std::optional { + return (user_dtypes.find(col_name) != std::end(user_dtypes)) + ? std::optional{{user_dtypes.find(col_name)->second}} + : std::optional{}; + }, + [col_name]( + std::map const& user_dtypes) -> std::optional { + return (user_dtypes.find(col_name) != std::end(user_dtypes)) + ? user_dtypes.find(col_name)->second + : std::optional{}; + }}, + options.get_dtypes()); +} + +// example schema and its path. +// "a": int {"a", int} +// "a": [ int ] {"a", list}, {"element", int} +// "a": { "b": int} {"a", struct}, {"b", int} +// "a": [ {"b": int }] {"a", list}, {"element", struct}, {"b", int} +// "a": [ null] {"a", list}, {"element", str} +// back() is root. +// front() is leaf. +std::optional get_path_data_type( + host_span> path, schema_element const& root) +{ + if (path.empty() || path.size() == 1) { + return root.type; + } else { + if (path.back().second == NC_STRUCT && root.type.id() == type_id::STRUCT) { + auto const child_name = path.first(path.size() - 1).back().first; + auto const child_schema_it = root.child_types.find(child_name); + return (child_schema_it != std::end(root.child_types)) + ? get_path_data_type(path.first(path.size() - 1), child_schema_it->second) + : std::optional{}; + } else if (path.back().second == NC_LIST && root.type.id() == type_id::LIST) { + auto const child_schema_it = root.child_types.find(list_child_name); + return (child_schema_it != std::end(root.child_types)) + ? get_path_data_type(path.first(path.size() - 1), child_schema_it->second) + : std::optional{}; + } + return std::optional{}; + } +} + +std::optional get_path_data_type( + host_span> path, + cudf::io::json_reader_options const& options) +{ + if (path.empty()) return {}; + std::optional col_schema = child_schema_element(path.back().first, options); + // check if it has value, then do recursive call and return. + if (col_schema.has_value()) { + return get_path_data_type(path, col_schema.value()); + } else { + return {}; + } +} + +// idea: write a memoizer using template and lambda?, then call recursively. +std::vector path_from_tree::get_path(NodeIndexT this_col_id) +{ + std::vector path; + // TODO Need to stop at row root. so, how to find row root? + while (this_col_id != parent_node_sentinel) { + auto type = column_categories[this_col_id]; + std::string name = ""; + // TODO make this ifelse into a separate lambda function, along with parent_col_id. + auto parent_col_id = column_parent_ids[this_col_id]; + if (parent_col_id == parent_node_sentinel || column_categories[parent_col_id] == NC_LIST) { + if (is_array_of_arrays && parent_col_id == row_array_parent_col_id) { + name = column_names[this_col_id]; + } else { + name = list_child_name; + } + } else if (column_categories[parent_col_id] == NC_FN) { + auto field_name_col_id = parent_col_id; + parent_col_id = column_parent_ids[parent_col_id]; + name = column_names[field_name_col_id]; + } + // "name": type/schema + path.emplace_back(name, type); + this_col_id = parent_col_id; + if (this_col_id == row_array_parent_col_id) return path; + } + return {}; +} + +} // namespace cudf::io::json::detail diff --git a/cpp/tests/io/json_test.cpp b/cpp/tests/io/json_test.cpp index 450ea550e99..0b70e5e3f93 100644 --- a/cpp/tests/io/json_test.cpp +++ b/cpp/tests/io/json_test.cpp @@ -2239,4 +2239,56 @@ TEST_F(JsonReaderTest, MixedTypes) expected_list); } +TEST_F(JsonReaderTest, MapTypes) +{ + using cudf::type_id; + // Testing function for mixed types in JSON (for spark json reader) + auto test_fn = [](std::string_view json_string, bool lines, std::vector types) { + std::map dtype_schema{ + {"foo1", {data_type{type_id::STRING}}}, // list won't be a string + {"foo2", {data_type{type_id::STRING}}}, // struct forced as a string + {"1", {data_type{type_id::STRING}}}, + {"2", {data_type{type_id::STRING}}}, + {"bar", {dtype()}}, + }; + + cudf::io::json_reader_options in_options = + cudf::io::json_reader_options::builder( + cudf::io::source_info{json_string.data(), json_string.size()}) + .dtypes(dtype_schema) + .mixed_types_as_string(true) + .lines(lines); + + cudf::io::table_with_metadata result = cudf::io::read_json(in_options); + EXPECT_EQ(result.tbl->num_columns(), types.size()); + int i = 0; + for (auto& col : result.tbl->view()) { + EXPECT_EQ(col.type().id(), types[i]) << "column[" << i << "].type"; + i++; + } + std::cout << "\n"; + }; + + // json + test_fn(R"([{ "foo1": [1,2,3], "bar": 123 }, + { "foo2": { "a": 1 }, "bar": 456 }])", + false, + {type_id::LIST, type_id::INT32, type_id::STRING}); + // jsonl + test_fn(R"( { "foo1": [1,2,3], "bar": 123 } + { "foo2": { "a": 1 }, "bar": 456 })", + true, + {type_id::LIST, type_id::INT32, type_id::STRING}); + // jsonl-array + test_fn(R"([123, [1,2,3]] + [456, null, { "a": 1 }])", + true, + {type_id::INT64, type_id::LIST, type_id::STRING}); + // json-array + test_fn(R"([[[1,2,3], null, 123], + [null, { "a": 1 }, 456 ]])", + false, + {type_id::LIST, type_id::STRING, type_id::STRING}); +} + CUDF_TEST_PROGRAM_MAIN() From c794ce4968b69e0cffc97b3db9496a1cdeab78bc Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 11 Mar 2024 09:41:32 -0500 Subject: [PATCH 384/384] Temporarily disable docs errors. (#15265) Currently there are some network issues affecting docs builds. To prevent this from causing complete CI blockage, we can temporarily allow errors in the docs build. This will allow us to monitor the network status and re-enable the docs builds when the network issues are resolved. Authors: - Bradley Dice (https://github.com/bdice) - Matthew Roeschke (https://github.com/mroeschke) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) - Jake Awe (https://github.com/AyodeAwe) URL: https://github.com/rapidsai/cudf/pull/15265 --- ci/build_docs.sh | 14 +++++++++++++- python/cudf/cudf/core/column/column.py | 7 +++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index b94c61cc184..4b6391be82c 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -34,6 +34,11 @@ rapids-mamba-retry install \ export RAPIDS_DOCS_DIR="$(mktemp -d)" +# TODO: Disable hard errors until the docs site is accessible (network problems) +EXITCODE=0 +trap "EXITCODE=1" ERR +set +e + rapids-logger "Build CPP docs" pushd cpp/doxygen aws s3 cp s3://rapidsai-docs/librmm/html/${RAPIDS_VERSION_NUMBER}/rmm.tag . || echo "Failed to download rmm Doxygen tag" @@ -66,4 +71,11 @@ if [[ "${RAPIDS_BUILD_TYPE}" != "pull-request" ]]; then fi popd -rapids-upload-docs +if [[ "${EXITCODE}" == "0" ]]; then + rapids-upload-docs +else + rapids-logger "Docs script had errors resulting in exit code $EXITCODE" +fi + +# TODO: Disable hard errors until the docs site is accessible (network problems) +exit 0 diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index b7080ff7a7c..3e0ec4b5cd7 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -975,9 +975,12 @@ def astype(self, dtype: Dtype, copy: bool = False) -> ColumnBase: # TODO: Figure out why `cudf.dtype("category")` # astype's different than just the string return col.as_categorical_column(dtype) - elif dtype == "interval" and isinstance( - self.dtype, cudf.IntervalDtype + elif ( + isinstance(dtype, str) + and dtype == "interval" + and isinstance(self.dtype, cudf.IntervalDtype) ): + # astype("interval") (the string only) should no-op return col was_object = dtype == object or dtype == np.dtype(object) dtype = cudf.dtype(dtype)