Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CLN: Enforce deprecations for EA.fillna #57983

Merged
merged 1 commit into from Mar 25, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion doc/source/whatsnew/v3.0.0.rst
Expand Up @@ -34,7 +34,6 @@ Other enhancements
- Allow dictionaries to be passed to :meth:`pandas.Series.str.replace` via ``pat`` parameter (:issue:`51748`)
- Support passing a :class:`Series` input to :func:`json_normalize` that retains the :class:`Series` :class:`Index` (:issue:`51452`)
- Users can globally disable any ``PerformanceWarning`` by setting the option ``mode.performance_warnings`` to ``False`` (:issue:`56920`)
-

.. ---------------------------------------------------------------------------
.. _whatsnew_300.notable_bug_fixes:
Expand Down Expand Up @@ -258,6 +257,7 @@ Removal of prior version deprecations/changes
- Unrecognized timezones when parsing strings to datetimes now raises a ``ValueError`` (:issue:`51477`)
- Removed the :class:`Grouper` attributes ``ax``, ``groups``, ``indexer``, and ``obj`` (:issue:`51206`, :issue:`51182`)
- Removed deprecated keyword ``verbose`` on :func:`read_csv` and :func:`read_table` (:issue:`56556`)
- Removed the ``method`` keyword in ``ExtensionArray.fillna``, implement ``ExtensionArray._pad_or_backfill`` instead (:issue:`53621`)
- Removed the attribute ``dtypes`` from :class:`.DataFrameGroupBy` (:issue:`51997`)

.. ---------------------------------------------------------------------------
Expand Down
32 changes: 6 additions & 26 deletions pandas/core/arrays/_mixins.py
Expand Up @@ -33,7 +33,6 @@
from pandas.util._decorators import doc
from pandas.util._validators import (
validate_bool_kwarg,
validate_fillna_kwargs,
validate_insert_loc,
)

Expand Down Expand Up @@ -336,13 +335,7 @@ def _pad_or_backfill(
return new_values

@doc(ExtensionArray.fillna)
def fillna(
self, value=None, method=None, limit: int | None = None, copy: bool = True
) -> Self:
value, method = validate_fillna_kwargs(
value, method, validate_scalar_dict_value=False
)

def fillna(self, value=None, limit: int | None = None, copy: bool = True) -> Self:
mask = self.isna()
# error: Argument 2 to "check_value_size" has incompatible type
# "ExtensionArray"; expected "ndarray"
Expand All @@ -353,25 +346,12 @@ def fillna(
)

if mask.any():
if method is not None:
# (for now) when self.ndim == 2, we assume axis=0
func = missing.get_fill_func(method, ndim=self.ndim)
npvalues = self._ndarray.T
if copy:
npvalues = npvalues.copy()
func(npvalues, limit=limit, mask=mask.T)
npvalues = npvalues.T

# TODO: NumpyExtensionArray didn't used to copy, need tests
# for this
new_values = self._from_backing_data(npvalues)
# fill with value
if copy:
new_values = self.copy()
else:
# fill with value
if copy:
new_values = self.copy()
else:
new_values = self[:]
new_values[mask] = value
new_values = self[:]
new_values[mask] = value
else:
# We validate the fill_value even if there is nothing to fill
if value is not None:
Expand Down
12 changes: 3 additions & 9 deletions pandas/core/arrays/arrow/array.py
Expand Up @@ -29,7 +29,6 @@
pa_version_under13p0,
)
from pandas.util._decorators import doc
from pandas.util._validators import validate_fillna_kwargs

from pandas.core.dtypes.cast import (
can_hold_element,
Expand Down Expand Up @@ -1068,6 +1067,7 @@ def _pad_or_backfill(
# a kernel for duration types.
pass

# TODO: Why do we no longer need the above cases?
# TODO(3.0): after EA.fillna 'method' deprecation is enforced, we can remove
# this method entirely.
return super()._pad_or_backfill(
Expand All @@ -1078,21 +1078,15 @@ def _pad_or_backfill(
def fillna(
self,
value: object | ArrayLike | None = None,
method: FillnaOptions | None = None,
limit: int | None = None,
copy: bool = True,
) -> Self:
value, method = validate_fillna_kwargs(value, method)

if not self._hasna:
# TODO(CoW): Not necessary anymore when CoW is the default
return self.copy()

if limit is not None:
return super().fillna(value=value, method=method, limit=limit, copy=copy)

if method is not None:
return super().fillna(method=method, limit=limit, copy=copy)
return super().fillna(value=value, limit=limit, copy=copy)

if isinstance(value, (np.ndarray, ExtensionArray)):
# Similar to check_value_size, but we do not mask here since we may
Expand All @@ -1118,7 +1112,7 @@ def fillna(
# a kernel for duration types.
pass

return super().fillna(value=value, method=method, limit=limit, copy=copy)
return super().fillna(value=value, limit=limit, copy=copy)

def isin(self, values: ArrayLike) -> npt.NDArray[np.bool_]:
# short-circuit to return all False array.
Expand Down
72 changes: 6 additions & 66 deletions pandas/core/arrays/base.py
Expand Up @@ -38,7 +38,6 @@
from pandas.util._exceptions import find_stack_level
from pandas.util._validators import (
validate_bool_kwarg,
validate_fillna_kwargs,
validate_insert_loc,
)

Expand Down Expand Up @@ -1007,31 +1006,6 @@ def _pad_or_backfill(
[<NA>, 2, 2, 3, <NA>, <NA>]
Length: 6, dtype: Int64
"""

# If a 3rd-party EA has implemented this functionality in fillna,
# we warn that they need to implement _pad_or_backfill instead.
if (
type(self).fillna is not ExtensionArray.fillna
and type(self)._pad_or_backfill is ExtensionArray._pad_or_backfill
):
# Check for _pad_or_backfill here allows us to call
# super()._pad_or_backfill without getting this warning
warnings.warn(
"ExtensionArray.fillna 'method' keyword is deprecated. "
"In a future version. arr._pad_or_backfill will be called "
"instead. 3rd-party ExtensionArray authors need to implement "
"_pad_or_backfill.",
DeprecationWarning,
stacklevel=find_stack_level(),
)
if limit_area is not None:
raise NotImplementedError(
f"{type(self).__name__} does not implement limit_area "
"(added in pandas 2.2). 3rd-party ExtnsionArray authors "
"need to add this argument to _pad_or_backfill."
)
return self.fillna(method=method, limit=limit)

mask = self.isna()

if mask.any():
Expand All @@ -1057,8 +1031,7 @@ def _pad_or_backfill(

def fillna(
self,
value: object | ArrayLike | None = None,
method: FillnaOptions | None = None,
value: object | ArrayLike,
limit: int | None = None,
copy: bool = True,
) -> Self:
Expand All @@ -1071,24 +1044,13 @@ def fillna(
If a scalar value is passed it is used to fill all missing values.
Alternatively, an array-like "value" can be given. It's expected
that the array-like have the same length as 'self'.
method : {'backfill', 'bfill', 'pad', 'ffill', None}, default None
Method to use for filling holes in reindexed Series:

* pad / ffill: propagate last valid observation forward to next valid.
* backfill / bfill: use NEXT valid observation to fill gap.

.. deprecated:: 2.1.0

limit : int, default None
If method is specified, this is the maximum number of consecutive
NaN values to forward/backward fill. In other words, if there is
a gap with more than this number of consecutive NaNs, it will only
be partially filled. If method is not specified, this is the
maximum number of entries along the entire axis where NaNs will be
filled.

.. deprecated:: 2.1.0

copy : bool, default True
Whether to make a copy of the data before filling. If False, then
the original should be modified and no new memory should be allocated.
Expand All @@ -1110,16 +1072,6 @@ def fillna(
[0, 0, 2, 3, 0, 0]
Length: 6, dtype: Int64
"""
if method is not None:
warnings.warn(
f"The 'method' keyword in {type(self).__name__}.fillna is "
"deprecated and will be removed in a future version.",
FutureWarning,
stacklevel=find_stack_level(),
)

value, method = validate_fillna_kwargs(value, method)

mask = self.isna()
# error: Argument 2 to "check_value_size" has incompatible type
# "ExtensionArray"; expected "ndarray"
Expand All @@ -1130,24 +1082,12 @@ def fillna(
)

if mask.any():
if method is not None:
meth = missing.clean_fill_method(method)

npmask = np.asarray(mask)
if meth == "pad":
indexer = libalgos.get_fill_indexer(npmask, limit=limit)
return self.take(indexer, allow_fill=True)
else:
# i.e. meth == "backfill"
indexer = libalgos.get_fill_indexer(npmask[::-1], limit=limit)[::-1]
return self[::-1].take(indexer, allow_fill=True)
# fill with value
if not copy:
new_values = self[:]
else:
# fill with value
if not copy:
new_values = self[:]
else:
new_values = self.copy()
new_values[mask] = value
new_values = self.copy()
new_values[mask] = value
else:
if not copy:
new_values = self[:]
Expand Down
24 changes: 1 addition & 23 deletions pandas/core/arrays/interval.py
Expand Up @@ -29,7 +29,6 @@
ArrayLike,
AxisInt,
Dtype,
FillnaOptions,
IntervalClosedType,
NpDtype,
PositionalIndexer,
Expand Down Expand Up @@ -894,23 +893,7 @@ def max(self, *, axis: AxisInt | None = None, skipna: bool = True) -> IntervalOr
indexer = obj.argsort()[-1]
return obj[indexer]

def _pad_or_backfill( # pylint: disable=useless-parent-delegation
self,
*,
method: FillnaOptions,
limit: int | None = None,
limit_area: Literal["inside", "outside"] | None = None,
copy: bool = True,
) -> Self:
# TODO(3.0): after EA.fillna 'method' deprecation is enforced, we can remove
# this method entirely.
return super()._pad_or_backfill(
method=method, limit=limit, limit_area=limit_area, copy=copy
)

def fillna(
self, value=None, method=None, limit: int | None = None, copy: bool = True
) -> Self:
def fillna(self, value=None, limit: int | None = None, copy: bool = True) -> Self:
"""
Fill NA/NaN values using the specified method.

Expand All @@ -921,9 +904,6 @@ def fillna(
Alternatively, a Series or dict can be used to fill in different
values for each index. The value should not be a list. The
value(s) passed should be either Interval objects or NA/NaN.
method : {'backfill', 'bfill', 'pad', 'ffill', None}, default None
(Not implemented yet for IntervalArray)
Method to use for filling holes in reindexed Series
limit : int, default None
(Not implemented yet for IntervalArray)
If method is specified, this is the maximum number of consecutive
Expand All @@ -944,8 +924,6 @@ def fillna(
"""
if copy is False:
raise NotImplementedError
if method is not None:
return super().fillna(value=value, method=method, limit=limit)

value_left, value_right = self._validate_scalar(value)

Expand Down
27 changes: 6 additions & 21 deletions pandas/core/arrays/masked.py
Expand Up @@ -38,7 +38,6 @@
)
from pandas.errors import AbstractMethodError
from pandas.util._decorators import doc
from pandas.util._validators import validate_fillna_kwargs

from pandas.core.dtypes.base import ExtensionDtype
from pandas.core.dtypes.common import (
Expand Down Expand Up @@ -237,32 +236,18 @@ def _pad_or_backfill(
return new_values

@doc(ExtensionArray.fillna)
def fillna(
self, value=None, method=None, limit: int | None = None, copy: bool = True
) -> Self:
value, method = validate_fillna_kwargs(value, method)

def fillna(self, value=None, limit: int | None = None, copy: bool = True) -> Self:
mask = self._mask

value = missing.check_value_size(value, mask, len(self))

if mask.any():
if method is not None:
func = missing.get_fill_func(method, ndim=self.ndim)
npvalues = self._data.T
new_mask = mask.T
if copy:
npvalues = npvalues.copy()
new_mask = new_mask.copy()
func(npvalues, limit=limit, mask=new_mask)
return self._simple_new(npvalues.T, new_mask.T)
# fill with value
if copy:
new_values = self.copy()
else:
# fill with value
if copy:
new_values = self.copy()
else:
new_values = self[:]
new_values[mask] = value
new_values = self[:]
new_values[mask] = value
else:
if copy:
new_values = self.copy()
Expand Down
13 changes: 0 additions & 13 deletions pandas/core/arrays/period.py
Expand Up @@ -847,19 +847,6 @@ def _pad_or_backfill(
else:
return self

def fillna(
self, value=None, method=None, limit: int | None = None, copy: bool = True
) -> Self:
if method is not None:
# view as dt64 so we get treated as timelike in core.missing,
# similar to dtl._period_dispatch
dta = self.view("M8[ns]")
result = dta.fillna(value=value, method=method, limit=limit, copy=copy)
# error: Incompatible return value type (got "Union[ExtensionArray,
# ndarray[Any, Any]]", expected "PeriodArray")
return result.view(self.dtype) # type: ignore[return-value]
return super().fillna(value=value, method=method, limit=limit, copy=copy)

# ------------------------------------------------------------------
# Arithmetic Methods

Expand Down