Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

REF: Use np.result_type instead of np.find_common_type #53343

Merged
merged 6 commits into from
May 23, 2023
Merged

REF: Use np.result_type instead of np.find_common_type #53343

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1f52516
REF: Use np.result_type instead of np.find_common_type
mroeschke May 22, 2023
7081bc1
Fall back to object
mroeschke May 22, 2023
cfb0522
Create a np_find_common_dtype
mroeschke May 22, 2023
a6d9ad3
Address circular import
mroeschke May 22, 2023
cfe1486
Use generator
mroeschke May 22, 2023
a0f176f
Typing
mroeschke May 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
7 changes: 5 additions & 2 deletions pandas/core/algorithms.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,10 @@
from pandas.util._decorators import doc
from pandas.util._exceptions import find_stack_level

from pandas.core.dtypes.cast import construct_1d_object_array_from_listlike
from pandas.core.dtypes.cast import (
construct_1d_object_array_from_listlike,
np_find_common_type,
)
from pandas.core.dtypes.common import (
ensure_float64,
ensure_object,
Expand Down Expand Up @@ -518,7 +521,7 @@ def f(c, v):
f = np.in1d

else:
common = np.find_common_type([values.dtype, comps_array.dtype], [])
common = np_find_common_type(values.dtype, comps_array.dtype)
values = values.astype(common, copy=False)
comps_array = comps_array.astype(common, copy=False)
f = htable.ismember
Expand Down
28 changes: 27 additions & 1 deletion pandas/core/dtypes/cast.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1328,6 +1328,32 @@ def common_dtype_categorical_compat(
return dtype


def np_find_common_type(*dtypes: np.dtype) -> np.dtype:
"""
np.find_common_type implementation pre-1.25 deprecation using np.result_type
https://github.com/pandas-dev/pandas/pull/49569#issuecomment-1308300065

Parameters
----------
dtypes : np.dtypes

Returns
-------
np.dtype
"""
try:
common_dtype = np.result_type(*dtypes)
if common_dtype.kind in "mMSU":
# NumPy promotion currently (1.25) misbehaves for for times and strings,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is there some numpy version at which we can remove this check?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure cc @seberg any thoughts?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have a PR that would fix it, so I hope next release this will happen. That was the PR which get stuck a bit on the discussion of what == should do.

# so fall back to object (find_common_dtype did unless there
# was only one dtype)
common_dtype = np.dtype("O")

except TypeError:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you give an example of what gets here?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

In [2]: np.result_type(np.dtype(float), np.dtype("M8"))
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[2], line 1
----> 1 np.result_type(np.dtype(float), np.dtype("M8"))

File <__array_function__ internals>:200, in result_type(*args, **kwargs)

TypeError: The DType <class 'numpy.dtype[float64]'> could not be promoted by <class 'numpy.dtype[datetime64]'>. This means that no common DType exists for the given inputs. For example they cannot be stored in a single array unless the dtype is `object`. The full list of DTypes is: (<class 'numpy.dtype[float64]'>, <class 'numpy.dtype[datetime64]'>)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

AFAICT pretty much anything that would resolve to object before would return a TypeError now

common_dtype = np.dtype("O")
return common_dtype


@overload
def find_common_type(types: list[np.dtype]) -> np.dtype:
...
Expand Down Expand Up @@ -1395,7 +1421,7 @@ def find_common_type(types):
if t.kind in "iufc":
return np.dtype("object")

return np.find_common_type(types, [])
return np_find_common_type(*types)


def construct_2d_arraylike_from_scalar(
Expand Down
9 changes: 4 additions & 5 deletions pandas/core/dtypes/concat.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from pandas.core.dtypes.cast import (
common_dtype_categorical_compat,
find_common_type,
np_find_common_type,
)
from pandas.core.dtypes.dtypes import CategoricalDtype
from pandas.core.dtypes.generic import (
Expand Down Expand Up @@ -156,11 +157,9 @@ def _get_result_dtype(
target_dtype = np.dtype(object)
kinds = {"o"}
else:
# Argument 1 to "list" has incompatible type "Set[Union[ExtensionDtype,
# Any]]"; expected "Iterable[Union[dtype[Any], None, Type[Any],
# _SupportsDType[dtype[Any]], str, Tuple[Any, Union[SupportsIndex,
# Sequence[SupportsIndex]]], List[Any], _DTypeDict, Tuple[Any, Any]]]"
target_dtype = np.find_common_type(list(dtypes), []) # type: ignore[arg-type]
# error: Argument 1 to "np_find_common_type" has incompatible type
# "*Set[Union[ExtensionDtype, Any]]"; expected "dtype[Any]"
target_dtype = np_find_common_type(*dtypes) # type: ignore[arg-type]

return any_ea, kinds, target_dtype

Expand Down
6 changes: 4 additions & 2 deletions pandas/core/dtypes/dtypes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1921,6 +1921,8 @@ def _subtype_with_str(self):
def _get_common_dtype(self, dtypes: list[DtypeObj]) -> DtypeObj | None:
# TODO for now only handle SparseDtypes and numpy dtypes => extend
# with other compatible extension dtypes
from pandas.core.dtypes.cast import np_find_common_type

if any(
isinstance(x, ExtensionDtype) and not isinstance(x, SparseDtype)
for x in dtypes
Expand All @@ -1943,8 +1945,8 @@ def _get_common_dtype(self, dtypes: list[DtypeObj]) -> DtypeObj | None:
stacklevel=find_stack_level(),
)

np_dtypes = [x.subtype if isinstance(x, SparseDtype) else x for x in dtypes]
return SparseDtype(np.find_common_type(np_dtypes, []), fill_value=fill_value)
np_dtypes = (x.subtype if isinstance(x, SparseDtype) else x for x in dtypes)
return SparseDtype(np_find_common_type(*np_dtypes), fill_value=fill_value)


@register_extension_dtype
Expand Down
3 changes: 2 additions & 1 deletion pandas/core/internals/array_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
ensure_dtype_can_hold_na,
find_common_type,
infer_dtype_from_scalar,
np_find_common_type,
)
from pandas.core.dtypes.common import (
ensure_platform_int,
Expand Down Expand Up @@ -1409,7 +1410,7 @@ def concat_arrays(to_concat: list) -> ArrayLike:
target_dtype = to_concat_no_proxy[0].dtype
elif all(x.kind in "iub" and isinstance(x, np.dtype) for x in dtypes):
# GH#42092
target_dtype = np.find_common_type(list(dtypes), [])
target_dtype = np_find_common_type(*dtypes)
else:
target_dtype = find_common_type([arr.dtype for arr in to_concat_no_proxy])

Expand Down
4 changes: 1 addition & 3 deletions pandas/tests/dtypes/test_inference.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -997,9 +997,7 @@ def test_maybe_convert_objects_itemsize(self, data0, data1):
data = [data0, data1]
arr = np.array(data, dtype="object")

common_kind = np.find_common_type(
[type(data0), type(data1)], scalar_types=[]
).kind
common_kind = np.result_type(type(data0), type(data1)).kind
kind0 = "python" if not hasattr(data0, "dtype") else data0.dtype.kind
kind1 = "python" if not hasattr(data1, "dtype") else data1.dtype.kind
if kind0 != "python" and kind1 != "python":
Expand Down