From 1f525160d91366e3eef98f88f22b59a8e0177683 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 22 May 2023 13:35:41 -0700 Subject: [PATCH 1/6] REF: Use np.result_type instead of np.find_common_type --- pandas/core/algorithms.py | 2 +- pandas/core/dtypes/cast.py | 2 +- pandas/core/dtypes/concat.py | 6 +----- pandas/core/dtypes/dtypes.py | 2 +- pandas/core/internals/array_manager.py | 2 +- pandas/tests/dtypes/test_inference.py | 4 +--- 6 files changed, 6 insertions(+), 12 deletions(-) diff --git a/pandas/core/algorithms.py b/pandas/core/algorithms.py index 37c1fa76fbbcf..933913acb012b 100644 --- a/pandas/core/algorithms.py +++ b/pandas/core/algorithms.py @@ -518,7 +518,7 @@ def f(c, v): f = np.in1d else: - common = np.find_common_type([values.dtype, comps_array.dtype], []) + common = np.result_type(values.dtype, comps_array.dtype) values = values.astype(common, copy=False) comps_array = comps_array.astype(common, copy=False) f = htable.ismember diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index e7a6692807685..3fc223a9248fa 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -1395,7 +1395,7 @@ def find_common_type(types): if t.kind in "iufc": return np.dtype("object") - return np.find_common_type(types, []) + return np.result_type(*types) def construct_2d_arraylike_from_scalar( diff --git a/pandas/core/dtypes/concat.py b/pandas/core/dtypes/concat.py index 4a25c3541a398..0c83af97ce2b6 100644 --- a/pandas/core/dtypes/concat.py +++ b/pandas/core/dtypes/concat.py @@ -156,11 +156,7 @@ def _get_result_dtype( target_dtype = np.dtype(object) kinds = {"o"} else: - # Argument 1 to "list" has incompatible type "Set[Union[ExtensionDtype, - # Any]]"; expected "Iterable[Union[dtype[Any], None, Type[Any], - # _SupportsDType[dtype[Any]], str, Tuple[Any, Union[SupportsIndex, - # Sequence[SupportsIndex]]], List[Any], _DTypeDict, Tuple[Any, Any]]]" - target_dtype = np.find_common_type(list(dtypes), []) # type: ignore[arg-type] + target_dtype = np.result_type(*dtypes) return any_ea, kinds, target_dtype diff --git a/pandas/core/dtypes/dtypes.py b/pandas/core/dtypes/dtypes.py index a3481cbe9eae1..a68712e6e7a13 100644 --- a/pandas/core/dtypes/dtypes.py +++ b/pandas/core/dtypes/dtypes.py @@ -1944,7 +1944,7 @@ def _get_common_dtype(self, dtypes: list[DtypeObj]) -> DtypeObj | None: ) np_dtypes = [x.subtype if isinstance(x, SparseDtype) else x for x in dtypes] - return SparseDtype(np.find_common_type(np_dtypes, []), fill_value=fill_value) + return SparseDtype(np.result_type(*np_dtypes), fill_value=fill_value) @register_extension_dtype diff --git a/pandas/core/internals/array_manager.py b/pandas/core/internals/array_manager.py index 75eaaa80a9961..ac80b903d1f03 100644 --- a/pandas/core/internals/array_manager.py +++ b/pandas/core/internals/array_manager.py @@ -1409,7 +1409,7 @@ def concat_arrays(to_concat: list) -> ArrayLike: target_dtype = to_concat_no_proxy[0].dtype elif all(x.kind in "iub" and isinstance(x, np.dtype) for x in dtypes): # GH#42092 - target_dtype = np.find_common_type(list(dtypes), []) + target_dtype = np.result_type(*dtypes) else: target_dtype = find_common_type([arr.dtype for arr in to_concat_no_proxy]) diff --git a/pandas/tests/dtypes/test_inference.py b/pandas/tests/dtypes/test_inference.py index 4f884afcb1a90..bbce40727c669 100644 --- a/pandas/tests/dtypes/test_inference.py +++ b/pandas/tests/dtypes/test_inference.py @@ -997,9 +997,7 @@ def test_maybe_convert_objects_itemsize(self, data0, data1): data = [data0, data1] arr = np.array(data, dtype="object") - common_kind = np.find_common_type( - [type(data0), type(data1)], scalar_types=[] - ).kind + common_kind = np.result_type(type(data0), type(data1)).kind kind0 = "python" if not hasattr(data0, "dtype") else data0.dtype.kind kind1 = "python" if not hasattr(data1, "dtype") else data1.dtype.kind if kind0 != "python" and kind1 != "python": From 7081bc132e377a40cf6245c3df925c55be97bfac Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 22 May 2023 14:51:06 -0700 Subject: [PATCH 2/6] Fall back to object --- pandas/core/dtypes/cast.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 3fc223a9248fa..fae727ce597be 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -1395,7 +1395,10 @@ def find_common_type(types): if t.kind in "iufc": return np.dtype("object") - return np.result_type(*types) + try: + return np.result_type(*types) + except TypeError: + return np.dtype(object) def construct_2d_arraylike_from_scalar( From cfb05227ce65fcb3e4e8146e8c182a9d564df6b8 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 22 May 2023 15:04:18 -0700 Subject: [PATCH 3/6] Create a np_find_common_dtype --- pandas/core/algorithms.py | 7 ++++-- pandas/core/dtypes/cast.py | 31 ++++++++++++++++++++++---- pandas/core/dtypes/concat.py | 3 ++- pandas/core/dtypes/dtypes.py | 3 ++- pandas/core/internals/array_manager.py | 3 ++- 5 files changed, 38 insertions(+), 9 deletions(-) diff --git a/pandas/core/algorithms.py b/pandas/core/algorithms.py index 933913acb012b..768265a5ce621 100644 --- a/pandas/core/algorithms.py +++ b/pandas/core/algorithms.py @@ -32,7 +32,10 @@ from pandas.util._decorators import doc from pandas.util._exceptions import find_stack_level -from pandas.core.dtypes.cast import construct_1d_object_array_from_listlike +from pandas.core.dtypes.cast import ( + construct_1d_object_array_from_listlike, + np_find_common_type, +) from pandas.core.dtypes.common import ( ensure_float64, ensure_object, @@ -518,7 +521,7 @@ def f(c, v): f = np.in1d else: - common = np.result_type(values.dtype, comps_array.dtype) + common = np_find_common_type(values.dtype, comps_array.dtype) values = values.astype(common, copy=False) comps_array = comps_array.astype(common, copy=False) f = htable.ismember diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index fae727ce597be..c863e5bb4dbd4 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -1328,6 +1328,32 @@ def common_dtype_categorical_compat( return dtype +def np_find_common_type(*dtypes: np.dtype) -> np.dtype: + """ + np.find_common_type implementation pre-1.25 deprecation using np.result_type + https://github.com/pandas-dev/pandas/pull/49569#issuecomment-1308300065 + + Parameters + ---------- + dtypes : np.dtypes + + Returns + ------- + np.dtype + """ + try: + common_dtype = np.result_type(*dtypes) + if common_dtype.kind in "mMSU": + # NumPy promotion currently (1.25) misbehaves for for times and strings, + # so fall back to object (find_common_dtype did unless there + # was only one dtype) + common_dtype = np.dtype("O") + + except TypeError: + common_dtype = np.dtype("O") + return common_dtype + + @overload def find_common_type(types: list[np.dtype]) -> np.dtype: ... @@ -1395,10 +1421,7 @@ def find_common_type(types): if t.kind in "iufc": return np.dtype("object") - try: - return np.result_type(*types) - except TypeError: - return np.dtype(object) + return np_find_common_type(*types) def construct_2d_arraylike_from_scalar( diff --git a/pandas/core/dtypes/concat.py b/pandas/core/dtypes/concat.py index 0c83af97ce2b6..7c80449411584 100644 --- a/pandas/core/dtypes/concat.py +++ b/pandas/core/dtypes/concat.py @@ -17,6 +17,7 @@ from pandas.core.dtypes.cast import ( common_dtype_categorical_compat, find_common_type, + np_find_common_type, ) from pandas.core.dtypes.dtypes import CategoricalDtype from pandas.core.dtypes.generic import ( @@ -156,7 +157,7 @@ def _get_result_dtype( target_dtype = np.dtype(object) kinds = {"o"} else: - target_dtype = np.result_type(*dtypes) + target_dtype = np_find_common_type(*dtypes) return any_ea, kinds, target_dtype diff --git a/pandas/core/dtypes/dtypes.py b/pandas/core/dtypes/dtypes.py index a68712e6e7a13..e07af4550d8d6 100644 --- a/pandas/core/dtypes/dtypes.py +++ b/pandas/core/dtypes/dtypes.py @@ -52,6 +52,7 @@ StorageExtensionDtype, register_extension_dtype, ) +from pandas.core.dtypes.cast import np_find_common_type from pandas.core.dtypes.generic import ( ABCCategoricalIndex, ABCIndex, @@ -1944,7 +1945,7 @@ def _get_common_dtype(self, dtypes: list[DtypeObj]) -> DtypeObj | None: ) np_dtypes = [x.subtype if isinstance(x, SparseDtype) else x for x in dtypes] - return SparseDtype(np.result_type(*np_dtypes), fill_value=fill_value) + return SparseDtype(np_find_common_type(*np_dtypes), fill_value=fill_value) @register_extension_dtype diff --git a/pandas/core/internals/array_manager.py b/pandas/core/internals/array_manager.py index ac80b903d1f03..098a78fc54b71 100644 --- a/pandas/core/internals/array_manager.py +++ b/pandas/core/internals/array_manager.py @@ -29,6 +29,7 @@ ensure_dtype_can_hold_na, find_common_type, infer_dtype_from_scalar, + np_find_common_type, ) from pandas.core.dtypes.common import ( ensure_platform_int, @@ -1409,7 +1410,7 @@ def concat_arrays(to_concat: list) -> ArrayLike: target_dtype = to_concat_no_proxy[0].dtype elif all(x.kind in "iub" and isinstance(x, np.dtype) for x in dtypes): # GH#42092 - target_dtype = np.result_type(*dtypes) + target_dtype = np_find_common_type(*dtypes) else: target_dtype = find_common_type([arr.dtype for arr in to_concat_no_proxy]) From a6d9ad3082dd9c93b18c34df339ac80c13d5f624 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 22 May 2023 15:18:49 -0700 Subject: [PATCH 4/6] Address circular import --- pandas/core/dtypes/dtypes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/core/dtypes/dtypes.py b/pandas/core/dtypes/dtypes.py index e07af4550d8d6..63e1f71d67c53 100644 --- a/pandas/core/dtypes/dtypes.py +++ b/pandas/core/dtypes/dtypes.py @@ -52,7 +52,6 @@ StorageExtensionDtype, register_extension_dtype, ) -from pandas.core.dtypes.cast import np_find_common_type from pandas.core.dtypes.generic import ( ABCCategoricalIndex, ABCIndex, @@ -1922,6 +1921,8 @@ def _subtype_with_str(self): def _get_common_dtype(self, dtypes: list[DtypeObj]) -> DtypeObj | None: # TODO for now only handle SparseDtypes and numpy dtypes => extend # with other compatible extension dtypes + from pandas.core.dtypes.cast import np_find_common_type + if any( isinstance(x, ExtensionDtype) and not isinstance(x, SparseDtype) for x in dtypes From cfe1486342d265c6b4f53ce02ff3d5779027a24e Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 22 May 2023 15:19:19 -0700 Subject: [PATCH 5/6] Use generator --- pandas/core/dtypes/dtypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/dtypes/dtypes.py b/pandas/core/dtypes/dtypes.py index 63e1f71d67c53..e6c1c70a2aff5 100644 --- a/pandas/core/dtypes/dtypes.py +++ b/pandas/core/dtypes/dtypes.py @@ -1945,7 +1945,7 @@ def _get_common_dtype(self, dtypes: list[DtypeObj]) -> DtypeObj | None: stacklevel=find_stack_level(), ) - np_dtypes = [x.subtype if isinstance(x, SparseDtype) else x for x in dtypes] + np_dtypes = (x.subtype if isinstance(x, SparseDtype) else x for x in dtypes) return SparseDtype(np_find_common_type(*np_dtypes), fill_value=fill_value) From a0f176f682c17b16823911f0b13dd62352669880 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 22 May 2023 16:21:01 -0700 Subject: [PATCH 6/6] Typing --- pandas/core/dtypes/concat.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/core/dtypes/concat.py b/pandas/core/dtypes/concat.py index 7c80449411584..cba7c44a219bf 100644 --- a/pandas/core/dtypes/concat.py +++ b/pandas/core/dtypes/concat.py @@ -157,7 +157,9 @@ def _get_result_dtype( target_dtype = np.dtype(object) kinds = {"o"} else: - target_dtype = np_find_common_type(*dtypes) + # error: Argument 1 to "np_find_common_type" has incompatible type + # "*Set[Union[ExtensionDtype, Any]]"; expected "dtype[Any]" + target_dtype = np_find_common_type(*dtypes) # type: ignore[arg-type] return any_ea, kinds, target_dtype