From 9d97999e3b477f0c3a1a04aa286d65aebf59604a Mon Sep 17 00:00:00 2001 From: Matthew Zeitlin Date: Thu, 17 Jun 2021 16:51:12 -0700 Subject: [PATCH 1/5] CLN: use fused type for lib.has_infs --- pandas/_libs/lib.pyi | 3 +-- pandas/_libs/lib.pyx | 37 +++++++++++-------------------------- pandas/core/nanops.py | 5 +---- 3 files changed, 13 insertions(+), 32 deletions(-) diff --git a/pandas/_libs/lib.pyi b/pandas/_libs/lib.pyi index 077d2e60cc3a4..d39c5ac5d2967 100644 --- a/pandas/_libs/lib.pyi +++ b/pandas/_libs/lib.pyi @@ -219,8 +219,7 @@ def array_equivalent_object( left: np.ndarray, # object[:] right: np.ndarray, # object[:] ) -> bool: ... -def has_infs_f8(arr: np.ndarray) -> bool: ... # const float64_t[:] -def has_infs_f4(arr: np.ndarray) -> bool: ... # const float32_t[:] +def has_infs(arr: np.ndarray) -> bool: ... # const floating[:] def get_reverse_indexer( indexer: np.ndarray, # const intp_t[:] length: int, diff --git a/pandas/_libs/lib.pyx b/pandas/_libs/lib.pyx index 506ad0102e157..bddbead969d10 100644 --- a/pandas/_libs/lib.pyx +++ b/pandas/_libs/lib.pyx @@ -25,6 +25,7 @@ from cpython.tuple cimport ( PyTuple_New, PyTuple_SET_ITEM, ) +from cython cimport floating PyDateTime_IMPORT @@ -517,38 +518,22 @@ def get_reverse_indexer(const intp_t[:] indexer, Py_ssize_t length) -> ndarray: return rev_indexer -@cython.wraparound(False) -@cython.boundscheck(False) -def has_infs_f4(const float32_t[:] arr) -> bool: +# Can add const once https://github.com/cython/cython/issues/1772 resolved +def has_infs(floating[:] arr) -> bool: cdef: Py_ssize_t i, n = len(arr) float32_t inf, neginf, val + bint ret = False inf = np.inf neginf = -inf - - for i in range(n): - val = arr[i] - if val == inf or val == neginf: - return True - return False - - -@cython.wraparound(False) -@cython.boundscheck(False) -def has_infs_f8(const float64_t[:] arr) -> bool: - cdef: - Py_ssize_t i, n = len(arr) - float64_t inf, neginf, val - - inf = np.inf - neginf = -inf - - for i in range(n): - val = arr[i] - if val == inf or val == neginf: - return True - return False + with nogil: + for i in range(n): + val = arr[i] + if val == inf or val == neginf: + ret = True + break + return ret def maybe_indices_to_slice(ndarray[intp_t] indices, int max_len): diff --git a/pandas/core/nanops.py b/pandas/core/nanops.py index c34944985f2b6..a3036bb1e3ae9 100644 --- a/pandas/core/nanops.py +++ b/pandas/core/nanops.py @@ -177,10 +177,7 @@ def _bn_ok_dtype(dtype: DtypeObj, name: str) -> bool: def _has_infs(result) -> bool: if isinstance(result, np.ndarray): - if result.dtype == "f8": - return lib.has_infs_f8(result.ravel("K")) - elif result.dtype == "f4": - return lib.has_infs_f4(result.ravel("K")) + return lib.has_infs(result.ravel("K")) try: return np.isinf(result).any() except (TypeError, NotImplementedError): From e11f16edee051e380e87ea3610e56db465686096 Mon Sep 17 00:00:00 2001 From: Matthew Zeitlin Date: Thu, 17 Jun 2021 16:53:03 -0700 Subject: [PATCH 2/5] Add back decorators --- pandas/_libs/lib.pyx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/_libs/lib.pyx b/pandas/_libs/lib.pyx index bddbead969d10..88a451ca721ae 100644 --- a/pandas/_libs/lib.pyx +++ b/pandas/_libs/lib.pyx @@ -518,6 +518,8 @@ def get_reverse_indexer(const intp_t[:] indexer, Py_ssize_t length) -> ndarray: return rev_indexer +@cython.wraparound(False) +@cython.boundscheck(False) # Can add const once https://github.com/cython/cython/issues/1772 resolved def has_infs(floating[:] arr) -> bool: cdef: From 1bdc8fc012c1440e43f4686dabc25e280227db4d Mon Sep 17 00:00:00 2001 From: Matthew Zeitlin Date: Thu, 17 Jun 2021 22:01:19 -0700 Subject: [PATCH 3/5] Some fixups --- pandas/_libs/lib.pyx | 2 +- pandas/core/nanops.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/_libs/lib.pyx b/pandas/_libs/lib.pyx index 88a451ca721ae..8d476489bffb3 100644 --- a/pandas/_libs/lib.pyx +++ b/pandas/_libs/lib.pyx @@ -524,7 +524,7 @@ def get_reverse_indexer(const intp_t[:] indexer, Py_ssize_t length) -> ndarray: def has_infs(floating[:] arr) -> bool: cdef: Py_ssize_t i, n = len(arr) - float32_t inf, neginf, val + floating inf, neginf, val bint ret = False inf = np.inf diff --git a/pandas/core/nanops.py b/pandas/core/nanops.py index a3036bb1e3ae9..802fd1c0efa11 100644 --- a/pandas/core/nanops.py +++ b/pandas/core/nanops.py @@ -177,7 +177,8 @@ def _bn_ok_dtype(dtype: DtypeObj, name: str) -> bool: def _has_infs(result) -> bool: if isinstance(result, np.ndarray): - return lib.has_infs(result.ravel("K")) + if is_float_dtype(result): + return lib.has_infs(result.ravel("K")) try: return np.isinf(result).any() except (TypeError, NotImplementedError): From 71c468d36682d32480f817c063ff7fb3b606fc38 Mon Sep 17 00:00:00 2001 From: Matthew Zeitlin Date: Fri, 18 Jun 2021 09:51:44 -0700 Subject: [PATCH 4/5] Float16 fix --- pandas/core/nanops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/nanops.py b/pandas/core/nanops.py index 802fd1c0efa11..1dc00f5e89400 100644 --- a/pandas/core/nanops.py +++ b/pandas/core/nanops.py @@ -177,7 +177,7 @@ def _bn_ok_dtype(dtype: DtypeObj, name: str) -> bool: def _has_infs(result) -> bool: if isinstance(result, np.ndarray): - if is_float_dtype(result): + if result.dtype in ("f8", "f4"): return lib.has_infs(result.ravel("K")) try: return np.isinf(result).any() From 8184185e3118d13b7f0d9f58fb3c675ad903b788 Mon Sep 17 00:00:00 2001 From: Matthew Zeitlin Date: Fri, 18 Jun 2021 16:41:33 -0700 Subject: [PATCH 5/5] mypy fixup --- pandas/core/nanops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/nanops.py b/pandas/core/nanops.py index 1dc00f5e89400..fb6580bbb7ea6 100644 --- a/pandas/core/nanops.py +++ b/pandas/core/nanops.py @@ -177,7 +177,7 @@ def _bn_ok_dtype(dtype: DtypeObj, name: str) -> bool: def _has_infs(result) -> bool: if isinstance(result, np.ndarray): - if result.dtype in ("f8", "f4"): + if result.dtype == "f8" or result.dtype == "f4": return lib.has_infs(result.ravel("K")) try: return np.isinf(result).any()