diff --git a/pandas/_libs/tslibs/offsets.pyi b/pandas/_libs/tslibs/offsets.pyi index 9410379b16ba2..058bbcdd346e0 100644 --- a/pandas/_libs/tslibs/offsets.pyi +++ b/pandas/_libs/tslibs/offsets.pyi @@ -108,6 +108,7 @@ def to_offset(freq: None) -> None: ... def to_offset(freq: timedelta | BaseOffset | str) -> BaseOffset: ... class Tick(SingleConstructorOffset): + _reso: int def __init__(self, n: int = ..., normalize: bool = ...) -> None: ... @property def delta(self) -> Timedelta: ... diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx index 05c91379cbdc5..5a21b1a3abb1d 100644 --- a/pandas/_libs/tslibs/offsets.pyx +++ b/pandas/_libs/tslibs/offsets.pyx @@ -968,42 +968,49 @@ cdef class Day(Tick): _nanos_inc = 24 * 3600 * 1_000_000_000 _prefix = "D" _period_dtype_code = PeriodDtypeCode.D + _reso = NPY_DATETIMEUNIT.NPY_FR_D cdef class Hour(Tick): _nanos_inc = 3600 * 1_000_000_000 _prefix = "H" _period_dtype_code = PeriodDtypeCode.H + _reso = NPY_DATETIMEUNIT.NPY_FR_h cdef class Minute(Tick): _nanos_inc = 60 * 1_000_000_000 _prefix = "T" _period_dtype_code = PeriodDtypeCode.T + _reso = NPY_DATETIMEUNIT.NPY_FR_m cdef class Second(Tick): _nanos_inc = 1_000_000_000 _prefix = "S" _period_dtype_code = PeriodDtypeCode.S + _reso = NPY_DATETIMEUNIT.NPY_FR_s cdef class Milli(Tick): _nanos_inc = 1_000_000 _prefix = "L" _period_dtype_code = PeriodDtypeCode.L + _reso = NPY_DATETIMEUNIT.NPY_FR_ms cdef class Micro(Tick): _nanos_inc = 1000 _prefix = "U" _period_dtype_code = PeriodDtypeCode.U + _reso = NPY_DATETIMEUNIT.NPY_FR_us cdef class Nano(Tick): _nanos_inc = 1 _prefix = "N" _period_dtype_code = PeriodDtypeCode.N + _reso = NPY_DATETIMEUNIT.NPY_FR_ns def delta_to_tick(delta: timedelta) -> Tick: diff --git a/pandas/_libs/tslibs/period.pyx b/pandas/_libs/tslibs/period.pyx index de44f0da8275e..69b3b719ae085 100644 --- a/pandas/_libs/tslibs/period.pyx +++ b/pandas/_libs/tslibs/period.pyx @@ -1680,14 +1680,17 @@ cdef class _Period(PeriodMixin): def _add_timedeltalike_scalar(self, other) -> "Period": cdef: - int64_t nanos, base_nanos + int64_t inc if is_tick_object(self.freq): - nanos = delta_to_nanoseconds(other) - base_nanos = self.freq.base.nanos - if nanos % base_nanos == 0: - ordinal = self.ordinal + (nanos // base_nanos) - return Period(ordinal=ordinal, freq=self.freq) + try: + inc = delta_to_nanoseconds(other, reso=self.freq._reso, round_ok=False) + except ValueError as err: + raise IncompatibleFrequency("Input cannot be converted to " + f"Period(freq={self.freqstr})") from err + # TODO: overflow-check here + ordinal = self.ordinal + inc + return Period(ordinal=ordinal, freq=self.freq) raise IncompatibleFrequency("Input cannot be converted to " f"Period(freq={self.freqstr})") diff --git a/pandas/_libs/tslibs/timedeltas.pxd b/pandas/_libs/tslibs/timedeltas.pxd index eb562e743fc01..e851665c49d89 100644 --- a/pandas/_libs/tslibs/timedeltas.pxd +++ b/pandas/_libs/tslibs/timedeltas.pxd @@ -5,7 +5,9 @@ from .np_datetime cimport NPY_DATETIMEUNIT # Exposed for tslib, not intended for outside use. -cpdef int64_t delta_to_nanoseconds(delta) except? -1 +cpdef int64_t delta_to_nanoseconds( + delta, NPY_DATETIMEUNIT reso=*, bint round_ok=*, bint allow_year_month=* +) except? -1 cdef convert_to_timedelta64(object ts, str unit) cdef bint is_any_td_scalar(object obj) diff --git a/pandas/_libs/tslibs/timedeltas.pyi b/pandas/_libs/tslibs/timedeltas.pyi index 57cc76811fea7..70770e1c0fdc9 100644 --- a/pandas/_libs/tslibs/timedeltas.pyi +++ b/pandas/_libs/tslibs/timedeltas.pyi @@ -72,7 +72,12 @@ def array_to_timedelta64( errors: str = ..., ) -> np.ndarray: ... # np.ndarray[m8ns] def parse_timedelta_unit(unit: str | None) -> UnitChoices: ... -def delta_to_nanoseconds(delta: np.timedelta64 | timedelta | Tick) -> int: ... +def delta_to_nanoseconds( + delta: np.timedelta64 | timedelta | Tick, + reso: int = ..., # NPY_DATETIMEUNIT + round_ok: bool = ..., + allow_year_month: bool = ..., +) -> int: ... class Timedelta(timedelta): min: ClassVar[Timedelta] diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index 593b31f0d9113..0fc1476b43c42 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -201,28 +201,76 @@ def ints_to_pytimedelta(ndarray m8values, box=False): # ---------------------------------------------------------------------- -cpdef int64_t delta_to_nanoseconds(delta) except? -1: - if is_tick_object(delta): - return delta.nanos - if isinstance(delta, _Timedelta): - if delta._reso == NPY_FR_ns: - return delta.value - raise NotImplementedError(delta._reso) - if is_timedelta64_object(delta): - return get_timedelta64_value(ensure_td64ns(delta)) +cpdef int64_t delta_to_nanoseconds( + delta, + NPY_DATETIMEUNIT reso=NPY_FR_ns, + bint round_ok=True, + bint allow_year_month=False, +) except? -1: + cdef: + _Timedelta td + NPY_DATETIMEUNIT in_reso + int64_t n + + if is_tick_object(delta): + n = delta.n + in_reso = delta._reso + if in_reso == reso: + return n + else: + td = Timedelta._from_value_and_reso(delta.n, reso=in_reso) + + elif isinstance(delta, _Timedelta): + td = delta + n = delta.value + in_reso = delta._reso + if in_reso == reso: + return n + + elif is_timedelta64_object(delta): + in_reso = get_datetime64_unit(delta) + n = get_timedelta64_value(delta) + if in_reso == reso: + return n + else: + # _from_value_and_reso does not support Year, Month, or unit-less, + # so we have special handling if speciifed + try: + td = Timedelta._from_value_and_reso(n, reso=in_reso) + except NotImplementedError: + if allow_year_month: + td64 = ensure_td64ns(delta) + return delta_to_nanoseconds(td64, reso=reso) + else: + raise - if PyDelta_Check(delta): + elif PyDelta_Check(delta): + in_reso = NPY_DATETIMEUNIT.NPY_FR_us try: - return ( + n = ( delta.days * 24 * 3600 * 1_000_000 + delta.seconds * 1_000_000 + delta.microseconds - ) * 1000 + ) except OverflowError as err: raise OutOfBoundsTimedelta(*err.args) from err - raise TypeError(type(delta)) + if in_reso == reso: + return n + else: + td = Timedelta._from_value_and_reso(n, reso=in_reso) + + else: + raise TypeError(type(delta)) + + try: + return td._as_reso(reso, round_ok=round_ok).value + except OverflowError as err: + unit_str = npy_unit_to_abbrev(reso) + raise OutOfBoundsTimedelta( + f"Cannot cast {str(delta)} to unit={unit_str} without overflow." + ) from err @cython.overflowcheck(True) @@ -1411,6 +1459,7 @@ cdef class _Timedelta(timedelta): else: mult = get_conversion_factor(self._reso, reso) with cython.overflowcheck(True): + # Note: caller is responsible for re-raising as OutOfBoundsTimedelta value = self.value * mult return type(self)._from_value_and_reso(value, reso=reso) diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index a1dc70397fbb9..8706d59b084b9 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -353,7 +353,17 @@ cdef class _Timestamp(ABCTimestamp): raise NotImplementedError(self._reso) if is_any_td_scalar(other): - nanos = delta_to_nanoseconds(other) + if ( + is_timedelta64_object(other) + and get_datetime64_unit(other) == NPY_DATETIMEUNIT.NPY_FR_GENERIC + ): + # TODO: deprecate allowing this? We only get here + # with test_timedelta_add_timestamp_interval + other = np.timedelta64(other.view("i8"), "ns") + # TODO: disallow round_ok, allow_year_month? + nanos = delta_to_nanoseconds( + other, reso=self._reso, round_ok=True, allow_year_month=True + ) try: result = type(self)(self.value + nanos, tz=self.tzinfo) except OverflowError: diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 98012bfa31943..a8772709935af 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -1120,28 +1120,9 @@ def _add_timedeltalike_scalar(self, other): new_values.fill(iNaT) return type(self)(new_values, dtype=self.dtype) - # FIXME: this may overflow with non-nano - inc = delta_to_nanoseconds(other) - - if not is_period_dtype(self.dtype): - # FIXME: don't hardcode 7, 8, 9, 10 here - # TODO: maybe patch delta_to_nanoseconds to take reso? - - # error: "DatetimeLikeArrayMixin" has no attribute "_reso" - reso = self._reso # type: ignore[attr-defined] - if reso == 10: - pass - elif reso == 9: - # microsecond - inc = inc // 1000 - elif reso == 8: - # millisecond - inc = inc // 1_000_000 - elif reso == 7: - # second - inc = inc // 1_000_000_000 - else: - raise NotImplementedError(reso) + # PeriodArray overrides, so we only get here with DTA/TDA + # error: "DatetimeLikeArrayMixin" has no attribute "_reso" + inc = delta_to_nanoseconds(other, reso=self._reso) # type: ignore[attr-defined] new_values = checked_add_with_arr(self.asi8, inc, arr_mask=self._isnan) new_values = new_values.view("i8") diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index f313d0d67c344..dfafb4ca410d6 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -782,6 +782,7 @@ def _add_offset(self, other: BaseOffset): self._require_matching_freq(other, base=True) return self._addsub_int_array_or_scalar(other.n, operator.add) + # TODO: can we de-duplicate with Period._add_timedeltalike_scalar? def _add_timedeltalike_scalar(self, other): """ Parameters @@ -797,10 +798,15 @@ def _add_timedeltalike_scalar(self, other): raise raise_on_incompatible(self, other) if notna(other): - # special handling for np.timedelta64("NaT"), avoid calling - # _check_timedeltalike_freq_compat as that would raise TypeError - other = self._check_timedeltalike_freq_compat(other) - other = np.timedelta64(other, "ns") + # Convert to an integer increment of our own freq, disallowing + # e.g. 30seconds if our freq is minutes. + try: + inc = delta_to_nanoseconds(other, reso=self.freq._reso, round_ok=False) + except ValueError as err: + # "Cannot losslessly convert units" + raise raise_on_incompatible(self, other) from err + + return self._addsub_int_array_or_scalar(inc, operator.add) return super()._add_timedeltalike_scalar(other) diff --git a/pandas/tests/scalar/timedelta/test_arithmetic.py b/pandas/tests/scalar/timedelta/test_arithmetic.py index 74aa7f045088e..c9e096bb3678e 100644 --- a/pandas/tests/scalar/timedelta/test_arithmetic.py +++ b/pandas/tests/scalar/timedelta/test_arithmetic.py @@ -103,6 +103,7 @@ def test_td_add_timestamp_overflow(self): with pytest.raises(OverflowError, match=msg): Timestamp("1700-01-01") + Timedelta(13 * 19999, unit="D") + msg = "Cannot cast 259987 days, 0:00:00 to unit=ns without overflow" with pytest.raises(OutOfBoundsTimedelta, match=msg): Timestamp("1700-01-01") + timedelta(days=13 * 19999) diff --git a/pandas/tests/scalar/timedelta/test_constructors.py b/pandas/tests/scalar/timedelta/test_constructors.py index 7fc7bd3a5a74d..3a12fa031545b 100644 --- a/pandas/tests/scalar/timedelta/test_constructors.py +++ b/pandas/tests/scalar/timedelta/test_constructors.py @@ -215,6 +215,7 @@ def test_overflow_on_construction(): with pytest.raises(OverflowError, match=msg): Timedelta(7 * 19999, unit="D") + msg = "Cannot cast 259987 days, 0:00:00 to unit=ns without overflow" with pytest.raises(OutOfBoundsTimedelta, match=msg): Timedelta(timedelta(days=13 * 19999)) diff --git a/pandas/tests/scalar/timestamp/test_arithmetic.py b/pandas/tests/scalar/timestamp/test_arithmetic.py index b46962fb82896..788d6f3504760 100644 --- a/pandas/tests/scalar/timestamp/test_arithmetic.py +++ b/pandas/tests/scalar/timestamp/test_arithmetic.py @@ -9,6 +9,7 @@ from pandas._libs.tslibs import ( OutOfBoundsDatetime, + OutOfBoundsTimedelta, Timedelta, Timestamp, offsets, @@ -45,16 +46,20 @@ def test_overflow_offset_raises(self): "will overflow" ) lmsg = "|".join( - ["Python int too large to convert to C long", "int too big to convert"] + [ + "Python int too large to convert to C (long|int)", + "int too big to convert", + ] ) + lmsg2 = r"Cannot cast <-?20169940 \* Days> to unit=ns without overflow" - with pytest.raises(OverflowError, match=lmsg): + with pytest.raises(OutOfBoundsTimedelta, match=lmsg2): stamp + offset_overflow with pytest.raises(OverflowError, match=msg): offset_overflow + stamp - with pytest.raises(OverflowError, match=lmsg): + with pytest.raises(OutOfBoundsTimedelta, match=lmsg2): stamp - offset_overflow # xref https://github.com/pandas-dev/pandas/issues/14080 diff --git a/pandas/tests/tools/test_to_datetime.py b/pandas/tests/tools/test_to_datetime.py index 0bd93a78227ff..c98aedc9a2cf0 100644 --- a/pandas/tests/tools/test_to_datetime.py +++ b/pandas/tests/tools/test_to_datetime.py @@ -1847,14 +1847,7 @@ def test_to_datetime_list_of_integers(self): def test_to_datetime_overflow(self): # gh-17637 # we are overflowing Timedelta range here - - msg = "|".join( - [ - "Python int too large to convert to C long", - "long too big to convert", - "int too big to convert", - ] - ) + msg = "Cannot cast 139999 days, 0:00:00 to unit=ns without overflow" with pytest.raises(OutOfBoundsTimedelta, match=msg): date_range(start="1/1/1700", freq="B", periods=100000)