From 92a2df086135259c9cf96cfbdaacc9d90629cfeb Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 26 Feb 2018 08:47:42 -0800 Subject: [PATCH 1/5] catch more cases explicitly, catch offsets more carefully --- pandas/core/indexes/datetimelike.py | 34 ++++++++++++++---- pandas/core/indexes/datetimes.py | 8 ++--- pandas/core/indexes/period.py | 35 ++++++++++++++++--- pandas/core/indexes/timedeltas.py | 6 ++++ .../indexes/datetimes/test_arithmetic.py | 12 +++++++ .../tests/indexes/period/test_arithmetic.py | 15 ++++++++ .../indexes/timedeltas/test_arithmetic.py | 12 +++++++ 7 files changed, 107 insertions(+), 15 deletions(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 8e56fc2775a56..4d748d65a2171 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -36,7 +36,7 @@ is_period_dtype, is_timedelta64_dtype) from pandas.core.dtypes.generic import ( - ABCIndex, ABCSeries, ABCPeriodIndex, ABCIndexClass) + ABCIndex, ABCSeries, ABCDataFrame, ABCPanel, ABCPeriodIndex, ABCIndexClass) from pandas.core.dtypes.missing import isna from pandas.core import common as com, algorithms, ops from pandas.core.algorithms import checked_add_with_arr @@ -47,6 +47,7 @@ from pandas.util._decorators import Appender, cache_readonly import pandas.core.dtypes.concat as _concat import pandas.tseries.frequencies as frequencies +from pandas.tseries.offsets import Tick, DateOffset import pandas.core.indexes.base as ibase _index_doc_kwargs = dict(ibase._index_doc_kwargs) @@ -643,6 +644,9 @@ def _sub_datelike(self, other): def _sub_period(self, other): return NotImplemented + def _add_offset(self, offset): + raise com.AbstractMethodError(self) + def _addsub_offset_array(self, other, op): """ Add or subtract array-like of DateOffset objects @@ -682,12 +686,15 @@ def __add__(self, other): from pandas import DateOffset other = lib.item_from_zerodim(other) - if isinstance(other, ABCSeries): + if isinstance(other, (ABCSeries, ABCDataFrame, ABCPanel)): return NotImplemented # scalar others - elif isinstance(other, (DateOffset, timedelta, np.timedelta64)): + elif isinstance(other, (Tick, timedelta, np.timedelta64)): result = self._add_delta(other) + elif isinstance(other, DateOffset): + # specifically _not_ a Tick + result = self._add_offset(other) elif isinstance(other, (datetime, np.datetime64)): result = self._add_datelike(other) elif is_integer(other): @@ -708,6 +715,12 @@ def __add__(self, other): elif is_integer_dtype(other) and self.freq is None: # GH#19123 raise NullFrequencyError("Cannot shift with no freq") + elif is_float_dtype(other): + # Explicitly catch invalid dtypes + raise TypeError("cannot add {dtype}-dtype to {cls}" + .format(dtype=other.dtype, + cls=type(self).__name__)) + else: # pragma: no cover return NotImplemented @@ -724,15 +737,18 @@ def __radd__(self, other): cls.__radd__ = __radd__ def __sub__(self, other): - from pandas import Index, DateOffset + from pandas import Index other = lib.item_from_zerodim(other) - if isinstance(other, ABCSeries): + if isinstance(other, (ABCSeries, ABCDataFrame, ABCPanel)): return NotImplemented # scalar others - elif isinstance(other, (DateOffset, timedelta, np.timedelta64)): + elif isinstance(other, (Tick, timedelta, np.timedelta64)): result = self._add_delta(-other) + elif isinstance(other, DateOffset): + # specifically _not_ a Tick + result = self._add_offset(-other) elif isinstance(other, (datetime, np.datetime64)): result = self._sub_datelike(other) elif is_integer(other): @@ -759,6 +775,12 @@ def __sub__(self, other): elif is_integer_dtype(other) and self.freq is None: # GH#19123 raise NullFrequencyError("Cannot shift with no freq") + + elif is_float_dtype(other): + # Explicitly catch invalid dtypes + raise TypeError("cannot subtract {dtype}-dtype from {cls}" + .format(dtype=other.dtype, + cls=type(self).__name__)) else: # pragma: no cover return NotImplemented diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 55d8b7c18a622..1bb6908a26c4a 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -943,8 +943,6 @@ def _add_delta(self, delta): if not isinstance(delta, TimedeltaIndex): delta = TimedeltaIndex(delta) new_values = self._add_delta_tdi(delta) - elif isinstance(delta, DateOffset): - new_values = self._add_offset(delta).asi8 else: new_values = self.astype('O') + delta @@ -955,6 +953,7 @@ def _add_delta(self, delta): return result def _add_offset(self, offset): + assert not isinstance(offset, Tick) try: if self.tz is not None: values = self.tz_localize(None) @@ -963,12 +962,13 @@ def _add_offset(self, offset): result = offset.apply_index(values) if self.tz is not None: result = result.tz_localize(self.tz) - return result except NotImplementedError: warnings.warn("Non-vectorized DateOffset being applied to Series " "or DatetimeIndex", PerformanceWarning) - return self.astype('O') + offset + result = self.astype('O') + offset + + return DatetimeIndex(result, freq='infer') def _format_native_types(self, na_rep='NaT', date_format=None, **kwargs): from pandas.io.formats.format import _get_format_datetime64_from_values diff --git a/pandas/core/indexes/period.py b/pandas/core/indexes/period.py index f0567c9c963af..10efd12defe13 100644 --- a/pandas/core/indexes/period.py +++ b/pandas/core/indexes/period.py @@ -22,11 +22,12 @@ import pandas.tseries.frequencies as frequencies from pandas.tseries.frequencies import get_freq_code as _gfc +from pandas.tseries.offsets import Tick, DateOffset + from pandas.core.indexes.datetimes import DatetimeIndex, Int64Index, Index from pandas.core.indexes.timedeltas import TimedeltaIndex from pandas.core.indexes.datetimelike import DatelikeOps, DatetimeIndexOpsMixin from pandas.core.tools.datetimes import parse_time_string -import pandas.tseries.offsets as offsets from pandas._libs.lib import infer_dtype from pandas._libs import tslib, index as libindex @@ -682,9 +683,9 @@ def to_timestamp(self, freq=None, how='start'): def _maybe_convert_timedelta(self, other): if isinstance( - other, (timedelta, np.timedelta64, offsets.Tick, np.ndarray)): + other, (timedelta, np.timedelta64, Tick, np.ndarray)): offset = frequencies.to_offset(self.freq.rule_code) - if isinstance(offset, offsets.Tick): + if isinstance(offset, Tick): if isinstance(other, np.ndarray): nanos = np.vectorize(delta_to_nanoseconds)(other) else: @@ -693,7 +694,7 @@ def _maybe_convert_timedelta(self, other): check = np.all(nanos % offset_nanos == 0) if check: return nanos // offset_nanos - elif isinstance(other, offsets.DateOffset): + elif isinstance(other, DateOffset): freqstr = other.rule_code base = frequencies.get_base_alias(freqstr) if base == self.freq.rule_code: @@ -705,7 +706,7 @@ def _maybe_convert_timedelta(self, other): return other elif is_timedelta64_dtype(other): offset = frequencies.to_offset(self.freq) - if isinstance(offset, offsets.Tick): + if isinstance(offset, Tick): nanos = delta_to_nanoseconds(other) offset_nanos = delta_to_nanoseconds(offset) if (nanos % offset_nanos).all() == 0: @@ -719,6 +720,30 @@ def _maybe_convert_timedelta(self, other): msg = "Input has different freq from PeriodIndex(freq={0})" raise IncompatibleFrequency(msg.format(self.freqstr)) + def _add_offset(self, other): + assert not isinstance(other, Tick) + base = frequencies.get_base_alias(other.rule_code) + if base != self.freq.rule_code: + msg = _DIFFERENT_FREQ_INDEX.format(self.freqstr, other.freqstr) + raise IncompatibleFrequency(msg) + return self.shift(other.n) + + def _add_delta_td(self, other): + assert isinstance(other, (timedelta, np.timedelta64, Tick)) + nanos = delta_to_nanoseconds(other) + own_offset = frequencies.to_offset(self.freq.rule_code) + + if isinstance(own_offset, Tick): + offset_nanos = delta_to_nanoseconds(own_offset) + if np.all(nanos % offset_nanos == 0): + return self.shift(nanos // offset_nanos) + + # raise when input doesn't have freq + raise IncompatibleFrequency("Input has different freq from " + "{cls}(freq={freqstr})" + .format(cls=type(self).__name__, + freqstr=self.freqstr)) + def _add_delta(self, other): ordinal_delta = self._maybe_convert_timedelta(other) return self.shift(ordinal_delta) diff --git a/pandas/core/indexes/timedeltas.py b/pandas/core/indexes/timedeltas.py index eebd52d7fb801..abd973556527c 100644 --- a/pandas/core/indexes/timedeltas.py +++ b/pandas/core/indexes/timedeltas.py @@ -353,6 +353,12 @@ def _maybe_update_attributes(self, attrs): attrs['freq'] = 'infer' return attrs + def _add_offset(self, other): + assert not isinstance(other, Tick) + raise TypeError("cannot add the type {typ} to a {cls}" + .format(typ=type(other).__name__, + cls=type(self).__name__)) + def _add_delta(self, delta): """ Add a timedelta-like, Tick, or TimedeltaIndex-like object diff --git a/pandas/tests/indexes/datetimes/test_arithmetic.py b/pandas/tests/indexes/datetimes/test_arithmetic.py index 0c56c6b16fb2f..84fa8d9265982 100644 --- a/pandas/tests/indexes/datetimes/test_arithmetic.py +++ b/pandas/tests/indexes/datetimes/test_arithmetic.py @@ -14,6 +14,7 @@ from pandas import (Timestamp, Timedelta, Series, DatetimeIndex, TimedeltaIndex, date_range) +from pandas.core import ops from pandas._libs import tslib from pandas._libs.tslibs.offsets import shift_months @@ -307,6 +308,17 @@ def test_dti_cmp_list(self): class TestDatetimeIndexArithmetic(object): + # ------------------------------------------------------------- + # Invalid Operations + + @pytest.mark.parametrize('other', [3.14, np.array([2.0, 3.0])]) + @pytest.mark.parametrize('op', [operator.add, ops.radd, + operator.sub, ops.rsub]) + def test_dti_add_sub_float(self, op, other): + dti = DatetimeIndex(['2011-01-01', '2011-01-02']) + with pytest.raises(TypeError): + op(dti, other) + def test_dti_add_timestamp_raises(self): idx = DatetimeIndex(['2011-01-01', '2011-01-02']) msg = "cannot add DatetimeIndex and Timestamp" diff --git a/pandas/tests/indexes/period/test_arithmetic.py b/pandas/tests/indexes/period/test_arithmetic.py index d7bf1e0210f62..20ce35e9762c9 100644 --- a/pandas/tests/indexes/period/test_arithmetic.py +++ b/pandas/tests/indexes/period/test_arithmetic.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- from datetime import timedelta +import operator + import pytest import numpy as np @@ -9,6 +11,7 @@ period_range, Period, PeriodIndex, _np_version_under1p10) import pandas.core.indexes.period as period +from pandas.core import ops from pandas.errors import PerformanceWarning @@ -256,6 +259,18 @@ def test_comp_nat(self, dtype): class TestPeriodIndexArithmetic(object): + # ------------------------------------------------------------- + # Invalid Operations + + @pytest.mark.parametrize('other', [3.14, np.array([2.0, 3.0])]) + @pytest.mark.parametrize('op', [operator.add, ops.radd, + operator.sub, ops.rsub]) + def test_tdi_add_sub_float(self, op, other): + dti = pd.DatetimeIndex(['2011-01-01', '2011-01-02']) + pi = dti.to_period('D') + with pytest.raises(TypeError): + op(pi, other) + # ----------------------------------------------------------------- # __add__/__sub__ with ndarray[datetime64] and ndarray[timedelta64] diff --git a/pandas/tests/indexes/timedeltas/test_arithmetic.py b/pandas/tests/indexes/timedeltas/test_arithmetic.py index 9ffffb6ff06d5..119a9c92dc49f 100644 --- a/pandas/tests/indexes/timedeltas/test_arithmetic.py +++ b/pandas/tests/indexes/timedeltas/test_arithmetic.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +import operator + import pytest import numpy as np from datetime import timedelta @@ -11,6 +13,7 @@ Series, Timestamp, Timedelta) from pandas.errors import PerformanceWarning, NullFrequencyError +from pandas.core import ops @pytest.fixture(params=[pd.offsets.Hour(2), timedelta(hours=2), @@ -270,6 +273,15 @@ class TestTimedeltaIndexArithmetic(object): # ------------------------------------------------------------- # Invalid Operations + @pytest.mark.parametrize('other', [3.14, np.array([2.0, 3.0])]) + @pytest.mark.parametrize('op', [operator.add, ops.radd, + operator.sub, ops.rsub]) + def test_tdi_add_sub_float(self, op, other): + dti = DatetimeIndex(['2011-01-01', '2011-01-02']) + tdi = dti - dti.shift(1) + with pytest.raises(TypeError): + op(tdi, other) + def test_tdi_add_str_invalid(self): # GH 13624 tdi = TimedeltaIndex(['1 day', '2 days']) From 92e22490259c648dfc7f51c985e555e59ac50a5f Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 26 Feb 2018 08:50:52 -0800 Subject: [PATCH 2/5] fixup copy/paste typo --- pandas/tests/indexes/period/test_arithmetic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/indexes/period/test_arithmetic.py b/pandas/tests/indexes/period/test_arithmetic.py index 20ce35e9762c9..71002f89e1acc 100644 --- a/pandas/tests/indexes/period/test_arithmetic.py +++ b/pandas/tests/indexes/period/test_arithmetic.py @@ -265,7 +265,7 @@ class TestPeriodIndexArithmetic(object): @pytest.mark.parametrize('other', [3.14, np.array([2.0, 3.0])]) @pytest.mark.parametrize('op', [operator.add, ops.radd, operator.sub, ops.rsub]) - def test_tdi_add_sub_float(self, op, other): + def test_pi_add_sub_float(self, op, other): dti = pd.DatetimeIndex(['2011-01-01', '2011-01-02']) pi = dti.to_period('D') with pytest.raises(TypeError): From f6a3fdda0145e6cdff3297499e3e45bc111fa427 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 26 Feb 2018 09:17:02 -0800 Subject: [PATCH 3/5] add missing freq --- pandas/tests/indexes/datetimes/test_arithmetic.py | 2 +- pandas/tests/indexes/period/test_arithmetic.py | 2 +- pandas/tests/indexes/timedeltas/test_arithmetic.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_arithmetic.py b/pandas/tests/indexes/datetimes/test_arithmetic.py index 84fa8d9265982..8f259a7e78897 100644 --- a/pandas/tests/indexes/datetimes/test_arithmetic.py +++ b/pandas/tests/indexes/datetimes/test_arithmetic.py @@ -315,7 +315,7 @@ class TestDatetimeIndexArithmetic(object): @pytest.mark.parametrize('op', [operator.add, ops.radd, operator.sub, ops.rsub]) def test_dti_add_sub_float(self, op, other): - dti = DatetimeIndex(['2011-01-01', '2011-01-02']) + dti = DatetimeIndex(['2011-01-01', '2011-01-02'], freq='D') with pytest.raises(TypeError): op(dti, other) diff --git a/pandas/tests/indexes/period/test_arithmetic.py b/pandas/tests/indexes/period/test_arithmetic.py index 71002f89e1acc..c75fdd35a974c 100644 --- a/pandas/tests/indexes/period/test_arithmetic.py +++ b/pandas/tests/indexes/period/test_arithmetic.py @@ -266,7 +266,7 @@ class TestPeriodIndexArithmetic(object): @pytest.mark.parametrize('op', [operator.add, ops.radd, operator.sub, ops.rsub]) def test_pi_add_sub_float(self, op, other): - dti = pd.DatetimeIndex(['2011-01-01', '2011-01-02']) + dti = pd.DatetimeIndex(['2011-01-01', '2011-01-02'], freq='D') pi = dti.to_period('D') with pytest.raises(TypeError): op(pi, other) diff --git a/pandas/tests/indexes/timedeltas/test_arithmetic.py b/pandas/tests/indexes/timedeltas/test_arithmetic.py index 119a9c92dc49f..9035434046ccb 100644 --- a/pandas/tests/indexes/timedeltas/test_arithmetic.py +++ b/pandas/tests/indexes/timedeltas/test_arithmetic.py @@ -277,7 +277,7 @@ class TestTimedeltaIndexArithmetic(object): @pytest.mark.parametrize('op', [operator.add, ops.radd, operator.sub, ops.rsub]) def test_tdi_add_sub_float(self, op, other): - dti = DatetimeIndex(['2011-01-01', '2011-01-02']) + dti = DatetimeIndex(['2011-01-01', '2011-01-02'], freq='D') tdi = dti - dti.shift(1) with pytest.raises(TypeError): op(tdi, other) From e42c0f2ed9d7d02aa5fa2ac4a34ea973770f61a6 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 26 Feb 2018 14:22:22 -0800 Subject: [PATCH 4/5] dummy commit to force CI --- pandas/core/indexes/datetimelike.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 4d748d65a2171..ec1729855cadc 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ Base and utility classes for tseries type pandas objects. """ From 58457d4eb49bb35eeb2dfa4aa3e8b75a2a52e3fb Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 28 Feb 2018 17:17:33 -0800 Subject: [PATCH 5/5] remove panel checks --- pandas/core/indexes/datetimelike.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index a2d969c1d76a9..e673bfe411cb4 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -37,7 +37,7 @@ is_period_dtype, is_timedelta64_dtype) from pandas.core.dtypes.generic import ( - ABCIndex, ABCSeries, ABCDataFrame, ABCPanel, ABCPeriodIndex, ABCIndexClass) + ABCIndex, ABCSeries, ABCDataFrame, ABCPeriodIndex, ABCIndexClass) from pandas.core.dtypes.missing import isna from pandas.core import common as com, algorithms, ops from pandas.core.algorithms import checked_add_with_arr @@ -709,7 +709,7 @@ def __add__(self, other): from pandas import DateOffset other = lib.item_from_zerodim(other) - if isinstance(other, (ABCSeries, ABCDataFrame, ABCPanel)): + if isinstance(other, (ABCSeries, ABCDataFrame)): return NotImplemented # scalar others @@ -769,7 +769,7 @@ def __sub__(self, other): from pandas import Index other = lib.item_from_zerodim(other) - if isinstance(other, (ABCSeries, ABCDataFrame, ABCPanel)): + if isinstance(other, (ABCSeries, ABCDataFrame)): return NotImplemented # scalar others