PERF: Datetime/Timestamp.normalize for timezone naive datetimes

asv_bench/benchmarks/timestamp.py

@@ -102,6 +102,9 @@ def time_replace_None(self, tz):
     def time_to_pydatetime(self, tz):
         self.ts.to_pydatetime()
 
+    def time_normalize(self, tz):
+        self.ts.normalize()
+
 
 class TimestampAcrossDst(object):
     def setup(self):

doc/source/whatsnew/v0.24.0.txt

@@ -1081,6 +1081,7 @@ Performance Improvements
 - Improved performance of :func:`IndexEngine.get_indexer_non_unique` for sorted, non-unique indexes (:issue:`9466`)
 - Improved performance of :func:`PeriodIndex.unique` (:issue:`23083`)
 - Improved performance of :func:`pd.concat` for `Series` objects (:issue:`23404`)
+- Improved performance of :meth:`DatetimeIndex.normalize` and :meth:`Timestamp.normalize` for timezone naive datetimes (:issue:`23634`)
 
 
 .. _whatsnew_0240.docs:

pandas/_libs/tslibs/conversion.pyx

@@ -22,7 +22,8 @@ from np_datetime cimport (check_dts_bounds,
                           npy_datetime,
                           dt64_to_dtstruct, dtstruct_to_dt64,
                           get_datetime64_unit, get_datetime64_value,
-                          pydatetime_to_dt64, NPY_DATETIMEUNIT, NPY_FR_ns)
+                          pydatetime_to_dt64, NPY_DATETIMEUNIT, NPY_FR_ns,
+                          DAY_S)
 from np_datetime import OutOfBoundsDatetime
 
 from util cimport (is_string_object,
@@ -41,7 +42,6 @@ from nattype cimport NPY_NAT, checknull_with_nat
 # ----------------------------------------------------------------------
 # Constants
 
-cdef int64_t DAY_NS = 86400000000000LL
 cdef int64_t HOURS_NS = 3600000000000
 NS_DTYPE = np.dtype('M8[ns]')
 TD_DTYPE = np.dtype('m8[ns]')
@@ -931,10 +931,10 @@ def tz_localize_to_utc(ndarray[int64_t] vals, object tz, object ambiguous=None,
     result_b[:] = NPY_NAT
 
     idx_shifted_left = (np.maximum(0, trans.searchsorted(
-        vals - DAY_NS, side='right') - 1)).astype(np.int64)
+        vals - DAY_S * 1000000000, side='right') - 1)).astype(np.int64)
 
     idx_shifted_right = (np.maximum(0, trans.searchsorted(
-        vals + DAY_NS, side='right') - 1)).astype(np.int64)
+        vals + DAY_S * 1000000000, side='right') - 1)).astype(np.int64)
 
     for i in range(n):
         val = vals[i]
@@ -1116,9 +1116,9 @@ def normalize_date(dt: object) -> datetime:
 @cython.boundscheck(False)
 def normalize_i8_timestamps(int64_t[:] stamps, object tz=None):
     """
-    Normalize each of the (nanosecond) timestamps in the given array by
-    rounding down to the beginning of the day (i.e. midnight).  If `tz`
-    is not None, then this is midnight for this timezone.
+    Normalize each of the (nanosecond) timezone aware timestamps in the given
+    array by rounding down to the beginning of the day (i.e. midnight).
+    This is midnight for timezone, `tz`.
 
     Parameters
     ----------
@@ -1130,21 +1130,11 @@ def normalize_i8_timestamps(int64_t[:] stamps, object tz=None):
     result : int64 ndarray of converted of normalized nanosecond timestamps
     """
     cdef:
-        Py_ssize_t i, n = len(stamps)
-        npy_datetimestruct dts
+        Py_ssize_t n = len(stamps)
         int64_t[:] result = np.empty(n, dtype=np.int64)
 
-    if tz is not None:
-        tz = maybe_get_tz(tz)
-        result = _normalize_local(stamps, tz)
-    else:
-        with nogil:
-            for i in range(n):
-                if stamps[i] == NPY_NAT:
-                    result[i] = NPY_NAT
-                    continue
-                dt64_to_dtstruct(stamps[i], &dts)
-                result[i] = _normalized_stamp(&dts)
+    tz = maybe_get_tz(tz)
+    result = _normalize_local(stamps, tz)
 
     return result.base  # .base to access underlying np.ndarray
 

pandas/_libs/tslibs/fields.pyx

@@ -16,7 +16,7 @@ from ccalendar import get_locale_names, MONTHS_FULL, DAYS_FULL
 from ccalendar cimport (get_days_in_month, is_leapyear, dayofweek,
                         get_week_of_year, get_day_of_year)
 from np_datetime cimport (npy_datetimestruct, pandas_timedeltastruct,
-                          dt64_to_dtstruct, td64_to_tdstruct)
+                          dt64_to_dtstruct, td64_to_tdstruct, DAY_S)
 from nattype cimport NPY_NAT
 
 
@@ -36,7 +36,7 @@ def get_time_micros(ndarray[int64_t] dtindex):
     cdef:
         ndarray[int64_t] micros
 
-    micros = np.mod(dtindex, 86400000000000, dtype=np.int64) // 1000LL
+    micros = np.mod(dtindex, DAY_S * 1000000000, dtype=np.int64) // 1000LL
     return micros
 
 

pandas/_libs/tslibs/np_datetime.pxd

@@ -54,6 +54,7 @@ cdef extern from "src/datetime/np_datetime.h":
                                            NPY_DATETIMEUNIT fr,
                                            npy_datetimestruct *result) nogil
 
+cdef int64_t DAY_S
 
 cdef int reverse_ops[6]
 

pandas/_libs/tslibs/np_datetime.pyx

@@ -37,6 +37,12 @@ cdef extern from "src/datetime/np_datetime_strings.h":
                                 npy_datetimestruct *out,
                                 int *out_local, int *out_tzoffset)
 
+# ----------------------------------------------------------------------
+# time constants
+
+cdef int64_t DAY_S = 86400
+
+
 # ----------------------------------------------------------------------
 # numpy object inspection
 

pandas/_libs/tslibs/timedeltas.pyx

@@ -29,7 +29,7 @@ from util cimport (is_timedelta64_object, is_datetime64_object,
                    is_string_object)
 
 from np_datetime cimport (cmp_scalar, reverse_ops, td64_to_tdstruct,
-                          pandas_timedeltastruct)
+                          pandas_timedeltastruct, DAY_S)
 
 from nattype import nat_strings, NaT
 from nattype cimport checknull_with_nat, NPY_NAT
@@ -38,8 +38,6 @@ from offsets cimport to_offset
 # ----------------------------------------------------------------------
 # Constants
 
-cdef int64_t DAY_NS = 86400000000000LL
-
 # components named tuple
 Components = collections.namedtuple('Components', [
     'days', 'hours', 'minutes', 'seconds',
@@ -266,10 +264,10 @@ cdef inline int64_t cast_from_unit(object ts, object unit) except? -1:
         m = 1000000000L * 2629746
         p = 9
     elif unit == 'W':
-        m = 1000000000L * 86400 * 7
+        m = 1000000000L * DAY_S * 7
         p = 9
     elif unit == 'D' or unit == 'd':
-        m = 1000000000L * 86400
+        m = 1000000000L * DAY_S
         p = 9
     elif unit == 'h':
         m = 1000000000L * 3600

pandas/_libs/tslibs/timestamps.pyx

@@ -40,6 +40,7 @@ from timezones cimport (
 # Constants
 _zero_time = datetime_time(0, 0)
 _no_input = object()
+cdef int64_t DAY_NS = 86400000000000
 
 
 # ----------------------------------------------------------------------
@@ -1285,6 +1286,8 @@ class Timestamp(_Timestamp):
         Normalize Timestamp to midnight, preserving
         tz information.
         """
+        if self.tz is None:
+            return Timestamp(self.value - (self.value % DAY_NS))
         normalized_value = normalize_i8_timestamps(
             np.array([self.value], dtype='i8'), tz=self.tz)[0]
         return Timestamp(normalized_value).tz_localize(self.tz)

pandas/core/arrays/datetimes.py

@@ -832,7 +832,14 @@ def normalize(self):
                        '2014-08-01 00:00:00+05:30'],
                        dtype='datetime64[ns, Asia/Calcutta]', freq=None)
         """
-        new_values = conversion.normalize_i8_timestamps(self.asi8, self.tz)
+        if self.tz is None:
+            not_null = self.notnull()
+            DAY_NS = 86400000000000
+            new_values = self.asi8.copy()
+            adjustment = (new_values[not_null] % DAY_NS)
+            new_values[not_null] = new_values[not_null] - adjustment
+        else:
+            new_values = conversion.normalize_i8_timestamps(self.asi8, self.tz)
         return type(self)(new_values, freq='infer').tz_localize(self.tz)
 
     def to_period(self, freq=None):

pandas/tests/indexes/datetimes/test_scalar_compat.py

@@ -235,6 +235,12 @@ def test_normalize(self):
         assert result.is_normalized
         assert not rng.is_normalized
 
+    def test_normalize_nat(self):
+        dti = DatetimeIndex([pd.NaT, Timestamp('2018-01-01 01:00:00')])
+        result = dti.normalize()
+        expected = DatetimeIndex([pd.NaT, Timestamp('2018-01-01')])
+        tm.assert_index_equal(result, expected)
+
 
 class TestDateTimeIndexToJulianDate(object):
 

pandas/tests/scalar/timestamp/test_unary_ops.py

@@ -328,6 +328,17 @@ def test_replace_dst_border(self):
         expected = Timestamp('2013-11-3 03:00:00', tz='America/Chicago')
         assert result == expected
 
+    # --------------------------------------------------------------
+    # Timestamp.normalize
+
+    @pytest.mark.parametrize('arg', ['2013-11-30', '2013-11-30 12:00:00'])
+    def test_normalize(self, tz_naive_fixture, arg):
+        tz = tz_naive_fixture
+        ts = Timestamp(arg, tz=tz)
+        result = ts.normalize()
+        expected = Timestamp('2013-11-30', tz=tz)
+        assert result == expected
+
     # --------------------------------------------------------------
 
     @td.skip_if_windows