[READY] perf improvements for strftime

asv_bench/benchmarks/io/csv.py

@@ -79,7 +79,13 @@ def setup(self):
         rng = date_range("1/1/2000", periods=1000)
         self.data = DataFrame(rng, index=rng)
 
-    def time_frame_date_formatting(self):
+    def time_frame_date_formatting_default(self):
+        self.data.to_csv(self.fname)
+
+    def time_frame_date_formatting_default_explicit(self):
+        self.data.to_csv(self.fname, date_format="%Y-%m-%d")
+
+    def time_frame_date_formatting_custom(self):
         self.data.to_csv(self.fname, date_format="%Y%m%d")
 
 
@@ -90,11 +96,14 @@ def setup(self):
         rng = date_range("2000", periods=100_000, freq="s")
         self.data = DataFrame({"a": 1}, index=rng)
 
-    def time_frame_date_formatting_index(self):
+    def time_frame_date_formatting_index_default(self):
+        self.data.to_csv(self.fname)
+
+    def time_frame_date_formatting_index_default_explicit(self):
         self.data.to_csv(self.fname, date_format="%Y-%m-%d %H:%M:%S")
 
-    def time_frame_date_no_format_index(self):
-        self.data.to_csv(self.fname)
+    def time_frame_date_formatting_index_custom(self):
+        self.data.to_csv(self.fname, date_format="%Y-%m-%d__%H:%M:%S")
 
 
 class ToCSVPeriod(BaseIO):
@@ -117,7 +126,7 @@ def time_frame_period_formatting_default(self, nobs, freq):
     def time_frame_period_formatting_default_explicit(self, nobs, freq):
         self.data.to_csv(self.fname, date_format=self.default_fmt)
 
-    def time_frame_period_formatting(self, nobs, freq):
+    def time_frame_period_formatting_custom(self, nobs, freq):
         # Nb: `date_format` is not actually taken into account here today, so the
         # performance is currently identical to `time_frame_period_formatting_default`
         # above. This timer is therefore expected to degrade when GH#51621 is fixed.
@@ -139,15 +148,15 @@ def setup(self, nobs, freq):
         elif freq == "h":
             self.default_fmt = "%Y-%m-%d %H:00"
 
-    def time_frame_period_formatting_index(self, nobs, freq):
-        self.data.to_csv(self.fname, date_format="%Y-%m-%d___%H:%M:%S")
-
     def time_frame_period_formatting_index_default(self, nobs, freq):
         self.data.to_csv(self.fname)
 
     def time_frame_period_formatting_index_default_explicit(self, nobs, freq):
         self.data.to_csv(self.fname, date_format=self.default_fmt)
 
+    def time_frame_period_formatting_index_custom(self, nobs, freq):
+        self.data.to_csv(self.fname, date_format="%Y-%m-%d___%H:%M:%S")
+
 
 class ToCSVDatetimeBig(BaseIO):
     fname = "__test__.csv"
@@ -166,9 +175,12 @@ def setup(self, nobs):
             }
         )
 
-    def time_frame(self, nobs):
+    def time_frame_formatting_default(self, nobs):
         self.data.to_csv(self.fname)
 
+    def time_frame_date_formatting_custom(self, nobs):
+        self.data.to_csv(self.fname, date_format="%Y%m%d__%H%M%S")
+
 
 class ToCSVIndexes(BaseIO):
     fname = "__test__.csv"

asv_bench/benchmarks/strftime.py

@@ -6,10 +6,10 @@
 
 class DatetimeStrftime:
     timeout = 1500
-    params = [1000, 10000]
-    param_names = ["nobs"]
+    params = ([1000, 10000], [False, True])
+    param_names = ["nobs", "tz_aware"]
 
-    def setup(self, nobs):
+    def setup(self, nobs, tz_aware):
         d = "2018-11-29"
         dt = "2018-11-26 11:18:27.0"
         self.data = pd.DataFrame(
@@ -19,37 +19,68 @@ def setup(self, nobs):
                 "r": [np.random.uniform()] * nobs,
             }
         )
+        if tz_aware:
+            self.data["dt"] = self.data["dt"].dt.tz_localize("UTC")
+            self.data["d"] = self.data["d"].dt.tz_localize("UTC")
+
+        self.data["i"] = self.data["dt"]
+        self.data.set_index("i", inplace=True)
 
-    def time_frame_date_to_str(self, nobs):
+    def time_frame_date_to_str(self, nobs, tz_aware):
         self.data["d"].astype(str)
 
-    def time_frame_date_formatting_default(self, nobs):
+    def time_frame_date_formatting_default(self, nobs, tz_aware):
         self.data["d"].dt.strftime(date_format=None)
 
-    def time_frame_date_formatting_default_explicit(self, nobs):
-        self.data["d"].dt.strftime(date_format="%Y-%m-%d")
+    def time_frame_date_formatting_index_to_str(self, nobs, tz_aware):
+        self.data.index.astype(str)
+
+    def time_frame_date_formatting_index_default(self, nobs, tz_aware):
+        self.data.index.strftime(date_format=None)
 
-    def time_frame_date_formatting_custom(self, nobs):
+    def time_frame_date_formatting_custom(self, nobs, tz_aware):
         self.data["d"].dt.strftime(date_format="%Y---%m---%d")
 
-    def time_frame_datetime_to_str(self, nobs):
+    def time_frame_date_formatting_index_custom(self, nobs, tz_aware):
+        self.data.index.strftime(date_format="%Y---%m---%d")
+
+    def time_frame_datetime_to_str(self, nobs, tz_aware):
         self.data["dt"].astype(str)
 
-    def time_frame_datetime_formatting_default(self, nobs):
+    def time_frame_datetime_formatting_default(self, nobs, tz_aware):
         self.data["dt"].dt.strftime(date_format=None)
 
-    def time_frame_datetime_formatting_default_explicit_date_only(self, nobs):
+    def time_frame_datetime_formatting_default_explicit_date_only(self, nobs, tz_aware):
         self.data["dt"].dt.strftime(date_format="%Y-%m-%d")
 
-    def time_frame_datetime_formatting_default_explicit(self, nobs):
+    def time_frame_datetime_formatting_default_explicit(self, nobs, tz_aware):
         self.data["dt"].dt.strftime(date_format="%Y-%m-%d %H:%M:%S")
 
-    def time_frame_datetime_formatting_default_with_float(self, nobs):
+    def time_frame_datetime_formatting_default_with_float(self, nobs, tz_aware):
         self.data["dt"].dt.strftime(date_format="%Y-%m-%d %H:%M:%S.%f")
 
-    def time_frame_datetime_formatting_custom(self, nobs):
+    def time_frame_datetime_formatting_index_to_str(self, nobs, tz_aware):
+        self.data.set_index("dt").index.astype(str)
+
+    def time_frame_datetime_formatting_index_default(self, nobs, tz_aware):
+        self.data.set_index("dt").index.strftime(date_format=None)
+
+    def time_frame_datetime_formatting_custom(self, nobs, tz_aware):
         self.data["dt"].dt.strftime(date_format="%Y-%m-%d --- %H:%M:%S")
 
+    def time_frame_datetime_formatting_index_custom(self, nobs, tz_aware):
+        self.data.set_index("dt").index.strftime(date_format="%Y-%m-%d --- %H:%M:%S")
+
+    def time_frame_datetime_formatting_iso8601_map(self, nobs, tz_aware):
+        self.data["dt"].map(lambda timestamp: timestamp.isoformat())
+
+    def time_frame_datetime_formatting_iso8601_strftime_Z(self, nobs, tz_aware):
+        self.data["dt"].dt.strftime(date_format="%Y-%m-%dT%H:%M:%SZ")
+
+    def time_frame_datetime_formatting_iso8601_strftime_offset(self, nobs, tz_aware):
+        """Not optimized yet as %z is not supported by `convert_strftime_format`"""
+        self.data["dt"].dt.strftime(date_format="%Y-%m-%dT%H:%M:%S%z")
+
 
 class PeriodStrftime:
     timeout = 1500
@@ -73,6 +104,12 @@ def setup(self, nobs, freq):
     def time_frame_period_to_str(self, nobs, freq):
         self.data["p"].astype(str)
 
+    def time_frame_period_str(self, nobs, freq):
+        self.data["p"].apply(str)
+
+    def time_frame_period_repr(self, nobs, freq):
+        self.data["p"].apply(repr)
+
     def time_frame_period_formatting_default(self, nobs, freq):
         self.data["p"].dt.strftime(date_format=None)
 

doc/source/whatsnew/v3.0.0.rst

@@ -295,6 +295,13 @@ Removal of prior version deprecations/changes
 
 Performance improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
+
+- Performance improvement in all datetime formatting procedures, achieved with using python string formatting instead of OS ``strftime`` (:issue:`51298`)::
+
+  - in :meth:`DatetimeLikeArrayMixin.strftime`. Classes :class:`DatetimeArray`, :class:`PeriodArray`, :class:`DatetimeIndex`, :class:`PeriodIndex` benefit from the improvement. :class:`TimedeltaArray.strftime` and :class:`TimedeltaArray.format` are not impacted as their ``date_format`` argument is currently ignored.
+  - in :meth:`NDFrame.to_csv`, :meth:`DataFrameRenderer.to_csv` and :class:`CSVFormatter`
+  - This is achieved thanks to new :func:`pd.tseries.api.convert_strftime_format` to convert a strftime formatting template into a python string formatting template. strftime templates that can not be converted to such a fast python string template continue to be processed with OS ``strftime`` as fallback.
+
 - :attr:`Categorical.categories` returns a :class:`RangeIndex` columns instead of an :class:`Index` if the constructed ``values`` was a ``range``. (:issue:`57787`)
 - :class:`DataFrame` returns a :class:`RangeIndex` columns when possible when ``data`` is a ``dict`` (:issue:`57943`)
 - :class:`Series` returns a :class:`RangeIndex` index when possible when ``data`` is a ``dict`` (:issue:`58118`)

pandas/_libs/__init__.py

@@ -5,6 +5,7 @@
     "Period",
     "Timedelta",
     "Timestamp",
+    "convert_strftime_format",
     "iNaT",
     "Interval",
 ]
@@ -23,5 +24,6 @@
     Period,
     Timedelta,
     Timestamp,
+    convert_strftime_format,
     iNaT,
 )

pandas/_libs/tslib.pyi

@@ -10,6 +10,7 @@ def format_array_from_datetime(
     format: str | None = ...,
     na_rep: str | float = ...,
     reso: int = ...,  # NPY_DATETIMEUNIT
+    fast_strftime: bool = ...,
 ) -> npt.NDArray[np.object_]: ...
 def array_with_unit_to_datetime(
     values: npt.NDArray[np.object_],

pandas/_libs/tslib.pyx

@@ -80,6 +80,10 @@ from pandas._libs.tslibs import (
     Resolution,
     get_resolution,
 )
+from pandas._libs.tslibs.strftime import (
+    UnsupportedStrFmtDirective,
+    convert_strftime_format,
+)
 from pandas._libs.tslibs.timestamps import Timestamp
 
 # Note: this is the only non-tslibs intra-pandas dependency here
@@ -121,6 +125,7 @@ def format_array_from_datetime(
     str format=None,
     na_rep: str | float = "NaT",
     NPY_DATETIMEUNIT reso=NPY_FR_ns,
+    fast_strftime=True,
 ) -> np.ndarray:
     """
     return a np object array of the string formatted values
@@ -134,18 +139,22 @@ def format_array_from_datetime(
     na_rep : optional, default is None
           a nat format
     reso : NPY_DATETIMEUNIT, default NPY_FR_ns
+    fast_strftime : bool, default True
+          If `True` (default) and the format permits it, a faster formatting
+          method will be used. See `convert_strftime_format`.
 
     Returns
     -------
     np.ndarray[object]
     """
     cdef:
-        int64_t val, ns, N = values.size
+        int64_t val, ns, y, h, N = values.size
         bint show_ms = False, show_us = False, show_ns = False
         bint basic_format = False, basic_format_day = False
         _Timestamp ts
         object res
         npy_datetimestruct dts
+        object str_format, locale_dt_strings
 
         # Note that `result` (and thus `result_flat`) is C-order and
         #  `it` iterates C-order as well, so the iteration matches
@@ -179,8 +188,24 @@ def format_array_from_datetime(
             # Default format for dates
             basic_format_day = True
 
+    # Sanity check - these flags are exclusive
     assert not (basic_format_day and basic_format)
 
+    if not basic_format_day and not basic_format and fast_strftime:
+        # Preprocessing for fast_strftime
+        if format is None:
+            # We'll fallback to the Timestamp.str method
+            fast_strftime = False
+        else:
+            try:
+                # Try to get the string formatting template for this format
+                str_format, locale_dt_strings = convert_strftime_format(
+                    format, target="datetime"
+                )
+            except UnsupportedStrFmtDirective:
+                # Unsupported directive: fallback to standard `strftime`
+                fast_strftime = False
+
     for i in range(N):
         # Analogous to: utc_val = values[i]
         val = (<int64_t*>cnp.PyArray_ITER_DATA(it))[0]
@@ -206,6 +231,35 @@ def format_array_from_datetime(
             elif show_ms:
                 res += f".{dts.us // 1000:03d}"
 
+        elif fast_strftime:
+
+            if tz is None:
+                pandas_datetime_to_datetimestruct(val, reso, &dts)
+
+                # Use string formatting for faster strftime
+                y = dts.year
+                shortyear = y % 100
+                if y < 0 and shortyear != 0:
+                    # Fix negative modulo to adopt C-style modulo
+                    shortyear -= 100
+                h = dts.hour
+                res = str_format % {
+                    "year": y,
+                    "shortyear": shortyear,
+                    "month": dts.month,
+                    "day": dts.day,
+                    "hour": h,
+                    "hour12": 12 if h in (0, 12) else (h % 12),
+                    "ampm": locale_dt_strings.pm if (h // 12) else locale_dt_strings.am,
+                    "min": dts.min,
+                    "sec": dts.sec,
+                    "us": dts.us,
+                }
+            else:
+                ts = Timestamp._from_value_and_reso(val, reso=reso, tz=tz)
+
+                # Use string formatting for faster strftime
+                res = ts._fast_strftime(str_format, locale_dt_strings)
         else:
 
             ts = Timestamp._from_value_and_reso(val, reso=reso, tz=tz)

pandas/_libs/tslibs/__init__.py

@@ -9,6 +9,8 @@
     "OutOfBoundsTimedelta",
     "IncompatibleFrequency",
     "Period",
+    "convert_strftime_format",
+    "UnsupportedStrFmtDirective",
     "Resolution",
     "Timedelta",
     "normalize_i8_timestamps",
@@ -69,6 +71,10 @@
     IncompatibleFrequency,
     Period,
 )
+from pandas._libs.tslibs.strftime import (
+    UnsupportedStrFmtDirective,
+    convert_strftime_format,
+)
 from pandas._libs.tslibs.timedeltas import (
     Timedelta,
     delta_to_nanoseconds,

pandas/_libs/tslibs/meson.build

@@ -51,6 +51,7 @@ sources_to_install = [
     'offsets.pyi',
     'parsing.pyi',
     'period.pyi',
+    'strftime.py',
     'strptime.pyi',
     'timedeltas.pyi',
     'timestamps.pyi',

pandas/_libs/tslibs/period.pyi

@@ -48,6 +48,7 @@ def period_array_strftime(
     dtype_code: int,
     na_rep,
     date_format: str | None,
+    fast_strftime: bool,
 ) -> npt.NDArray[np.object_]: ...
 
 # exposed for tests