diff --git a/xarray/coding/times.py b/xarray/coding/times.py
index 4acfe55d8b0..08fef07fc1a 100644
--- a/xarray/coding/times.py
+++ b/xarray/coding/times.py
@@ -249,17 +249,11 @@ def _decode_datetime_with_pandas(
     if flat_num_dates.dtype.kind in "iu":
         flat_num_dates = flat_num_dates.astype(np.int64)
 
-    # Cast input ordinals to integers of nanoseconds because pd.to_timedelta
-    # works much faster when dealing with integers (GH 1399).
-    flat_num_dates_ns_int = (flat_num_dates * _NS_PER_TIME_DELTA[delta]).astype(
-        np.int64
-    )
-
     # Use pd.to_timedelta to safely cast integer values to timedeltas,
     # and add those to a Timestamp to safely produce a DatetimeIndex.  This
     # ensures that we do not encounter integer overflow at any point in the
     # process without raising OutOfBoundsDatetime.
-    return (pd.to_timedelta(flat_num_dates_ns_int, "ns") + ref_date).values
+    return (pd.to_timedelta(flat_num_dates, delta) + ref_date).values
 
 
 def decode_cf_datetime(