diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py
index b439a3e2dfbc85..5b13e13bb20bae 100644
--- a/pandas/core/dtypes/cast.py
+++ b/pandas/core/dtypes/cast.py
@@ -367,16 +367,24 @@ def maybe_promote(dtype, fill_value=np.nan):
             except (TypeError, ValueError):
                 dtype = np.dtype(np.object_)
     elif issubclass(dtype.type, np.timedelta64):
-        try:
-            fv = tslibs.Timedelta(fill_value)
-        except ValueError:
+        if (
+            is_integer(fill_value)
+            or (is_float(fill_value) and not np.isnan(fill_value))
+            or isinstance(fill_value, str)
+        ):
+            # TODO: What about str that can be a timedelta?
             dtype = np.dtype(np.object_)
         else:
-            if fv is NaT:
-                # NaT has no `to_timedelta64` method
-                fill_value = np.timedelta64("NaT", "ns")
+            try:
+                fv = tslibs.Timedelta(fill_value)
+            except ValueError:
+                dtype = np.dtype(np.object_)
             else:
-                fill_value = fv.to_timedelta64()
+                if fv is NaT:
+                    # NaT has no `to_timedelta64` method
+                    fill_value = np.timedelta64("NaT", "ns")
+                else:
+                    fill_value = fv.to_timedelta64()
     elif is_datetime64tz_dtype(dtype):
         if isna(fill_value):
             fill_value = NaT
diff --git a/pandas/core/internals/concat.py b/pandas/core/internals/concat.py
index 121c61d8d36237..aa372af4aceb8b 100644
--- a/pandas/core/internals/concat.py
+++ b/pandas/core/internals/concat.py
@@ -338,6 +338,7 @@ def get_empty_dtype_and_na(join_units):
     if not upcast_classes:
         upcast_classes = null_upcast_classes
 
+    # TODO: de-duplicate with maybe_promote?
     # create the result
     if "object" in upcast_classes:
         return np.dtype(np.object_), np.nan
@@ -356,7 +357,7 @@ def get_empty_dtype_and_na(join_units):
     elif "datetime" in upcast_classes:
         return np.dtype("M8[ns]"), tslibs.iNaT
     elif "timedelta" in upcast_classes:
-        return np.dtype("m8[ns]"), tslibs.iNaT
+        return np.dtype("m8[ns]"), np.timedelta64("NaT", "ns")
     else:  # pragma
         try:
             g = np.find_common_type(upcast_classes, [])
diff --git a/pandas/tests/dtypes/cast/test_promote.py b/pandas/tests/dtypes/cast/test_promote.py
index 7c926b30766c53..e4e5a22ea6ca07 100644
--- a/pandas/tests/dtypes/cast/test_promote.py
+++ b/pandas/tests/dtypes/cast/test_promote.py
@@ -821,8 +821,6 @@ def test_maybe_promote_timedelta64_with_any(
     else:
         if boxed and box_dtype is None:
             pytest.xfail("does not upcast to object")
-        if not boxed:
-            pytest.xfail("does not upcast to object or raises")
 
     # create array of given dtype; casts "1" to correct dtype
     fill_value = np.array([1], dtype=fill_dtype)[0]