Backport PR #55189 on branch 2.1.x (Revert "DEPR: Deprecate returning…

… a DataFrame in SeriesApply.apply_standard) (#55205)

Backport PR #55189: Revert "DEPR: Deprecate returning a DataFrame in SeriesApply.apply_standard

Co-authored-by: Patrick Hoefler <61934744+phofl@users.noreply.github.com>

doc/source/user_guide/cookbook.rst

@@ -794,12 +794,12 @@ Apply
        index=["I", "II", "III"],
    )
 
-   def make_df(ser):
-       new_vals = [pd.Series(value, name=name) for name, value in ser.items()]
-       return pd.DataFrame(new_vals)
-
-   df_orgz = pd.concat({ind: row.pipe(make_df) for ind, row in df.iterrows()})
+   def SeriesFromSubList(aList):
+       return pd.Series(aList)
 
+   df_orgz = pd.concat(
+       {ind: row.apply(SeriesFromSubList) for ind, row in df.iterrows()}
+   )
    df_orgz
 
 `Rolling apply with a DataFrame returning a Series

doc/source/user_guide/groupby.rst

@@ -1207,6 +1207,19 @@ The dimension of the returned result can also change:
 
     grouped.apply(f)
 
+``apply`` on a Series can operate on a returned value from the applied function
+that is itself a series, and possibly upcast the result to a DataFrame:
+
+.. ipython:: python
+
+    def f(x):
+        return pd.Series([x, x ** 2], index=["x", "x^2"])
+
+
+    s = pd.Series(np.random.rand(5))
+    s
+    s.apply(f)
+
 Similar to :ref:`groupby.aggregate.agg`, the resulting dtype will reflect that of the
 apply function. If the results from different groups have different dtypes, then
 a common dtype will be determined in the same way as ``DataFrame`` construction.

doc/source/whatsnew/v0.10.0.rst

@@ -244,26 +244,15 @@ Convenience methods ``ffill`` and  ``bfill`` have been added:
   function, that is itself a series, and possibly upcast the result to a
   DataFrame
 
-  .. code-block:: python
-
-     >>> def f(x):
-     ...     return pd.Series([x, x ** 2], index=["x", "x^2"])
-     >>>
-     >>> s = pd.Series(np.random.rand(5))
-     >>> s
-     0    0.340445
-     1    0.984729
-     2    0.919540
-     3    0.037772
-     4    0.861549
-     dtype: float64
-     >>> s.apply(f)
-               x       x^2
-     0  0.340445  0.115903
-     1  0.984729  0.969691
-     2  0.919540  0.845555
-     3  0.037772  0.001427
-     4  0.861549  0.742267
+  .. ipython:: python
+
+      def f(x):
+          return pd.Series([x, x ** 2], index=["x", "x^2"])
+
+
+      s = pd.Series(np.random.rand(5))
+      s
+      s.apply(f)
 
 - New API functions for working with pandas options (:issue:`2097`):
 

doc/source/whatsnew/v2.1.0.rst

@@ -555,7 +555,6 @@ Other Deprecations
 - Deprecated behavior of :func:`concat` when :class:`DataFrame` has columns that are all-NA, in a future version these will not be discarded when determining the resulting dtype (:issue:`40893`)
 - Deprecated behavior of :meth:`Series.dt.to_pydatetime`, in a future version this will return a :class:`Series` containing python ``datetime`` objects instead of an ``ndarray`` of datetimes; this matches the behavior of other :attr:`Series.dt` properties (:issue:`20306`)
 - Deprecated logical operations (``|``, ``&``, ``^``) between pandas objects and dtype-less sequences (e.g. ``list``, ``tuple``), wrap a sequence in a :class:`Series` or NumPy array before operating instead (:issue:`51521`)
-- Deprecated making :meth:`Series.apply` return a :class:`DataFrame` when the passed-in callable returns a :class:`Series` object. In the future this will return a :class:`Series` whose values are themselves :class:`Series`. This pattern was very slow and it's recommended to use alternative methods to archive the same goal (:issue:`52116`)
 - Deprecated parameter ``convert_type`` in :meth:`Series.apply` (:issue:`52140`)
 - Deprecated passing a dictionary to :meth:`.SeriesGroupBy.agg`; pass a list of aggregations instead (:issue:`50684`)
 - Deprecated the ``fastpath`` keyword in :class:`Categorical` constructor, use :meth:`Categorical.from_codes` instead (:issue:`20110`)

doc/source/whatsnew/v2.1.1.rst

@@ -44,7 +44,7 @@ Bug fixes
 
 Other
 ~~~~~
--
+- Reverted the deprecation that disallowed :meth:`Series.apply` returning a :class:`DataFrame` when the passed-in callable returns a :class:`Series` object (:issue:`52116`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_211.contributors:

pandas/core/apply.py

@@ -1289,14 +1289,6 @@ def curried(x):
         )
 
         if len(mapped) and isinstance(mapped[0], ABCSeries):
-            warnings.warn(
-                "Returning a DataFrame from Series.apply when the supplied function "
-                "returns a Series is deprecated and will be removed in a future "
-                "version.",
-                FutureWarning,
-                stacklevel=find_stack_level(),
-            )  # GH52116
-
             # GH#43986 Need to do list(mapped) in order to get treated as nested
             #  See also GH#25959 regarding EA support
             return obj._constructor_expanddim(list(mapped), index=obj.index)

pandas/core/series.py

@@ -4634,11 +4634,6 @@ def apply(
         """
         Invoke function on values of Series.
 
-        .. deprecated:: 2.1.0
-
-            If the result from ``func`` is a ``Series``, wrapping the output in a
-            ``DataFrame`` instead of a ``Series`` has been deprecated.
-
         Can be ufunc (a NumPy function that applies to the entire Series)
         or a Python function that only works on single values.
 

pandas/tests/apply/test_series_apply.py

@@ -420,15 +420,20 @@ def test_agg_evaluate_lambdas(string_series):
 def test_with_nested_series(datetime_series, op_name):
     # GH 2316
     # .agg with a reducer and a transform, what to do
-    msg = "Returning a DataFrame from Series.apply when the supplied function"
-    with tm.assert_produces_warning(FutureWarning, match=msg):
+    msg = "cannot aggregate"
+    warning = FutureWarning if op_name == "agg" else None
+    with tm.assert_produces_warning(warning, match=msg):
         # GH52123
         result = getattr(datetime_series, op_name)(
             lambda x: Series([x, x**2], index=["x", "x^2"])
         )
     expected = DataFrame({"x": datetime_series, "x^2": datetime_series**2})
     tm.assert_frame_equal(result, expected)
 
+    with tm.assert_produces_warning(FutureWarning, match=msg):
+        result = datetime_series.agg(lambda x: Series([x, x**2], index=["x", "x^2"]))
+    tm.assert_frame_equal(result, expected)
+
 
 def test_replicate_describe(string_series):
     # this also tests a result set that is all scalars
@@ -512,10 +517,7 @@ def test_apply_series_on_date_time_index_aware_series(dti, exp, aware):
         index = dti.tz_localize("UTC").index
     else:
         index = dti.index
-    msg = "Returning a DataFrame from Series.apply when the supplied function"
-    with tm.assert_produces_warning(FutureWarning, match=msg):
-        # GH52123
-        result = Series(index).apply(lambda x: Series([1, 2]))
+    result = Series(index).apply(lambda x: Series([1, 2]))
     tm.assert_frame_equal(result, exp)
 
 
@@ -662,19 +664,7 @@ def test_apply_dictlike_lambda(ops, by_row, expected):
 def test_apply_retains_column_name(by_row):
     # GH 16380
     df = DataFrame({"x": range(3)}, Index(range(3), name="x"))
-    func = lambda x: Series(range(x + 1), Index(range(x + 1), name="y"))
-
-    if not by_row:
-        # GH53400
-        msg = "'Series' object cannot be interpreted as an integer"
-        with pytest.raises(TypeError, match=msg):
-            df.x.apply(func, by_row=by_row)
-        return
-
-    msg = "Returning a DataFrame from Series.apply when the supplied function"
-    with tm.assert_produces_warning(FutureWarning, match=msg):
-        # GH52123
-        result = df.x.apply(func, by_row=by_row)
+    result = df.x.apply(lambda x: Series(range(x + 1), Index(range(x + 1), name="y")))
     expected = DataFrame(
         [[0.0, np.nan, np.nan], [0.0, 1.0, np.nan], [0.0, 1.0, 2.0]],
         columns=Index(range(3), name="y"),
@@ -689,3 +679,11 @@ def test_apply_type():
     result = s.apply(type)
     expected = Series([int, str, type], index=["a", "b", "c"])
     tm.assert_series_equal(result, expected)
+
+
+def test_series_apply_unpack_nested_data():
+    # GH#55189
+    ser = Series([[1, 2, 3], [4, 5, 6, 7]])
+    result = ser.apply(lambda x: Series(x))
+    expected = DataFrame({0: [1.0, 4.0], 1: [2.0, 5.0], 2: [3.0, 6.0], 3: [np.nan, 7]})
+    tm.assert_frame_equal(result, expected)