diff --git a/sktime/performance_metrics/forecasting/_classes.py b/sktime/performance_metrics/forecasting/_classes.py
index 57f4486a479..1c8f5a84c39 100644
--- a/sktime/performance_metrics/forecasting/_classes.py
+++ b/sktime/performance_metrics/forecasting/_classes.py
@@ -575,6 +575,7 @@ def _evaluate(self, y_true, y_pred, **kwargs):
         if getfullargspec(func).varkw is None:
             func_params = signature(func).parameters.keys()
             func_params = set(func_params).difference(["y_true", "y_pred"])
+            func_params = func_params.intersection(params.keys())
             params = {key: params[key] for key in func_params}
 
         res = func(y_true=y_true, y_pred=y_pred, **params)
@@ -657,12 +658,12 @@ def make_forecasting_scorer(
     multioutput="uniform_average",
     multilevel="uniform_average",
 ):
-    """Create a metric class from a metric functions.
+    """Create a metric class from a metric function.
 
     Parameters
     ----------
-    func
-        Function to convert to a forecasting scorer class.
+    func : callable
+        Callable to convert to a forecasting scorer class.
         Score function (or loss function) with signature ``func(y, y_pred, **kwargs)``.
     name : str, default=None
         Name to use for the forecasting scorer loss class.
diff --git a/sktime/performance_metrics/forecasting/tests/test_metrics.py b/sktime/performance_metrics/forecasting/tests/test_metrics.py
index ea82b9381b4..b8144528260 100644
--- a/sktime/performance_metrics/forecasting/tests/test_metrics.py
+++ b/sktime/performance_metrics/forecasting/tests/test_metrics.py
@@ -4,6 +4,7 @@
 # since the numpy output print changes between versions
 
 import numpy as np
+import pandas as pd
 
 
 def test_gmse_class():
@@ -106,3 +107,18 @@ def test_linex_function():
     assert np.allclose(
         mean_linex_error(y_true, y_pred, multioutput=[0.3, 0.7]), 0.30917568000716666
     )
+
+
+def test_make_scorer():
+    """Test make_forecasting_scorer and the failure case in #4827."""
+    import functools
+
+    from sklearn.metrics import mean_squared_log_error
+
+    from sktime.performance_metrics.forecasting import make_forecasting_scorer
+
+    rmsle = functools.partial(mean_squared_log_error, squared=False)
+
+    scorer = make_forecasting_scorer(rmsle, name="RMSLE")
+
+    scorer.evaluate(pd.Series([1, 2, 3]), pd.Series([1, 2, 4]))