From ab1768d605b6690b112f807719ff721779c501d8 Mon Sep 17 00:00:00 2001 From: Omswastik-11 Date: Thu, 20 Nov 2025 19:56:37 +0530 Subject: [PATCH 1/9] Integrated Skforecast --- examples/skforecast/skforecast_example.py | 63 +++++++ pyproject.toml | 9 + .../experiment/integrations/__init__.py | 4 + .../integrations/skforecast_forecasting.py | 146 ++++++++++++++++ .../integrations/skforecast/__init__.py | 5 + .../skforecast/skforecast_opt_cv.py | 157 ++++++++++++++++++ .../skforecast/tests/test_skforecast.py | 56 +++++++ 7 files changed, 440 insertions(+) create mode 100644 examples/skforecast/skforecast_example.py create mode 100644 src/hyperactive/experiment/integrations/skforecast_forecasting.py create mode 100644 src/hyperactive/integrations/skforecast/__init__.py create mode 100644 src/hyperactive/integrations/skforecast/skforecast_opt_cv.py create mode 100644 src/hyperactive/integrations/skforecast/tests/test_skforecast.py diff --git a/examples/skforecast/skforecast_example.py b/examples/skforecast/skforecast_example.py new file mode 100644 index 00000000..9a39090b --- /dev/null +++ b/examples/skforecast/skforecast_example.py @@ -0,0 +1,63 @@ +""" +Skforecast Integration Example - Hyperparameter Tuning for Time Series Forecasting + +This example demonstrates how to use Hyperactive to tune hyperparameters of a +skforecast ForecasterRecursive model. It uses the SkforecastOptCV class which +provides a familiar sklearn-like API for integrating skforecast models with +Hyperactive's optimization algorithms. + +Characteristics: +- Integration with skforecast's backtesting functionality +- Tuning of regressor hyperparameters (e.g., RandomForestRegressor) +- Uses HillClimbing optimizer (can be swapped for any Hyperactive optimizer) +- Time series cross-validation via backtesting +""" + +import numpy as np +import pandas as pd +from skforecast.recursive import ForecasterRecursive +from sklearn.ensemble import RandomForestRegressor +from hyperactive.opt import HillClimbing +from hyperactive.integrations.skforecast import SkforecastOptCV + +# Generate synthetic data +data = pd.Series( + np.random.randn(100), + index=pd.date_range(start="2020-01-01", periods=100, freq="D"), + name="y", +) + +# Define forecaster +forecaster = ForecasterRecursive( + regressor=RandomForestRegressor(random_state=123), lags=5 +) + +# Define optimizer +optimizer = HillClimbing( + search_space={ + "n_estimators": list(range(10, 100, 10)), + "max_depth": list(range(2, 10)), + }, + n_iter=10, +) + +# Define SkforecastOptCV +opt_cv = SkforecastOptCV( + forecaster=forecaster, + optimizer=optimizer, + steps=5, + metric="mean_squared_error", + initial_train_size=50, + verbose=True, +) + +# Fit +print("Fitting...") +opt_cv.fit(y=data) + +# Predict +print("Predicting...") +predictions = opt_cv.predict(steps=5) +print("Predictions:") +print(predictions) +print("Best params:", opt_cv.best_params_) diff --git a/pyproject.toml b/pyproject.toml index a11b720e..7b505927 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,15 @@ sktime-integration = [ "skpro", 'sktime; python_version < "3.14"', ] +skforecast-integration = [ + "skforecast", +] +integrations = [ + "scikit-learn <1.8.0", + "skpro", + 'sktime; python_version < "3.14"', + "skforecast", +] build = [ "setuptools", "build", diff --git a/src/hyperactive/experiment/integrations/__init__.py b/src/hyperactive/experiment/integrations/__init__.py index c302e25a..341a7837 100644 --- a/src/hyperactive/experiment/integrations/__init__.py +++ b/src/hyperactive/experiment/integrations/__init__.py @@ -11,6 +11,9 @@ from hyperactive.experiment.integrations.sktime_forecasting import ( SktimeForecastingExperiment, ) +from hyperactive.experiment.integrations.skforecast_forecasting import ( + SkforecastExperiment, +) from hyperactive.experiment.integrations.torch_lightning_experiment import ( TorchExperiment, ) @@ -20,5 +23,6 @@ "SkproProbaRegExperiment", "SktimeClassificationExperiment", "SktimeForecastingExperiment", + "SkforecastExperiment", "TorchExperiment", ] diff --git a/src/hyperactive/experiment/integrations/skforecast_forecasting.py b/src/hyperactive/experiment/integrations/skforecast_forecasting.py new file mode 100644 index 00000000..f12c26ee --- /dev/null +++ b/src/hyperactive/experiment/integrations/skforecast_forecasting.py @@ -0,0 +1,146 @@ +"""Experiment adapter for skforecast backtesting experiments.""" +# copyright: hyperactive developers, MIT License (see LICENSE file) + +import copy +import numpy as np +from hyperactive.base import BaseExperiment + + +class SkforecastExperiment(BaseExperiment): + """Experiment adapter for skforecast backtesting experiments. + + This class is used to perform backtesting experiments using a given + skforecast forecaster. It allows for hyperparameter tuning and evaluation of + the model's performance. + + Parameters + ---------- + forecaster : skforecast forecaster + skforecast forecaster to benchmark + + y : pandas Series + Target time series used in the evaluation experiment + + steps : int + Number of steps to predict + + metric : str or callable + Metric used to quantify the goodness of fit of the model + + initial_train_size : int + Number of samples in the initial training set + + exog : pandas Series or DataFrame, optional + Exogenous variable/s used in the evaluation experiment + + refit : bool, optional + Whether to re-fit the forecaster in each iteration + + fixed_train_size : bool, optional + If True, the train size doesn't increase but moves by `steps` in each iteration + + gap : int, optional + Number of samples to exclude from the end of each training set and the start of the test set + + allow_incomplete_fold : bool, optional + If True, the last fold is allowed to have fewer samples than `steps` + + return_best : bool, optional + If True, the best model is returned + + n_jobs : int or 'auto', optional + Number of jobs to run in parallel + + verbose : bool, optional + Print summary figures + + show_progress : bool, optional + Whether to show a progress bar + """ + + def __init__( + self, + forecaster, + y, + steps, + metric, + initial_train_size, + exog=None, + refit=False, + fixed_train_size=False, + gap=0, + allow_incomplete_fold=True, + return_best=False, + n_jobs="auto", + verbose=False, + show_progress=False, + ): + self.forecaster = forecaster + self.y = y + self.steps = steps + self.metric = metric + self.initial_train_size = initial_train_size + self.exog = exog + self.refit = refit + self.fixed_train_size = fixed_train_size + self.gap = gap + self.allow_incomplete_fold = allow_incomplete_fold + self.return_best = return_best + self.n_jobs = n_jobs + self.verbose = verbose + self.show_progress = show_progress + + super().__init__() + + def _evaluate(self, params): + """Evaluate the parameters. + + Parameters + ---------- + params : dict with string keys + Parameters to evaluate. + + Returns + ------- + float + The value of the parameters as per evaluation. + dict + Additional metadata about the search. + """ + from skforecast.model_selection import backtesting_forecaster + from skforecast.model_selection import TimeSeriesFold + + forecaster = copy.deepcopy(self.forecaster) + forecaster.set_params(params) + + cv = TimeSeriesFold( + steps=self.steps, + initial_train_size=self.initial_train_size, + refit=self.refit, + fixed_train_size=self.fixed_train_size, + gap=self.gap, + allow_incomplete_fold=self.allow_incomplete_fold, + ) + + results, _ = backtesting_forecaster( + forecaster=forecaster, + y=self.y, + cv=cv, + metric=self.metric, + exog=self.exog, + n_jobs=self.n_jobs, + verbose=self.verbose, + show_progress=self.show_progress, + ) + + if isinstance(self.metric, str): + metric_name = self.metric + else: + metric_name = ( + self.metric.__name__ if hasattr(self.metric, "__name__") else "score" + ) + + # backtesting_forecaster returns a DataFrame + res_float = results[metric_name].iloc[0] + + return res_float, {"results": results} diff --git a/src/hyperactive/integrations/skforecast/__init__.py b/src/hyperactive/integrations/skforecast/__init__.py new file mode 100644 index 00000000..90b237a5 --- /dev/null +++ b/src/hyperactive/integrations/skforecast/__init__.py @@ -0,0 +1,5 @@ +# copyright: hyperactive developers, MIT License (see LICENSE file) + +from .skforecast_opt_cv import SkforecastOptCV + +__all__ = ["SkforecastOptCV"] diff --git a/src/hyperactive/integrations/skforecast/skforecast_opt_cv.py b/src/hyperactive/integrations/skforecast/skforecast_opt_cv.py new file mode 100644 index 00000000..4bb3f871 --- /dev/null +++ b/src/hyperactive/integrations/skforecast/skforecast_opt_cv.py @@ -0,0 +1,157 @@ +# copyright: hyperactive developers, MIT License (see LICENSE file) + +import copy +import numpy as np +from sklearn.base import BaseEstimator + +from hyperactive.experiment.integrations.skforecast_forecasting import ( + SkforecastExperiment, +) + + +class SkforecastOptCV(BaseEstimator): + """Tune a skforecast forecaster via any optimizer in the hyperactive toolbox. + + Parameters + ---------- + forecaster : skforecast forecaster + The forecaster to tune. + + optimizer : hyperactive BaseOptimizer + The optimizer to be used for hyperparameter search. + + steps : int + Number of steps to predict + + metric : str or callable + Metric used to quantify the goodness of fit of the model + + initial_train_size : int + Number of samples in the initial training set + + exog : pandas Series or DataFrame, optional + Exogenous variable/s used in the evaluation experiment + + refit : bool, optional + Whether to re-fit the forecaster in each iteration + + fixed_train_size : bool, optional + If True, the train size doesn't increase but moves by `steps` in each iteration + + gap : int, optional + Number of samples to exclude from the end of each training set and the start of the test set + + allow_incomplete_fold : bool, optional + If True, the last fold is allowed to have fewer samples than `steps` + + return_best : bool, optional + If True, the best model is returned + + n_jobs : int or 'auto', optional + Number of jobs to run in parallel + + verbose : bool, optional + Print summary figures + + show_progress : bool, optional + Whether to show a progress bar + """ + + def __init__( + self, + forecaster, + optimizer, + steps, + metric, + initial_train_size, + exog=None, + refit=False, + fixed_train_size=False, + gap=0, + allow_incomplete_fold=True, + return_best=False, + n_jobs="auto", + verbose=False, + show_progress=False, + ): + self.forecaster = forecaster + self.optimizer = optimizer + self.steps = steps + self.metric = metric + self.initial_train_size = initial_train_size + self.exog = exog + self.refit = refit + self.fixed_train_size = fixed_train_size + self.gap = gap + self.allow_incomplete_fold = allow_incomplete_fold + self.return_best = return_best + self.n_jobs = n_jobs + self.verbose = verbose + self.show_progress = show_progress + + def fit(self, y, exog=None): + """Fit to training data. + + Parameters + ---------- + y : pandas Series + Target time series to which to fit the forecaster. + exog : pandas Series or DataFrame, optional + Exogenous variables. + + Returns + ------- + self : returns an instance of self. + """ + current_exog = exog if exog is not None else self.exog + + experiment = SkforecastExperiment( + forecaster=self.forecaster, + y=y, + steps=self.steps, + metric=self.metric, + initial_train_size=self.initial_train_size, + exog=current_exog, + refit=self.refit, + fixed_train_size=self.fixed_train_size, + gap=self.gap, + allow_incomplete_fold=self.allow_incomplete_fold, + return_best=self.return_best, + n_jobs=self.n_jobs, + verbose=self.verbose, + show_progress=self.show_progress, + ) + + if hasattr(self.optimizer, "clone"): + optimizer = self.optimizer.clone() + else: + optimizer = copy.deepcopy(self.optimizer) + + optimizer.set_params(experiment=experiment) + best_params = optimizer.solve() + + self.best_params_ = best_params + self.best_forecaster_ = copy.deepcopy(self.forecaster) + self.best_forecaster_.set_params(best_params) + + # Refit model with best parameters on the whole dataset + self.best_forecaster_.fit(y=y, exog=current_exog) + + return self + + def predict(self, steps, exog=None, **kwargs): + """Forecast time series at future horizon. + + Parameters + ---------- + steps : int + Number of steps to predict. + exog : pandas Series or DataFrame, optional + Exogenous variables. + + Returns + ------- + predictions : pandas Series + Predicted values. + """ + return self.best_forecaster_.predict(steps=steps, exog=exog, **kwargs) diff --git a/src/hyperactive/integrations/skforecast/tests/test_skforecast.py b/src/hyperactive/integrations/skforecast/tests/test_skforecast.py new file mode 100644 index 00000000..b2e82b26 --- /dev/null +++ b/src/hyperactive/integrations/skforecast/tests/test_skforecast.py @@ -0,0 +1,56 @@ +import pytest +import numpy as np +import pandas as pd +from sklearn.ensemble import RandomForestRegressor +from hyperactive.opt import HillClimbing +from hyperactive.integrations.skforecast import SkforecastOptCV + +try: + from skforecast.recursive import ForecasterRecursive +except ImportError: + pass + + +@pytest.fixture +def data(): + return pd.Series( + np.random.randn(100), + index=pd.date_range(start="2020-01-01", periods=100, freq="D"), + name="y", + ) + + +def test_skforecast_opt_cv(data): + try: + import skforecast + except ImportError: + pytest.skip("skforecast not installed", allow_module_level=True) + + forecaster = ForecasterRecursive( + regressor=RandomForestRegressor(random_state=123), lags=5 + ) + + optimizer = HillClimbing( + search_space={ + "n_estimators": [10, 20], + "max_depth": [2, 5], + }, + n_iter=2, + ) + + opt_cv = SkforecastOptCV( + forecaster=forecaster, + optimizer=optimizer, + steps=5, + metric="mean_squared_error", + initial_train_size=50, + verbose=False, + ) + + opt_cv.fit(y=data) + predictions = opt_cv.predict(steps=5) + + assert len(predictions) == 5 + assert isinstance(predictions, pd.Series) + assert "n_estimators" in opt_cv.best_params_ + assert "max_depth" in opt_cv.best_params_ From bbb153ff6beab17a9180077df8d39b53872a64ee Mon Sep 17 00:00:00 2001 From: Omswastik-11 Date: Sat, 22 Nov 2025 22:36:24 +0530 Subject: [PATCH 2/9] added get_test_params method and improved docstrings --- .../experiment/integrations/__init__.py | 6 +- .../integrations/skforecast_forecasting.py | 115 ++++++++++++++---- .../integrations/skforecast/__init__.py | 1 + .../skforecast/skforecast_opt_cv.py | 87 +++++++++---- .../skforecast/tests/test_skforecast.py | 14 ++- 5 files changed, 165 insertions(+), 58 deletions(-) diff --git a/src/hyperactive/experiment/integrations/__init__.py b/src/hyperactive/experiment/integrations/__init__.py index 341a7837..e7e5659b 100644 --- a/src/hyperactive/experiment/integrations/__init__.py +++ b/src/hyperactive/experiment/integrations/__init__.py @@ -1,6 +1,9 @@ """Integrations with packages for tuning.""" # copyright: hyperactive developers, MIT License (see LICENSE file) +from hyperactive.experiment.integrations.skforecast_forecasting import ( + SkforecastExperiment, +) from hyperactive.experiment.integrations.sklearn_cv import SklearnCvExperiment from hyperactive.experiment.integrations.skpro_probareg import ( SkproProbaRegExperiment, @@ -11,9 +14,6 @@ from hyperactive.experiment.integrations.sktime_forecasting import ( SktimeForecastingExperiment, ) -from hyperactive.experiment.integrations.skforecast_forecasting import ( - SkforecastExperiment, -) from hyperactive.experiment.integrations.torch_lightning_experiment import ( TorchExperiment, ) diff --git a/src/hyperactive/experiment/integrations/skforecast_forecasting.py b/src/hyperactive/experiment/integrations/skforecast_forecasting.py index f12c26ee..a134a657 100644 --- a/src/hyperactive/experiment/integrations/skforecast_forecasting.py +++ b/src/hyperactive/experiment/integrations/skforecast_forecasting.py @@ -2,7 +2,7 @@ # copyright: hyperactive developers, MIT License (see LICENSE file) import copy -import numpy as np + from hyperactive.base import BaseExperiment @@ -16,46 +16,50 @@ class SkforecastExperiment(BaseExperiment): Parameters ---------- forecaster : skforecast forecaster - skforecast forecaster to benchmark + skforecast forecaster to benchmark. y : pandas Series - Target time series used in the evaluation experiment + Target time series used in the evaluation experiment. + + exog : pandas Series or DataFrame, default=None + Exogenous variable/s used in the evaluation experiment. steps : int - Number of steps to predict + Number of steps to predict. metric : str or callable - Metric used to quantify the goodness of fit of the model + Metric used to quantify the goodness of fit of the model. + If string, it must be a metric name allowed by skforecast + (e.g., 'mean_squared_error'). + If callable, it must take (y_true, y_pred) and return a float. initial_train_size : int - Number of samples in the initial training set - - exog : pandas Series or DataFrame, optional - Exogenous variable/s used in the evaluation experiment + Number of samples in the initial training set. - refit : bool, optional - Whether to re-fit the forecaster in each iteration + refit : bool, default=False + Whether to re-fit the forecaster in each iteration. - fixed_train_size : bool, optional - If True, the train size doesn't increase but moves by `steps` in each iteration + fixed_train_size : bool, default=False + If True, the train size doesn't increase but moves by `steps` in each iteration. - gap : int, optional - Number of samples to exclude from the end of each training set and the start of the test set + gap : int, default=0 + Number of samples to exclude from the end of each training set and the + start of the test set. - allow_incomplete_fold : bool, optional - If True, the last fold is allowed to have fewer samples than `steps` + allow_incomplete_fold : bool, default=True + If True, the last fold is allowed to have fewer samples than `steps`. - return_best : bool, optional - If True, the best model is returned + return_best : bool, default=False + If True, the best model is returned. - n_jobs : int or 'auto', optional - Number of jobs to run in parallel + n_jobs : int or 'auto', default="auto" + Number of jobs to run in parallel. - verbose : bool, optional - Print summary figures + verbose : bool, default=False + Print summary figures. - show_progress : bool, optional - Whether to show a progress bar + show_progress : bool, default=False + Whether to show a progress bar. """ def __init__( @@ -92,6 +96,64 @@ def __init__( super().__init__() + @classmethod + def get_test_params(cls, parameter_set="default"): + """Return testing parameter settings for the estimator. + + Parameters + ---------- + parameter_set : str, default="default" + Name of the parameter set to return. + + Returns + ------- + params : dict or list of dict, default = {} + Parameters to create testing instances of the class + Each dict are parameters to construct an "interesting" test instance, + i.e., MyClass(**params) or MyClass(**params[i]) creates a valid test + instance. + create_test_instance uses the first (or only) dictionary in `params` + """ + import numpy as np + import pandas as pd + from skforecast.recursive import ForecasterRecursive + from sklearn.ensemble import RandomForestRegressor + + forecaster = ForecasterRecursive( + regressor=RandomForestRegressor(random_state=123), + lags=2, + ) + + y = pd.Series( + np.random.randn(20), + index=pd.date_range(start="2020-01-01", periods=20, freq="D"), + name="y", + ) + + params = { + "forecaster": forecaster, + "y": y, + "steps": 3, + "metric": "mean_squared_error", + "initial_train_size": 10, + } + return [params] + + @classmethod + def _get_score_params(cls): + """Return settings for testing score/evaluate functions. Used in tests only. + + Returns a list, the i-th element should be valid arguments for + self.evaluate and self.score, of an instance constructed with + self.get_test_params()[i]. + + Returns + ------- + list of dict + The parameters to be used for scoring. + """ + return [{"n_estimators": 5}] + def _evaluate(self, params): """Evaluate the parameters. @@ -107,8 +169,7 @@ def _evaluate(self, params): dict Additional metadata about the search. """ - from skforecast.model_selection import backtesting_forecaster - from skforecast.model_selection import TimeSeriesFold + from skforecast.model_selection import TimeSeriesFold, backtesting_forecaster forecaster = copy.deepcopy(self.forecaster) forecaster.set_params(params) diff --git a/src/hyperactive/integrations/skforecast/__init__.py b/src/hyperactive/integrations/skforecast/__init__.py index 90b237a5..4c03cd3a 100644 --- a/src/hyperactive/integrations/skforecast/__init__.py +++ b/src/hyperactive/integrations/skforecast/__init__.py @@ -1,3 +1,4 @@ +"""Skforecast integration package.""" # copyright: hyperactive developers, MIT License (see LICENSE file) from .skforecast_opt_cv import SkforecastOptCV diff --git a/src/hyperactive/integrations/skforecast/skforecast_opt_cv.py b/src/hyperactive/integrations/skforecast/skforecast_opt_cv.py index 4bb3f871..f4fdf9da 100644 --- a/src/hyperactive/integrations/skforecast/skforecast_opt_cv.py +++ b/src/hyperactive/integrations/skforecast/skforecast_opt_cv.py @@ -1,7 +1,8 @@ +"""Skforecast integration for hyperactive.""" # copyright: hyperactive developers, MIT License (see LICENSE file) import copy -import numpy as np + from sklearn.base import BaseEstimator from hyperactive.experiment.integrations.skforecast_forecasting import ( @@ -21,40 +22,44 @@ class SkforecastOptCV(BaseEstimator): The optimizer to be used for hyperparameter search. steps : int - Number of steps to predict + Number of steps to predict. metric : str or callable - Metric used to quantify the goodness of fit of the model + Metric used to quantify the goodness of fit of the model. + If string, it must be a metric name allowed by skforecast + (e.g., 'mean_squared_error'). + If callable, it must take (y_true, y_pred) and return a float. initial_train_size : int - Number of samples in the initial training set + Number of samples in the initial training set. - exog : pandas Series or DataFrame, optional - Exogenous variable/s used in the evaluation experiment + exog : pandas Series or DataFrame, default=None + Exogenous variable/s used in the evaluation experiment. - refit : bool, optional - Whether to re-fit the forecaster in each iteration + refit : bool, default=False + Whether to re-fit the forecaster in each iteration. - fixed_train_size : bool, optional - If True, the train size doesn't increase but moves by `steps` in each iteration + fixed_train_size : bool, default=False + If True, the train size doesn't increase but moves by `steps` in each iteration. - gap : int, optional - Number of samples to exclude from the end of each training set and the start of the test set + gap : int, default=0 + Number of samples to exclude from the end of each training set and the + start of the test set. - allow_incomplete_fold : bool, optional - If True, the last fold is allowed to have fewer samples than `steps` + allow_incomplete_fold : bool, default=True + If True, the last fold is allowed to have fewer samples than `steps`. - return_best : bool, optional - If True, the best model is returned + return_best : bool, default=False + If True, the best model is returned. - n_jobs : int or 'auto', optional - Number of jobs to run in parallel + n_jobs : int or 'auto', default="auto" + Number of jobs to run in parallel. - verbose : bool, optional - Print summary figures + verbose : bool, default=False + Print summary figures. - show_progress : bool, optional - Whether to show a progress bar + show_progress : bool, default=False + Whether to show a progress bar. """ def __init__( @@ -89,6 +94,44 @@ def __init__( self.verbose = verbose self.show_progress = show_progress + @classmethod + def get_test_params(cls, parameter_set="default"): + """Return testing parameter settings for the estimator. + + Parameters + ---------- + parameter_set : str, default="default" + Name of the parameter set to return. + + Returns + ------- + params : dict or list of dict, default = {} + Parameters to create testing instances of the class + Each dict are parameters to construct an "interesting" test instance, + i.e., MyClass(**params) or MyClass(**params[i]) creates a valid test + instance. + create_test_instance uses the first (or only) dictionary in `params` + """ + from skforecast.recursive import ForecasterRecursive + from sklearn.ensemble import RandomForestRegressor + + from hyperactive import HillClimbingOptimizer + + forecaster = ForecasterRecursive( + regressor=RandomForestRegressor(random_state=123), + lags=2, + ) + optimizer = HillClimbingOptimizer() + + params = { + "forecaster": forecaster, + "optimizer": optimizer, + "steps": 3, + "metric": "mean_squared_error", + "initial_train_size": 10, + } + return [params] + def fit(self, y, exog=None): """Fit to training data. diff --git a/src/hyperactive/integrations/skforecast/tests/test_skforecast.py b/src/hyperactive/integrations/skforecast/tests/test_skforecast.py index b2e82b26..ec8b5c6a 100644 --- a/src/hyperactive/integrations/skforecast/tests/test_skforecast.py +++ b/src/hyperactive/integrations/skforecast/tests/test_skforecast.py @@ -1,9 +1,12 @@ -import pytest +"""Test skforecast integration.""" + import numpy as np import pandas as pd +import pytest from sklearn.ensemble import RandomForestRegressor -from hyperactive.opt import HillClimbing + from hyperactive.integrations.skforecast import SkforecastOptCV +from hyperactive.opt import HillClimbing try: from skforecast.recursive import ForecasterRecursive @@ -13,6 +16,7 @@ @pytest.fixture def data(): + """Create test data.""" return pd.Series( np.random.randn(100), index=pd.date_range(start="2020-01-01", periods=100, freq="D"), @@ -21,10 +25,8 @@ def data(): def test_skforecast_opt_cv(data): - try: - import skforecast - except ImportError: - pytest.skip("skforecast not installed", allow_module_level=True) + """Test SkforecastOptCV.""" + pytest.importorskip("skforecast") forecaster = ForecasterRecursive( regressor=RandomForestRegressor(random_state=123), lags=5 From fbcbba3fc03b9c48c5ebc50db56d52ab1151bf7b Mon Sep 17 00:00:00 2001 From: Omswastik-11 Date: Sun, 23 Nov 2025 14:20:24 +0530 Subject: [PATCH 3/9] Corrected no disk space problem in CI builds and added skforecast to python env --- .github/workflows/test.yml | 9 +++++++++ pyproject.toml | 1 + 2 files changed, 10 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5476c0cf..4c35cdc4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -108,6 +108,15 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Free Disk Space (Ubuntu) + if: runner.os == 'Linux' + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL + sudo docker image prune --all --force + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: diff --git a/pyproject.toml b/pyproject.toml index 7b505927..8abf244a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ sklearn-integration = [ sktime-integration = [ "skpro", 'sktime; python_version < "3.14"', + "skforecast", ] skforecast-integration = [ "skforecast", From 3d92878e05a64569ffc33eb960fded1a3726f18e Mon Sep 17 00:00:00 2001 From: Omswastik-11 Date: Thu, 27 Nov 2025 16:19:54 +0530 Subject: [PATCH 4/9] changed the versions --- .github/workflows/test.yml | 2 +- Makefile | 3 +++ pyproject.toml | 11 ++++++++--- src/hyperactive/experiment/integrations/__init__.py | 3 --- .../experiment/integrations/skforecast_forecasting.py | 6 ++++++ src/hyperactive/integrations/skforecast/__init__.py | 2 -- .../integrations/skforecast/skforecast_opt_cv.py | 6 ++++++ 7 files changed, 24 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4c35cdc4..fb45af17 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -206,7 +206,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install build - make install-all-extras-for-test + make install-examples-for-test - name: Show dependencies if: steps.check-examples.outputs.examples_changed == 'true' diff --git a/Makefile b/Makefile index 3801fe58..eac1017d 100644 --- a/Makefile +++ b/Makefile @@ -92,6 +92,9 @@ install-no-extras-for-test: install-all-extras-for-test: python -m pip install .[all_extras,test,test_parallel_backends,sktime-integration] +install-examples-for-test: + python -m pip install .[test_examples] + install-editable: pip install -e . diff --git a/pyproject.toml b/pyproject.toml index 8abf244a..a8c8e5b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,16 +52,16 @@ sklearn-integration = [ sktime-integration = [ "skpro", 'sktime; python_version < "3.14"', - "skforecast", + 'skforecast; python_version < "3.14"', ] skforecast-integration = [ - "skforecast", + 'skforecast; python_version < "3.14"', ] integrations = [ "scikit-learn <1.8.0", "skpro", 'sktime; python_version < "3.14"', - "skforecast", + 'skforecast; python_version < "3.14"', ] build = [ "setuptools", @@ -86,6 +86,11 @@ all_extras = [ "optuna<5", "lightning", ] +test_examples = [ + "pytest == 9.0.1", + "hyperactive[integrations]", + "optuna<5", +] [project.urls] diff --git a/src/hyperactive/experiment/integrations/__init__.py b/src/hyperactive/experiment/integrations/__init__.py index e7e5659b..0ecd50c1 100644 --- a/src/hyperactive/experiment/integrations/__init__.py +++ b/src/hyperactive/experiment/integrations/__init__.py @@ -1,9 +1,6 @@ """Integrations with packages for tuning.""" # copyright: hyperactive developers, MIT License (see LICENSE file) -from hyperactive.experiment.integrations.skforecast_forecasting import ( - SkforecastExperiment, -) from hyperactive.experiment.integrations.sklearn_cv import SklearnCvExperiment from hyperactive.experiment.integrations.skpro_probareg import ( SkproProbaRegExperiment, diff --git a/src/hyperactive/experiment/integrations/skforecast_forecasting.py b/src/hyperactive/experiment/integrations/skforecast_forecasting.py index a134a657..10806b7b 100644 --- a/src/hyperactive/experiment/integrations/skforecast_forecasting.py +++ b/src/hyperactive/experiment/integrations/skforecast_forecasting.py @@ -62,6 +62,12 @@ class SkforecastExperiment(BaseExperiment): Whether to show a progress bar. """ + _tags = { + "authors": "Omswastik-11", + "maintainers": ["Omswastik-11", "fkiraly", "JoaquinAmatRodrigo", "SimonBlanke"], + "python_dependencies": "skforecast", + } + def __init__( self, forecaster, diff --git a/src/hyperactive/integrations/skforecast/__init__.py b/src/hyperactive/integrations/skforecast/__init__.py index 4c03cd3a..9f1e558d 100644 --- a/src/hyperactive/integrations/skforecast/__init__.py +++ b/src/hyperactive/integrations/skforecast/__init__.py @@ -1,6 +1,4 @@ """Skforecast integration package.""" # copyright: hyperactive developers, MIT License (see LICENSE file) -from .skforecast_opt_cv import SkforecastOptCV - __all__ = ["SkforecastOptCV"] diff --git a/src/hyperactive/integrations/skforecast/skforecast_opt_cv.py b/src/hyperactive/integrations/skforecast/skforecast_opt_cv.py index f4fdf9da..c07c5e85 100644 --- a/src/hyperactive/integrations/skforecast/skforecast_opt_cv.py +++ b/src/hyperactive/integrations/skforecast/skforecast_opt_cv.py @@ -62,6 +62,12 @@ class SkforecastOptCV(BaseEstimator): Whether to show a progress bar. """ + _tags = { + "authors": "Omswastik-11", + "maintainers": ["Omswastik-11", "fkiraly", "JoaquinAmatRodrigo", "SimonBlanke"], + "python_dependencies": "skforecast", + } + def __init__( self, forecaster, From 7c26c0cb4ae1c006b9c468ca9fd9388ab0b7e721 Mon Sep 17 00:00:00 2001 From: Omswastik-11 Date: Thu, 27 Nov 2025 17:28:08 +0530 Subject: [PATCH 5/9] added tags and inferred metrices in constructor --- src/hyperactive/experiment/integrations/__init__.py | 3 +++ .../integrations/skforecast_forecasting.py | 12 +++++++++++- src/hyperactive/integrations/skforecast/__init__.py | 2 ++ .../integrations/skforecast/skforecast_opt_cv.py | 7 ++++++- 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/hyperactive/experiment/integrations/__init__.py b/src/hyperactive/experiment/integrations/__init__.py index 0ecd50c1..e7e5659b 100644 --- a/src/hyperactive/experiment/integrations/__init__.py +++ b/src/hyperactive/experiment/integrations/__init__.py @@ -1,6 +1,9 @@ """Integrations with packages for tuning.""" # copyright: hyperactive developers, MIT License (see LICENSE file) +from hyperactive.experiment.integrations.skforecast_forecasting import ( + SkforecastExperiment, +) from hyperactive.experiment.integrations.sklearn_cv import SklearnCvExperiment from hyperactive.experiment.integrations.skpro_probareg import ( SkproProbaRegExperiment, diff --git a/src/hyperactive/experiment/integrations/skforecast_forecasting.py b/src/hyperactive/experiment/integrations/skforecast_forecasting.py index 10806b7b..10b0eb11 100644 --- a/src/hyperactive/experiment/integrations/skforecast_forecasting.py +++ b/src/hyperactive/experiment/integrations/skforecast_forecasting.py @@ -63,7 +63,7 @@ class SkforecastExperiment(BaseExperiment): """ _tags = { - "authors": "Omswastik-11", + "authors": ["Omswastik-11", "JoaquinAmatRodrigo"], "maintainers": ["Omswastik-11", "fkiraly", "JoaquinAmatRodrigo", "SimonBlanke"], "python_dependencies": "skforecast", } @@ -102,6 +102,11 @@ def __init__( super().__init__() + # Infer higher_or_lower_is_better from metric + # All standard skforecast/sklearn regression metrics are "lower is better" + # (MSE, MAE, MAPE, etc.). Custom callables are assumed lower is better. + self.set_tags(**{"property:higher_or_lower_is_better": "lower"}) + @classmethod def get_test_params(cls, parameter_set="default"): """Return testing parameter settings for the estimator. @@ -120,6 +125,11 @@ def get_test_params(cls, parameter_set="default"): instance. create_test_instance uses the first (or only) dictionary in `params` """ + from skbase.utils.dependencies import _check_soft_dependencies + + if not _check_soft_dependencies("skforecast", severity="none"): + return [] + import numpy as np import pandas as pd from skforecast.recursive import ForecasterRecursive diff --git a/src/hyperactive/integrations/skforecast/__init__.py b/src/hyperactive/integrations/skforecast/__init__.py index 9f1e558d..99eb49ce 100644 --- a/src/hyperactive/integrations/skforecast/__init__.py +++ b/src/hyperactive/integrations/skforecast/__init__.py @@ -1,4 +1,6 @@ """Skforecast integration package.""" # copyright: hyperactive developers, MIT License (see LICENSE file) +from hyperactive.integrations.skforecast.skforecast_opt_cv import SkforecastOptCV + __all__ = ["SkforecastOptCV"] diff --git a/src/hyperactive/integrations/skforecast/skforecast_opt_cv.py b/src/hyperactive/integrations/skforecast/skforecast_opt_cv.py index c07c5e85..fb3e6c83 100644 --- a/src/hyperactive/integrations/skforecast/skforecast_opt_cv.py +++ b/src/hyperactive/integrations/skforecast/skforecast_opt_cv.py @@ -63,7 +63,7 @@ class SkforecastOptCV(BaseEstimator): """ _tags = { - "authors": "Omswastik-11", + "authors": ["Omswastik-11", "JoaquinAmatRodrigo"], "maintainers": ["Omswastik-11", "fkiraly", "JoaquinAmatRodrigo", "SimonBlanke"], "python_dependencies": "skforecast", } @@ -118,6 +118,11 @@ def get_test_params(cls, parameter_set="default"): instance. create_test_instance uses the first (or only) dictionary in `params` """ + from skbase.utils.dependencies import _check_soft_dependencies + + if not _check_soft_dependencies("skforecast", severity="none"): + return [] + from skforecast.recursive import ForecasterRecursive from sklearn.ensemble import RandomForestRegressor From 87da9db7ad152589badd19f795674590db3c3959 Mon Sep 17 00:00:00 2001 From: Omswastik-11 Date: Thu, 27 Nov 2025 17:29:57 +0530 Subject: [PATCH 6/9] update skforecast_forecasting.py --- .../experiment/integrations/skforecast_forecasting.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hyperactive/experiment/integrations/skforecast_forecasting.py b/src/hyperactive/experiment/integrations/skforecast_forecasting.py index 10b0eb11..0b93890e 100644 --- a/src/hyperactive/experiment/integrations/skforecast_forecasting.py +++ b/src/hyperactive/experiment/integrations/skforecast_forecasting.py @@ -102,9 +102,8 @@ def __init__( super().__init__() - # Infer higher_or_lower_is_better from metric # All standard skforecast/sklearn regression metrics are "lower is better" - # (MSE, MAE, MAPE, etc.). Custom callables are assumed lower is better. + # (MSE, MAE, MAPE, RMSE, etc.). Custom callables are assumed lower is better. self.set_tags(**{"property:higher_or_lower_is_better": "lower"}) @classmethod From ca013a4c83ce583a8fa0766dd6416cb7e2953e5c Mon Sep 17 00:00:00 2001 From: Omswastik-11 Date: Thu, 27 Nov 2025 22:59:53 +0530 Subject: [PATCH 7/9] added a 'higher_is_better' parameter in constructor --- .../integrations/skforecast_forecasting.py | 14 +++++++++++--- .../integrations/skforecast/skforecast_opt_cv.py | 9 +++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/hyperactive/experiment/integrations/skforecast_forecasting.py b/src/hyperactive/experiment/integrations/skforecast_forecasting.py index 0b93890e..e6de8fb5 100644 --- a/src/hyperactive/experiment/integrations/skforecast_forecasting.py +++ b/src/hyperactive/experiment/integrations/skforecast_forecasting.py @@ -60,6 +60,12 @@ class SkforecastExperiment(BaseExperiment): show_progress : bool, default=False Whether to show a progress bar. + + higher_is_better : bool, default=False + Whether higher metric values indicate better performance. + Set to False (default) for error metrics like MSE, MAE, MAPE where + lower values are better. Set to True for metrics like R2 where + higher values indicate better model performance. """ _tags = { @@ -84,6 +90,7 @@ def __init__( n_jobs="auto", verbose=False, show_progress=False, + higher_is_better=False, ): self.forecaster = forecaster self.y = y @@ -99,12 +106,13 @@ def __init__( self.n_jobs = n_jobs self.verbose = verbose self.show_progress = show_progress + self.higher_is_better = higher_is_better super().__init__() - # All standard skforecast/sklearn regression metrics are "lower is better" - # (MSE, MAE, MAPE, RMSE, etc.). Custom callables are assumed lower is better. - self.set_tags(**{"property:higher_or_lower_is_better": "lower"}) + # Set the optimization direction based on higher_is_better parameter + higher_or_lower = "higher" if higher_is_better else "lower" + self.set_tags(**{"property:higher_or_lower_is_better": higher_or_lower}) @classmethod def get_test_params(cls, parameter_set="default"): diff --git a/src/hyperactive/integrations/skforecast/skforecast_opt_cv.py b/src/hyperactive/integrations/skforecast/skforecast_opt_cv.py index fb3e6c83..56ee0351 100644 --- a/src/hyperactive/integrations/skforecast/skforecast_opt_cv.py +++ b/src/hyperactive/integrations/skforecast/skforecast_opt_cv.py @@ -60,6 +60,12 @@ class SkforecastOptCV(BaseEstimator): show_progress : bool, default=False Whether to show a progress bar. + + higher_is_better : bool, default=False + Whether higher metric values indicate better performance. + Set to False (default) for error metrics like MSE, MAE, MAPE where + lower values are better. Set to True for metrics like R2 where + higher values indicate better model performance. """ _tags = { @@ -84,6 +90,7 @@ def __init__( n_jobs="auto", verbose=False, show_progress=False, + higher_is_better=False, ): self.forecaster = forecaster self.optimizer = optimizer @@ -99,6 +106,7 @@ def __init__( self.n_jobs = n_jobs self.verbose = verbose self.show_progress = show_progress + self.higher_is_better = higher_is_better @classmethod def get_test_params(cls, parameter_set="default"): @@ -174,6 +182,7 @@ def fit(self, y, exog=None): n_jobs=self.n_jobs, verbose=self.verbose, show_progress=self.show_progress, + higher_is_better=self.higher_is_better, ) if hasattr(self.optimizer, "clone"): From 3e7f96a97140cb196e49166f9ebfeec765fc02f2 Mon Sep 17 00:00:00 2001 From: Omswastik-11 Date: Fri, 28 Nov 2025 12:32:52 +0530 Subject: [PATCH 8/9] revert the changes in CI --- .github/workflows/test.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fb45af17..525659e5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -108,15 +108,6 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Free Disk Space (Ubuntu) - if: runner.os == 'Linux' - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - sudo rm -rf /opt/hostedtoolcache/CodeQL - sudo docker image prune --all --force - - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: From 59d05ce72ee63034dcf73caf220ee3fab37ae240 Mon Sep 17 00:00:00 2001 From: Omswastik-11 Date: Tue, 2 Dec 2025 21:11:50 +0530 Subject: [PATCH 9/9] revert depset changes and clean runner before testing --- .github/workflows/test.yml | 20 +++++++++++++++++++- Makefile | 3 --- pyproject.toml | 6 ------ 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 525659e5..9e495401 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -108,6 +108,15 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Free Disk Space (Ubuntu) + if: runner.os == 'Linux' + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL + sudo docker image prune --all --force + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -167,6 +176,15 @@ jobs: with: fetch-depth: 0 + - name: Free Disk Space (Ubuntu) + if: runner.os == 'Linux' + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL + sudo docker image prune --all --force + - name: Set up Python 3.11 uses: actions/setup-python@v5 with: @@ -197,7 +215,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install build - make install-examples-for-test + make install-all-extras-for-test - name: Show dependencies if: steps.check-examples.outputs.examples_changed == 'true' diff --git a/Makefile b/Makefile index eac1017d..3801fe58 100644 --- a/Makefile +++ b/Makefile @@ -92,9 +92,6 @@ install-no-extras-for-test: install-all-extras-for-test: python -m pip install .[all_extras,test,test_parallel_backends,sktime-integration] -install-examples-for-test: - python -m pip install .[test_examples] - install-editable: pip install -e . diff --git a/pyproject.toml b/pyproject.toml index a8c8e5b3..1f654d76 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,12 +86,6 @@ all_extras = [ "optuna<5", "lightning", ] -test_examples = [ - "pytest == 9.0.1", - "hyperactive[integrations]", - "optuna<5", -] - [project.urls] "Homepage" = "https://github.com/SimonBlanke/Hyperactive"