diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index da52e7e4..8956a396 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -59,6 +59,9 @@ jobs:
matrix:
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
os: ["macos-latest", "windows-latest", "ubuntu-latest"]
+ exclude:
+ - os: "windows-latest"
+ python-version: "3.13"
fail-fast: false
@@ -93,6 +96,9 @@ jobs:
matrix:
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
os: ["macos-latest", "windows-latest", "ubuntu-latest"]
+ exclude:
+ - os: "windows-latest"
+ python-version: "3.13"
fail-fast: false
diff --git a/README.md b/README.md
index 8ee02d1c..3f1960d6 100644
--- a/README.md
+++ b/README.md
@@ -1,15 +1,16 @@
-
- 
-
-
-
----
+## Welcome to hyperactive
-An optimization and data collection toolbox for convenient and fast prototyping of computationally expensive models.
+
+
+
-
+**A unified interface for optimization algorithms and problems.**
+* [easy sklearn-like interface](#hyperactive-is-very-easy-to-use), [versatile and configurable](./examples/optimization_applications/search_space_example.py)
+- collection of [optimization algorithms](#overview), integrates with major [ML frameworks](#overview) such as `scikit-learn`
+- [memory-efficient](./examples/optimization_applications/memory.py) native implementations of [gradient-free optimizers](https://github.com/SimonBlanke/Gradient-Free-Optimizers)
+- unified API to popular optimization packages such as `optuna`
@@ -19,8 +20,7 @@
@@ -32,35 +32,120 @@
| **CI/CD** | [](https://github.com/SimonBlanke/hyperactive/actions/workflows/test.yml) [](https://www.hyperactive.net/en/latest/?badge=latest)
| **Code** | [](https://pypi.org/project/hyperactive/) [](https://www.python.org/) [](https://github.com/psf/black) |
+## Installation
-
+```console
+pip install hyperactive
+```
-## Hyperactive:
+## :zap: Quickstart
-- is [very easy](#hyperactive-is-very-easy-to-use) to learn but [extremly versatile](./examples/optimization_applications/search_space_example.py)
+### Maximizing a custom function
-- provides intelligent [optimization algorithms](#overview), support for all major [machine-learning frameworks](#overview) and many interesting [applications](#overview)
+```python
+import numpy as np
-- makes optimization [data collection](./examples/optimization_applications/meta_data_collection.py) simple
+# function to be maximized
+def problem(opt):
+ x = opt["x"]
+ y = opt["y"]
-- saves your [computation time](./examples/optimization_applications/memory.py)
+ return - x ** 2 + - y ** 2
-- supports [parallel computing](./examples/tested_and_supported_packages/multiprocessing_example.py)
+# discrete search space: dict of iterable, scikit-learn like grid space
+# (valid search space types depends on optimizer)
+grid = {
+ "x": np.arange(-1, 1, 0.01),
+ "y": np.arange(-1, 2, 0.1),
+}
+from hyperactive.opt import HillClimbing
+hillclimbing = HillClimbing(
+ experiment=problem,
+ search_space=grid,
+ n_iter=100,
+)
+# running the hill climbing search:
+best_params = hillclimbing.run()
+```
-
-
-
+### experiment abstraction - example: scikit-learn CV experiment
+"experiment" abstraction = parametrized optimization problem
-As its name suggests Hyperactive started as a hyperparameter optimization package, but it has been generalized to solve expensive gradient-free optimization problems. It uses the [Gradient-Free-Optimizers](https://github.com/SimonBlanke/Gradient-Free-Optimizers) package as an optimization-backend and expands on it with additional features and tools.
+`hyperactive` provides a number of common experiments, e.g.,
+`scikit-learn` cross-validation experiments:
----
+```python
+from hyperactive.experiment.integrations import SklearnCvExperiment
+from sklearn.datasets import load_iris
+from sklearn.svm import SVC
+from sklearn.metrics import accuracy_score
+from sklearn.model_selection import KFold
+
+X, y = load_iris(return_X_y=True)
+
+# create experiment
+sklearn_exp = SklearnCvExperiment(
+ estimator=SVC(),
+ scoring=accuracy_score,
+ cv=KFold(n_splits=3, shuffle=True),
+ X=X,
+ y=y,
+)
+
+# experiments can be evaluated via "score
+params = {"C": 1.0, "kernel": "linear"}
+score, add_info = sklearn_exp.score(params)
+
+# they can be used in optimizers like above
+from hyperactive.opt import HillClimbing
+
+hillclimbing = HillClimbing(
+ experiment=problem,
+ search_space={
+ "C": np.logspace(0.01, 100, num=10),
+ "kernel": ["linear", "rbf"],
+ }
+ n_iter=100,
+)
-
+best_params = hillclimbing.run()
+```
+
+### full ML toolbox integration - example: scikit-learn
+
+Any `hyperactive` optimizer can be combined with the ML toolbox integrations!
+
+`OptCV` for tuning `scikit-learn` estimators with any `hyperactive` optimizer:
+
+```python
+# 1. defining the tuned estimator:
+from sklearn.svm import SVC
+from hyperactive.integrations.sklearn import OptCV
+from hyperactive.opt import HillClimbing
+
+param_grid = {"kernel": ["linear", "rbf"], "C": [1, 10]}
+tuned_svc = OptCV(SVC(), HillClimbing(param_grid))
+# 2. fitting the tuned estimator:
+from sklearn.datasets import load_iris
+from sklearn.model_selection import train_test_split
+X, y = load_iris(return_X_y=True)
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+
+tuned_svc.fit(X_train, y_train)
+
+y_pred = tuned_svc.predict(X_test)
+
+# 3. obtaining best parameters and best estimator
+best_params = tuned_svc.best_params_
+best_estimator = tuned_svc.best_estimator_
+```
+
+
## Overview
@@ -202,799 +287,6 @@ The following packages are designed to support Hyperactive and expand its use ca
| [Search-Data-Collector](https://github.com/SimonBlanke/search-data-collector) | Simple tool to save search-data during or after the optimization run into csv-files. |
| [Search-Data-Explorer](https://github.com/SimonBlanke/search-data-explorer) | Visualize search-data with plotly inside a streamlit dashboard.
-If you want news about Hyperactive and related projects you can follow me on [twitter](https://twitter.com/blanke_simon).
-
-
-
-
-## Notebooks and Tutorials
-
-- [Introduction to Hyperactive](https://nbviewer.org/github/SimonBlanke/hyperactive-tutorial/blob/main/notebooks/hyperactive_tutorial.ipynb)
-
-
-
-
-## Installation
-
-The most recent version of Hyperactive is available on PyPi:
-
-[](https://pypi.org/project/hyperactive)
-[](https://pypi.org/project/hyperactive/)
-[](https://pypi.org/project/hyperactive/)
-
-```console
-pip install hyperactive
-```
-
-
-
-
-
-## Example
-
-```python
-from sklearn.model_selection import cross_val_score
-from sklearn.ensemble import GradientBoostingRegressor
-from sklearn.datasets import load_diabetes
-from hyperactive import Hyperactive
-
-data = load_diabetes()
-X, y = data.data, data.target
-
-# define the model in a function
-def model(opt):
- # pass the suggested parameter to the machine learning model
- gbr = GradientBoostingRegressor(
- n_estimators=opt["n_estimators"], max_depth=opt["max_depth"]
- )
- scores = cross_val_score(gbr, X, y, cv=4)
-
- # return a single numerical value
- return scores.mean()
-
-# search space determines the ranges of parameters you want the optimizer to search through
-search_space = {
- "n_estimators": list(range(10, 150, 5)),
- "max_depth": list(range(2, 12)),
-}
-
-# start the optimization run
-hyper = Hyperactive()
-hyper.add_search(model, search_space, n_iter=50)
-hyper.run()
-
-```
-
-
-
-## Hyperactive API reference
-
-
-
-
-### Basic Usage
-
-
- Hyperactive(verbosity, distribution, n_processes)
-
-- verbosity = ["progress_bar", "print_results", "print_times"]
- - Possible parameter types: (list, False)
- - The verbosity list determines what part of the optimization information will be printed in the command line.
-
-- distribution = "multiprocessing"
- - Possible parameter types: ("multiprocessing", "joblib", "pathos")
- - Determine, which distribution service you want to use. Each library uses different packages to pickle objects:
- - multiprocessing uses pickle
- - joblib uses dill
- - pathos uses cloudpickle
-
-
-- n_processes = "auto",
- - Possible parameter types: (str, int)
- - The maximum number of processes that are allowed to run simultaneously. If n_processes is of int-type there will only run n_processes-number of jobs simultaneously instead of all at once. So if n_processes=10 and n_jobs_total=35, then the schedule would look like this 10 - 10 - 10 - 5. This saves computational resources if there is a large number of n_jobs. If "auto", then n_processes is the sum of all n_jobs (from .add_search(...)).
-
-
-
-
-
- .add_search(objective_function, search_space, n_iter, optimizer, n_jobs, initialize, pass_through, callbacks, catch, max_score, early_stopping, random_state, memory, memory_warm_start)
-
-
-- objective_function
- - Possible parameter types: (callable)
- - The objective function defines the optimization problem. The optimization algorithm will try to maximize the numerical value that is returned by the objective function by trying out different parameters from the search space.
-
-
-- search_space
- - Possible parameter types: (dict)
- - Defines the space were the optimization algorithm can search for the best parameters for the given objective function.
-
-
-- n_iter
- - Possible parameter types: (int)
- - The number of iterations that will be performed during the optimization run. The entire iteration consists of the optimization-step, which decides the next parameter that will be evaluated and the evaluation-step, which will run the objective function with the chosen parameter and return the score.
-
-
-- optimizer = "default"
- - Possible parameter types: ("default", initialized optimizer object)
- - Instance of optimization class that can be imported from Hyperactive. "default" corresponds to the random search optimizer. The imported optimization classes from hyperactive are different from gfo. They only accept optimizer-specific-parameters. The following classes can be imported and used:
-
- - HillClimbingOptimizer
- - StochasticHillClimbingOptimizer
- - RepulsingHillClimbingOptimizer
- - SimulatedAnnealingOptimizer
- - DownhillSimplexOptimizer
- - RandomSearchOptimizer
- - GridSearchOptimizer
- - RandomRestartHillClimbingOptimizer
- - RandomAnnealingOptimizer
- - PowellsMethod
- - PatternSearch
- - ParallelTemperingOptimizer
- - ParticleSwarmOptimizer
- - SpiralOptimization
- - GeneticAlgorithmOptimizer
- - EvolutionStrategyOptimizer
- - DifferentialEvolutionOptimizer
- - BayesianOptimizer
- - LipschitzOptimizer
- - DirectAlgorithm
- - TreeStructuredParzenEstimators
- - ForestOptimizer
-
- - Example:
- ```python
- ...
-
- opt_hco = HillClimbingOptimizer(epsilon=0.08)
- hyper = Hyperactive()
- hyper.add_search(..., optimizer=opt_hco)
- hyper.run()
-
- ...
- ```
-
-
-- n_jobs = 1
- - Possible parameter types: (int)
- - Number of jobs to run in parallel. Those jobs are optimization runs that work independent from another (no information sharing). If n_jobs == -1 the maximum available number of cpu cores is used.
-
-
-- initialize = {"grid": 4, "random": 2, "vertices": 4}
- - Possible parameter types: (dict)
- - The initialization dictionary automatically determines a number of parameters that will be evaluated in the first n iterations (n is the sum of the values in initialize). The initialize keywords are the following:
- - grid
- - Initializes positions in a grid like pattern. Positions that cannot be put into a grid are randomly positioned. For very high dimensional search spaces (>30) this pattern becomes random.
- - vertices
- - Initializes positions at the vertices of the search space. Positions that cannot be put into a new vertex are randomly positioned.
-
- - random
- - Number of random initialized positions
-
- - warm_start
- - List of parameter dictionaries that marks additional start points for the optimization run.
-
- Example:
- ```python
- ...
- search_space = {
- "x1": list(range(10, 150, 5)),
- "x2": list(range(2, 12)),
- }
-
- ws1 = {"x1": 10, "x2": 2}
- ws2 = {"x1": 15, "x2": 10}
-
- hyper = Hyperactive()
- hyper.add_search(
- model,
- search_space,
- n_iter=30,
- initialize={"grid": 4, "random": 10, "vertices": 4, "warm_start": [ws1, ws2]},
- )
- hyper.run()
- ```
-
-
-- pass_through = {}
- - Possible parameter types: (dict)
- - The pass_through accepts a dictionary that contains information that will be passed to the objective-function argument. This information will not change during the optimization run, unless the user does so by himself (within the objective-function).
-
- Example:
- ```python
- ...
- def objective_function(para):
- para.pass_through["stuff1"] # <--- this variable is 1
- para.pass_through["stuff2"] # <--- this variable is 2
-
- score = -para["x1"] * para["x1"]
- return score
-
- pass_through = {
- "stuff1": 1,
- "stuff2": 2,
- }
-
- hyper = Hyperactive()
- hyper.add_search(
- model,
- search_space,
- n_iter=30,
- pass_through=pass_through,
- )
- hyper.run()
- ```
-
-
-- callbacks = {}
- - Possible parameter types: (dict)
- - The callbacks enables you to pass functions to hyperactive that are called every iteration during the optimization run. The function has access to the same argument as the objective-function. You can decide if the functions are called before or after the objective-function is evaluated via the keys of the callbacks-dictionary. The values of the dictionary are lists of the callback-functions. The following example should show they way to use callbacks:
-
-
- Example:
- ```python
- ...
-
- def callback_1(access):
- # do some stuff
-
- def callback_2(access):
- # do some stuff
-
- def callback_3(access):
- # do some stuff
-
- hyper = Hyperactive()
- hyper.add_search(
- objective_function,
- search_space,
- n_iter=100,
- callbacks={
- "after": [callback_1, callback_2],
- "before": [callback_3]
- },
- )
- hyper.run()
- ```
-
-
-- catch = {}
- - Possible parameter types: (dict)
- - The catch parameter provides a way to handle exceptions that occur during the evaluation of the objective-function or the callbacks. It is a dictionary that accepts the exception class as a key and the score that is returned instead as the value. This way you can handle multiple types of exceptions and return different scores for each.
- In the case of an exception it often makes sense to return `np.nan` as a score. You can see an example of this in the following code-snippet:
-
- Example:
- ```python
- ...
-
- hyper = Hyperactive()
- hyper.add_search(
- objective_function,
- search_space,
- n_iter=100,
- catch={
- ValueError: np.nan,
- },
- )
- hyper.run()
- ```
-
-
-- max_score = None
- - Possible parameter types: (float, None)
- - Maximum score until the optimization stops. The score will be checked after each completed iteration.
-
-
-- early_stopping=None
- - (dict, None)
- - Stops the optimization run early if it did not achive any score-improvement within the last iterations. The early_stopping-parameter enables to set three parameters:
- - `n_iter_no_change`: Non-optional int-parameter. This marks the last n iterations to look for an improvement over the iterations that came before n. If the best score of the entire run is within those last n iterations the run will continue (until other stopping criteria are met), otherwise the run will stop.
- - `tol_abs`: Optional float-paramter. The score must have improved at least this absolute tolerance in the last n iterations over the best score in the iterations before n. This is an absolute value, so 0.1 means an imporvement of 0.8 -> 0.9 is acceptable but 0.81 -> 0.9 would stop the run.
- - `tol_rel`: Optional float-paramter. The score must have imporved at least this relative tolerance (in percentage) in the last n iterations over the best score in the iterations before n. This is a relative value, so 10 means an imporvement of 0.8 -> 0.88 is acceptable but 0.8 -> 0.87 would stop the run.
-
- - random_state = None
- - Possible parameter types: (int, None)
- - Random state for random processes in the random, numpy and scipy module.
-
-
-- memory = "share"
- - Possible parameter types: (bool, "share")
- - Whether or not to use the "memory"-feature. The memory is a dictionary, which gets filled with parameters and scores during the optimization run. If the optimizer encounters a parameter that is already in the dictionary it just extracts the score instead of reevaluating the objective function (which can take a long time). If memory is set to "share" and there are multiple jobs for the same objective function then the memory dictionary is automatically shared between the different processes.
-
-- memory_warm_start = None
- - Possible parameter types: (pandas dataframe, None)
- - Pandas dataframe that contains score and parameter information that will be automatically loaded into the memory-dictionary.
-
- example:
-
-
-
-
- | score |
- x1 |
- x2 |
- x... |
-
-
-
-
- | 0.756 |
- 0.1 |
- 0.2 |
- ... |
-
-
- | 0.823 |
- 0.3 |
- 0.1 |
- ... |
-
-
- | ... |
- ... |
- ... |
- ... |
-
-
- | ... |
- ... |
- ... |
- ... |
-
-
-
-
-
-
-
-
-
-
- .run(max_time)
-
-- max_time = None
- - Possible parameter types: (float, None)
- - Maximum number of seconds until the optimization stops. The time will be checked after each completed iteration.
-
-
-
-
-
-
-
-### Special Parameters
-
-
- Objective Function
-
-Each iteration consists of two steps:
- - The optimization step: decides what position in the search space (parameter set) to evaluate next
- - The evaluation step: calls the objective function, which returns the score for the given position in the search space
-
-The objective function has one argument that is often called "para", "params", "opt" or "access".
-This argument is your access to the parameter set that the optimizer has selected in the
-corresponding iteration.
-
-```python
-def objective_function(opt):
- # get x1 and x2 from the argument "opt"
- x1 = opt["x1"]
- x2 = opt["x2"]
-
- # calculate the score with the parameter set
- score = -(x1 * x1 + x2 * x2)
-
- # return the score
- return score
-```
-
-The objective function always needs a score, which shows how "good" or "bad" the current parameter set is. But you can also return some additional information with a dictionary:
-
-```python
-def objective_function(opt):
- x1 = opt["x1"]
- x2 = opt["x2"]
-
- score = -(x1 * x1 + x2 * x2)
-
- other_info = {
- "x1 squared" : x1**2,
- "x2 squared" : x2**2,
- }
-
- return score, other_info
-```
-
-When you take a look at the results (a pandas dataframe with all iteration information) after the run has ended you will see the additional information in it. The reason we need a dictionary for this is because Hyperactive needs to know the names of the additonal parameters. The score does not need that, because it is always called "score" in the results. You can run [this example script](https://github.com/SimonBlanke/Hyperactive/blob/main/examples/optimization_applications/multiple_scores.py) if you want to give it a try.
-
-
-
-
-
- Search Space Dictionary
-
-The search space defines what values the optimizer can select during the search. These selected values will be inside the objective function argument and can be accessed like in a dictionary. The values in each search space dimension should always be in a list. If you use np.arange you should put it in a list afterwards:
-
-```python
-search_space = {
- "x1": list(np.arange(-100, 101, 1)),
- "x2": list(np.arange(-100, 101, 1)),
-}
-```
-
-A special feature of Hyperactive is shown in the next example. You can put not just numeric values into the search space dimensions, but also strings and functions. This enables a very high flexibility in how you can create your studies.
-
-```python
-def func1():
- # do stuff
- return stuff
-
-
-def func2():
- # do stuff
- return stuff
-
-
-search_space = {
- "x": list(np.arange(-100, 101, 1)),
- "str": ["a string", "another string"],
- "function" : [func1, func2],
-}
-```
-
-If you want to put other types of variables (like numpy arrays, pandas dataframes, lists, ...) into the search space you can do that via functions:
-
-```python
-def array1():
- return np.array([1, 2, 3])
-
-
-def array2():
- return np.array([3, 2, 1])
-
-
-search_space = {
- "x": list(np.arange(-100, 101, 1)),
- "str": ["a string", "another string"],
- "numpy_array" : [array1, array2],
-}
-```
-
-The functions contain the numpy arrays and returns them. This way you can use them inside the objective function.
-
-
-
-
-
-
- Optimizer Classes
-
-Each of the following optimizer classes can be initialized and passed to the "add_search"-method via the "optimizer"-argument. During this initialization the optimizer class accepts **only optimizer-specific-paramters** (no random_state, initialize, ... ):
-
- ```python
- optimizer = HillClimbingOptimizer(epsilon=0.1, distribution="laplace", n_neighbours=4)
- ```
-
- for the default parameters you can just write:
-
- ```python
- optimizer = HillClimbingOptimizer()
- ```
-
- and pass it to Hyperactive:
-
- ```python
- hyper = Hyperactive()
- hyper.add_search(model, search_space, optimizer=optimizer, n_iter=100)
- hyper.run()
- ```
-
- So the optimizer-classes are **different** from Gradient-Free-Optimizers. A more detailed explanation of the optimization-algorithms and the optimizer-specific-paramters can be found in the [Optimization Tutorial](https://github.com/SimonBlanke/optimization-tutorial).
-
-- HillClimbingOptimizer
-- RepulsingHillClimbingOptimizer
-- SimulatedAnnealingOptimizer
-- DownhillSimplexOptimizer
-- RandomSearchOptimizer
-- GridSearchOptimizer
-- RandomRestartHillClimbingOptimizer
-- RandomAnnealingOptimizer
-- PowellsMethod
-- PatternSearch
-- ParallelTemperingOptimizer
-- ParticleSwarmOptimizer
-- GeneticAlgorithmOptimizer
-- EvolutionStrategyOptimizer
-- DifferentialEvolutionOptimizer
-- BayesianOptimizer
-- TreeStructuredParzenEstimators
-- ForestOptimizer
-
-
-
-
-
-
-
-### Result Attributes
-
-
-
- .best_para(objective_function)
-
-- objective_function
- - (callable)
-- returnes: dictionary
-- Parameter dictionary of the best score of the given objective_function found in the previous optimization run.
-
- example:
- ```python
- {
- 'x1': 0.2,
- 'x2': 0.3,
- }
- ```
-
-
-
-
-
- .best_score(objective_function)
-
-- objective_function
- - (callable)
-- returns: int or float
-- Numerical value of the best score of the given objective_function found in the previous optimization run.
-
-
-
-
-
- .search_data(objective_function, times=False)
-
-- objective_function
- - (callable)
-- returns: Pandas dataframe
-- The dataframe contains score and parameter information of the given objective_function found in the optimization run. If the parameter `times` is set to True the evaluation- and iteration- times are added to the dataframe.
-
- example:
-
-
-
-
- | score |
- x1 |
- x2 |
- x... |
-
-
-
-
- | 0.756 |
- 0.1 |
- 0.2 |
- ... |
-
-
- | 0.823 |
- 0.3 |
- 0.1 |
- ... |
-
-
- | ... |
- ... |
- ... |
- ... |
-
-
- | ... |
- ... |
- ... |
- ... |
-
-
-
-
-
-
-
-
-
-
-
-## Roadmap
-
-
-
-v2.0.0 :heavy_check_mark:
-
- - [x] Change API
-
-
-
-
-v2.1.0 :heavy_check_mark:
-
- - [x] Save memory of evaluations for later runs (long term memory)
- - [x] Warm start sequence based optimizers with long term memory
- - [x] Gaussian process regressors from various packages (gpy, sklearn, GPflow, ...) via wrapper
-
-
-
-
-v2.2.0 :heavy_check_mark:
-
- - [x] Add basic dataset meta-features to long term memory
- - [x] Add helper-functions for memory
- - [x] connect two different model/dataset hashes
- - [x] split two different model/dataset hashes
- - [x] delete memory of model/dataset
- - [x] return best known model for dataset
- - [x] return search space for best model
- - [x] return best parameter for best model
-
-
-
-
-v2.3.0 :heavy_check_mark:
-
- - [x] Tree-structured Parzen Estimator
- - [x] Decision Tree Optimizer
- - [x] add "max_sample_size" and "skip_retrain" parameter for sbom to decrease optimization time
-
-
-
-
-v3.0.0 :heavy_check_mark:
-
- - [x] New API
- - [x] expand usage of objective-function
- - [x] No passing of training data into Hyperactive
- - [x] Removing "long term memory"-support (better to do in separate package)
- - [x] More intuitive selection of optimization strategies and parameters
- - [x] Separate optimization algorithms into other package
- - [x] expand api so that optimizer parameter can be changed at runtime
- - [x] add extensive testing procedure (similar to Gradient-Free-Optimizers)
-
-
-
-
-
-v3.1.0 :heavy_check_mark:
-
- - [x] Decouple number of runs from active processes (Thanks to [PartiallyTyped](https://github.com/PartiallyTyped))
-
-
-
-
-
-v3.2.0 :heavy_check_mark:
-
- - [x] Dashboard for visualization of search-data at runtime via streamlit (Progress-Board)
-
-
-
-
-
-v3.3.0 :heavy_check_mark:
-
- - [x] Early stopping
- - [x] Shared memory dictionary between processes with the same objective function
-
-
-
-
-
-v4.0.0 :heavy_check_mark:
-
- - [x] small adjustments to API
- - [x] move optimization strategies into sub-module "optimizers"
- - [x] preparation for future add ons (long-term-memory, meta-learn, ...) from separate repositories
- - [x] separate progress board into separate repository
-
-
-
-
-
-v4.1.0 :heavy_check_mark:
-
- - [x] add python 3.9 to testing
- - [x] add pass_through-parameter
- - [x] add v1 GFO optimization algorithms
-
-
-
-
-
-v4.2.0 :heavy_check_mark:
-
- - [x] add callbacks-parameter
- - [x] add catch-parameter
- - [x] add option to add eval- and iter- times to search-data
-
-
-
-
-
-v4.3.0 :heavy_check_mark:
-
- - [x] add new features from GFO
- - [x] add Spiral Optimization
- - [x] add Lipschitz Optimizer
- - [x] add DIRECT Optimizer
- - [x] print the random seed for reproducibility
-
-
-
-
-
-v4.4.0 :heavy_check_mark:
-
- - [x] add Optimization-Strategies
- - [x] redesign progress-bar
-
-
-
-
-
-v4.5.0 :heavy_check_mark:
-
- - [x] add early stopping feature to custom optimization strategies
- - [x] display additional outputs from objective-function in results in command-line
- - [x] add type hints to hyperactive-api
-
-
-
-
-
-v4.6.0 :heavy_check_mark:
-
- - [x] add support for constrained optimization
-
-
-
-
-
-v4.7.0 :heavy_check_mark:
-
- - [x] add Genetic algorithm optimizer
- - [x] add Differential evolution optimizer
-
-
-
-
-
-v4.8.0 :heavy_check_mark:
-
- - [x] add support for numpy v2
- - [x] add support for pandas v2
- - [x] add support for python 3.12
- - [x] transfer setup.py to pyproject.toml
- - [x] change project structure to src-layout
-
-
-
-
-
-v4.9.0
-
- - [ ] add sklearn integration
-
-
-
-
-
-
-
-
-Future releases
-
- - [ ] new optimization algorithms from [Gradient-Free-Optimizers](https://github.com/SimonBlanke/Gradient-Free-Optimizers) will always be added to Hyperactive
- - [ ] add "prune_search_space"-method to custom optimization strategy class
-
-
-
diff --git a/examples/optuna/gp_sampler_example.py b/examples/optuna/gp_sampler_example.py
index 930c0b23..16fc8257 100644
--- a/examples/optuna/gp_sampler_example.py
+++ b/examples/optuna/gp_sampler_example.py
@@ -14,10 +14,8 @@
- Can handle constraints and noisy observations
"""
-import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.svm import SVC
-from sklearn.model_selection import cross_val_score
from hyperactive.experiment.integrations import SklearnCvExperiment
from hyperactive.opt.optuna import GPOptimizer
diff --git a/examples/optuna/grid_sampler_example.py b/examples/optuna/grid_sampler_example.py
index 846f8f87..3acb83ca 100644
--- a/examples/optuna/grid_sampler_example.py
+++ b/examples/optuna/grid_sampler_example.py
@@ -14,10 +14,8 @@
- Interpretable and deterministic results
"""
-import numpy as np
from sklearn.datasets import load_iris
from sklearn.neighbors import KNeighborsClassifier
-from sklearn.model_selection import cross_val_score
from hyperactive.experiment.integrations import SklearnCvExperiment
from hyperactive.opt.optuna import GridOptimizer
diff --git a/examples/optuna/nsga_ii_sampler_example.py b/examples/optuna/nsga_ii_sampler_example.py
index 88f844ae..83b66343 100644
--- a/examples/optuna/nsga_ii_sampler_example.py
+++ b/examples/optuna/nsga_ii_sampler_example.py
@@ -33,7 +33,7 @@ def __init__(self, X, y):
self.X = X
self.y = y
- def __call__(self, **params):
+ def __call__(self, params):
# Create model with parameters
model = RandomForestClassifier(random_state=42, **params)
diff --git a/examples/optuna/nsga_iii_sampler_example.py b/examples/optuna/nsga_iii_sampler_example.py
index c0c67772..4966cff0 100644
--- a/examples/optuna/nsga_iii_sampler_example.py
+++ b/examples/optuna/nsga_iii_sampler_example.py
@@ -34,7 +34,7 @@ def __init__(self, X, y):
self.X = X
self.y = y
- def __call__(self, **params):
+ def __call__(self, params):
# Create model with parameters
model = DecisionTreeClassifier(random_state=42, **params)
diff --git a/examples/optuna/qmc_sampler_example.py b/examples/optuna/qmc_sampler_example.py
index 70b6b846..33cc5bc3 100644
--- a/examples/optuna/qmc_sampler_example.py
+++ b/examples/optuna/qmc_sampler_example.py
@@ -19,10 +19,8 @@
- Baseline optimization comparisons
"""
-import numpy as np
from sklearn.datasets import load_wine
from sklearn.linear_model import LogisticRegression
-from sklearn.model_selection import cross_val_score
from hyperactive.experiment.integrations import SklearnCvExperiment
from hyperactive.opt.optuna import QMCOptimizer
diff --git a/examples/optuna/random_sampler_example.py b/examples/optuna/random_sampler_example.py
index 14733188..d97bfd1e 100644
--- a/examples/optuna/random_sampler_example.py
+++ b/examples/optuna/random_sampler_example.py
@@ -15,10 +15,8 @@
- Good when objective function is noisy
"""
-import numpy as np
from sklearn.datasets import load_digits
from sklearn.svm import SVC
-from sklearn.model_selection import cross_val_score
from hyperactive.experiment.integrations import SklearnCvExperiment
from hyperactive.opt.optuna import RandomOptimizer
diff --git a/examples/optuna/tpe_sampler_example.py b/examples/optuna/tpe_sampler_example.py
index ba4736dd..98e5a348 100644
--- a/examples/optuna/tpe_sampler_example.py
+++ b/examples/optuna/tpe_sampler_example.py
@@ -13,10 +13,8 @@
- Default choice for most hyperparameter optimization tasks
"""
-import numpy as np
from sklearn.datasets import load_wine
from sklearn.ensemble import RandomForestClassifier
-from sklearn.model_selection import cross_val_score
from hyperactive.experiment.integrations import SklearnCvExperiment
from hyperactive.opt.optuna import TPEOptimizer
diff --git a/src/hyperactive/base/_experiment.py b/src/hyperactive/base/_experiment.py
index db5d2aeb..5add7401 100644
--- a/src/hyperactive/base/_experiment.py
+++ b/src/hyperactive/base/_experiment.py
@@ -21,9 +21,9 @@ class BaseExperiment(BaseObject):
def __init__(self):
super().__init__()
- def __call__(self, **kwargs):
- """Score parameters, with kwargs call. Same as score call."""
- score, _ = self.score(kwargs)
+ def __call__(self, params):
+ """Score parameters. Same as score call, returns only only a first element."""
+ score, _ = self.score(params)
return score
@property
diff --git a/src/hyperactive/experiment/bench/_ackley.py b/src/hyperactive/experiment/bench/_ackley.py
index ece7d3af..1fe03f44 100644
--- a/src/hyperactive/experiment/bench/_ackley.py
+++ b/src/hyperactive/experiment/bench/_ackley.py
@@ -39,10 +39,10 @@ class Ackley(BaseExperiment):
>>> from hyperactive.experiment.bench import Ackley
>>> ackley = Ackley(a=20)
>>> params = {"x0": 1, "x1": 2}
- >>> score, add_info = ackley.score(params)
+ >>> score, metadata = ackley.score(params)
Quick call without metadata return or dictionary:
- >>> score = ackley(x0=1, x1=2)
+ >>> score = ackley({"x0": 1, "x1": 2})
""" # noqa: E501
_tags = {
diff --git a/src/hyperactive/experiment/bench/_parabola.py b/src/hyperactive/experiment/bench/_parabola.py
index aa19ba2f..e14655fe 100644
--- a/src/hyperactive/experiment/bench/_parabola.py
+++ b/src/hyperactive/experiment/bench/_parabola.py
@@ -33,10 +33,10 @@ class Parabola(BaseExperiment):
>>> from hyperactive.experiment.bench import Parabola
>>> parabola = Parabola(a=1.0, b=0.0, c=0.0)
>>> params = {"x": 1, "y": 2}
- >>> score, add_info = parabola.score(params)
+ >>> score, metadata = parabola.score(params)
Quick call without metadata return or dictionary:
- >>> score = parabola(x=1, y=2)
+ >>> score = parabola({"x": 1, "y": 2})
"""
_tags = {
diff --git a/src/hyperactive/experiment/bench/_sphere.py b/src/hyperactive/experiment/bench/_sphere.py
index 159a6f5d..fcc7ef74 100644
--- a/src/hyperactive/experiment/bench/_sphere.py
+++ b/src/hyperactive/experiment/bench/_sphere.py
@@ -36,14 +36,14 @@ class Sphere(BaseExperiment):
>>> from hyperactive.experiment.bench import Sphere
>>> sphere = Sphere(const=0, n_dim=3)
>>> params = {"x0": 1, "x1": 2, "x2": 3}
- >>> score, add_info = sphere.score(params)
+ >>> score, metadata = sphere.score(params)
Quick call without metadata return or dictionary:
- >>> score = sphere(x0=1, x1=2, x2=3)
+ >>> score = sphere({"x0": 1, "x1": 2, "x2": 3})
Different number of dimensions changes the parameter names:
>>> sphere4D = Sphere(const=0, n_dim=4)
- >>> score4D = sphere4D(x0=1, x1=2, x2=3, x3=4)
+ >>> score4D = sphere4D({"x0": 1, "x1": 2, "x2": 3, "x3": 4})
"""
_tags = {
diff --git a/src/hyperactive/experiment/integrations/__init__.py b/src/hyperactive/experiment/integrations/__init__.py
index 1b600df2..4ed0ce52 100644
--- a/src/hyperactive/experiment/integrations/__init__.py
+++ b/src/hyperactive/experiment/integrations/__init__.py
@@ -2,8 +2,15 @@
# copyright: hyperactive developers, MIT License (see LICENSE file)
from hyperactive.experiment.integrations.sklearn_cv import SklearnCvExperiment
+from hyperactive.experiment.integrations.sktime_classification import (
+ SktimeClassificationExperiment,
+)
from hyperactive.experiment.integrations.sktime_forecasting import (
SktimeForecastingExperiment,
)
-__all__ = ["SklearnCvExperiment", "SktimeForecastingExperiment"]
+__all__ = [
+ "SklearnCvExperiment",
+ "SktimeClassificationExperiment",
+ "SktimeForecastingExperiment",
+]
diff --git a/src/hyperactive/experiment/integrations/_skl_metrics.py b/src/hyperactive/experiment/integrations/_skl_metrics.py
new file mode 100644
index 00000000..1bd130b1
--- /dev/null
+++ b/src/hyperactive/experiment/integrations/_skl_metrics.py
@@ -0,0 +1,128 @@
+"""Integration utilities for sklearn metrics with Hyperactive."""
+
+__all__ = ["_coerce_to_scorer", "_guess_sign_of_sklmetric"]
+
+
+def _coerce_to_scorer(scoring, estimator):
+ """Coerce scoring argument into a sklearn scorer.
+
+ Parameters
+ ----------
+ scoring : str, callable, or None
+ The scoring strategy to use.
+ estimator : estimator object or str
+ The estimator to use for default scoring if scoring is None.
+
+ If str, indicates estimator type, should be one of {"classifier", "regressor"}.
+
+ Returns
+ -------
+ scorer : callable
+ A sklearn scorer callable.
+ Follows the unified sklearn scorer interface
+ """
+ from sklearn.metrics import check_scoring
+
+ # check if scoring is a scorer by checking for "estimator" in signature
+ if scoring is None:
+ if isinstance(estimator, str):
+ if estimator == "classifier":
+ from sklearn.metrics import accuracy_score
+
+ scoring = accuracy_score
+ elif estimator == "regressor":
+ from sklearn.metrics import r2_score
+
+ scoring = r2_score
+ else:
+ return check_scoring(estimator)
+
+ # check using inspect.signature for "estimator" in signature
+ if callable(scoring):
+ from inspect import signature
+
+ if "estimator" in signature(scoring).parameters:
+ return scoring
+ else:
+ from sklearn.metrics import make_scorer
+
+ return make_scorer(scoring)
+ else:
+ # scoring is a string (scorer name)
+ return check_scoring(estimator, scoring=scoring)
+
+
+def _guess_sign_of_sklmetric(scorer):
+ """Guess the sign of a sklearn metric scorer.
+
+ Parameters
+ ----------
+ scorer : callable
+ The sklearn metric scorer to guess the sign for.
+
+ Returns
+ -------
+ int
+ 1 if higher scores are better, -1 if lower scores are better.
+ """
+ HIGHER_IS_BETTER = {
+ # Classification
+ "accuracy_score": True,
+ "auc": True,
+ "average_precision_score": True,
+ "balanced_accuracy_score": True,
+ "brier_score_loss": False,
+ "class_likelihood_ratios": False,
+ "cohen_kappa_score": True,
+ "d2_log_loss_score": True,
+ "dcg_score": True,
+ "f1_score": True,
+ "fbeta_score": True,
+ "hamming_loss": False,
+ "hinge_loss": False,
+ "jaccard_score": True,
+ "log_loss": False,
+ "matthews_corrcoef": True,
+ "ndcg_score": True,
+ "precision_score": True,
+ "recall_score": True,
+ "roc_auc_score": True,
+ "top_k_accuracy_score": True,
+ "zero_one_loss": False,
+ # Regression
+ "d2_absolute_error_score": True,
+ "d2_pinball_score": True,
+ "d2_tweedie_score": True,
+ "explained_variance_score": True,
+ "max_error": False,
+ "mean_absolute_error": False,
+ "mean_absolute_percentage_error": False,
+ "mean_gamma_deviance": False,
+ "mean_pinball_loss": False,
+ "mean_poisson_deviance": False,
+ "mean_squared_error": False,
+ "mean_squared_log_error": False,
+ "mean_tweedie_deviance": False,
+ "median_absolute_error": False,
+ "r2_score": True,
+ "root_mean_squared_error": False,
+ "root_mean_squared_log_error": False,
+ }
+
+ scorer_name = getattr(scorer, "__name__", None)
+
+ if hasattr(scorer, "greater_is_better"):
+ return 1 if scorer.greater_is_better else -1
+ elif scorer_name in HIGHER_IS_BETTER:
+ return 1 if HIGHER_IS_BETTER[scorer_name] else -1
+ elif scorer_name.endswith("_score"):
+ # If the scorer name ends with "_score", we assume higher is better
+ return 1
+ elif scorer_name.endswith("_loss") or scorer_name.endswith("_deviance"):
+ # If the scorer name ends with "_loss", we assume lower is better
+ return -1
+ elif scorer_name.endswith("_error"):
+ return -1
+ else:
+ # If we cannot determine the sign, we assume lower is better
+ return -1
diff --git a/src/hyperactive/experiment/integrations/sklearn_cv.py b/src/hyperactive/experiment/integrations/sklearn_cv.py
index 3a649801..051edfde 100644
--- a/src/hyperactive/experiment/integrations/sklearn_cv.py
+++ b/src/hyperactive/experiment/integrations/sklearn_cv.py
@@ -3,11 +3,14 @@
# copyright: hyperactive developers, MIT License (see LICENSE file)
from sklearn import clone
-from sklearn.metrics import check_scoring
from sklearn.model_selection import cross_validate
from sklearn.utils.validation import _num_samples
from hyperactive.base import BaseExperiment
+from hyperactive.experiment.integrations._skl_metrics import (
+ _coerce_to_scorer,
+ _guess_sign_of_sklmetric,
+)
class SklearnCvExperiment(BaseExperiment):
@@ -62,7 +65,7 @@ class SklearnCvExperiment(BaseExperiment):
... y=y,
... )
>>> params = {"C": 1.0, "kernel": "linear"}
- >>> score, add_info = sklearn_exp.score(params)
+ >>> score, metadata = sklearn_exp.score(params)
For default choices of ``scoring`` and ``cv``:
>>> sklearn_exp = SklearnCvExperiment(
@@ -71,10 +74,10 @@ class SklearnCvExperiment(BaseExperiment):
... y=y,
... )
>>> params = {"C": 1.0, "kernel": "linear"}
- >>> score, add_info = sklearn_exp.score(params)
+ >>> score, metadata = sklearn_exp.score(params)
Quick call without metadata return or dictionary:
- >>> score = sklearn_exp(C=1.0, kernel="linear")
+ >>> score = sklearn_exp({"C": 1.0, "kernel": "linear"})
"""
def __init__(self, estimator, X, y, scoring=None, cv=None):
@@ -97,22 +100,7 @@ def __init__(self, estimator, X, y, scoring=None, cv=None):
else:
self._cv = cv
- # check if scoring is a scorer by checking for "estimator" in signature
- if scoring is None:
- self._scoring = check_scoring(self.estimator)
- # check using inspect.signature for "estimator" in signature
- elif callable(scoring):
- from inspect import signature
-
- if "estimator" in signature(scoring).parameters:
- self._scoring = scoring
- else:
- from sklearn.metrics import make_scorer
-
- self._scoring = make_scorer(scoring)
- else:
- # scoring is a string (scorer name)
- self._scoring = check_scoring(self.estimator, scoring=scoring)
+ self._scoring = _coerce_to_scorer(scoring, self.estimator)
self.scorer_ = self._scoring
# Set the sign of the scoring function
@@ -158,13 +146,13 @@ def _evaluate(self, params):
cv=self._cv,
)
- add_info_d = {
+ metadata = {
"score_time": cv_results["score_time"],
"fit_time": cv_results["fit_time"],
"n_test_samples": _num_samples(self.X),
}
- return cv_results["test_score"].mean(), add_info_d
+ return cv_results["test_score"].mean(), metadata
@classmethod
def get_test_params(cls, parameter_set="default"):
@@ -281,79 +269,3 @@ def _get_score_params(self):
score_params_defaults,
]
return params
-
-
-def _guess_sign_of_sklmetric(scorer):
- """Guess the sign of a sklearn metric scorer.
-
- Parameters
- ----------
- scorer : callable
- The sklearn metric scorer to guess the sign for.
-
- Returns
- -------
- int
- 1 if higher scores are better, -1 if lower scores are better.
- """
- HIGHER_IS_BETTER = {
- # Classification
- "accuracy_score": True,
- "auc": True,
- "average_precision_score": True,
- "balanced_accuracy_score": True,
- "brier_score_loss": False,
- "class_likelihood_ratios": False,
- "cohen_kappa_score": True,
- "d2_log_loss_score": True,
- "dcg_score": True,
- "f1_score": True,
- "fbeta_score": True,
- "hamming_loss": False,
- "hinge_loss": False,
- "jaccard_score": True,
- "log_loss": False,
- "matthews_corrcoef": True,
- "ndcg_score": True,
- "precision_score": True,
- "recall_score": True,
- "roc_auc_score": True,
- "top_k_accuracy_score": True,
- "zero_one_loss": False,
- # Regression
- "d2_absolute_error_score": True,
- "d2_pinball_score": True,
- "d2_tweedie_score": True,
- "explained_variance_score": True,
- "max_error": False,
- "mean_absolute_error": False,
- "mean_absolute_percentage_error": False,
- "mean_gamma_deviance": False,
- "mean_pinball_loss": False,
- "mean_poisson_deviance": False,
- "mean_squared_error": False,
- "mean_squared_log_error": False,
- "mean_tweedie_deviance": False,
- "median_absolute_error": False,
- "r2_score": True,
- "root_mean_squared_error": False,
- "root_mean_squared_log_error": False,
- }
-
- scorer_name = getattr(scorer, "__name__", None)
-
- if hasattr(scorer, "greater_is_better"):
- return 1 if scorer.greater_is_better else -1
- elif scorer_name in HIGHER_IS_BETTER:
- return 1 if HIGHER_IS_BETTER[scorer_name] else -1
- elif scorer_name.endswith("_score"):
- # If the scorer name ends with "_score", we assume higher is better
- return 1
- elif scorer_name.endswith("_loss") or scorer_name.endswith("_deviance"):
- # If the scorer name ends with "_loss", we assume lower is better
- return -1
- elif scorer_name.endswith("_error"):
- return -1
- else:
- # If we cannot determine the sign, we assume lower is better
- return -1
diff --git a/src/hyperactive/experiment/integrations/sktime_classification.py b/src/hyperactive/experiment/integrations/sktime_classification.py
new file mode 100644
index 00000000..ab4622b8
--- /dev/null
+++ b/src/hyperactive/experiment/integrations/sktime_classification.py
@@ -0,0 +1,314 @@
+"""Experiment adapter for sktime backtesting experiments."""
+# copyright: hyperactive developers, MIT License (see LICENSE file)
+
+import numpy as np
+
+from hyperactive.base import BaseExperiment
+from hyperactive.experiment.integrations._skl_metrics import (
+ _coerce_to_scorer,
+ _guess_sign_of_sklmetric,
+)
+
+
+class SktimeClassificationExperiment(BaseExperiment):
+ """Experiment adapter for time series classification experiments.
+
+ This class is used to perform cross-validation experiments using a given
+ sktime classifier. It allows for hyperparameter tuning and evaluation of
+ the model's performance.
+
+ The score returned is the summary backtesting score,
+ of applying ``sktime`` ``evaluate`` to ``estimator`` with the parameters given in
+ ``score`` ``params``.
+
+ The backtesting performed is specified by the ``cv`` parameter,
+ and the scoring metric is specified by the ``scoring`` parameter.
+ The ``X`` and ``y`` parameters are the input data and target values,
+ which are used in fit/predict cross-validation.
+
+ Parameters
+ ----------
+ estimator : sktime BaseClassifier descendant (concrete classifier)
+ sktime classifier to benchmark
+
+ X : sktime-compatible panel data (Panel scitype)
+ Panel data container. Supported formats include:
+
+ - ``pd.DataFrame`` with MultiIndex [instance, time] and variable columns
+ - 3D ``np.array`` with shape ``[n_instances, n_dimensions, series_length]``
+ - Other formats listed in ``datatypes.SCITYPE_REGISTER``
+
+ y : sktime-compatible tabular data (Table scitype)
+ Target variable, typically a 1D ``np.ndarray`` or ``pd.Series``
+ of shape ``[n_instances]``.
+
+ cv : int, sklearn cross-validation generator or an iterable, default=3-fold CV
+ Determines the cross-validation splitting strategy.
+ Possible inputs for cv are:
+
+ - None = default = ``KFold(n_splits=3, shuffle=True)``
+ - integer, number of folds folds in a ``KFold`` splitter, ``shuffle=True``
+ - An iterable yielding (train, test) splits as arrays of indices.
+
+ For integer/None inputs, if the estimator is a classifier and ``y`` is
+ either binary or multiclass, :class:`StratifiedKFold` is used. In all
+ other cases, :class:`KFold` is used. These splitters are instantiated
+ with ``shuffle=False`` so the splits will be the same across calls.
+
+ scoring : str, callable, default=None
+ Strategy to evaluate the performance of the cross-validated model on
+ the test set. Can be:
+
+ - a single string resolvable to an sklearn scorer
+ - a callable that returns a single value;
+ - ``None`` = default = ``accuracy_score``
+
+ error_score : "raise" or numeric, default=np.nan
+ Value to assign to the score if an exception occurs in estimator fitting. If set
+ to "raise", the exception is raised. If a numeric value is given,
+ FitFailedWarning is raised.
+
+ backend : string, by default "None".
+ Parallelization backend to use for runs.
+ Runs parallel evaluate if specified and ``strategy="refit"``.
+
+ - "None": executes loop sequentially, simple list comprehension
+ - "loky", "multiprocessing" and "threading": uses ``joblib.Parallel`` loops
+ - "joblib": custom and 3rd party ``joblib`` backends, e.g., ``spark``
+ - "dask": uses ``dask``, requires ``dask`` package in environment
+ - "dask_lazy": same as "dask",
+ but changes the return to (lazy) ``dask.dataframe.DataFrame``.
+ - "ray": uses ``ray``, requires ``ray`` package in environment
+
+ Recommendation: Use "dask" or "loky" for parallel evaluate.
+ "threading" is unlikely to see speed ups due to the GIL and the serialization
+ backend (``cloudpickle``) for "dask" and "loky" is generally more robust
+ than the standard ``pickle`` library used in "multiprocessing".
+
+ backend_params : dict, optional
+ additional parameters passed to the backend as config.
+ Directly passed to ``utils.parallel.parallelize``.
+ Valid keys depend on the value of ``backend``:
+
+ - "None": no additional parameters, ``backend_params`` is ignored
+ - "loky", "multiprocessing" and "threading": default ``joblib`` backends
+ any valid keys for ``joblib.Parallel`` can be passed here, e.g., ``n_jobs``,
+ with the exception of ``backend`` which is directly controlled by ``backend``.
+ If ``n_jobs`` is not passed, it will default to ``-1``, other parameters
+ will default to ``joblib`` defaults.
+ - "joblib": custom and 3rd party ``joblib`` backends, e.g., ``spark``.
+ any valid keys for ``joblib.Parallel`` can be passed here, e.g., ``n_jobs``,
+ ``backend`` must be passed as a key of ``backend_params`` in this case.
+ If ``n_jobs`` is not passed, it will default to ``-1``, other parameters
+ will default to ``joblib`` defaults.
+ - "dask": any valid keys for ``dask.compute`` can be passed,
+ e.g., ``scheduler``
+
+ - "ray": The following keys can be passed:
+
+ - "ray_remote_args": dictionary of valid keys for ``ray.init``
+ - "shutdown_ray": bool, default=True; False prevents ``ray`` from shutting
+ down after parallelization.
+ - "logger_name": str, default="ray"; name of the logger to use.
+ - "mute_warnings": bool, default=False; if True, suppresses warnings
+
+ Example
+ -------
+ >>> from hyperactive.experiment.integrations import SktimeClassificationExperiment
+ >>> from sklearn.model_selection import KFold
+ >>> from sklearn.metrics import accuracy_score
+ >>> from sktime.datasets import load_unit_test
+ >>> from sktime.classification.dummy import DummyClassifier
+ >>>
+ >>> X, y = load_unit_test()
+ >>>
+ >>> sktime_exp = SktimeClassificationExperiment(
+ ... estimator=DummyClassifier(),
+ ... scoring=accuracy_score,
+ ... cv=KFold(n_splits=2),
+ ... X=X,
+ ... y=y,
+ ... )
+ >>> params = {"strategy": "most_frequent"}
+ >>> score, add_info = sktime_exp.score(params)
+
+ For default choices of ``scoring`` and ``cv``:
+ >>> sktime_exp = SktimeClassificationExperiment(
+ ... estimator=DummyClassifier(),
+ ... X=X,
+ ... y=y,
+ ... )
+ >>> params = {"strategy": "most_frequent"}
+ >>> score, add_info = sktime_exp.score(params)
+
+ Quick call without metadata return or dictionary:
+ >>> score = sktime_exp({"strategy": "most_frequent"})
+ """
+
+ _tags = {
+ "authors": "fkiraly",
+ "maintainers": "fkiraly",
+ "python_dependencies": "sktime", # python dependencies
+ }
+
+ def __init__(
+ self,
+ estimator,
+ X,
+ y,
+ cv=None,
+ scoring=None,
+ error_score=np.nan,
+ backend=None,
+ backend_params=None,
+ ):
+ self.estimator = estimator
+ self.X = X
+ self.y = y
+ self.scoring = scoring
+ self.cv = cv
+ self.error_score = error_score
+ self.backend = backend
+ self.backend_params = backend_params
+
+ super().__init__()
+
+ self._scoring = _coerce_to_scorer(scoring, "classifier")
+
+ # Set the sign of the scoring function
+ if hasattr(self._scoring, "_score"):
+ score_func = self._scoring._score_func
+ _sign = _guess_sign_of_sklmetric(score_func)
+ _sign_str = "higher" if _sign == 1 else "lower"
+ self.set_tags(**{"property:higher_or_lower_is_better": _sign_str})
+
+ # default handling for cv
+ if isinstance(cv, int):
+ from sklearn.model_selection import KFold
+
+ self._cv = KFold(n_splits=cv, shuffle=True)
+ elif cv is None:
+ from sklearn.model_selection import KFold
+
+ self._cv = KFold(n_splits=3, shuffle=True)
+ else:
+ self._cv = cv
+
+ def _paramnames(self):
+ """Return the parameter names of the search.
+
+ Returns
+ -------
+ list of str
+ The parameter names of the search parameters.
+ """
+ return list(self.estimator.get_params().keys())
+
+ def _evaluate(self, params):
+ """Evaluate the parameters.
+
+ Parameters
+ ----------
+ params : dict with string keys
+ Parameters to evaluate.
+
+ Returns
+ -------
+ float
+ The value of the parameters as per evaluation.
+ dict
+ Additional metadata about the search.
+ """
+ from sktime.classification.model_evaluation import evaluate
+
+ estimator = self.estimator.clone().set_params(**params)
+
+ results = evaluate(
+ estimator,
+ cv=self._cv,
+ X=self.X,
+ y=self.y,
+ scoring=self._scoring._score_func,
+ error_score=self.error_score,
+ backend=self.backend,
+ backend_params=self.backend_params,
+ )
+
+ metric = self._scoring._score_func
+ result_name = f"test_{metric.__name__}"
+
+ res_float = results[result_name].mean()
+
+ return res_float, {"results": results}
+
+ @classmethod
+ def get_test_params(cls, parameter_set="default"):
+ """Return testing parameter settings for the skbase object.
+
+ ``get_test_params`` is a unified interface point to store
+ parameter settings for testing purposes. This function is also
+ used in ``create_test_instance`` and ``create_test_instances_and_names``
+ to construct test instances.
+
+ ``get_test_params`` should return a single ``dict``, or a ``list`` of ``dict``.
+
+ Each ``dict`` is a parameter configuration for testing,
+ and can be used to construct an "interesting" test instance.
+ A call to ``cls(**params)`` should
+ be valid for all dictionaries ``params`` in the return of ``get_test_params``.
+
+ The ``get_test_params`` need not return fixed lists of dictionaries,
+ it can also return dynamic or stochastic parameter settings.
+
+ Parameters
+ ----------
+ parameter_set : str, default="default"
+ Name of the set of test parameters to return, for use in tests. If no
+ special parameters are defined for a value, will return `"default"` set.
+
+ Returns
+ -------
+ params : dict or list of dict, default = {}
+ Parameters to create testing instances of the class
+ Each dict are parameters to construct an "interesting" test instance, i.e.,
+ `MyClass(**params)` or `MyClass(**params[i])` creates a valid test instance.
+ `create_test_instance` uses the first (or only) dictionary in `params`
+ """
+ from sklearn.metrics import brier_score_loss
+ from sklearn.model_selection import KFold
+ from sktime.classification.dummy import DummyClassifier
+ from sktime.datasets import load_unit_test
+
+ X, y = load_unit_test(return_X_y=True, return_type="pd-multiindex")
+ params0 = {
+ "estimator": DummyClassifier(strategy="most_frequent"),
+ "X": X,
+ "y": y,
+ }
+
+ params1 = {
+ "estimator": DummyClassifier(strategy="stratified"),
+ "cv": KFold(n_splits=2),
+ "X": X,
+ "y": y,
+ "scoring": brier_score_loss,
+ }
+
+ return [params0, params1]
+
+ @classmethod
+ def _get_score_params(self):
+ """Return settings for testing score/evaluate functions. Used in tests only.
+
+ Returns a list, the i-th element should be valid arguments for
+ self.evaluate and self.score, of an instance constructed with
+ self.get_test_params()[i].
+
+ Returns
+ -------
+ list of dict
+ The parameters to be used for scoring.
+ """
+ val0 = {}
+ val1 = {"strategy": "most_frequent"}
+ return [val0, val1]
diff --git a/src/hyperactive/experiment/integrations/sktime_forecasting.py b/src/hyperactive/experiment/integrations/sktime_forecasting.py
index 0f876d70..f335f6b2 100644
--- a/src/hyperactive/experiment/integrations/sktime_forecasting.py
+++ b/src/hyperactive/experiment/integrations/sktime_forecasting.py
@@ -121,7 +121,7 @@ class SktimeForecastingExperiment(BaseExperiment):
... y=y,
... )
>>> params = {"strategy": "mean"}
- >>> score, add_info = sktime_exp.score(params)
+ >>> score, metadata = sktime_exp.score(params)
For default choices of ``scoring``:
>>> sktime_exp = SktimeForecastingExperiment(
@@ -130,10 +130,10 @@ class SktimeForecastingExperiment(BaseExperiment):
... y=y,
... )
>>> params = {"strategy": "mean"}
- >>> score, add_info = sktime_exp.score(params)
+ >>> score, metadata = sktime_exp.score(params)
Quick call without metadata return or dictionary:
- >>> score = sktime_exp(strategy="mean")
+ >>> score = sktime_exp({"strategy": "mean"})
"""
_tags = {
diff --git a/src/hyperactive/integrations/sktime/__init__.py b/src/hyperactive/integrations/sktime/__init__.py
index a88ca2f0..256d03ea 100644
--- a/src/hyperactive/integrations/sktime/__init__.py
+++ b/src/hyperactive/integrations/sktime/__init__.py
@@ -1,5 +1,6 @@
"""Integrations for sktime with Hyperactive."""
+from hyperactive.integrations.sktime._classification import TSCOptCV
from hyperactive.integrations.sktime._forecasting import ForecastingOptCV
-__all__ = ["ForecastingOptCV"]
+__all__ = ["TSCOptCV", "ForecastingOptCV"]
diff --git a/src/hyperactive/integrations/sktime/_classification.py b/src/hyperactive/integrations/sktime/_classification.py
new file mode 100644
index 00000000..72674c57
--- /dev/null
+++ b/src/hyperactive/integrations/sktime/_classification.py
@@ -0,0 +1,349 @@
+# copyright: hyperactive developers, MIT License (see LICENSE file)
+
+import numpy as np
+from skbase.utils.dependencies import _check_soft_dependencies
+
+if _check_soft_dependencies("sktime", severity="none"):
+ from sktime.classification._delegate import _DelegatedClassifier
+else:
+ from skbase.base import BaseEstimator as _DelegatedClassifier
+
+from hyperactive.experiment.integrations.sktime_classification import (
+ SktimeClassificationExperiment,
+)
+
+
+class TSCOptCV(_DelegatedClassifier):
+ """Tune an sktime classifier via any optimizer in the hyperactive toolbox.
+
+ ``TSCOptCV`` uses any available tuning engine from ``hyperactive``
+ to tune a classifier by backtesting.
+
+ It passes backtesting results as scores to the tuning engine,
+ which identifies the best hyperparameters.
+
+ Any available tuning engine from hyperactive can be used, for example:
+
+ * grid search - ``from hyperactive.opt import GridSearchSk as GridSearch``,
+ this results in the same algorithm as ``TSCGridSearchCV``
+ * hill climbing - ``from hyperactive.opt import HillClimbing``
+ * optuna parzen-tree search - ``from hyperactive.opt.optuna import TPEOptimizer``
+
+ Configuration of the tuning engine is as per the respective documentation.
+
+ Formally, ``TSCOptCV`` does the following:
+
+ In ``fit``:
+
+ * wraps the ``estimator``, ``scoring``, and other parameters
+ into a ``SktimeClassificationExperiment`` instance, which is passed to the
+ optimizer ``optimizer`` as the ``experiment`` argument.
+ * Optimal parameters are then obtained from ``optimizer.solve``, and set
+ as ``best_params_`` and ``best_estimator_`` attributes.
+ * If ``refit=True``, ``best_estimator_`` is fitted to the entire ``y`` and ``X``.
+
+ In ``predict`` and ``predict``-like methods, calls the respective method
+ of the ``best_estimator_`` if ``refit=True``.
+
+ Parameters
+ ----------
+ estimator : sktime classifier, BaseClassifier instance or interface compatible
+ The classifier to tune, must implement the sktime classifier interface.
+
+ optimizer : hyperactive BaseOptimizer
+ The optimizer to be used for hyperparameter search.
+
+ cv : int, sklearn cross-validation generator or an iterable, default=3-fold CV
+ Determines the cross-validation splitting strategy.
+ Possible inputs for cv are:
+
+ - None = default = ``KFold(n_splits=3, shuffle=True)``
+ - integer, number of folds folds in a ``KFold`` splitter, ``shuffle=True``
+ - An iterable yielding (train, test) splits as arrays of indices.
+
+ For integer/None inputs, if the estimator is a classifier and ``y`` is
+ either binary or multiclass, :class:`StratifiedKFold` is used. In all
+ other cases, :class:`KFold` is used. These splitters are instantiated
+ with ``shuffle=False`` so the splits will be the same across calls.
+
+ scoring : str, callable, default=None
+ Strategy to evaluate the performance of the cross-validated model on
+ the test set. Can be:
+
+ - a single string resolvable to an sklearn scorer
+ - a callable that returns a single value;
+ - ``None`` = default = ``accuracy_score``
+
+ refit : bool, optional (default=True)
+ True = refit the forecaster with the best parameters on the entire data in fit
+ False = no refitting takes place. The forecaster cannot be used to predict.
+ This is to be used to tune the hyperparameters, and then use the estimator
+ as a parameter estimator, e.g., via get_fitted_params or PluginParamsForecaster.
+
+ error_score : "raise" or numeric, default=np.nan
+ Value to assign to the score if an exception occurs in estimator fitting. If set
+ to "raise", the exception is raised. If a numeric value is given,
+ FitFailedWarning is raised.
+
+ backend : string, by default "None".
+ Parallelization backend to use for runs.
+ Runs parallel evaluate if specified and ``strategy="refit"``.
+
+ - "None": executes loop sequentially, simple list comprehension
+ - "loky", "multiprocessing" and "threading": uses ``joblib.Parallel`` loops
+ - "joblib": custom and 3rd party ``joblib`` backends, e.g., ``spark``
+ - "dask": uses ``dask``, requires ``dask`` package in environment
+ - "dask_lazy": same as "dask",
+ but changes the return to (lazy) ``dask.dataframe.DataFrame``.
+ - "ray": uses ``ray``, requires ``ray`` package in environment
+
+ Recommendation: Use "dask" or "loky" for parallel evaluate.
+ "threading" is unlikely to see speed ups due to the GIL and the serialization
+ backend (``cloudpickle``) for "dask" and "loky" is generally more robust
+ than the standard ``pickle`` library used in "multiprocessing".
+
+ backend_params : dict, optional
+ additional parameters passed to the backend as config.
+ Directly passed to ``utils.parallel.parallelize``.
+ Valid keys depend on the value of ``backend``:
+
+ - "None": no additional parameters, ``backend_params`` is ignored
+ - "loky", "multiprocessing" and "threading": default ``joblib`` backends
+ any valid keys for ``joblib.Parallel`` can be passed here, e.g., ``n_jobs``,
+ with the exception of ``backend`` which is directly controlled by ``backend``.
+ If ``n_jobs`` is not passed, it will default to ``-1``, other parameters
+ will default to ``joblib`` defaults.
+ - "joblib": custom and 3rd party ``joblib`` backends, e.g., ``spark``.
+ any valid keys for ``joblib.Parallel`` can be passed here, e.g., ``n_jobs``,
+ ``backend`` must be passed as a key of ``backend_params`` in this case.
+ If ``n_jobs`` is not passed, it will default to ``-1``, other parameters
+ will default to ``joblib`` defaults.
+ - "dask": any valid keys for ``dask.compute`` can be passed,
+ e.g., ``scheduler``
+
+ - "ray": The following keys can be passed:
+
+ - "ray_remote_args": dictionary of valid keys for ``ray.init``
+ - "shutdown_ray": bool, default=True; False prevents ``ray`` from shutting
+ down after parallelization.
+ - "logger_name": str, default="ray"; name of the logger to use.
+ - "mute_warnings": bool, default=False; if True, suppresses warnings
+
+ Example
+ -------
+ Any available tuning engine from hyperactive can be used, for example:
+
+ * grid search - ``from hyperactive.opt import GridSearchSk as GridSearch``
+ * hill climbing - ``from hyperactive.opt import HillClimbing``
+ * optuna parzen-tree search - ``from hyperactive.opt.optuna import TPEOptimizer``
+
+ For illustration, we use grid search, this can be replaced by any other optimizer.
+
+ 1. defining the tuned estimator:
+ >>> from sktime.classification.dummy import DummyClassifier
+ >>> from sklearn.model_selection import KFold
+ >>> from hyperactive.integrations.sktime import TSCOptCV
+ >>> from hyperactive.opt import GridSearchSk as GridSearch
+ >>>
+ >>> param_grid = {"strategy": ["most_frequent", "stratified"]}
+ >>> tuned_naive = TSCOptCV(
+ ... DummyClassifier(),
+ ... GridSearch(param_grid),
+ ... cv=KFold(n_splits=2, shuffle=False),
+ ... )
+
+ 2. fitting the tuned estimator:
+ >>> from sktime.datasets import load_unit_test
+ >>> X_train, y_train = load_unit_test(
+ ... return_X_y=True, split="TRAIN", return_type="pd-multiindex"
+ ... )
+ >>> X_test, _ = load_unit_test(
+ ... return_X_y=True, split="TEST", return_type="pd-multiindex"
+ ... )
+ >>>
+ >>> tuned_naive.fit(X_train, y_train)
+ TSCOptCV(...)
+ >>> y_pred = tuned_naive.predict(X_test)
+
+ 3. obtaining best parameters and best estimator
+ >>> best_params = tuned_naive.best_params_
+ >>> best_classifier = tuned_naive.best_estimator_
+ """
+
+ _tags = {
+ "authors": "fkiraly",
+ "maintainers": "fkiraly",
+ "python_dependencies": "sktime",
+ }
+
+ # attribute for _DelegatedClassifier, which then delegates
+ # all non-overridden methods are same as of getattr(self, _delegate_name)
+ # see further details in _DelegatedClassifier docstring
+ _delegate_name = "best_estimator_"
+
+ def __init__(
+ self,
+ estimator,
+ optimizer,
+ cv=None,
+ scoring=None,
+ refit=True,
+ error_score=np.nan,
+ backend=None,
+ backend_params=None,
+ ):
+ self.estimator = estimator
+ self.optimizer = optimizer
+ self.cv = cv
+ self.scoring = scoring
+ self.refit = refit
+ self.error_score = error_score
+ self.backend = backend
+ self.backend_params = backend_params
+ super().__init__()
+
+ def _fit(self, X, y):
+ """Fit time series classifier to training data.
+
+ private _fit containing the core logic, called from fit
+
+ Writes to self:
+ Sets fitted model attributes ending in "_".
+
+ Parameters
+ ----------
+ X : guaranteed to be of a type in self.get_tag("X_inner_mtype")
+ if self.get_tag("X_inner_mtype") = "numpy3D":
+ 3D np.ndarray of shape = [n_instances, n_dimensions, series_length]
+ if self.get_tag("X_inner_mtype") = "pd-multiindex:":
+ pd.DataFrame with columns = variables,
+ index = pd.MultiIndex with first level = instance indices,
+ second level = time indices
+ for list of other mtypes, see datatypes.SCITYPE_REGISTER
+ for specifications, see examples/AA_datatypes_and_datasets.ipynb
+ y : guaranteed to be of a type in self.get_tag("y_inner_mtype")
+ 1D iterable, of shape [n_instances]
+ or 2D iterable, of shape [n_instances, n_dimensions]
+ class labels for fitting
+ if self.get_tag("capaility:multioutput") = False, guaranteed to be 1D
+ if self.get_tag("capaility:multioutput") = True, guaranteed to be 2D
+
+ Returns
+ -------
+ self : Reference to self.
+ """
+ from sklearn.dummy import DummyClassifier
+ from sklearn.metrics import check_scoring
+
+ estimator = self.estimator.clone()
+
+ # use dummy classifier from sklearn to get default coercion behaviour
+ # for classificatoin metrics
+ scoring = check_scoring(DummyClassifier(), self.scoring)
+ # scoring_name = f"test_{scoring.name}"
+
+ experiment = SktimeClassificationExperiment(
+ estimator=estimator,
+ scoring=scoring,
+ cv=self.cv,
+ X=X,
+ y=y,
+ error_score=self.error_score,
+ backend=self.backend,
+ backend_params=self.backend_params,
+ )
+
+ optimizer = self.optimizer.clone()
+ optimizer.set_params(experiment=experiment)
+ best_params = optimizer.solve()
+
+ self.best_params_ = best_params
+ self.best_estimator_ = estimator.set_params(**best_params)
+
+ # Refit model with best parameters.
+ if self.refit:
+ self.best_estimator_.fit(X=X, y=y)
+
+ return self
+
+ def _predict(self, X):
+ """Predict labels for sequences in X.
+
+ private _predict containing the core logic, called from predict
+
+ State required:
+ Requires state to be "fitted".
+
+ Accesses in self:
+ Fitted model attributes ending in "_"
+
+ Parameters
+ ----------
+ X : guaranteed to be of a type in self.get_tag("X_inner_mtype")
+ if self.get_tag("X_inner_mtype") = "numpy3D":
+ 3D np.ndarray of shape = [n_instances, n_dimensions, series_length]
+ if self.get_tag("X_inner_mtype") = "nested_univ":
+ pd.DataFrame with each column a dimension, each cell a pd.Series
+ for list of other mtypes, see datatypes.SCITYPE_REGISTER
+ for specifications, see examples/AA_datatypes_and_datasets.ipynb
+
+ Returns
+ -------
+ y : 1D np.array of int, of shape [n_instances] - predicted class labels
+ indices correspond to instance indices in X
+ """
+ if not self.refit:
+ raise RuntimeError(
+ f"In {self.__class__.__name__}, refit must be True to make predictions,"
+ f" but found refit=False. If refit=False, {self.__class__.__name__} can"
+ " be used only to tune hyper-parameters, as a parameter estimator."
+ )
+ return super()._predict(X=X)
+
+ @classmethod
+ def get_test_params(cls, parameter_set="default"):
+ """Return testing parameter settings for the estimator.
+
+ Parameters
+ ----------
+ parameter_set : str, default="default"
+ Name of the set of test parameters to return, for use in tests. If no
+ special parameters are defined for a value, will return ``"default"`` set.
+
+ Returns
+ -------
+ params : dict or list of dict
+ """
+ from sklearn.metrics import accuracy_score
+ from sklearn.model_selection import KFold
+ from sktime.classification.dummy import DummyClassifier
+
+ from hyperactive.opt.gfo import HillClimbing
+ from hyperactive.opt.gridsearch import GridSearchSk
+ from hyperactive.opt.random_search import RandomSearchSk
+
+ params_gridsearch = {
+ "estimator": DummyClassifier(),
+ "optimizer": GridSearchSk(
+ param_grid={"strategy": ["most_frequent", "stratified"]}
+ ),
+ }
+ params_randomsearch = {
+ "estimator": DummyClassifier(),
+ "cv": 2,
+ "optimizer": RandomSearchSk(
+ param_distributions={"strategy": ["most_frequent", "stratified"]},
+ ),
+ "scoring": accuracy_score,
+ }
+ params_hillclimb = {
+ "estimator": DummyClassifier(strategy="stratified"),
+ "cv": KFold(n_splits=2, shuffle=False),
+ "optimizer": HillClimbing(
+ search_space={"strategy": ["most_frequent", "stratified"]},
+ n_iter=10,
+ n_neighbours=5,
+ ),
+ "scoring": "cross-entropy",
+ }
+ return [params_gridsearch, params_randomsearch, params_hillclimb]
diff --git a/src/hyperactive/opt/_adapters/_base_optuna_adapter.py b/src/hyperactive/opt/_adapters/_base_optuna_adapter.py
index c17dbb6d..8dd3171f 100644
--- a/src/hyperactive/opt/_adapters/_base_optuna_adapter.py
+++ b/src/hyperactive/opt/_adapters/_base_optuna_adapter.py
@@ -110,7 +110,7 @@ def _objective(self, trial):
The objective value
"""
params = self._suggest_params(trial, self.param_space)
- score = self.experiment(**params)
+ score = self.experiment(params)
# Handle early stopping based on max_score
if self.max_score is not None and score >= self.max_score:
@@ -133,7 +133,7 @@ def _setup_initial_positions(self, study):
# For warm start, we manually add trials to the study history
# instead of using suggest methods to avoid distribution conflicts
for point in warm_start_points:
- self.experiment(**point)
+ self.experiment(point)
study.enqueue_trial(point)
def _solve(self, experiment, param_space, n_trials, **kwargs):
diff --git a/src/hyperactive/tests/test_all_objects.py b/src/hyperactive/tests/test_all_objects.py
index c78061c1..8009e1c7 100644
--- a/src/hyperactive/tests/test_all_objects.py
+++ b/src/hyperactive/tests/test_all_objects.py
@@ -225,7 +225,7 @@ def test_score_function(self, object_class):
msg = f"Score and eval calls do not match: |{e_score}| != |{score}|"
assert abs(e_score) == abs(score), msg
- call_sc = inst(**obj)
+ call_sc = inst(obj)
assert isinstance(call_sc, float), f"Score is not a float: {call_sc}"
if det_tag == "deterministic":
msg = f"Score does not match: {score} != {call_sc}"