error in example doc with scikit-optimize

[vladiscripts]

The example doc "Parameter Heatmap", section "Model-based optimization". https://kernc.github.io/backtesting.py/doc/examples/Parameter%20Heatmap%20&%20Optimization.html

When I run this snippet I get an error.

stats_skopt, heatmap, optimize_result = backtest.optimize(
     n1=[10, 100], # Note: For method="skopt", we
     n2=[20, 200], # only need interval end-points
     n_enter=[10, 40],
     n_exit=[10, 30],
     constraint=lambda p: p.n_exit < p.n_enter < p.n1 < p.n2,
     maximize='Equity Final [$]',
     method='skopt',
     max_tries=200,
     random_state=0,
     return_heatmap=True,
     return_optimization=True)

---------------------------------------------------------------------------
InvalidParameterError                     Traceback (most recent call last)
File <timed exec>:1, in <module>

File ~/anaconda3/envs/trade/lib/python3.8/site-packages/backtesting/backtesting.py:1490, in Backtest.optimize(self, maximize, method, max_tries, constraint, return_heatmap, return_optimization, random_state, **kwargs)
   1488     output = _optimize_grid()
   1489 elif method == 'skopt':
-> 1490     output = _optimize_skopt()
   1491 else:
   1492     raise ValueError(f"Method should be 'grid' or 'skopt', not {method!r}")

File ~/anaconda3/envs/trade/lib/python3.8/site-packages/backtesting/backtesting.py:1456, in Backtest.optimize.<locals>._optimize_skopt()
   1452 with warnings.catch_warnings():
   1453     warnings.filterwarnings(
   1454         'ignore', 'The objective has been evaluated at this point before.')
-> 1456     res = forest_minimize(
   1457         func=objective_function,
   1458         dimensions=dimensions,
   1459         n_calls=max_tries,
   1460         base_estimator=ExtraTreesRegressor(n_estimators=20, min_samples_leaf=2),
   1461         acq_func='LCB',
   1462         kappa=3,
   1463         n_initial_points=min(max_tries, 20 + 3 * len(kwargs)),
   1464         initial_point_generator='lhs',  # 'sobel' requires n_initial_points ~ 2**N
   1465         callback=DeltaXStopper(9e-7),
   1466         random_state=random_state)
   1468 stats = self.run(**dict(zip(kwargs.keys(), res.x)))
   1469 output = [stats]

File ~/anaconda3/envs/trade/lib/python3.8/site-packages/skopt/optimizer/forest.py:186, in forest_minimize(func, dimensions, base_estimator, n_calls, n_random_starts, n_initial_points, acq_func, initial_point_generator, x0, y0, random_state, verbose, callback, n_points, xi, kappa, n_jobs, model_queue_size)
     10 def forest_minimize(func, dimensions, base_estimator="ET", n_calls=100,
     11                     n_random_starts=None, n_initial_points=10, acq_func="EI",
     12                     initial_point_generator="random",
     13                     x0=None, y0=None, random_state=None, verbose=False,
     14                     callback=None, n_points=10000, xi=0.01, kappa=1.96,
     15                     n_jobs=1, model_queue_size=None):
     16     """Sequential optimisation using decision trees.
     17 
     18     A tree based regression model is used to model the expensive to evaluate
   (...)
    184         :class:`skopt.dummy_minimize`, :class:`skopt.gbrt_minimize`
    185     """
--> 186     return base_minimize(func, dimensions, base_estimator,
    187                          n_calls=n_calls, n_points=n_points,
    188                          n_random_starts=n_random_starts,
    189                          n_initial_points=n_initial_points,
    190                          initial_point_generator=initial_point_generator,
    191                          x0=x0, y0=y0, random_state=random_state,
    192                          n_jobs=n_jobs,
    193                          acq_func=acq_func,
    194                          xi=xi, kappa=kappa, verbose=verbose,
    195                          callback=callback, acq_optimizer="sampling",
    196                          model_queue_size=model_queue_size)

File ~/anaconda3/envs/trade/lib/python3.8/site-packages/skopt/optimizer/base.py:300, in base_minimize(func, dimensions, base_estimator, n_calls, n_random_starts, n_initial_points, initial_point_generator, acq_func, acq_optimizer, x0, y0, random_state, verbose, callback, n_points, n_restarts_optimizer, xi, kappa, n_jobs, model_queue_size)
    298 next_x = optimizer.ask()
    299 next_y = func(next_x)
--> 300 result = optimizer.tell(next_x, next_y)
    301 result.specs = specs
    302 if eval_callbacks(callbacks, result):

File ~/anaconda3/envs/trade/lib/python3.8/site-packages/skopt/optimizer/optimizer.py:493, in Optimizer.tell(self, x, y, fit)
    490         y = list(y)
    491         y[1] = log(y[1])
--> 493 return self._tell(x, y, fit=fit)

File ~/anaconda3/envs/trade/lib/python3.8/site-packages/skopt/optimizer/optimizer.py:536, in Optimizer._tell(self, x, y, fit)
    534 with warnings.catch_warnings():
    535     warnings.simplefilter("ignore")
--> 536     est.fit(self.space.transform(self.Xi), self.yi)
    538 if hasattr(self, "next_xs_") and self.acq_func == "gp_hedge":
    539     self.gains_ -= est.predict(np.vstack(self.next_xs_))

File ~/anaconda3/envs/trade/lib/python3.8/site-packages/sklearn/ensemble/_forest.py:341, in BaseForest.fit(self, X, y, sample_weight)
    314 def fit(self, X, y, sample_weight=None):
    315     """
    316     Build a forest of trees from the training set (X, y).
    317 
   (...)
    339         Fitted estimator.
    340     """
--> 341     self._validate_params()
    343     # Validate or convert input data
    344     if issparse(y):

File ~/anaconda3/envs/trade/lib/python3.8/site-packages/sklearn/base.py:570, in BaseEstimator._validate_params(self)
    562 def _validate_params(self):
    563     """Validate types and values of constructor parameters
    564 
    565     The expected type and values must be defined in the `_parameter_constraints`
   (...)
    568     accepted constraints.
    569     """
--> 570     validate_parameter_constraints(
    571         self._parameter_constraints,
    572         self.get_params(deep=False),
    573         caller_name=self.__class__.__name__,
    574     )

File ~/anaconda3/envs/trade/lib/python3.8/site-packages/sklearn/utils/_param_validation.py:97, in validate_parameter_constraints(parameter_constraints, params, caller_name)
     91 else:
     92     constraints_str = (
     93         f"{', '.join([str(c) for c in constraints[:-1]])} or"
     94         f" {constraints[-1]}"
     95     )
---> 97 raise InvalidParameterError(
     98     f"The {param_name!r} parameter of {caller_name} must be"
     99     f" {constraints_str}. Got {param_val!r} instead."
    100 )

InvalidParameterError: The 'criterion' parameter of ExtraTreesRegressor must be a str among {'squared_error', 'absolute_error', 'poisson', 'friedman_mse'}. Got 'mse' instead.

• Backtesting version: 0.3.3
• bokeh.__version__: Python 3.11, bokeh 2.4.3; Python 3.8.11, bokeh 3.0.3;
• scikit-optimize 0.9.0
• OS: Ubuntu 20.04

[sshquack]

Hi there, I am following the tutorial at https://kernc.github.io/backtesting.py/doc/examples/Parameter%20Heatmap%20&%20Optimization.html and I run into the same problem. Here are the details:

Throws on the following code:

stats_skopt, heatmap, optimize_result = backtest.optimize(
        n1=[10, 100],  # Note: For method="skopt", we
        n2=[20, 200],  # only need interval end-points
        n_enter=[10, 40],
        n_exit=[10, 30],
        constraint=lambda p: p.n_exit < p.n_enter < p.n1 < p.n2,
        maximize='Equity Final [$]',
        max_tries=200,
        random_state=0,
        return_heatmap=True,
        method='skopt',
        return_optimization=True)

With the following deps on Python 3.11:

Backtesting~=0.3.3
bokeh~=2.4.3
pandas~=1.5.2
seaborn~=0.12.1
scikit-optimize~=0.9.0
numpy~=1.23.0

With the following error:

  File "/lib/python3.11/site-packages/sklearn/utils/_param_validation.py", line 97, in validate_parameter_constraints
    raise InvalidParameterError(
sklearn.utils._param_validation.InvalidParameterError: The 'criterion' parameter of ExtraTreesRegressor must be a str among {'squared_error', 'poisson', 'friedman_mse', 'absolute_error'}. Got 'mse' instead.

When I upgrade the numpy version to 1.24.0:

Backtesting~=0.3.3
bokeh~=2.4.3
pandas~=1.5.2
seaborn~=0.12.1
scikit-optimize~=0.9.0
numpy~=1.24.0

I see the following error:

  File "/lib/python3.11/site-packages/skopt/space/transformers.py", line 275, in inverse_transform
    return np.round(X_orig).astype(np.int)
                                   ^^^^^^
  File "/lib/python3.11/site-packages/numpy/__init__.py", line 284, in __getattr__
    raise AttributeError("module {!r} has no attribute "
AttributeError: module 'numpy' has no attribute 'int'. Did you mean: 'inf'?

Please help! Thanks in advance.

[sshquack]

Full Python 3.11 stacktrace is here https://pastebin.com/4865192r Please let me know if you need any other info.

[sshquack]

This appears to be a problem in scikit-optimize. Hence scikit-learn here d5db8be See scikit-optimize/scikit-optimize#1123 which fixes the bug and scikit-optimize/scikit-optimize#1138

[kiann00]

Hi @sshquack , apologies mate, I am getting this error in one of my remote servers...

Can I check how this error is actually fixed?

Is there any particular version of numpy, or scikit-learn, or scikit-optimize we need to install/upgrade/downgrade to?

I read this thread [( https://github.com/scikit-optimize/scikit-optimize/pull/1123)], but I am still not sure where/how the fix is..

[eervin123]

@kernc I saw you put in a pr for the skopt but it seems they have gone dormant over there. Is there any anticipated fix for this? Perhaps something we can do based on your pr?

Or can we possibly downgrade numpy to a lower version?