add example for nevergrad and some comments+typing

examples/basic/regression.py

@@ -4,8 +4,6 @@
 from photonai.base import Hyperpipe, PipelineElement, OutputSettings
 from photonai.optimization import IntegerRange
 
-
-# WE USE THE BREAST CANCER SET FROM SKLEARN
 X, y = load_boston(return_X_y=True)
 
 # DESIGN YOUR PIPELINE

examples/neural_networks/keras_cnn_classification.py

@@ -11,7 +11,7 @@
 from keras.layers import Dropout
 from keras.layers.convolutional import Conv1D
 from keras.layers.convolutional import MaxPooling1D
-from sklearn.model_selection import KFold
+from sklearn.model_selection import StratifiedShuffleSplit
 
 from examples.neural_networks.dataset import load_har
 
@@ -49,8 +49,8 @@
                     optimizer_params={},
                     metrics=['accuracy'],
                     best_config_metric='accuracy',
-                    outer_cv=KFold(n_splits=3),
-                    inner_cv=KFold(n_splits=2),
+                    outer_cv=StratifiedShuffleSplit(n_splits=3, test_size=0.2),
+                    inner_cv=StratifiedShuffleSplit(n_splits=2, test_size=0.2),
                     verbosity=1,
                     output_settings=settings)
 

examples/neural_networks/keras_cnn_optimization.py

@@ -13,7 +13,7 @@
 from keras.layers import Dropout
 from keras.layers.convolutional import Conv1D
 from keras.layers.convolutional import MaxPooling1D
-from sklearn.model_selection import KFold
+from sklearn.model_selection import StratifiedShuffleSplit
 from sklearn.preprocessing import StandardScaler
 from sklearn.base import BaseEstimator
 
@@ -134,8 +134,8 @@ def build_model(n_filters, kernel_size, n_timesteps, n_features, n_outputs):
                         optimizer_params={'n_configurations': 25},
                         metrics=['accuracy'],
                         best_config_metric='accuracy',
-                        outer_cv=KFold(n_splits=5),
-                        inner_cv=KFold(n_splits=5),
+                        outer_cv=StratifiedShuffleSplit(n_splits=5, test_size=0.2),
+                        inner_cv=StratifiedShuffleSplit(n_splits=5, test_size=0.2),
                         verbosity=1,
                         output_settings=OutputSettings(project_folder='./tmp/'))
 

examples/optimizer/nevergrad_example.py

@@ -0,0 +1,34 @@
+from sklearn.datasets import load_boston
+from sklearn.model_selection import ShuffleSplit
+import nevergrad as ng
+
+from photonai.base import Hyperpipe, PipelineElement, OutputSettings
+from photonai.optimization import BooleanSwitch, FloatRange
+
+X, y = load_boston(return_X_y=True)
+
+# list of all available nevergrad optimizer
+print(list(ng.optimizers.registry.values()))
+
+my_pipe = Hyperpipe('basic_svm_pipe_no_performance',
+                    optimizer='nevergrad',
+                    optimizer_params={'facade': 'NGO', 'n_configurations': 30},
+                    metrics=['mean_squared_error', 'pearson_correlation', 'mean_absolute_error', 'explained_variance'],
+                    best_config_metric='mean_squared_error',
+                    outer_cv=ShuffleSplit(n_splits=3, test_size=0.2),
+                    inner_cv=ShuffleSplit(n_splits=3, test_size=0.2),
+                    verbosity=0,
+                    output_settings=OutputSettings(project_folder='./tmp/'))
+
+# ADD ELEMENTS TO YOUR PIPELINE
+# first normalize all features
+my_pipe += PipelineElement('StandardScaler')
+
+my_pipe += PipelineElement('PCA', n_components='mle')
+
+my_pipe += PipelineElement('Ridge',
+                           hyperparameters={'alpha': FloatRange(0.1, 100),
+                                            'fit_intercept': BooleanSwitch(),
+                                            'normalize': BooleanSwitch()})
+
+my_pipe.fit(X, y)

photonai/optimization/base_optimizer.py

@@ -1,14 +1,20 @@
+from typing import Callable
+
 
 class PhotonSlaveOptimizer(object):
-    """
-    The PhotonSlaveOptimizer is controlled by PHOTON. By ask-tell principle PHOTON get new configs.
-    It terminates by some specific criteria with an aks -> empty yield.
+    """PhotonSlaveOptimizer
+
+    The PhotonSlaveOptimizer is controlled by PHOTONAI.
+    With the ask-tell principle PHOTONAI gets new configs.
+    It terminates by some specific criteria leads to empty yield ask.
+
     """
 
-    def prepare(self, pipeline_elements: list, maximize_metric: bool):
-        """
-        Initializes hyperparameter search.
-        Assembles all hyperparameters of the pipeline_element list in order to prepare the hyperparameter search space.
+    def prepare(self, pipeline_elements: list, maximize_metric: bool) -> None:
+        """Initializes hyperparameter search.
+
+        Assembles all hyperparameters of the pipeline_element
+        list in order to prepare the hyperparameter search space.
         Hyperparameters can be accessed via pipe_element.hyperparameters.
 
         Parameters
@@ -20,38 +26,46 @@ def prepare(self, pipeline_elements: list, maximize_metric: bool):
         """
         pass
 
-    def ask(self):
+    def ask(self) -> dict:
         """
-        When called, returns the next configuration that should be tested.
+        When ask is called it returns the next configuration to be tested.
 
         Returns
         -------
-        next config to test
+        * _ [dict]:
+            config_dict, the next config to be tested
         """
         pass
 
-    def tell(self, config, performance):
+    def tell(self, config: dict, performance: float) -> None:
         """
+        Provide result for optimizer to calculate new ones.
+
         Parameters
         ----------
         * 'config' [dict]:
-            The configuration that has been trained and tested
+            The configuration that has been trained and tested.
         * 'performance' [dict]:
             Metrics about the configuration's generalization capabilities.
         """
         pass
 
 
 class PhotonMasterOptimizer(object):
+    """PhotonMasterOptimizer
+
+    The PhotonMasterOptimizer controls PHOTONAI.
+    PHOTONAI creates an objective function that is used by the optimizer.
+    The limitation of runs and configs of the
+    objective function is up to PhotonMasterOptimizer.
+
     """
-        The PhotonMasterOptimizer controls PHOTON. PHOTON provides an objective_function.
-        The runs and configs of the objective_function is up to PhotonMasterOptimizer.
-        """
 
-    def prepare(self, pipeline_elements: list, maximize_metric: bool, objective_function):
-        """
-        Initializes hyperparameter search.
-        Assembles all hyperparameters of the pipeline_element list in order to prepare the hyperparameter search space.
+    def prepare(self, pipeline_elements: list, maximize_metric: bool, objective_function: Callable) -> None:
+        """Initializes hyperparameter search.
+
+        Assembles all hyperparameters of the pipeline_element
+        list in order to prepare the hyperparameter search space.
         Hyperparameters can be accessed via pipe_element.hyperparameters.
 
         Parameters
@@ -60,12 +74,12 @@ def prepare(self, pipeline_elements: list, maximize_metric: bool, objective_func
             List of all pipeline_elements to create hyperparameter_space.
         * `maximize_metric` [bool]:
             Boolean for distinguish between score and error.
-        * `objective_function` [callable]:
+        * `objective_function` [Callable]:
             The cost or objective function.
         """
         pass
 
-    def optimize(self):
+    def optimize(self) -> None:
         """
         Start optimization over objective_function.
         """

photonai/optimization/grid_search/grid_search.py

@@ -1,6 +1,7 @@
 import datetime
 
 import numpy as np
+from typing import Union, Generator
 
 from photonai.optimization.base_optimizer import PhotonSlaveOptimizer
 from photonai.optimization.config_grid import create_global_config_grid
@@ -11,43 +12,64 @@ class GridSearchOptimizer(PhotonSlaveOptimizer):
     """
     Searches for the best configuration by iteratively testing all possible hyperparameter combinations.
     """
+
     def __init__(self):
-        super(GridSearchOptimizer, self).__init__()
         self.param_grid = []
         self.pipeline_elements = None
         self.parameter_iterable = None
         self.ask = self.next_config_generator()
 
-    def prepare(self, pipeline_elements, maximize_metric):
+    def prepare(self, pipeline_elements: list, maximize_metric: bool) -> None:
+        """Initializes grid based hyperparameter search.
+
+        Creates a grid from input pipeline_elements.
+        Hyperparameters can be accessed via pipe_element.hyperparameters.
+
+        Parameters
+        ----------
+        * `pipeline_elements` [list]:
+            List of all pipeline_elements to create hyperparameter_space.
+        * `maximize_metric` [bool]:
+            Boolean for distinguish between score and error.
+
+        """
         self.pipeline_elements = pipeline_elements
         self.ask = self.next_config_generator()
         self.param_grid = create_global_config_grid(self.pipeline_elements)
         logger.info("Grid Search generated " + str(len(self.param_grid)) + " configurations")
 
-    def next_config_generator(self):
+    def next_config_generator(self) -> Generator:
+        """
+        Creates a param-generator dealing with the entire grid.
+        """
         for parameters in self.param_grid:
             yield parameters
 
-    def tell(self, config, performance):
-        # influence return value of next_config
-        pass
-
 
 class RandomGridSearchOptimizer(GridSearchOptimizer):
     """
-     Searches for the best configuration by randomly testing k possible hyperparameter combinations.
+    Searches for the best configuration by randomly testing k possible hyperparameter combinations.
+
+    Parameter
+    ---------
+    * `n_configurations` [Union[int, None], default: 25]:
+        Number of configurations to be calculated.
+
     """
 
-    def __init__(self, n_configurations=25):
+    def __init__(self, n_configurations: Union[int, None] = 25):
         super(RandomGridSearchOptimizer, self).__init__()
         self._k = n_configurations
         self.n_configurations = self._k
 
-    def prepare(self, pipeline_elements, maximize_metric):
+    def prepare(self, pipeline_elements: list, maximize_metric: bool) -> None:
+        """
+        Initializes hyperparameter search. Same functionality as in GridSearchOptimizer.
+        """
         super(RandomGridSearchOptimizer, self).prepare(pipeline_elements, maximize_metric)
         self.n_configurations = self._k
         self.param_grid = list(self.param_grid)
-        # create random chaos in list
+        # create random order in list
         np.random.shuffle(self.param_grid)
         if self.n_configurations is not None:
             # k is maximal all grid items
@@ -59,19 +81,33 @@ def prepare(self, pipeline_elements, maximize_metric):
 class TimeBoxedRandomGridSearchOptimizer(RandomGridSearchOptimizer):
     """
     Iteratively tests k possible hyperparameter configurations until a certain time limit is reached.
+
+    Parameter
+    ---------
+    * `limit_in_minutes` [int, default: 60]:
+        Total time in minutes.
+    * `n_configurations` [Union[int, None], default: 25]:
+        Number of configurations to be calculated.
+
     """
 
-    def __init__(self, limit_in_minutes=60, n_configurations=None):
+    def __init__(self, limit_in_minutes: float = 60, n_configurations: Union[int, None] = None):
         super(TimeBoxedRandomGridSearchOptimizer, self).__init__(n_configurations)
         self.limit_in_minutes = limit_in_minutes
         self.start_time = None
         self.end_time = None
 
-    def prepare(self, pipeline_elements, maximize_metric):
+    def prepare(self, pipeline_elements: list, maximize_metric: bool) -> None:
+        """
+        Initializes hyperparameter search. Same functionality as in GridSearchOptimizer.
+        """
         super(TimeBoxedRandomGridSearchOptimizer, self).prepare(pipeline_elements, maximize_metric)
         self.start_time = None
 
-    def next_config_generator(self):
+    def next_config_generator(self) -> Generator:
+        """
+        Creates new configs until stopping criteria. Same functionality as in GridSearchOptimizer.
+        """
         if self.start_time is None:
             self.start_time = datetime.datetime.now()
             self.end_time = self.start_time + datetime.timedelta(minutes=self.limit_in_minutes)