Fixing test cases; updating metalearning with updated hyperparameters

autosklearn/pipeline/base.py

@@ -460,4 +460,5 @@ def _validate_include_exclude_params(self):
         steps = [step[0] for step in self.steps]
         for node in list(self.include.keys()) + list(self.exclude):
             if node not in steps:
-                raise ValueError("Component {0} in not part for pipeline steps {1}".format(node, str(steps)))
+                raise ValueError("Component {0} in not part for pipeline steps {1}"
+                                 .format(node, str(steps)))

autosklearn/pipeline/components/data_preprocessing/__init__.py

@@ -111,11 +111,12 @@ def get_hyperparameter_search_space(self, dataset_properties=None,
                     break
 
         preprocessor = CategoricalHyperparameter('__choice__',
-                                                 list(available_preprocessors.keys()), default_value=default)
+                                                 list(available_preprocessors.keys()),
+                                                 default_value=default)
         cs.add_hyperparameter(preprocessor)
         for name in available_preprocessors:
-            preprocessor_configuration_space = available_preprocessors[name](dataset_properties=dataset_properties,
-                                                                             include=include, exclude=exclude). \
+            preprocessor_configuration_space = available_preprocessors[name](
+                dataset_properties=dataset_properties, include=include, exclude=exclude). \
                 get_hyperparameter_search_space(dataset_properties)
             parent_hyperparameter = {'parent': preprocessor, 'value': name}
             cs.add_configuration_space(name, preprocessor_configuration_space,

autosklearn/pipeline/components/data_preprocessing/feature_type.py

@@ -125,6 +125,7 @@ def get_properties(dataset_properties=None):
                 'handles_classification': True,
                 'handles_multiclass': True,
                 'handles_multilabel': True,
+                'handles_multioutput': True,
                 # TODO find out of this is right!
                 'handles_sparse': True,
                 'handles_dense': True,

autosklearn/smbo.py

@@ -22,8 +22,7 @@
 import autosklearn.metalearning
 from autosklearn.constants import MULTILABEL_CLASSIFICATION, \
     BINARY_CLASSIFICATION, TASK_TYPES_TO_STRING, CLASSIFICATION_TASKS, \
-    REGRESSION_TASKS, MULTICLASS_CLASSIFICATION, REGRESSION, \
-    MULTIOUTPUT_REGRESSION
+    MULTICLASS_CLASSIFICATION, REGRESSION, MULTIOUTPUT_REGRESSION
 from autosklearn.ensemble_builder import EnsembleBuilderManager
 from autosklearn.metalearning.mismbo import suggest_via_metalearning
 from autosklearn.data.abstract_data_manager import AbstractDataManager
@@ -410,7 +409,8 @@ def run_smbo(self):
         if self.include is not None and self.exclude is not None:
             for node in self.include.keys():
                 if node in self.exclude.keys():
-                    raise ValueError('Cannot specify include and exclude for same step {0}.'.format(node))
+                    raise ValueError('Cannot specify include and exclude for same step {0}.'
+                                     .format(node))
 
         ta_kwargs = dict(
             backend=copy.deepcopy(self.backend),

autosklearn/util/pipeline.py

@@ -7,7 +7,6 @@
 
 from autosklearn.constants import (
     BINARY_CLASSIFICATION,
-    CLASSIFICATION_TASKS,
     MULTICLASS_CLASSIFICATION,
     MULTILABEL_CLASSIFICATION,
     MULTIOUTPUT_REGRESSION,
@@ -25,23 +24,26 @@
 
 
 def get_configuration_space(info: Dict[str, Any],
-                            include: Optional[dict] = None,
-                            exclude: Optional[dict] = None,
+                            include: Optional[Dict[str, List[str]]] = None,
+                            exclude: Optional[Dict[str, List[str]]] = None,
                             ) -> ConfigurationSpace:
 
     if include is not None and exclude is not None:
         for node in include.keys():
             if node in exclude.keys():
-                raise ValueError('Cannot specify include and exclude for same step {0}.'.format(node))
+                raise ValueError('Cannot specify include and exclude for same step {0}.'
+                                 .format(node))
 
     if info['task'] in REGRESSION_TASKS:
         return _get_regression_configuration_space(info, include, exclude)
     else:
         return _get_classification_configuration_space(info, include, exclude)
 
 
-def _get_regression_configuration_space(info: Dict[str, Any], include: Dict[str, List[str]],
-                                        exclude: Dict[str, List[str]]) -> ConfigurationSpace:
+def _get_regression_configuration_space(info: Dict[str, Any],
+                                        include: Optional[Dict[str, List[str]]],
+                                        exclude: Optional[Dict[str, List[str]]]
+                                        ) -> ConfigurationSpace:
     task_type = info['task']
     sparse = False
     multioutput = False
@@ -64,8 +66,10 @@ def _get_regression_configuration_space(info: Dict[str, Any], include: Dict[str,
     return configuration_space
 
 
-def _get_classification_configuration_space(info: Dict[str, Any], include: Dict[str, List[str]],
-                                            exclude: Dict[str, List[str]]) -> ConfigurationSpace:
+def _get_classification_configuration_space(info: Dict[str, Any],
+                                            include: Optional[Dict[str, List[str]]],
+                                            exclude: Optional[Dict[str, List[str]]]
+                                            ) -> ConfigurationSpace:
     task_type = info['task']
 
     multilabel = False

scripts/run_auto-sklearn_for_metadata_generation.py

@@ -70,13 +70,11 @@
     if is_test:
         automl_arguments['resampling_strategy_arguments'] = {'folds': 2}
         if task_type == 'classification':
-            automl_arguments['include_estimators'] = ['libsvm_svc']
-            automl_arguments['include_preprocessors'] = ['no_preprocessing']
             include = {'classifier': ['libsvm_svc'], 'feature_preprocessor': ['no_preprocessing']}
+            automl_arguments['include'] = include
         elif task_type == 'regression':
-            automl_arguments['include_estimators'] = ['extra_trees']
-            automl_arguments['include_preprocessors'] = ['no_preprocessing']
             include = {'regressor': ['extra_trees'], 'feature_preprocessor': ['no_preprocessing']}
+            automl_arguments['include'] = include
         else:
             raise ValueError('Unsupported task type: %s' % str(task_type))
     else:

test/test_automl/test_automl.py

@@ -204,8 +204,8 @@ def test_delete_non_candidate_models(backend, dask_client):
         seed=seed,
         initial_configurations_via_metalearning=0,
         resampling_strategy='holdout',
-        include_estimators=['sgd'],
-        include_preprocessors=['no_preprocessing'],
+        include={'classifier': ['sgd'],
+                 'feature_preprocessor': ['no_preprocessing']},
         metric=accuracy,
         dask_client=dask_client,
         # Force model to be deleted. That is, from 50 which is the
@@ -255,8 +255,8 @@ def test_binary_score_and_include(backend, dask_client):
 
     automl = autosklearn.automl.AutoML(
         backend, 20, 5,
-        include_estimators=['sgd'],
-        include_preprocessors=['no_preprocessing'],
+        include={'classifier': ['sgd'],
+                 'feature_preprocessor': ['no_preprocessing']},
         metric=accuracy,
         dask_client=dask_client,
     )

test/test_automl/test_estimators.py

@@ -61,8 +61,8 @@ def __call__(self, *args, **kwargs):
         initial_configurations_via_metalearning=0,
         ensemble_size=5,
         n_jobs=2,
-        include_estimators=['sgd'],
-        include_preprocessors=['no_preprocessing'],
+        include={'classifier': ['sgd'],
+                 'feature_preprocessor': ['no_preprocessing']},
         get_smac_object_callback=get_smac_object_wrapper_instance,
         max_models_on_disc=None,
     )
@@ -453,7 +453,7 @@ def test_classification_pandas_support(tmp_dir, output_dir, dask_client):
     automl = AutoSklearnClassifier(
         time_left_for_this_task=30,
         per_run_time_limit=5,
-        exclude_estimators=['libsvm_svc'],
+        exclude={'classifier': ['libsvm_svc']},
         dask_client=dask_client,
         seed=5,
         tmp_folder=tmp_dir,
@@ -561,7 +561,7 @@ def test_autosklearn_classification_methods_returns_self(dask_client):
                                    per_run_time_limit=10,
                                    ensemble_size=0,
                                    dask_client=dask_client,
-                                   exclude_preprocessors=['fast_ica'])
+                                   exclude={'feature_preprocessor': ['fast_ica']})
 
     automl_fitted = automl.fit(X_train, y_train)
     assert automl is automl_fitted

test/test_evaluation/test_train_evaluator.py

@@ -2403,7 +2403,7 @@ def test_eval_holdout_budget_iterations(self):
 
     def test_eval_holdout_budget_iterations_converged(self):
         configuration = get_configuration_space(
-            exclude_estimators=['random_forest', 'liblinear_svc'],
+            exclude={'classifier': ['random_forest', 'liblinear_svc']},
             info={'task': MULTICLASS_CLASSIFICATION, 'is_sparse': False},
         ).get_default_configuration()
         eval_holdout(
@@ -2486,7 +2486,7 @@ def test_eval_holdout_budget_mixed_iterations(self):
 
     def test_eval_holdout_budget_mixed_subsample(self):
         configuration = get_configuration_space(
-            exclude_estimators=['random_forest'],
+            exclude={'classifier': ['random_forest']},
             info={'task': MULTICLASS_CLASSIFICATION, 'is_sparse': False},
         ).get_default_configuration()
         self.assertEqual(configuration['classifier:__choice__'], 'liblinear_svc')