Replace allowed_pipelines with allowed_component_graphs

docs/source/release_notes.rst

@@ -9,6 +9,7 @@ Release Notes
         * Added change for ``k_neighbors`` parameter in SMOTE Oversamplers to automatically handle small samples :pr:`2375`
         * Changed naming for ``Logistic Regression Classifier`` file :pr:`2399`
     * Changes
+        * Replaced `allowed_pipelines` with `allowed_component_graphs` :pr:`2364`
         * Removed private method ``_compute_features_during_fit`` from ``PipelineBase`` :pr:`2359`
     * Documentation Changes
         * Fixed start page code and description dataset naming discrepancy :pr:`2370`
@@ -23,6 +24,7 @@ Release Notes
 .. warning::
 
     **Breaking Changes**
+        * `AutoMLSearch` will accept `allowed_component_graphs` instead of `allowed_pipelines` :pr:`2364`
 
 
 **v0.26.0 Jun. 08, 2021**

docs/source/user_guide/automl.ipynb

@@ -220,7 +220,7 @@
    "source": [
     "### Using custom pipelines\n",
     "\n",
-    "EvalML's AutoML algorithm generates a set of pipelines to search with. To provide a custom set instead, set allowed_pipelines to a list of custom pipeline instances. Note: this will prevent AutoML from generating other pipelines to search over."
+    "EvalML's AutoML algorithm generates a set of pipelines to search with. To provide a custom set instead, set allowed_component_graphs to a dictionary of custom component graphs. `AutoMLSearch` will use these to generate `Pipeline` instances. Note: this will prevent AutoML from generating other pipelines to search over."
    ]
   },
   {
@@ -235,7 +235,8 @@
     "automl_custom = evalml.automl.AutoMLSearch(X_train=X_train,\n",
     "                                           y_train=y_train,\n",
     "                                           problem_type='multiclass',\n",
-    "                                           allowed_pipelines=[MulticlassClassificationPipeline(component_graph=['Simple Imputer', 'Random Forest Classifier'])])"
+    "                                           allowed_component_graphs={\"My_pipeline\": ['Simple Imputer', 'Random Forest Classifier'],\n",
+    "                                                                     \"My_other_pipeline\": ['One Hot Encoder', 'Random Forest Classifier']})"
    ]
   },
   {

evalml/automl/automl_algorithm/iterative_algorithm.py

@@ -37,7 +37,6 @@ def __init__(
         text_in_ensembling=False,
         pipeline_params=None,
         custom_hyperparameters=None,
-        _frozen_pipeline_parameters=None,
         _estimator_family_order=None,
     ):
         """An automl algorithm which first fits a base round of pipelines with default parameters, then does a round of parameter tuning on each pipeline in order of performance.
@@ -54,7 +53,6 @@ def __init__(
             text_in_ensembling (boolean): If True and ensembling is True, then n_jobs will be set to 1 to avoid downstream sklearn stacking issues related to nltk.
             pipeline_params (dict or None): Pipeline-level parameters that should be passed to the proposed pipelines.
             custom_hyperparameters (dict or None): Custom hyperparameter ranges specified for pipelines to iterate over.
-            _frozen_pipeline_parameters (dict or None): Pipeline-level parameters are frozen and used in the proposed pipelines.
             _estimator_family_order (list(ModelFamily) or None): specify the sort order for the first batch. Defaults to _ESTIMATOR_FAMILY_ORDER.
         """
         self._estimator_family_order = (
@@ -95,7 +93,6 @@ def __init__(
         self.text_in_ensembling = text_in_ensembling
         self._pipeline_params = pipeline_params or {}
         self._custom_hyperparameters = custom_hyperparameters or {}
-        self._frozen_pipeline_parameters = _frozen_pipeline_parameters or {}
 
         if custom_hyperparameters and not isinstance(custom_hyperparameters, dict):
             raise ValueError(
@@ -136,7 +133,7 @@ def next_batch(self):
         if self._batch_number == 0:
             next_batch = [
                 pipeline.new(
-                    parameters=self._combine_parameters(pipeline, {}),
+                    parameters=self._transform_parameters(pipeline, {}),
                     random_seed=self.random_seed,
                 )
                 for pipeline in self.allowed_pipelines
@@ -152,7 +149,7 @@ def next_batch(self):
             for pipeline_dict in self._best_pipeline_info.values():
                 pipeline = pipeline_dict["pipeline"]
                 pipeline_params = pipeline_dict["parameters"]
-                parameters = self._combine_parameters(pipeline, pipeline_params)
+                parameters = self._transform_parameters(pipeline, pipeline_params)
                 input_pipelines.append(
                     pipeline.new(parameters=parameters, random_seed=self.random_seed)
                 )
@@ -175,21 +172,14 @@ def next_batch(self):
             pipeline = self._first_batch_results[idx][1]
             for i in range(self.pipelines_per_batch):
                 proposed_parameters = self._tuners[pipeline.name].propose()
-                parameters = self._combine_parameters(pipeline, proposed_parameters)
+                parameters = self._transform_parameters(pipeline, proposed_parameters)
                 next_batch.append(
                     pipeline.new(parameters=parameters, random_seed=self.random_seed)
                 )
         self._pipeline_number += len(next_batch)
         self._batch_number += 1
         return next_batch
 
-    def _combine_parameters(self, pipeline, proposed_parameters):
-        """Helper function for logic to transform proposed parameters and frozen parameters."""
-        return {
-            **self._transform_parameters(pipeline, proposed_parameters),
-            **self._frozen_pipeline_parameters,
-        }
-
     def add_result(self, score_to_minimize, pipeline, trained_pipeline_results):
         """Register results from evaluating a pipeline
 

evalml/automl/automl_search.py

@@ -20,6 +20,7 @@
     check_all_pipeline_names_unique,
     get_best_sampler_for_data,
     get_default_primary_search_objective,
+    get_pipelines_from_component_graphs,
     make_data_splitter,
 )
 from evalml.data_checks import DefaultDataChecks
@@ -36,8 +37,8 @@
 )
 from evalml.pipelines import (
     BinaryClassificationPipeline,
+    ComponentGraph,
     MulticlassClassificationPipeline,
-    PipelineBase,
     RegressionPipeline,
     TimeSeriesBinaryClassificationPipeline,
     TimeSeriesMulticlassClassificationPipeline,
@@ -137,7 +138,7 @@ def __init__(
         patience=None,
         tolerance=None,
         data_splitter=None,
-        allowed_pipelines=None,
+        allowed_component_graphs=None,
         allowed_model_families=None,
         start_iteration_callback=None,
         add_result_callback=None,
@@ -190,10 +191,14 @@ def __init__(
             tolerance (float): Minimum percentage difference to qualify as score improvement for early stopping.
                 Only applicable if patience is not None. Defaults to None.
 
-            allowed_pipelines (list(class)): A list of PipelineBase subclasses indicating the pipelines allowed in the search.
-                The default of None indicates all pipelines for this problem type are allowed. Setting this field will cause
+            allowed_component_graphs (dict): A dictionary of lists or ComponentGraphs indicating the component graphs allowed in the search.
+                The format should follow { "Name_0": [list_of_components], "Name_1": [ComponentGraph(...)] }
+
+                The default of None indicates all pipeline component graphs for this problem type are allowed. Setting this field will cause
                 allowed_model_families to be ignored.
 
+                e.g. allowed_component_graphs = { "My_Graph": ["Imputer", "One Hot Encoder", "Random Forest Classifier"] }
+
             allowed_model_families (list(str, ModelFamily)): The model families to search. The default of None searches over all
                 model families. Run evalml.pipelines.components.utils.allowed_model_families("binary") to see options. Change `binary`
                 to `multiclass` or `regression` depending on the problem type. Note that if allowed_pipelines is provided,
@@ -399,18 +404,17 @@ def __init__(
             logger.warning(
                 "Unable to import plotly; skipping pipeline search plotting\n"
             )
-
-        if allowed_pipelines is not None and not isinstance(allowed_pipelines, list):
-            raise ValueError(
-                "Parameter allowed_pipelines must be either None or a list!"
-            )
-        if allowed_pipelines is not None and not all(
-            isinstance(p, PipelineBase) for p in allowed_pipelines
-        ):
-            raise ValueError(
-                "Every element of allowed_pipelines must an instance of PipelineBase!"
-            )
-        self.allowed_pipelines = allowed_pipelines
+        if allowed_component_graphs is not None:
+            if not isinstance(allowed_component_graphs, dict):
+                raise ValueError(
+                    "Parameter allowed_component_graphs must be either None or a dictionary!"
+                )
+            for graph_name, graph in allowed_component_graphs.items():
+                if not isinstance(graph, (list, dict, ComponentGraph)):
+                    raise ValueError(
+                        "Every component graph passed must be of type list, dictionary, or ComponentGraph!"
+                    )
+        self.allowed_component_graphs = allowed_component_graphs
         self.allowed_model_families = allowed_model_families
         self._automl_algorithm = None
         self._start = 0.0
@@ -442,15 +446,11 @@ def __init__(
         self.custom_hyperparameters = custom_hyperparameters or {}
         self.search_iteration_plot = None
         self._interrupted = False
-        self._frozen_pipeline_parameters = {}
 
         parameters = copy.copy(self.pipeline_parameters)
 
         if self.problem_configuration:
             parameters.update({"pipeline": self.problem_configuration})
-            self._frozen_pipeline_parameters.update(
-                {"pipeline": self.problem_configuration}
-            )
 
         self.sampler_method = sampler_method
         self.sampler_balanced_ratio = sampler_balanced_ratio
@@ -473,11 +473,8 @@ def __init__(
                 parameters[self._sampler_name].update(
                     {"sampling_ratio": self.sampler_balanced_ratio}
                 )
-            self._frozen_pipeline_parameters[self._sampler_name] = parameters[
-                self._sampler_name
-            ]
 
-        if self.allowed_pipelines is None:
+        if self.allowed_component_graphs is None:
             logger.info("Generating pipelines to search over...")
             allowed_estimators = get_estimators(
                 self.problem_type, self.allowed_model_families
@@ -492,26 +489,30 @@ def __init__(
             )
             index_columns = list(self.X_train.ww.select("index").columns)
             if len(index_columns) > 0 and drop_columns is None:
-                self._frozen_pipeline_parameters["Drop Columns Transformer"] = {
-                    "columns": index_columns
-                }
+                parameters["Drop Columns Transformer"] = {"columns": index_columns}
             self.allowed_pipelines = [
                 make_pipeline(
                     self.X_train,
                     self.y_train,
                     estimator,
                     self.problem_type,
-                    parameters=self._frozen_pipeline_parameters,
+                    parameters=parameters,
                     sampler_name=self._sampler_name,
                 )
                 for estimator in allowed_estimators
             ]
+        else:
+            self.allowed_pipelines = get_pipelines_from_component_graphs(
+                self.allowed_component_graphs,
+                self.problem_type,
+                parameters,
+                self.random_seed,
+            )
 
         if self.allowed_pipelines == []:
             raise ValueError("No allowed pipelines to search")
 
         logger.info(f"{len(self.allowed_pipelines)} pipelines ready for search.")
-        check_all_pipeline_names_unique(self.allowed_pipelines)
 
         run_ensembling = self.ensembling
         text_in_ensembling = len(self.X_train.ww.select("natural_language").columns) > 0
@@ -605,8 +606,7 @@ def __init__(
             ensembling=run_ensembling,
             text_in_ensembling=text_in_ensembling,
             pipeline_params=parameters,
-            custom_hyperparameters=self.custom_hyperparameters,
-            _frozen_pipeline_parameters=self._frozen_pipeline_parameters,
+            custom_hyperparameters=custom_hyperparameters,
         )
 
     def _get_batch_number(self):
@@ -937,14 +937,6 @@ def _validate_problem_type(self):
                     )
                 )
 
-        for pipeline in self.allowed_pipelines or []:
-            if pipeline.problem_type != self.problem_type:
-                raise ValueError(
-                    "Given pipeline {} is not compatible with problem_type {}.".format(
-                        pipeline.name, self.problem_type.value
-                    )
-                )
-
     def _get_baseline_pipeline(self):
         """Creates a baseline pipeline instance."""
         if self.problem_type == ProblemTypes.BINARY:

evalml/automl/utils.py

@@ -5,7 +5,15 @@
 from sklearn.model_selection import KFold, StratifiedKFold
 
 from evalml.objectives import get_objective
-from evalml.pipelines import ComponentGraph
+from evalml.pipelines import (
+    BinaryClassificationPipeline,
+    ComponentGraph,
+    MulticlassClassificationPipeline,
+    RegressionPipeline,
+    TimeSeriesBinaryClassificationPipeline,
+    TimeSeriesMulticlassClassificationPipeline,
+    TimeSeriesRegressionPipeline,
+)
 from evalml.preprocessing.data_splitters import (
     TimeSeriesSplit,
     TrainingValidationSplit,
@@ -220,3 +228,39 @@ def get_hyperparameter_ranges(component_graph, custom_hyperparameters):
             component_hyperparameters.update(custom_hyperparameters[component_name])
         hyperparameter_ranges[component_name] = component_hyperparameters
     return hyperparameter_ranges
+
+
+def get_pipelines_from_component_graphs(
+    component_graphs_dict, problem_type, parameters=None, random_seed=0
+):
+    """
+    Returns created pipelines from passed component graphs based on the specified problem type.
+
+    Arguments:
+        component_graphs_dict (dict): The dict of component graphs.
+        problem_type (str or ProblemType): The problem type for which pipelines will be created.
+        parameters (dict or None): Pipeline-level parameters that should be passed to the proposed pipelines.
+        random_seed (int): Random seed.
+
+    Returns:
+        list: List of pipelines made from the passed component graphs.
+    """
+    pipeline_class = {
+        ProblemTypes.BINARY: BinaryClassificationPipeline,
+        ProblemTypes.MULTICLASS: MulticlassClassificationPipeline,
+        ProblemTypes.REGRESSION: RegressionPipeline,
+        ProblemTypes.TIME_SERIES_BINARY: TimeSeriesBinaryClassificationPipeline,
+        ProblemTypes.TIME_SERIES_MULTICLASS: TimeSeriesMulticlassClassificationPipeline,
+        ProblemTypes.TIME_SERIES_REGRESSION: TimeSeriesRegressionPipeline,
+    }[handle_problem_types(problem_type)]
+    created_pipelines = []
+    for graph_name, component_graph in component_graphs_dict.items():
+        created_pipelines.append(
+            pipeline_class(
+                component_graph=component_graph,
+                parameters=parameters,
+                custom_name=graph_name,
+                random_seed=random_seed,
+            )
+        )
+    return created_pipelines

evalml/pipelines/time_series_regression_pipeline.py

@@ -38,7 +38,7 @@ def __init__(
                  Pipeline(parameters={"pipeline": {"date_index": "Date", "max_delay": 4, "gap": 2}}).
             random_seed (int): Seed for the random number generator. Defaults to 0.
         """
-        if "pipeline" not in parameters:
+        if not parameters or "pipeline" not in parameters:
             raise ValueError(
                 "date_index, gap, and max_delay parameters cannot be omitted from the parameters dict. "
                 "Please specify them as a dictionary with the key 'pipeline'."

evalml/tests/automl_tests/dask_tests/test_automl_dask.py

@@ -3,6 +3,7 @@
 from dask.distributed import Client, LocalCluster
 
 from evalml.automl import AutoMLSearch
+from evalml.automl.automl_algorithm import IterativeAlgorithm
 from evalml.automl.callbacks import raise_error_callback
 from evalml.automl.engine import DaskEngine, SequentialEngine
 from evalml.tests.automl_tests.dask_test_utils import (
@@ -11,6 +12,7 @@
     TestPipelineWithFitError,
     TestPipelineWithScoreError,
 )
+from evalml.tuners import SKOptTuner
 
 
 @pytest.fixture
@@ -107,8 +109,9 @@ def test_automl_train_dask_error_callback(X_y_binary_cls, cluster, caplog):
             problem_type="binary",
             engine=parallel_engine,
             max_iterations=2,
-            allowed_pipelines=pipelines,
         )
+        automl.allowed_pipelines = pipelines
+
         automl.train_pipelines(pipelines)
         assert "Train error for PipelineWithError: Yikes" in caplog.text
 
@@ -127,8 +130,9 @@ def test_automl_score_dask_error_callback(X_y_binary_cls, cluster, caplog):
             problem_type="binary",
             engine=parallel_engine,
             max_iterations=2,
-            allowed_pipelines=pipelines,
         )
+        automl.allowed_pipelines = pipelines
+
         automl.score_pipelines(
             pipelines, X, y, objectives=["Log Loss Binary", "F1", "AUC"]
         )
@@ -153,10 +157,22 @@ def test_automl_immediate_quit(X_y_binary_cls, cluster, caplog):
             problem_type="binary",
             engine=parallel_engine,
             max_iterations=4,
-            allowed_pipelines=pipelines,
             error_callback=raise_error_callback,
             optimize_thresholds=False,
         )
+        automl._automl_algorithm = IterativeAlgorithm(
+            max_iterations=4,
+            allowed_pipelines=pipelines,
+            tuner_class=SKOptTuner,
+            random_seed=0,
+            n_jobs=-1,
+            number_features=X.shape[1],
+            pipelines_per_batch=5,
+            ensembling=False,
+            text_in_ensembling=False,
+            pipeline_params={},
+            custom_hyperparameters=None,
+        )
 
         # Ensure the broken pipeline raises the error
         with pytest.raises(Exception, match="Yikes"):