Implement stacked ensemble classes

docs/source/release_notes.rst

@@ -4,6 +4,7 @@ Release Notes
 **Future Releases**
     * Enhancements
         * Added `output_format` field to explain predictions functions :pr:`1107`
+        * Added stacked ensemble component classes (StackedEnsembleClassifier, StackedEnsembleRegressor) :pr:`1134`
         * Modified `get_objective` and `get_objectives` to be able to return any objective in `evalml.objectives` :pr:`1132`
         * Added a `return_instance` boolean parameter to `get_objective` :pr:`1132`
         * Added `ClassImbalanceDataCheck` to determine whether target imbalance falls below a given threshold :pr:`1135`

evalml/exceptions/exceptions.py

@@ -38,6 +38,11 @@ class AutoMLSearchException(Exception):
     pass
 
 
+class EnsembleMissingEstimatorsError(Exception):
+    """An exception raised when an ensemble is missing `estimators` (list) as a parameter."""
+    pass
+
+
 class PipelineScoreError(Exception):
     """An exception raised when a pipeline errors while scoring any objective in a list of objectives.
 

evalml/model_family/model_family.py

@@ -9,7 +9,8 @@ class ModelFamily(Enum):
     LINEAR_MODEL = 'linear_model'
     CATBOOST = 'catboost'
     EXTRA_TREES = 'extra_trees'
-    BASELINE = 'baseline'
+    BASELINE = 'baseline',
+    ENSEMBLE = 'ensemble',
     NONE = 'none'
 
     def __str__(self):
@@ -20,6 +21,7 @@ def __str__(self):
                              ModelFamily.CATBOOST.name: "CatBoost",
                              ModelFamily.EXTRA_TREES.name: "Extra Trees",
                              ModelFamily.BASELINE.name: "Baseline",
+                             ModelFamily.ENSEMBLE.name: "Ensemble",
                              ModelFamily.NONE.name: "None"}
         return model_family_dict[self.name]
 

evalml/pipelines/components/__init__.py

@@ -36,3 +36,7 @@
     TextFeaturizer,
     LSA,
     )
+from .ensemble import (
+    StackedEnsembleClassifier,
+    StackedEnsembleRegressor
+)

evalml/pipelines/components/ensemble/__init__.py

@@ -0,0 +1,4 @@
+# flake8:noqa
+from .ensemble_base import EnsembleBase
+from .stacked_ensemble_classifier import StackedEnsembleClassifier
+from .stacked_ensemble_regressor import StackedEnsembleRegressor

evalml/pipelines/components/ensemble/ensemble_base.py

@@ -0,0 +1,5 @@
+from evalml.pipelines.components.estimators import Estimator
+
+
+class EnsembleBase(Estimator):
+    pass

evalml/pipelines/components/ensemble/stacked_ensemble_classifier.py

@@ -0,0 +1,59 @@
+from sklearn.ensemble import StackingClassifier
+
+from evalml.exceptions import EnsembleMissingEstimatorsError
+from evalml.model_family import ModelFamily
+from evalml.pipelines.components import LogisticRegressionClassifier
+from evalml.pipelines.components.ensemble import EnsembleBase
+from evalml.problem_types import ProblemTypes
+from evalml.utils.gen_utils import _nonstackable_model_families
+
+
+class StackedEnsembleClassifier(EnsembleBase):
+    """Stacked Ensemble Classifier."""
+    name = "Stacked Ensemble Classifier"
+    model_family = ModelFamily.ENSEMBLE
+    supported_problem_types = [ProblemTypes.BINARY, ProblemTypes.MULTICLASS]
+    hyperparameter_ranges = {}
+
+    def __init__(self, final_estimator=None, cv=None, n_jobs=-1, random_state=0, **kwargs):
+        """Stacked ensemble classifier.
+
+        Arguments:
+            final_estimator (Estimator or subclass): The classifier used to combine the base estimators. If None, uses LogisticRegressionClassifier.
+            cv (int, cross-validation generator or an iterable): Determines the cross-validation splitting strategy used to train final_estimator.
+                For int/None inputs, if the estimator is a classifier and y is either binary or multiclass, StratifiedKFold is used. In all other cases, KFold is used.
+                Possible inputs for cv are:
+                    - None: 5-fold cross validation
+                    - int: the number of folds in a (Stratified) KFold
+                    - An scikit-learn cross-validation generator object
+                    - An iterable yielding (train, test) splits
+            n_jobs (int or None): Non-negative integer describing level of parallelism used for pipelines.
+                None and 1 are equivalent. If set to -1, all CPUs are used. For n_jobs below -1, (n_cpus + 1 + n_jobs) are used.
+            random_state (int, np.random.RandomState): seed for the random number generator
+            **kwargs: 'estimators' containing a list of Estimator objects must be passed as a keyword argument, or else EnsembleMissingEstimatorsError will be raised
+        """
+        if 'estimators' not in kwargs:
+            raise EnsembleMissingEstimatorsError("`estimators` must be passed to the constructor as a keyword argument")
+        estimators = kwargs.get('estimators')
+        parameters = {
+            "estimators": estimators,
+            "final_estimator": final_estimator,
+            "cv": cv,
+            "n_jobs": n_jobs
+        }
+        contains_non_stackable = [estimator for estimator in estimators if estimator.model_family in _nonstackable_model_families]
+        if contains_non_stackable:
+            raise ValueError("Classifiers with any of the following model families cannot be used as base estimators in StackedEnsembleClassifier: {}".format(_nonstackable_model_families))
+        sklearn_parameters = parameters.copy()
+        parameters.update(kwargs)
+        if final_estimator is None:
+            final_estimator = LogisticRegressionClassifier()
+        sklearn_parameters.update({"final_estimator": final_estimator._component_obj})
+        sklearn_parameters.update({"estimators": [(estimator.name + f"({idx})", estimator._component_obj) for idx, estimator in enumerate(estimators)]})
+        super().__init__(parameters=parameters,
+                         component_obj=StackingClassifier(**sklearn_parameters),
+                         random_state=random_state)
+
+    @property
+    def feature_importance(self):
+        raise NotImplementedError("feature_importance is not implemented for StackedEnsembleClassifier")

evalml/pipelines/components/ensemble/stacked_ensemble_regressor.py

@@ -0,0 +1,80 @@
+from sklearn.ensemble import StackingRegressor
+
+from evalml.exceptions import EnsembleMissingEstimatorsError
+from evalml.model_family import ModelFamily
+from evalml.pipelines.components import LinearRegressor
+from evalml.pipelines.components.ensemble import EnsembleBase
+from evalml.problem_types import ProblemTypes
+from evalml.utils.gen_utils import _nonstackable_model_families
+
+
+class StackedEnsembleRegressor(EnsembleBase):
+    """Stacked Ensemble Regressor."""
+    name = "Stacked Ensemble Regressor"
+    model_family = ModelFamily.ENSEMBLE
+    supported_problem_types = [ProblemTypes.REGRESSION]
+    hyperparameter_ranges = {}
+
+    def __init__(self, final_estimator=None, cv=None, n_jobs=-1, random_state=0, **kwargs):
+        """Stacked ensemble regressor.
+
+        Arguments:
+            final_estimator (Estimator or subclass): The regressor used to combine the base estimators. If None, uses LinearRegressor.
+            cv (int, cross-validation generator or an iterable): Determines the cross-validation splitting strategy used to train final_estimator.
+                For int/None inputs, if the estimator is a classifier and y is either binary or multiclass, StratifiedKFold is used. In all other cases, KFold is used.
+                Possible inputs for cv are:
+                    - None: 5-fold cross validation
+                    - int: the number of folds in a (Stratified) KFold
+                    - An scikit-learn cross-validation generator object
+                    - An iterable yielding (train, test) splits
+            n_jobs (int or None): Non-negative integer describing level of parallelism used for pipelines.
+                None and 1 are equivalent. If set to -1, all CPUs are used. For n_jobs below -1, (n_cpus + 1 + n_jobs) are used.
+            random_state (int, np.random.RandomState): seed for the random number generator
+            **kwargs: 'estimators' containing a list of Estimator objects must be passed as a keyword argument, or else EnsembleMissingEstimatorsError will be raised
+        """
+        # if 'estimators' not in kwargs:
+        #     raise EnsembleMissingEstimatorsError("`estimators` must be passed to the constructor as a keyword argument")
+        # estimators = kwargs.get('estimators')
+        # parameters = {
+        #     "estimators": estimators,
+        #     "final_estimator": final_estimator,
+        #     "cv": cv,
+        #     "n_jobs": n_jobs
+        # }
+        # contains_non_stackable = [estimator for estimator in estimators if estimator.model_family in _nonstackable_model_families]
+        # if contains_non_stackable:
+        #     raise ValueError("Regressors with any of the following model families cannot be used as base estimators in StackedEnsembleRegressor: {}".format(_nonstackable_model_families))
+        # sklearn_parameters = parameters.copy()
+        # parameters.update(kwargs)
+        # if final_estimator is None:
+        #     final_estimator = LinearRegressor()
+        # sklearn_parameters.update({"final_estimator": final_estimator._component_obj})
+        # sklearn_parameters.update({"estimators": [(estimator.name + f"({idx})", estimator._component_obj) for idx, estimator in enumerate(estimators)]})
+        # super().__init__(parameters=parameters,
+        #                  component_obj=StackingRegressor(**sklearn_parameters),
+        #                  random_state=random_state)
+        if 'estimators' not in kwargs:
+            raise EnsembleMissingEstimatorsError("`estimators` must be passed to the constructor as a keyword argument")
+        estimators = kwargs.get('estimators')
+        parameters = {
+            "estimators": estimators,
+            "final_estimator": final_estimator,
+            "cv": cv,
+            "n_jobs": n_jobs
+        }
+        contains_non_stackable = [estimator for estimator in estimators if estimator.model_family in _nonstackable_model_families]
+        if contains_non_stackable:
+            raise ValueError("Regressors with any of the following model families cannot be used as base estimators in StackedEnsembleRegressor: {}".format(_nonstackable_model_families))
+        sklearn_parameters = parameters.copy()
+        parameters.update(kwargs)
+        if final_estimator is None:
+            final_estimator = LinearRegressor()
+        sklearn_parameters.update({"final_estimator": final_estimator._component_obj})
+        sklearn_parameters.update({"estimators": [(estimator.name + f"({idx})", estimator._component_obj) for idx, estimator in enumerate(estimators)]})
+        super().__init__(parameters=parameters,
+                         component_obj=StackingRegressor(**sklearn_parameters),
+                         random_state=random_state)
+
+    @property
+    def feature_importance(self):
+        raise NotImplementedError("feature_importance is not implemented for StackedEnsembleRegressor")

evalml/tests/component_tests/test_components.py

@@ -11,6 +11,7 @@
 
 from evalml.exceptions import (
     ComponentNotYetFittedError,
+    EnsembleMissingEstimatorsError,
     MethodPropertyNotFoundError
 )
 from evalml.model_family import ModelFamily
@@ -36,6 +37,10 @@
     Transformer,
     XGBoostClassifier
 )
+from evalml.pipelines.components.ensemble import (
+    StackedEnsembleClassifier,
+    StackedEnsembleRegressor
+)
 from evalml.pipelines.components.utils import (
     _all_estimators,
     _all_estimators_used_in_search,
@@ -353,7 +358,11 @@ def test_component_parameters_getter(test_classes):
 def test_component_parameters_init():
     for component_class in all_components():
         print('Testing component {}'.format(component_class.name))
-        component = component_class()
+        try:
+            component = component_class()
+        except EnsembleMissingEstimatorsError:
+            component = component_class(estimators=[])
+
         parameters = component.parameters
 
         component2 = component_class(**parameters)
@@ -403,7 +412,10 @@ def test_clone_fitted(X_y_binary):
 
 def test_components_init_kwargs():
     for component_class in all_components():
-        component = component_class()
+        try:
+            component = component_class()
+        except EnsembleMissingEstimatorsError:
+            continue
         if component._component_obj is None:
             continue
 
@@ -516,13 +528,12 @@ def test_estimator_predict_output_type(X_y_binary):
             assert (predict_proba_output.columns == y_cols_expected).all()
 
 
-@pytest.mark.parametrize("cls", all_components())
+@pytest.mark.parametrize("cls", [cls for cls in all_components() if cls not in [StackedEnsembleRegressor, StackedEnsembleClassifier]])
 def test_default_parameters(cls):
-
     assert cls.default_parameters == cls().parameters, f"{cls.__name__}'s default parameters don't match __init__."
 
 
-@pytest.mark.parametrize("cls", all_components())
+@pytest.mark.parametrize("cls", [cls for cls in all_components() if cls not in [StackedEnsembleRegressor, StackedEnsembleClassifier]])
 def test_default_parameters_raise_no_warnings(cls):
     with warnings.catch_warnings(record=True) as w:
         warnings.simplefilter("always")
@@ -701,7 +712,8 @@ def test_all_transformers_check_fit(X_y_binary):
 
 def test_all_estimators_check_fit(X_y_binary, test_estimator_needs_fitting_false):
     X, y = X_y_binary
-    for component_class in _all_estimators() + [test_estimator_needs_fitting_false]:
+    estimators_to_check = [estimator for estimator in _all_estimators() if estimator not in [StackedEnsembleClassifier, StackedEnsembleRegressor]] + [test_estimator_needs_fitting_false]
+    for component_class in estimators_to_check:
         if not component_class.needs_fitting:
             continue
 
@@ -739,20 +751,38 @@ def test_no_fitting_required_components(X_y_binary, test_estimator_needs_fitting
 def test_serialization(X_y_binary, tmpdir):
     X, y = X_y_binary
     path = os.path.join(str(tmpdir), 'component.pkl')
-
     for component_class in all_components():
         print('Testing serialization of component {}'.format(component_class.name))
+        try:
+            component = component_class()
+        except EnsembleMissingEstimatorsError:
+            if (component_class == StackedEnsembleClassifier):
+                component = component_class(estimators=[RandomForestClassifier(n_estimators=2)])
+            elif (component_class == StackedEnsembleRegressor):
+                component = component_class(estimators=[RandomForestRegressor(n_estimators=2)])
 
-        component = component_class()
         component.fit(X, y)
 
         for pickle_protocol in range(cloudpickle.DEFAULT_PROTOCOL + 1):
             component.save(path, pickle_protocol=pickle_protocol)
             loaded_component = ComponentBase.load(path)
-            assert component.parameters == loaded_component.parameters
-            assert component.describe(return_dict=True) == loaded_component.describe(return_dict=True)
-            if issubclass(component_class, Estimator):
-                assert (component.feature_importance == loaded_component.feature_importance).all()
+            if isinstance(component, StackedEnsembleClassifier) or isinstance(component, StackedEnsembleRegressor):
+                # test all parameters except "estimators"
+                params_without_estimators = {key: value for key, value in component.parameters.items() if key != "estimators"}
+                loaded_params_without_estimators = {key: value for key, value in loaded_component.parameters.items() if key != "estimators"}
+                assert params_without_estimators == loaded_params_without_estimators
+                estimators = component.parameters['estimators']
+                loaded_estimators = loaded_component.parameters['estimators']
+                # test equality for each estimator in estimators
+                for est, loaded_est in zip(estimators, loaded_estimators):
+                    assert est.parameters == loaded_est.parameters
+                    assert est.describe(return_dict=True) == loaded_est.describe(return_dict=True)
+
+            else:
+                assert component.parameters == loaded_component.parameters
+                assert component.describe(return_dict=True) == loaded_component.describe(return_dict=True)
+                if issubclass(component_class, Estimator):
+                    assert (component.feature_importance == loaded_component.feature_importance).all()
 
 
 @patch('cloudpickle.dump')

evalml/tests/component_tests/test_lgbm_classifier.py

@@ -2,7 +2,7 @@
 
 import numpy as np
 import pandas as pd
-from pandas._testing import assert_frame_equal, assert_series_equal
+from pandas.testing import assert_frame_equal, assert_series_equal
 from pytest import importorskip
 
 from evalml.model_family import ModelFamily