Implement Early Stopping for AutoML

docs/source/changelog.rst

@@ -4,6 +4,7 @@ Changelog
 ---------
 **Future Releases**
     * Enhancements
+        * Add early stopping to AutoML :pr:`241`
         * Added ROC and confusion matrix metrics and plot for classification problems and introduce PipelineSearchPlots class :pr:`242`
     * Fixes
         * Lower botocore requirement :pr:`235`

evalml/models/auto_base.py

@@ -23,13 +23,14 @@ class AutoBase:
     plot = PipelineSearchPlots
 
     def __init__(self, problem_type, tuner, cv, objective, max_pipelines, max_time,
-                 model_types, detect_label_leakage, start_iteration_callback,
+                 patience, model_types, detect_label_leakage, start_iteration_callback,
                  add_result_callback, additional_objectives, random_state, verbose):
         if tuner is None:
             tuner = SKOptTuner
         self.objective = get_objective(objective)
         self.problem_type = problem_type
         self.max_pipelines = max_pipelines
+        self.patience = patience
         self.model_types = model_types
         self.detect_label_leakage = detect_label_leakage
         self.start_iteration_callback = start_iteration_callback
@@ -58,6 +59,11 @@ def __init__(self, problem_type, tuner, cv, objective, max_pipelines, max_time,
             self.max_time = convert_to_seconds(max_time)
         else:
             raise TypeError("max_time must be a float, int, or string. Received a {}.".format(type(max_time)))
+
+        if self.patience:
+            if (not isinstance(self.patience, int)) or self.patience < 0:
+                raise ValueError("patience value must be a positive integer. Received {} instead".format(self.patience))
+
         self.results = {}
         self.trained_pipelines = {}
         self.random_state = random_state
@@ -94,7 +100,6 @@ def fit(self, X, y, feature_types=None, raise_errors=False):
 
             self
         """
-        # make everything pandas objects
         if not isinstance(X, pd.DataFrame):
             X = pd.DataFrame(X)
 
@@ -130,26 +135,60 @@ def fit(self, X, y, feature_types=None, raise_errors=False):
                 self.logger.log("WARNING: Possible label leakage: %s" % ", ".join(leaked))
 
         if self.max_pipelines is None:
-            start = time.time()
             pbar = tqdm(total=self.max_time, disable=not self.verbose, file=stdout, bar_format='{desc} |    Elapsed:{elapsed}')
             pbar._instances.clear()
-            while time.time() - start <= self.max_time:
-                self._do_iteration(X, y, pbar, raise_errors)
-            pbar.close()
         else:
             pbar = tqdm(range(self.max_pipelines), disable=not self.verbose, file=stdout, bar_format='{desc}   {percentage:3.0f}%|{bar}| Elapsed:{elapsed}')
             pbar._instances.clear()
-            start = time.time()
-            for n in pbar:
-                elapsed = time.time() - start
-                if self.max_time and elapsed > self.max_time:
-                    pbar.close()
-                    self.logger.log("\n\nMax time elapsed. Stopping search early.")
-                    break
-                self._do_iteration(X, y, pbar, raise_errors)
-            pbar.close()
-
-        self.logger.log("\n✔ Optimization finished")
+
+        start = time.time()
+        self._do_iteration(X, y, pbar, raise_errors)
+        pbar.update(1)
+        while self._check_stopping_condition(start):
+            self._do_iteration(X, y, pbar, raise_errors)
+            pbar.update(1)
+
+        desc = "✔ Optimization finished"
+        desc = desc.ljust(self._MAX_NAME_LEN)
+        pbar.set_description_str(desc=desc, refresh=True)
+        pbar.close()
+
+    def _check_stopping_condition(self, start):
+        cont = True
+        msg = None
+
+        # check max_time and max_pipelines
+        elapsed = time.time() - start
+        if self.max_time and elapsed >= self.max_time:
+            cont = False
+        elif self.max_pipelines and len(self.results) >= self.max_pipelines:
+            cont = False
+
+        # check patience
+        curr_id = max(self.results, key=int)
+        if self.objective.greater_is_better:
+            best_id = max(self.results, key=lambda x: self.results[x]['score'])
+        else:
+            best_id = min(self.results, key=lambda x: self.results[x]['score'])
+
+        best_score = self.results[best_id]['score']
+        if self.patience is not None and curr_id >= best_id + self.patience:
+            ids_to_check = [i for i in range(best_id, best_id + self.patience + 1)]
+            scores_to_check = [self.results[id]['score'] for id in ids_to_check]
+            without_improvement = 0
+            for score in scores_to_check:
+                if self.objective.greater_is_better:
+                    if score <= best_score:
+                        without_improvement += 1
+                else:
+                    if score >= best_score:
+                        without_improvement += 1
+            if without_improvement >= self.patience:
+                cont = False
+                msg = "\n\n{} iterations without improvement. Stopping search early...".format(self.patience)
+        if not cont and msg:
+            self.logger.log(msg)
+        return cont
 
     def check_multiclass(self, y):
         if y.nunique() <= 2:
@@ -226,6 +265,9 @@ def _do_iteration(self, X, y, pbar, raise_errors):
         if self.verbose:  # To force new line between progress bar iterations
             print('')
 
+        # return average CV score
+        return score
+
     def _select_pipeline(self):
         return random.choice(self.possible_pipelines)
 

evalml/models/auto_classifier.py

@@ -17,6 +17,7 @@ def __init__(self,
                  multiclass=False,
                  max_pipelines=None,
                  max_time=None,
+                 patience=None,
                  model_types=None,
                  cv=None,
                  tuner=None,
@@ -41,6 +42,9 @@ def __init__(self,
                 has elapsed. If it is an integer, then the time will be in seconds.
                 For strings, time can be specified as seconds, minutes, or hours.
 
+            patience (int): Number of iterations without improvement to stop search early. Must be positive.
+                If None, early stopping is disabled. Defaults to None.
+
             model_types (list): The model types to search. By default searches over all
                 model_types. Run evalml.list_model_types("classification") to see options.
 
@@ -84,6 +88,7 @@ def __init__(self,
             cv=cv,
             max_pipelines=max_pipelines,
             max_time=max_time,
+            patience=patience,
             model_types=model_types,
             problem_type=problem_type,
             detect_label_leakage=detect_label_leakage,

evalml/models/auto_regressor.py

@@ -14,6 +14,7 @@ def __init__(self,
                  objective=None,
                  max_pipelines=None,
                  max_time=None,
+                 patience=None,
                  model_types=None,
                  cv=None,
                  tuner=None,
@@ -39,6 +40,9 @@ def __init__(self,
             model_types (list): The model types to search. By default searches over all
                 model_types. Run evalml.list_model_types("regression") to see options.
 
+            patience (int): Number of iterations without improvement to stop search early. Must be positive.
+                If None, early stopping is disabled. Defaults to None.
+
             cv: cross validation method to use. By default StratifiedKFold
 
             tuner: the tuner class to use. Defaults to scikit-optimize tuner
@@ -74,6 +78,7 @@ def __init__(self,
             cv=cv,
             max_pipelines=max_pipelines,
             max_time=max_time,
+            patience=patience,
             model_types=model_types,
             problem_type=problem_type,
             detect_label_leakage=detect_label_leakage,

evalml/tests/automl_tests/test_autoclassifier.py

@@ -269,3 +269,15 @@ def test_max_time_units():
 
     with pytest.raises(TypeError, match="max_time must be a float, int, or string. Received a <class 'tuple'>."):
         AutoClassifier(objective='F1', max_time=(30, 'minutes'))
+
+
+def test_early_stopping(capsys, X_y):
+    X, y = X_y
+
+    with pytest.raises(ValueError, match='patience value must be a positive integer.'):
+        clf = AutoClassifier(objective='AUC', max_pipelines=5, model_types=['linear_model'], patience=-1, random_state=0)
+
+    clf = AutoClassifier(objective='AUC', max_pipelines=5, model_types=['linear_model'], patience=1, random_state=0)
+    clf.fit(X, y)
+    out, _ = capsys.readouterr()
+    assert "1 iterations without improvement. Stopping search early." in out

evalml/tests/automl_tests/test_autoregressor.py

@@ -81,3 +81,24 @@ def add_result_callback(results, trained_pipeline, counts=counts):
 
     assert counts["start_iteration_callback"] == max_pipelines
     assert counts["add_result_callback"] == max_pipelines
+
+
+def test_early_stopping(capsys, X_y):
+    X, y = X_y
+    clf = AutoRegressor(objective='r2', max_pipelines=5, patience=1, model_types=['linear_model'], random_state=0)
+    clf.fit(X, y)
+
+    out, _ = capsys.readouterr()
+    assert "1 iterations without improvement. Stopping search early." in out
+
+    clf = AutoRegressor(objective='r2', max_time='60 seconds', patience=1, model_types=['linear_model'], random_state=0)
+    clf.fit(X, y)
+
+    out, _ = capsys.readouterr()
+    assert "1 iterations without improvement. Stopping search early." in out
+
+    clf = AutoRegressor(objective='mse', max_time='60 seconds', patience=1, model_types=['linear_model'], random_state=0)
+    clf.fit(X, y)
+
+    out, _ = capsys.readouterr()
+    assert "1 iterations without improvement. Stopping search early." in out