Internal target check to ensure no class missing from train/val

docs/source/release_notes.rst

@@ -7,6 +7,7 @@ Release Notes
         * Added `get_feature_names` on `OneHotEncoder` :pr:`1193`
         * Added LightGBM to AutoMLSearch :pr:`1199`
         * Updates scikit-learn and scikit-optimize to use latest versions - 0.23.2 and 0.8.1 respectively :pr:`1141`
+        * Included internal target check for both training and validation data in AutoMLSearch :pr:`1226`
         * Add `ProblemTypes.all_problem_types` helper to get list of supported problem types :pr:`1219`
     * Fixes
         * Updated GitHub URL after migration to Alteryx GitHub org :pr:`1207`

evalml/automl/automl_search.py

@@ -607,10 +607,18 @@ def _compute_cv_scores(self, pipeline, X, y):
         start = time.time()
         cv_data = []
         logger.info("\tStarting cross validation")
+        warnings.filterwarnings("ignore", lineno=665)
         for i, (train, test) in enumerate(self.data_split.split(X, y)):
             logger.debug(f"\t\tTraining and scoring on fold {i}")
             X_train, X_test = X.iloc[train], X.iloc[test]
             y_train, y_test = y.iloc[train], y.iloc[test]
+            if self.problem_type in [ProblemTypes.BINARY, ProblemTypes.MULTICLASS]:
+                diff_train = set(y).difference(set(y_train))
+                diff_test = set(y).difference(set(y_test))
+                diff_string = f"Missing target values in the training set: {diff_train}. " if diff_train else ""
+                diff_string += f"Missing target values in the test set: {diff_test}." if diff_test else ""
+                if diff_string:
+                    raise Exception(diff_string)
             objectives_to_score = [self.objective] + self.additional_objectives
             cv_pipeline = None
             try:

evalml/tests/automl_tests/test_automl.py

@@ -1080,3 +1080,54 @@ def test_max_batches_must_be_non_negative(max_batches):
 
 def test_can_print_out_automl_objective_names():
     AutoMLSearch.print_objective_names_allowed_in_automl()
+
+
+def test_data_split_binary(X_y_binary):
+    X, y = X_y_binary
+    y[:] = 0
+    y[0] = 1
+
+    automl = AutoMLSearch(problem_type='binary')
+    with pytest.raises(Exception, match="Missing target values in the"):
+        automl.search(X, y)
+    with pytest.raises(Exception, match="Missing target values in the"):
+        automl.search(X, y, data_checks="disabled")
+
+    y[1] = 1
+    with pytest.raises(Exception, match="Missing target values in the"):
+        automl.search(X, y)
+    with pytest.raises(Exception, match="Missing target values in the"):
+        automl.search(X, y, data_checks="disabled")
+
+    y[2] = 1
+    automl.search(X, y, data_checks="disabled")
+
+
+def test_data_split_multi(X_y_multi):
+    X, y = X_y_multi
+    y[:] = 1
+    y[0] = 0
+
+    automl = AutoMLSearch(problem_type='multiclass')
+    with pytest.raises(Exception, match="Missing target values"):
+        automl.search(X, y)
+    with pytest.raises(Exception, match="Missing target values"):
+        automl.search(X, y, data_checks="disabled")
+
+    y[1] = 2
+    with pytest.raises(Exception, match=r"(\{\d?\}.+\{\d?\})|(\{.+\,.+\})"):
+        automl.search(X, y)
+    with pytest.raises(Exception, match=r"(\{\d?\}.+\{\d?\})|(\{.+\,.+\})"):
+        automl.search(X, y, data_checks="disabled")
+
+    y[1] = 0
+    y[2:4] = 2
+    with pytest.raises(Exception, match="Missing target values"):
+        automl.search(X, y, data_checks="disabled")
+
+    y[4] = 2
+    with pytest.raises(Exception, match="Missing target values"):
+        automl.search(X, y, data_checks="disabled")
+
+    y[5] = 0
+    automl.search(X, y, data_checks="disabled")