Raise all warnings in _catch_warnings

docs/source/release_notes.rst

@@ -3,6 +3,7 @@ Release Notes
 **Future Releases**
     * Enhancements
     * Fixes
+        * Fixed bug where warnings during ``make_pipeline`` were not being raised to the user :pr:`2765`
     * Changes
     * Testing Changes
 

evalml/automl/automl_search.py

@@ -579,7 +579,7 @@ def __init__(
         self.random_seed = random_seed
         self.n_jobs = n_jobs
 
-        if not self.plot:
+        if not self.plot and self.verbose:
             self.logger.warning(
                 "Unable to import plotly; skipping pipeline search plotting\n"
             )
@@ -847,19 +847,28 @@ def close_engine(self):
         self._engine.close()
 
     def _catch_warnings(self, warning_list):
-        if len(warning_list) == len(self.allowed_pipelines) and len(warning_list) > 0:
-            # we find the value(s) that we must throw any ParameterNotUsedWarnings for
+        parameter_not_used_warnings = []
+        raised_messages = []
+        for msg in warning_list:
+            if isinstance(msg.message, ParameterNotUsedWarning):
+                parameter_not_used_warnings.append(msg.message)
+            # Raise non-PNU warnings immediately, but only once per warning
+            elif str(msg.message) not in raised_messages:
+                warnings.warn(msg.message)
+                raised_messages.append(str(msg.message))
+
+        # Raise PNU warnings, iff the warning was raised in every pipeline
+        if len(parameter_not_used_warnings) == len(self.allowed_pipelines) and len(
+            parameter_not_used_warnings
+        ):
             final_message = set([])
-            for idx, msg in enumerate(warning_list):
-                if isinstance(msg.message, ParameterNotUsedWarning):
-                    if idx == 0:
-                        final_message = final_message.union(msg.message.components)
-                    else:
-                        final_message = final_message.intersection(
-                            msg.message.components
-                        )
-            if len(final_message):
-                warnings.warn(ParameterNotUsedWarning(final_message))
+            for msg in parameter_not_used_warnings:
+                if len(final_message) == 0:
+                    final_message = final_message.union(msg.components)
+                else:
+                    final_message = final_message.intersection(msg.components)
+
+            warnings.warn(ParameterNotUsedWarning(final_message))
 
     def _get_batch_number(self):
         batch_number = 1

evalml/tests/automl_tests/test_automl.py

@@ -5102,6 +5102,26 @@ def test_pipeline_parameter_warnings_only_parameternotused(
     assert str(warning) == "Big bad warning"
 
 
+@pytest.mark.parametrize("verbose", [True, False])
+def test_pipeline_parameter_warnings_with_other_types(verbose):
+    # Data that will trigger a UserWarning from `TargetLeakageDataCheck`
+    X = pd.DataFrame({"a": [1, 2, 2, 1, 2, 1, 2, 2, 1, 1] * 505})
+    y = [0.946, 0.972, 1.154, 0.954, 0.969, 1.222, 1.038, 0.999, 0.973, 0.897] * 505
+
+    with pytest.warns(None) as w:
+        AutoMLSearch(
+            X_train=X,
+            y_train=y,
+            problem_type="regression",
+            allowed_model_families=["random_forest", "linear_model"],
+            pipeline_parameters={"Decision Tree Classifier": {"max_depth": 1}},
+            verbose=verbose,
+        )
+    assert len(w) == 2
+    assert isinstance(w[0].message, UserWarning)
+    assert isinstance(w[1].message, ParameterNotUsedWarning)
+
+
 @pytest.mark.parametrize("nans", [None, pd.NA, np.nan])
 @patch("evalml.pipelines.components.Estimator.fit")
 @patch(