diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst index 14d30a0f37..b1e19aab6f 100644 --- a/docs/source/release_notes.rst +++ b/docs/source/release_notes.rst @@ -8,6 +8,7 @@ Release Notes * Added `explain_predictions` and `explain_predictions_best_worst` for explaining multiple predictions with SHAP :pr:`1016` * Added new LSA component for text featurization :pr:`1022` * Added guide on installing with conda :pr:`1041` + * Added `percent_better_than_baseline` to Automl search rankings and full rankings table :pr:`1050` * Fixes * Updated TextFeaturizer component to no longer require an internet connection to run :pr:`1022` * Fixed non-deterministic element of TextFeaturizer transformations :pr:`1022` diff --git a/evalml/automl/automl_search.py b/evalml/automl/automl_search.py index 7c2476c24f..0131967401 100644 --- a/evalml/automl/automl_search.py +++ b/evalml/automl/automl_search.py @@ -197,6 +197,7 @@ def __init__(self, self.allowed_model_families = allowed_model_families self._automl_algorithm = None self._start = None + self._baseline_score = None self._validate_problem_type() @@ -528,7 +529,7 @@ def _add_baseline_pipelines(self, X, y): self._start) baseline_results = self._compute_cv_scores(baseline, X, y) - self.baseline_score = baseline_results["cv_score_mean"] + self._baseline_score = baseline_results["cv_score_mean"] self._add_result(trained_pipeline=baseline, parameters=baseline.parameters, training_time=baseline_results['training_time'], @@ -615,7 +616,7 @@ def _compute_cv_scores(self, pipeline, X, y): def _add_result(self, trained_pipeline, parameters, training_time, cv_data, cv_scores): cv_score = cv_scores.mean() - percent_better = np.round(self.objective.calculate_percent_difference(cv_score, self.baseline_score), 2) + percent_better = np.round(self.objective.calculate_percent_difference(cv_score, self._baseline_score), 2) # calculate high_variance_cv # if the coefficient of variance is greater than .2 with warnings.catch_warnings(): diff --git a/evalml/tests/automl_tests/test_automl.py b/evalml/tests/automl_tests/test_automl.py index 1fa9f00041..cb9581869b 100644 --- a/evalml/tests/automl_tests/test_automl.py +++ b/evalml/tests/automl_tests/test_automl.py @@ -55,7 +55,8 @@ def test_search_results(X_y_regression, X_y_binary, X_y_multi, automl_type): assert automl.results['search_order'] == [0, 1] assert len(automl.results['pipeline_results']) == 2 for pipeline_id, results in automl.results['pipeline_results'].items(): - assert results.keys() == {'id', 'pipeline_name', 'pipeline_class', 'pipeline_summary', 'parameters', 'score', 'high_variance_cv', 'training_time', 'cv_data'} + assert results.keys() == {'id', 'pipeline_name', 'pipeline_class', 'pipeline_summary', 'parameters', 'score', 'high_variance_cv', 'training_time', + 'cv_data', 'percent_better_than_baseline'} assert results['id'] == pipeline_id assert isinstance(results['pipeline_name'], str) assert issubclass(results['pipeline_class'], expected_pipeline_class) @@ -77,11 +78,11 @@ def test_search_results(X_y_regression, X_y_binary, X_y_multi, automl_type): assert isinstance(automl.rankings, pd.DataFrame) assert isinstance(automl.full_rankings, pd.DataFrame) assert np.all(automl.rankings.dtypes == pd.Series( - [np.dtype('int64'), np.dtype('O'), np.dtype('float64'), np.dtype('bool'), np.dtype('O')], - index=['id', 'pipeline_name', 'score', 'high_variance_cv', 'parameters'])) + [np.dtype('int64'), np.dtype('O'), np.dtype('float64'), np.dtype('float64'), np.dtype('bool'), np.dtype('O')], + index=['id', 'pipeline_name', 'score', 'percent_better_than_baseline', 'high_variance_cv', 'parameters'])) assert np.all(automl.full_rankings.dtypes == pd.Series( - [np.dtype('int64'), np.dtype('O'), np.dtype('float64'), np.dtype('bool'), np.dtype('O')], - index=['id', 'pipeline_name', 'score', 'high_variance_cv', 'parameters'])) + [np.dtype('int64'), np.dtype('O'), np.dtype('float64'), np.dtype('float64'), np.dtype('bool'), np.dtype('O')], + index=['id', 'pipeline_name', 'score', 'percent_better_than_baseline', 'high_variance_cv', 'parameters'])) @pytest.mark.parametrize("automl_type", [ProblemTypes.BINARY, ProblemTypes.MULTICLASS, ProblemTypes.REGRESSION]) @@ -729,7 +730,8 @@ def test_no_search(): assert isinstance(automl.rankings, pd.DataFrame) assert isinstance(automl.full_rankings, pd.DataFrame) - df_columns = ["id", "pipeline_name", "score", "high_variance_cv", "parameters"] + df_columns = ["id", "pipeline_name", "score", "percent_better_than_baseline", + "high_variance_cv", "parameters"] assert (automl.rankings.columns == df_columns).all() assert (automl.full_rankings.columns == df_columns).all()