Consolidate graphing methods (#1060)

* init

* release notes and fix test

* fix docs

* actually fixing docs

* more fix docs

* remove for codecov

* update hierarchy

* delete old files

* more fixes

* update api ref

* update api ref order

docs/source/api_reference.rst

@@ -105,21 +105,6 @@ Regression Pipelines
     MeanBaselineRegressionPipeline
 
 
-Pipeline Graph Utils
-~~~~~~~~~~~~~~~~~~~~
-.. autosummary::
-    :toctree: generated
-    :nosignatures:
-
-    precision_recall_curve
-    graph_precision_recall_curve
-    roc_curve
-    graph_roc_curve
-    graph_confusion_matrix
-    calculate_permutation_importance
-    graph_permutation_importance
-
-
 .. currentmodule:: evalml.pipelines.utils
 
 Pipeline Utils
@@ -224,11 +209,33 @@ Regressors are components that output a predicted target value.
     XGBoostRegressor
     BaselineRegressor
 
+.. currentmodule:: evalml.model_understanding
 
-.. currentmodule:: evalml.pipelines.prediction_explanations
+Model Understanding
+===================
+
+Graph Utils
+~~~~~~~~~~~
+.. autosummary::
+    :toctree: generated
+    :nosignatures:
+
+    confusion_matrix
+    normalize_confusion_matrix
+    precision_recall_curve
+    graph_precision_recall_curve
+    roc_curve
+    graph_roc_curve
+    graph_confusion_matrix
+    calculate_permutation_importance
+    graph_permutation_importance
+
+
+.. currentmodule:: evalml.model_understanding.prediction_explanations
 
 Prediction Explanations
-========================
+~~~~~~~~~~~~~~~~~~~~~~~
+
 
 .. autosummary::
     :toctree: generated
@@ -258,7 +265,6 @@ Objective Base Classes
     RegressionObjective
 
 
-
 Domain-Specific Objectives
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -450,12 +456,3 @@ General Utils
     get_random_state
     get_random_seed
 
-
-Graph Utils
-~~~~~~~~~~~
-.. autosummary::
-    :toctree: generated
-    :nosignatures:
-
-    confusion_matrix
-    normalize_confusion_matrix

docs/source/release_notes.rst

@@ -18,6 +18,7 @@ Release Notes
         * Updated references to data types to use datatype lists defined in `evalml.utils.gen_utils` :pr:`1039`
         * Remove maximum version limit for SciPy dependency :pr:`1051`
         * Moved `all_components` and other component importers into runtime methods :pr:`1045`
+        * Consolidated graphing utility methods under `evalml.utils.graph_utils` :pr:`1060`
     * Documentation Changes
         * Update setup.py URL to point to the github repo :pr:`1037`
     * Testing Changes
@@ -29,7 +30,7 @@ Release Notes
 
     **Breaking Changes**
         * ``confusion_matrix`` and ``normalize_confusion_matrix`` have been moved to `evalml.utils` :pr:`1038`
-
+        * All graph utility methods previously under ``evalml.pipelines.graph_utils`` have been moved to ``evalml.utils.graph_utils`` :pr:`1060`
 
 
 **v0.12.2 Aug. 6, 2020**

docs/source/user_guide/model_understanding.ipynb

@@ -85,7 +85,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "evalml.pipelines.calculate_permutation_importance(pipeline, X, y, 'log_loss_binary')"
+    "from evalml.model_understanding.graphs import calculate_permutation_importance\n",
+    "calculate_permutation_importance(pipeline, X, y, 'log_loss_binary')"
    ]
   },
   {
@@ -94,7 +95,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "evalml.pipelines.graph_permutation_importance(pipeline, X, y, 'log_loss_binary')"
+    "from evalml.model_understanding.graphs import graph_permutation_importance\n",
+    "graph_permutation_importance(pipeline, X, y, 'log_loss_binary')"
    ]
   },
   {
@@ -112,8 +114,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "from evalml.model_understanding.graphs import graph_confusion_matrix\n",
     "y_pred = pipeline.predict(X)\n",
-    "evalml.pipelines.graph_utils.graph_confusion_matrix(y, y_pred)"
+    "graph_confusion_matrix(y, y_pred)"
    ]
   },
   {
@@ -131,10 +134,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "from evalml.model_understanding.graphs import graph_precision_recall_curve\n",
     "# get the predicted probabilities associated with the \"true\" label\n",
     "y_encoded = y.map({'malignant': 0, 'benign': 1})\n",
     "y_pred_proba = pipeline.predict_proba(X)[\"benign\"]\n",
-    "evalml.pipelines.graph_utils.graph_precision_recall_curve(y_encoded, y_pred_proba)"
+    "graph_precision_recall_curve(y_encoded, y_pred_proba)"
    ]
   },
   {
@@ -152,9 +156,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "from evalml.model_understanding.graphs import graph_roc_curve\n",
     "# get the predicted probabilities associated with the \"benign\" label\n",
     "y_pred_proba = pipeline.predict_proba(X)[\"benign\"]\n",
-    "evalml.pipelines.graph_utils.graph_roc_curve(y_encoded, y_pred_proba)"
+    "graph_roc_curve(y_encoded, y_pred_proba)"
    ]
   },
   {
@@ -163,7 +168,7 @@
    "source": [
     "## Explaining Individual Predictions\n",
     "\n",
-    "We can explain why the model made an individual prediction with the [explain_prediction](../generated/evalml.pipelines.prediction_explanations.explain_prediction.ipynb) function. This will use the [Shapley Additive Explanations (SHAP)](https://github.com/slundberg/shap) algorithms to identify the top features that explain the predicted value. \n",
+    "We can explain why the model made an individual prediction with the [explain_prediction](../generated/evalml.model_understanding.prediction_explanations.explain_prediction.ipynb) function. This will use the [Shapley Additive Explanations (SHAP)](https://github.com/slundberg/shap) algorithms to identify the top features that explain the predicted value. \n",
     "\n",
     "This function can explain both classification and regression models - all you need to do is provide the pipeline, the input features (must correspond to one row of the input data) and the training data. The function will return a table that you can print summarizing the top 3 most positive and negative contributing features to the predicted value.\n",
     "\n",
@@ -176,7 +181,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from evalml.pipelines.prediction_explanations import explain_prediction\n",
+    "from evalml.model_understanding.prediction_explanations import explain_prediction\n",
     "\n",
     "table = explain_prediction(pipeline=pipeline, input_features=X.iloc[3:4],\n",
     "                           training_data=X, include_shap_values=True)\n",
@@ -198,9 +203,9 @@
    "source": [
     "## Explaining Multiple Predictions\n",
     "\n",
-    "When debugging machine learning models, it is often useful to analyze the best and worst predictions the model made. The [explain_predictions_best_worst](../generated/evalml.pipelines.prediction_explanations.explain_predictions_best_worst.ipynb) function can help us with this.\n",
+    "When debugging machine learning models, it is often useful to analyze the best and worst predictions the model made. The [explain_predictions_best_worst](../generated/evalml.model_understanding.prediction_explanations.explain_predictions_best_worst.ipynb) function can help us with this.\n",
     "\n",
-    "This function will display the output of [explain_prediction](../generated/evalml.pipelines.prediction_explanations.explain_prediction.ipynb) for the best 2 and worst 2 predictions. By default, the best and worst predictions are determined by the absolute error for regression problems and [cross entropy](https://en.wikipedia.org/wiki/Cross_entropy) for classification problems.\n",
+    "This function will display the output of [explain_prediction](../generated/evalml.model_understanding.prediction_explanations.explain_prediction.ipynb) for the best 2 and worst 2 predictions. By default, the best and worst predictions are determined by the absolute error for regression problems and [cross entropy](https://en.wikipedia.org/wiki/Cross_entropy) for classification problems.\n",
     "\n",
     "We can specify our own ranking function by passing in a function to the `metric` parameter. This function will be called on `y_true` and `y_pred`. By convention, lower scores are better.\n",
     "\n",
@@ -213,7 +218,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from evalml.pipelines.prediction_explanations import explain_predictions_best_worst\n",
+    "from evalml.model_understanding.prediction_explanations import explain_predictions_best_worst\n",
     "\n",
     "report = explain_predictions_best_worst(pipeline=pipeline, input_features=X, y_true=y,\n",
     "                                        include_shap_values=True, num_to_explain=2)\n",
@@ -248,7 +253,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "We can also manually explain predictions on any subset of the training data with the [explain_predictions](../generated/evalml.pipelines.prediction_explanations.explain_predictions.ipynb) function. Below, we explain the predictions on the first, fifth, and tenth row of the data."
+    "We can also manually explain predictions on any subset of the training data with the [explain_predictions](../generated/evalml.model_understanding.prediction_explanations.explain_predictions.ipynb) function. Below, we explain the predictions on the first, fifth, and tenth row of the data."
    ]
   },
   {
@@ -257,7 +262,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from evalml.pipelines.prediction_explanations import explain_predictions\n",
+    "from evalml.model_understanding.prediction_explanations import explain_predictions\n",
     "\n",
     "report = explain_predictions(pipeline=pipeline, input_features=X.iloc[[0, 4, 9]], include_shap_values=True)\n",
     "print(report)"
@@ -285,4 +290,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}

evalml/model_understanding/__init__.py

@@ -0,0 +1,13 @@
+# flake8:noqa
+from .graphs import (
+    precision_recall_curve,
+    graph_precision_recall_curve,
+    roc_curve,
+    graph_roc_curve,
+    graph_confusion_matrix,
+    calculate_permutation_importance,
+    graph_permutation_importance,
+    confusion_matrix,
+    normalize_confusion_matrix
+)
+from .prediction_explanations import explain_prediction, explain_predictions_best_worst, explain_predictions

evalml/pipelines/graph_utils.py → evalml/model_understanding/graphs.py

@@ -1,16 +1,68 @@
+
+import warnings
+
 import numpy as np
 import pandas as pd
 from sklearn.inspection import \
     permutation_importance as sk_permutation_importance
 from sklearn.metrics import auc as sklearn_auc
+from sklearn.metrics import confusion_matrix as sklearn_confusion_matrix
 from sklearn.metrics import \
     precision_recall_curve as sklearn_precision_recall_curve
 from sklearn.metrics import roc_curve as sklearn_roc_curve
 from sklearn.preprocessing import LabelBinarizer
+from sklearn.utils.multiclass import unique_labels
 
-from evalml.objectives import get_objective
+from evalml.objectives.utils import get_objective
 from evalml.utils import import_or_raise
-from evalml.utils.graph_utils import confusion_matrix
+
+
+def confusion_matrix(y_true, y_predicted, normalize_method='true'):
+    """Confusion matrix for binary and multiclass classification.
+
+    Arguments:
+        y_true (pd.Series or np.array): true binary labels.
+        y_pred (pd.Series or np.array): predictions from a binary classifier.
+        normalize_method ({'true', 'pred', 'all'}): Normalization method. Supported options are: 'true' to normalize by row, 'pred' to normalize by column, or 'all' to normalize by all values. Defaults to 'true'.
+
+    Returns:
+        pd.DataFrame: confusion matrix
+    """
+    if isinstance(y_true, pd.Series):
+        y_true = y_true.to_numpy()
+    if isinstance(y_predicted, pd.Series):
+        y_predicted = y_predicted.to_numpy()
+
+    labels = unique_labels(y_true, y_predicted)
+    conf_mat = sklearn_confusion_matrix(y_true, y_predicted)
+    conf_mat = pd.DataFrame(conf_mat, columns=labels)
+    if normalize_method is not None:
+        return normalize_confusion_matrix(conf_mat, normalize_method=normalize_method)
+    return conf_mat
+
+
+def normalize_confusion_matrix(conf_mat, normalize_method='true'):
+    """Normalizes a confusion matrix.
+
+    Arguments:
+        conf_mat (pd.DataFrame or np.array): confusion matrix to normalize.
+        normalize_method ({'true', 'pred', 'all'}): Normalization method. Supported options are: 'true' to normalize by row, 'pred' to normalize by column, or 'all' to normalize by all values. Defaults to 'true'.
+
+    Returns:
+        pd.DataFrame: normalized version of the input confusion matrix.
+    """
+    with warnings.catch_warnings(record=True) as w:
+        if normalize_method == 'true':
+            conf_mat = conf_mat.astype('float') / conf_mat.sum(axis=1)[:, np.newaxis]
+        elif normalize_method == 'pred':
+            conf_mat = conf_mat.astype('float') / conf_mat.sum(axis=0)
+        elif normalize_method == 'all':
+            conf_mat = conf_mat.astype('float') / conf_mat.sum().sum()
+        else:
+            raise ValueError('Invalid value provided for "normalize_method": %s'.format(normalize_method))
+        if w and "invalid value encountered in" in str(w[0].message):
+            raise ValueError("Sum of given axis is 0 and normalization is not possible. Please select another option.")
+    return conf_mat
 
 
 def precision_recall_curve(y_true, y_pred_proba):

evalml/pipelines/prediction_explanations/_user_interface.py → evalml/model_understanding/prediction_explanations/_user_interface.py

@@ -3,7 +3,7 @@
 import pandas as pd
 from texttable import Texttable
 
-from evalml.pipelines.prediction_explanations._algorithms import (
+from evalml.model_understanding.prediction_explanations._algorithms import (
     _compute_shap_values,
     _normalize_shap_values
 )

evalml/pipelines/prediction_explanations/explainers.py → evalml/model_understanding/prediction_explanations/explainers.py

@@ -5,7 +5,7 @@
 import pandas as pd
 
 from evalml.exceptions import PipelineScoreError
-from evalml.pipelines.prediction_explanations._user_interface import (
+from evalml.model_understanding.prediction_explanations._user_interface import (
     _ClassificationPredictedValuesMaker,
     _EmptyPredictedValuesMaker,
     _HeadingMaker,

evalml/objectives/cost_benefit_matrix.py

@@ -3,7 +3,7 @@
 
 from .binary_classification_objective import BinaryClassificationObjective
 
-from evalml.utils.graph_utils import confusion_matrix
+from evalml.model_understanding.graphs import confusion_matrix
 
 
 class CostBenefitMatrix(BinaryClassificationObjective):

evalml/pipelines/__init__.py

@@ -41,13 +41,3 @@
     BaselineRegressionPipeline,
     MeanBaselineRegressionPipeline
 )
-
-from .graph_utils import (
-    precision_recall_curve,
-    graph_precision_recall_curve,
-    roc_curve,
-    graph_roc_curve,
-    graph_confusion_matrix,
-    calculate_permutation_importance,
-    graph_permutation_importance
-)