[MRG] Enhancement: Add MAPE as an evaluation metric

doc/modules/classes.rst

@@ -836,6 +836,7 @@ details.
 
    metrics.explained_variance_score
    metrics.mean_absolute_error
+   metrics.mean_absolute_percentage_error
    metrics.mean_squared_error
    metrics.mean_squared_log_error
    metrics.median_absolute_error

doc/modules/model_evaluation.rst

@@ -85,6 +85,7 @@ Scoring                           Function
 **Regression**
 'explained_variance'              :func:`metrics.explained_variance_score`
 'neg_mean_absolute_error'         :func:`metrics.mean_absolute_error`
+'neg_mape'                        :func:`metrics.mean_absolute_percentage_error`
 'neg_mean_squared_error'          :func:`metrics.mean_squared_error`
 'neg_mean_squared_log_error'      :func:`metrics.mean_squared_log_error`
 'neg_median_absolute_error'       :func:`metrics.median_absolute_error`
@@ -104,7 +105,7 @@ Usage examples:
     >>> model = svm.SVC()
     >>> cross_val_score(model, X, y, scoring='wrong_choice')
     Traceback (most recent call last):
-    ValueError: 'wrong_choice' is not a valid scoring value. Valid options are ['accuracy', 'adjusted_mutual_info_score', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'brier_score_loss', 'completeness_score', 'explained_variance', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'fowlkes_mallows_score', 'homogeneity_score', 'mutual_info_score', 'neg_log_loss', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'normalized_mutual_info_score', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc', 'v_measure_score']
+    ValueError: 'wrong_choice' is not a valid scoring value. Valid options are ['accuracy', 'adjusted_mutual_info_score', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'brier_score_loss', 'completeness_score', 'explained_variance', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'fowlkes_mallows_score', 'homogeneity_score', 'mutual_info_score', 'neg_log_loss', 'neg_mape', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'normalized_mutual_info_score', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc', 'v_measure_score']
 
 .. note::
 
@@ -1551,6 +1552,32 @@ Here is a small example of usage of the :func:`mean_absolute_error` function::
   ... # doctest: +ELLIPSIS
   0.849...
 
+.. _mean_absolute_percentage_error:
+
+Mean absolute error
+-------------------
+
+The :func:`mean_absolute_percentage_error` function computes `mean absolute
+percentage error <https://en.wikipedia.org/wiki/Mean_absolute_percentage_error>`_, a risk
+metric corresponding to the expected value of the absolute percentage error loss or
+:math:`l1`-norm of percentage loss.
+
+If :math:`\hat{y}_i` is the predicted value of the :math:`i`-th sample,
+and :math:`y_i` is the corresponding true value, then the mean absolute percentage error
+(MAPE) estimated over :math:`n_{\text{samples}}` is defined as
+
+.. math::
+
+  \text{MAPE}(y, \hat{y}) = \frac{100}{n_{\text{samples}}} \sum_{i=0}^{n_{\text{samples}}-1} \left| \frac{y_i - \hat{y}_i}{y_i} \right|.
+
+Here is a small example of usage of the :func:`mean_absolute_percentage_error` function::
+
+  >>> from sklearn.metrics import mean_absolute_percentage_error
+  >>> y_true = [3, -0.5, 2, 7]
+  >>> y_pred = [2.5, 0.0, 2, 8]
+  >>> mean_absolute_percentage_error(y_true, y_pred)
+  32.738...
+
 .. _mean_squared_error:
 
 Mean squared error

doc/whats_new/v0.20.rst

@@ -88,6 +88,8 @@ Model evaluation
 - Added the :func:`metrics.balanced_accuracy_score` metric and a corresponding
   ``'balanced_accuracy'`` scorer for binary classification.
   :issue:`8066` by :user:`xyguo` and :user:`Aman Dalmia <dalmia>`.
+- Added the :func:`metrics.mean_absolute_percentage_error` metric and the associated
+  scorer for regression problems. :issue:`10711` by :user:`Mohamed Ali Jamaoui <mohamed-ali>`
 
 Decomposition, manifold learning and clustering
 

sklearn/metrics/__init__.py

@@ -54,6 +54,7 @@
 
 from .regression import explained_variance_score
 from .regression import mean_absolute_error
+from .regression import mean_absolute_percentage_error
 from .regression import mean_squared_error
 from .regression import mean_squared_log_error
 from .regression import median_absolute_error
@@ -97,6 +98,7 @@
     'make_scorer',
     'matthews_corrcoef',
     'mean_absolute_error',
+    'mean_absolute_percentage_error',
     'mean_squared_error',
     'mean_squared_log_error',
     'median_absolute_error',

sklearn/metrics/regression.py

@@ -19,6 +19,7 @@
 #          Manoj Kumar <manojkumarsivaraj334@gmail.com>
 #          Michael Eickenberg <michael.eickenberg@gmail.com>
 #          Konstantin Shmelkov <konstantin.shmelkov@polytechnique.edu>
+#          Mohamed Ali Jamaoui <m.ali.jamaoui@gmail.com>
 # License: BSD 3 clause
 
 from __future__ import division
@@ -32,6 +33,7 @@
 
 __ALL__ = [
     "mean_absolute_error",
+    "mean_absolute_percentage_error",
     "mean_squared_error",
     "mean_squared_log_error",
     "median_absolute_error",
@@ -181,6 +183,46 @@ def mean_absolute_error(y_true, y_pred,
     return np.average(output_errors, weights=multioutput)
 
 
+def mean_absolute_percentage_error(y_true, y_pred):
+    """Mean absolute percentage error regression loss
+
+    Read more in the :ref:`User Guide <mean_absolute_percentage_error>`.
+
+    Parameters
+    ----------
+    y_true : array-like of shape = (n_samples)
+        Ground truth (correct) target values.
+
+    y_pred : array-like of shape = (n_samples)
+        Estimated target values.
+
+    Returns
+    -------
+    loss : float
+        A positive floating point value (the best value is 0.0).
+
+    Examples
+    --------
+    >>> from sklearn.metrics import mean_absolute_percentage_error
+    >>> y_true = [3, -0.5, 2, 7]
+    >>> y_pred = [2.5, 0.0, 2, 8]
+    >>> mean_absolute_percentage_error(y_true, y_pred)
+    32.738...
+    """
+    y_type, y_true, y_pred, _ = _check_reg_targets(y_true, y_pred,
+                                                   'uniform_average')
+
+    if y_type == 'continuous-multioutput':
+        raise ValueError("Multioutput not supported "
+                         "in mean_absolute_percentage_error")
+
+    if (y_true == 0).any():
+        raise ValueError("mean_absolute_percentage_error requires"
+                         " y_true to not include zeros")
+
+    return np.mean(np.abs((y_true - y_pred) / y_true)) * 100
+
+
 def mean_squared_error(y_true, y_pred,
                        sample_weight=None,
                        multioutput='uniform_average'):

sklearn/metrics/scorer.py

@@ -24,9 +24,11 @@
 import numpy as np
 
 from . import (r2_score, median_absolute_error, mean_absolute_error,
-               mean_squared_error, mean_squared_log_error, accuracy_score,
+               mean_squared_error, mean_absolute_percentage_error,
+               mean_squared_log_error, accuracy_score,
                f1_score, roc_auc_score, average_precision_score,
-               precision_score, recall_score, log_loss, balanced_accuracy_score,
+               precision_score, recall_score, log_loss,
+               balanced_accuracy_score,
                explained_variance_score, brier_score_loss)
 
 from .cluster import adjusted_rand_score
@@ -487,6 +489,9 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False,
 mean_absolute_error_scorer = make_scorer(mean_absolute_error,
                                          greater_is_better=False)
 mean_absolute_error_scorer._deprecation_msg = deprecation_msg
+neg_mape_scorer = make_scorer(mean_absolute_percentage_error,
+                              greater_is_better=False)
+
 neg_median_absolute_error_scorer = make_scorer(median_absolute_error,
                                                greater_is_better=False)
 deprecation_msg = ('Scoring method median_absolute_error was renamed to '
@@ -536,6 +541,7 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False,
 
 SCORERS = dict(explained_variance=explained_variance_scorer,
                r2=r2_scorer,
+               neg_mape=neg_mape_scorer,
                neg_median_absolute_error=neg_median_absolute_error_scorer,
                neg_mean_absolute_error=neg_mean_absolute_error_scorer,
                neg_mean_squared_error=neg_mean_squared_error_scorer,

sklearn/metrics/tests/test_common.py

@@ -42,6 +42,7 @@
 from sklearn.metrics import log_loss
 from sklearn.metrics import matthews_corrcoef
 from sklearn.metrics import mean_absolute_error
+from sklearn.metrics import mean_absolute_percentage_error
 from sklearn.metrics import mean_squared_error
 from sklearn.metrics import median_absolute_error
 from sklearn.metrics import precision_score
@@ -93,6 +94,7 @@
 
 REGRESSION_METRICS = {
     "mean_absolute_error": mean_absolute_error,
+    "mean_absolute_percentage_error": mean_absolute_percentage_error,
     "mean_squared_error": mean_squared_error,
     "median_absolute_error": median_absolute_error,
     "explained_variance_score": explained_variance_score,
@@ -366,7 +368,9 @@
     "weighted_precision_score",
 
     "macro_f0.5_score", "macro_f2_score", "macro_precision_score",
-    "macro_recall_score", "log_loss", "hinge_loss"
+    "macro_recall_score", "log_loss", "hinge_loss",
+
+    "mean_absolute_percentage_error"
 ]
 
 
@@ -378,15 +382,21 @@
                         # confusion_matrix with sample_weight is in
                         # test_classification.py
     "median_absolute_error",
+    "mean_absolute_percentage_error"
+]
+
+# Metrics that only support non-zero y
+METRICS_WITH_NON_ZERO_Y = [
+    "mean_absolute_percentage_error"
 ]
 
 
 @ignore_warnings
 def test_symmetry():
     # Test the symmetry of score and loss functions
     random_state = check_random_state(0)
-    y_true = random_state.randint(0, 2, size=(20, ))
-    y_pred = random_state.randint(0, 2, size=(20, ))
+    y_true = random_state.randint(1, 3, size=(20, ))
+    y_pred = random_state.randint(1, 3, size=(20, ))
 
     # We shouldn't forget any metrics
     assert_equal(set(SYMMETRIC_METRICS).union(
@@ -415,8 +425,8 @@ def test_symmetry():
 @ignore_warnings
 def test_sample_order_invariance():
     random_state = check_random_state(0)
-    y_true = random_state.randint(0, 2, size=(20, ))
-    y_pred = random_state.randint(0, 2, size=(20, ))
+    y_true = random_state.randint(1, 3, size=(20, ))
+    y_pred = random_state.randint(1, 3, size=(20, ))
     y_true_shuffle, y_pred_shuffle = shuffle(y_true, y_pred, random_state=0)
 
     for name, metric in ALL_METRICS.items():
@@ -432,8 +442,6 @@ def test_sample_order_invariance():
 @ignore_warnings
 def test_sample_order_invariance_multilabel_and_multioutput():
     random_state = check_random_state(0)
-
-    # Generate some data
     y_true = random_state.randint(0, 2, size=(20, 25))
     y_pred = random_state.randint(0, 2, size=(20, 25))
     y_score = random_state.normal(size=y_true.shape)
@@ -472,8 +480,8 @@ def test_sample_order_invariance_multilabel_and_multioutput():
 @ignore_warnings
 def test_format_invariance_with_1d_vectors():
     random_state = check_random_state(0)
-    y1 = random_state.randint(0, 2, size=(20, ))
-    y2 = random_state.randint(0, 2, size=(20, ))
+    y1 = random_state.randint(1, 3, size=(20, ))
+    y2 = random_state.randint(1, 3, size=(20, ))
 
     y1_list = list(y1)
     y2_list = list(y2)
@@ -653,8 +661,8 @@ def check_single_sample(name):
     metric = ALL_METRICS[name]
 
     # assert that no exception is thrown
-    for i, j in product([0, 1], repeat=2):
-        metric([i], [j])
+    for i, j in product([1, 2], repeat=2):
+            metric([i], [j])
 
 
 @ignore_warnings

sklearn/metrics/tests/test_regression.py

@@ -11,6 +11,7 @@
 
 from sklearn.metrics import explained_variance_score
 from sklearn.metrics import mean_absolute_error
+from sklearn.metrics import mean_absolute_percentage_error
 from sklearn.metrics import mean_squared_error
 from sklearn.metrics import mean_squared_log_error
 from sklearn.metrics import median_absolute_error
@@ -28,6 +29,11 @@ def test_regression_metrics(n_samples=50):
                         mean_squared_error(np.log(1 + y_true),
                                            np.log(1 + y_pred)))
     assert_almost_equal(mean_absolute_error(y_true, y_pred), 1.)
+    # comparing (y_true + 1) and (y_pred + 1) instead of
+    # y_true and y_pred to avoid division by zero
+    assert_almost_equal(mean_absolute_percentage_error(1 + y_true,
+                                                       1 + y_pred),
+                        8.998, 2)
     assert_almost_equal(median_absolute_error(y_true, y_pred), 1.)
     assert_almost_equal(r2_score(y_true, y_pred),  0.995, 2)
     assert_almost_equal(explained_variance_score(y_true, y_pred), 1.)
@@ -72,7 +78,6 @@ def test_regression_metrics_at_limits():
                         mean_squared_log_error, [1., -2., 3.], [1., 2., 3.])
 
 
-
 def test__check_reg_targets():
     # All of length 3
     EXAMPLES = [

sklearn/metrics/tests/test_score_objects.py

@@ -41,8 +41,8 @@
 from sklearn.externals import joblib
 
 
-REGRESSION_SCORERS = ['explained_variance', 'r2',
-                      'neg_mean_absolute_error', 'neg_mean_squared_error',
+REGRESSION_SCORERS = ['explained_variance', 'r2', 'neg_mean_absolute_error',
+                      'neg_mape', 'neg_mean_squared_error',
                       'neg_mean_squared_log_error',
                       'neg_median_absolute_error', 'mean_absolute_error',
                       'mean_squared_error', 'median_absolute_error']
@@ -66,6 +66,7 @@
 
 MULTILABEL_ONLY_SCORERS = ['precision_samples', 'recall_samples', 'f1_samples']
 
+NONZERO_Y_SCORERS = ['neg_mape']
 
 def _make_estimators(X_train, y_train, y_ml_train):
     # Make estimators that make sense to test various scoring methods
@@ -486,6 +487,8 @@ def check_scorer_memmap(scorer_name):
     scorer, estimator = SCORERS[scorer_name], ESTIMATORS[scorer_name]
     if scorer_name in MULTILABEL_ONLY_SCORERS:
         score = scorer(estimator, X_mm, y_ml_mm)
+    elif scorer_name in NONZERO_Y_SCORERS:
+        score = scorer(estimator, X_mm, y_mm + 1)
     else:
         score = scorer(estimator, X_mm, y_mm)
     assert isinstance(score, numbers.Number), scorer_name