[MRG+1] Add new regression metric - Mean Squared Log Error (scikit-le…

…arn#7655)

* ENH Implement mean squared log error in sklearn.metrics.regression

* TST Add tests for mean squared log error.

* DOC Write user guide and docstring about mean squared log error.

* ENH Add neg_mean_squared_log_error in metrics.scorer

doc/modules/classes.rst

@@ -844,6 +844,7 @@ details.
    metrics.explained_variance_score
    metrics.mean_absolute_error
    metrics.mean_squared_error
+   metrics.mean_squared_log_error
    metrics.median_absolute_error
    metrics.r2_score
 
@@ -1418,4 +1419,4 @@ To be removed in 0.20
    cross_validation.cross_val_score
    cross_validation.check_cv
    cross_validation.permutation_test_score
-   cross_validation.train_test_split
+   cross_validation.train_test_split

doc/modules/model_evaluation.rst

@@ -77,6 +77,7 @@ Scoring                         Function                                      Co
 **Regression**
 'neg_mean_absolute_error'       :func:`metrics.mean_absolute_error`
 'neg_mean_squared_error'        :func:`metrics.mean_squared_error`
+'neg_mean_squared_log_error'    :func:`metrics.mean_squared_log_error`
 'neg_median_absolute_error'     :func:`metrics.median_absolute_error`
 'r2'                            :func:`metrics.r2_score`
 ===========================     =========================================     ==================================
@@ -93,7 +94,7 @@ Usage examples:
     >>> model = svm.SVC()
     >>> cross_val_score(model, X, y, scoring='wrong_choice')
     Traceback (most recent call last):
-    ValueError: 'wrong_choice' is not a valid scoring value. Valid options are ['accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'neg_log_loss', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc']
+    ValueError: 'wrong_choice' is not a valid scoring value. Valid options are ['accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'neg_log_loss', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc']
 
 .. note::
 
@@ -1360,7 +1361,7 @@ Mean squared error
 
 The :func:`mean_squared_error` function computes `mean square
 error <https://en.wikipedia.org/wiki/Mean_squared_error>`_, a risk
-metric corresponding to the expected value of the squared (quadratic) error loss or
+metric corresponding to the expected value of the squared (quadratic) error or
 loss.
 
 If :math:`\hat{y}_i` is the predicted value of the :math:`i`-th sample,
@@ -1390,6 +1391,43 @@ function::
     for an example of mean squared error usage to
     evaluate gradient boosting regression.
 
+.. _mean_squared_log_error:
+
+Mean squared logarithmic error
+------------------------------
+
+The :func:`mean_squared_log_error` function computes a risk metric
+corresponding to the expected value of the squared logarithmic (quadratic)
+error or loss.
+
+If :math:`\hat{y}_i` is the predicted value of the :math:`i`-th sample,
+and :math:`y_i` is the corresponding true value, then the mean squared
+logarithmic error (MSLE) estimated over :math:`n_{\text{samples}}` is
+defined as
+
+.. math::
+
+  \text{MSLE}(y, \hat{y}) = \frac{1}{n_\text{samples}} \sum_{i=0}^{n_\text{samples} - 1} (\log_e (1 + y_i) - \log_e (1 + \hat{y}_i) )^2.
+
+Where :math:`\log_e (x)` means the natural logarithm of :math:`x`. This metric
+is best to use when targets having exponential growth, such as population
+counts, average sales of a commodity over a span of years etc. Note that this
+metric penalizes an under-predicted estimate greater than an over-predicted
+estimate.
+
+Here is a small example of usage of the :func:`mean_squared_log_error`
+function::
+
+  >>> from sklearn.metrics import mean_squared_log_error
+  >>> y_true = [3, 5, 2.5, 7]
+  >>> y_pred = [2.5, 5, 4, 8]
+  >>> mean_squared_log_error(y_true, y_pred)  # doctest: +ELLIPSIS
+  0.039...
+  >>> y_true = [[0.5, 1], [1, 2], [7, 6]]
+  >>> y_pred = [[0.5, 2], [1, 2.5], [8, 8]]
+  >>> mean_squared_log_error(y_true, y_pred)  # doctest: +ELLIPSIS
+  0.044...
+
 .. _median_absolute_error:
 
 Median absolute error

sklearn/metrics/__init__.py

@@ -54,6 +54,7 @@
 from .regression import explained_variance_score
 from .regression import mean_absolute_error
 from .regression import mean_squared_error
+from .regression import mean_squared_log_error
 from .regression import median_absolute_error
 from .regression import r2_score
 
@@ -90,6 +91,7 @@
     'matthews_corrcoef',
     'mean_absolute_error',
     'mean_squared_error',
+    'mean_squared_log_error',
     'median_absolute_error',
     'mutual_info_score',
     'normalized_mutual_info_score',

sklearn/metrics/regression.py

@@ -14,6 +14,7 @@
 #          Jochen Wersdorfer <jochen@wersdoerfer.de>
 #          Lars Buitinck
 #          Joel Nothman <joel.nothman@gmail.com>
+#          Karan Desai <karandesai281196@gmail.com>
 #          Noel Dawe <noel@dawe.me>
 #          Manoj Kumar <manojkumarsivaraj334@gmail.com>
 #          Michael Eickenberg <michael.eickenberg@gmail.com>
@@ -33,6 +34,7 @@
 __ALL__ = [
     "mean_absolute_error",
     "mean_squared_error",
+    "mean_squared_log_error",
     "median_absolute_error",
     "r2_score",
     "explained_variance_score"
@@ -241,6 +243,73 @@ def mean_squared_error(y_true, y_pred,
     return np.average(output_errors, weights=multioutput)
 
 
+def mean_squared_log_error(y_true, y_pred,
+                           sample_weight=None,
+                           multioutput='uniform_average'):
+    """Mean squared logarithmic error regression loss
+
+    Read more in the :ref:`User Guide <mean_squared_log_error>`.
+
+    Parameters
+    ----------
+    y_true : array-like of shape = (n_samples) or (n_samples, n_outputs)
+        Ground truth (correct) target values.
+
+    y_pred : array-like of shape = (n_samples) or (n_samples, n_outputs)
+        Estimated target values.
+
+    sample_weight : array-like of shape = (n_samples), optional
+        Sample weights.
+
+    multioutput : string in ['raw_values', 'uniform_average'] \
+            or array-like of shape = (n_outputs)
+
+        Defines aggregating of multiple output values.
+        Array-like value defines weights used to average errors.
+
+        'raw_values' :
+            Returns a full set of errors when the input is of multioutput
+            format.
+
+        'uniform_average' :
+            Errors of all outputs are averaged with uniform weight.
+
+    Returns
+    -------
+    loss : float or ndarray of floats
+        A non-negative floating point value (the best value is 0.0), or an
+        array of floating point values, one for each individual target.
+
+    Examples
+    --------
+    >>> from sklearn.metrics import mean_squared_log_error
+    >>> y_true = [3, 5, 2.5, 7]
+    >>> y_pred = [2.5, 5, 4, 8]
+    >>> mean_squared_log_error(y_true, y_pred)  # doctest: +ELLIPSIS
+    0.039...
+    >>> y_true = [[0.5, 1], [1, 2], [7, 6]]
+    >>> y_pred = [[0.5, 2], [1, 2.5], [8, 8]]
+    >>> mean_squared_log_error(y_true, y_pred)  # doctest: +ELLIPSIS
+    0.044...
+    >>> mean_squared_log_error(y_true, y_pred, multioutput='raw_values')
+    ... # doctest: +ELLIPSIS
+    array([ 0.004...,  0.083...])
+    >>> mean_squared_log_error(y_true, y_pred, multioutput=[0.3, 0.7])
+    ... # doctest: +ELLIPSIS
+    0.060...
+
+    """
+    y_type, y_true, y_pred, multioutput = _check_reg_targets(
+        y_true, y_pred, multioutput)
+
+    if not (y_true >= 0).all() and not (y_pred >= 0).all():
+        raise ValueError("Mean Squared Logarithmic Error cannot be used when "
+                         "targets contain negative values.")
+
+    return mean_squared_error(np.log(y_true + 1), np.log(y_pred + 1),
+                              sample_weight, multioutput)
+
+
 def median_absolute_error(y_true, y_pred):
     """Median absolute error regression loss
 

sklearn/metrics/scorer.py

@@ -24,8 +24,8 @@
 import numpy as np
 
 from . import (r2_score, median_absolute_error, mean_absolute_error,
-               mean_squared_error, accuracy_score, f1_score,
-               roc_auc_score, average_precision_score,
+               mean_squared_error, mean_squared_log_error, accuracy_score,
+               f1_score, roc_auc_score, average_precision_score,
                precision_score, recall_score, log_loss)
 from .cluster import adjusted_rand_score
 from ..utils.multiclass import type_of_target
@@ -349,6 +349,8 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False,
 mean_squared_error_scorer = make_scorer(mean_squared_error,
                                         greater_is_better=False)
 mean_squared_error_scorer._deprecation_msg = deprecation_msg
+neg_mean_squared_log_error_scorer = make_scorer(mean_squared_log_error,
+                                                greater_is_better=False)
 neg_mean_absolute_error_scorer = make_scorer(mean_absolute_error,
                                              greater_is_better=False)
 deprecation_msg = ('Scoring method mean_absolute_error was renamed to '
@@ -396,6 +398,7 @@ def make_scorer(score_func, greater_is_better=True, needs_proba=False,
                neg_median_absolute_error=neg_median_absolute_error_scorer,
                neg_mean_absolute_error=neg_mean_absolute_error_scorer,
                neg_mean_squared_error=neg_mean_squared_error_scorer,
+               neg_mean_squared_log_error=neg_mean_squared_log_error_scorer,
                median_absolute_error=median_absolute_error_scorer,
                mean_absolute_error=mean_absolute_error_scorer,
                mean_squared_error=mean_squared_error_scorer,

sklearn/metrics/tests/test_regression.py

@@ -3,7 +3,7 @@
 import numpy as np
 from itertools import product
 
-from sklearn.utils.testing import assert_raises
+from sklearn.utils.testing import assert_raises, assert_raises_regex
 from sklearn.utils.testing import assert_equal
 from sklearn.utils.testing import assert_almost_equal
 from sklearn.utils.testing import assert_array_equal
@@ -12,6 +12,7 @@
 from sklearn.metrics import explained_variance_score
 from sklearn.metrics import mean_absolute_error
 from sklearn.metrics import mean_squared_error
+from sklearn.metrics import mean_squared_log_error
 from sklearn.metrics import median_absolute_error
 from sklearn.metrics import r2_score
 
@@ -23,6 +24,9 @@ def test_regression_metrics(n_samples=50):
     y_pred = y_true + 1
 
     assert_almost_equal(mean_squared_error(y_true, y_pred), 1.)
+    assert_almost_equal(mean_squared_log_error(y_true, y_pred),
+                        mean_squared_error(np.log(1 + y_true),
+                                           np.log(1 + y_pred)))
     assert_almost_equal(mean_absolute_error(y_true, y_pred), 1.)
     assert_almost_equal(median_absolute_error(y_true, y_pred), 1.)
     assert_almost_equal(r2_score(y_true, y_pred),  0.995, 2)
@@ -36,6 +40,9 @@ def test_multioutput_regression():
     error = mean_squared_error(y_true, y_pred)
     assert_almost_equal(error, (1. / 3 + 2. / 3 + 2. / 3) / 4.)
 
+    error = mean_squared_log_error(y_true, y_pred)
+    assert_almost_equal(error, 0.200, decimal=2)
+
     # mean_absolute_error and mean_squared_error are equal because
     # it is a binary problem.
     error = mean_absolute_error(y_true, y_pred)
@@ -49,10 +56,14 @@ def test_multioutput_regression():
 
 def test_regression_metrics_at_limits():
     assert_almost_equal(mean_squared_error([0.], [0.]), 0.00, 2)
+    assert_almost_equal(mean_squared_log_error([0.], [0.]), 0.00, 2)
     assert_almost_equal(mean_absolute_error([0.], [0.]), 0.00, 2)
     assert_almost_equal(median_absolute_error([0.], [0.]), 0.00, 2)
     assert_almost_equal(explained_variance_score([0.], [0.]), 1.00, 2)
     assert_almost_equal(r2_score([0., 1], [0., 1]), 1.00, 2)
+    assert_raises_regex(ValueError, "Mean Squared Logarithmic Error cannot be "
+                        "used when targets contain negative values.",
+                        mean_squared_log_error, [-1.], [-1.])
 
 
 def test__check_reg_targets():
@@ -127,6 +138,14 @@ def test_regression_multioutput_array():
     assert_array_almost_equal(evs, [1., -3.], decimal=2)
     assert_equal(np.mean(evs), explained_variance_score(y_true, y_pred))
 
+    # Handling msle separately as it does not accept negative inputs.
+    y_true = np.array([[0.5, 1], [1, 2], [7, 6]])
+    y_pred = np.array([[0.5, 2], [1, 2.5], [8, 8]])
+    msle = mean_squared_log_error(y_true, y_pred, multioutput='raw_values')
+    msle2 = mean_squared_error(np.log(1 + y_true), np.log(1 + y_pred),
+                               multioutput='raw_values')
+    assert_array_almost_equal(msle, msle2, decimal=2)
+
 
 def test_regression_custom_weights():
     y_true = [[1, 2], [2.5, -1], [4.5, 3], [5, 7]]
@@ -141,3 +160,11 @@ def test_regression_custom_weights():
     assert_almost_equal(maew, 0.475, decimal=3)
     assert_almost_equal(rw, 0.94, decimal=2)
     assert_almost_equal(evsw, 0.94, decimal=2)
+
+    # Handling msle separately as it does not accept negative inputs.
+    y_true = np.array([[0.5, 1], [1, 2], [7, 6]])
+    y_pred = np.array([[0.5, 2], [1, 2.5], [8, 8]])
+    msle = mean_squared_log_error(y_true, y_pred, multioutput=[0.3, 0.7])
+    msle2 = mean_squared_error(np.log(1 + y_true), np.log(1 + y_pred),
+                               multioutput=[0.3, 0.7])
+    assert_almost_equal(msle, msle2, decimal=2)

sklearn/metrics/tests/test_score_objects.py

@@ -39,8 +39,8 @@
 
 
 REGRESSION_SCORERS = ['r2', 'neg_mean_absolute_error',
-                      'neg_mean_squared_error', 'neg_median_absolute_error',
-                      'mean_absolute_error',
+                      'neg_mean_squared_error', 'neg_mean_squared_log_error',
+                      'neg_median_absolute_error', 'mean_absolute_error',
                       'mean_squared_error', 'median_absolute_error']
 
 CLF_SCORERS = ['accuracy', 'f1', 'f1_weighted', 'f1_macro', 'f1_micro',