[MRG] Fix selection of solver in ridge_regression when solver=='auto'

doc/whats_new/v0.21.rst

@@ -177,9 +177,9 @@ Support for Python 3.4 and below has been officially dropped.
 - |Enhancement| Minimized the validation of X in
   :class:`ensemble.AdaBoostClassifier` and :class:`ensemble.AdaBoostRegressor`
   :issue:`13174` by :user:`Christos Aridas <chkoar>`.
-  
+
 - |Enhancement| :class:`ensemble.IsolationForest` now exposes ``warm_start``
-  parameter, allowing iterative addition of trees to an isolation 
+  parameter, allowing iterative addition of trees to an isolation
   forest. :issue:`13496` by :user:`Peter Marko <petibear>`.
 
 - |Efficiency| Make :class:`ensemble.IsolationForest` more memory efficient
@@ -340,6 +340,16 @@ Support for Python 3.4 and below has been officially dropped.
   deterministic when trained in a multi-class setting on several threads.
   :issue:`13422` by :user:`Clément Doumouro <ClemDoum>`.
 
+- |Fix| Fixed bug in :func:`linear_model.ridge.ridge_regression` that
+  caused unhandled exception for arguments ``return_intercept=True`` and
+  ``solver=auto`` (default) or any other solver different from ``sag``.
+  :issue:`13363` by :user:`Bartosz Telenczuk <btel>`
+
+- |API| :func:`linear_model.ridge.ridge_regression` will choose ``sparse_cg``
+  solver for sparse inputs when ``solver=auto`` and ``sample_weight``
+  is provided (previously `cholesky` solver was selected). :issue:`13363`
+  by :user:`Bartosz Telenczuk <btel>`
+
 :mod:`sklearn.manifold`
 ............................
 

sklearn/linear_model/ridge.py

@@ -368,11 +368,25 @@ def _ridge_regression(X, y, alpha, sample_weight=None, solver='auto',
                       return_n_iter=False, return_intercept=False,
                       X_scale=None, X_offset=None):
 
-    if return_intercept and sparse.issparse(X) and solver != 'sag':
-        if solver != 'auto':
-            warnings.warn("In Ridge, only 'sag' solver can currently fit the "
-                          "intercept when X is sparse. Solver has been "
-                          "automatically changed into 'sag'.")
+    has_sw = sample_weight is not None
+
+    if solver == 'auto':
+        if return_intercept:
+            # only sag supports fitting intercept directly
+            solver = "sag"
+        elif not sparse.issparse(X):
+            solver = "cholesky"
+        else:
+            solver = "sparse_cg"
+
+    if solver not in ('sparse_cg', 'cholesky', 'svd', 'lsqr', 'sag', 'saga'):
+        raise ValueError("Known solvers are 'sparse_cg', 'cholesky', 'svd'"
+                         " 'lsqr', 'sag' or 'saga'. Got %s." % solver)
+
+    if return_intercept and solver != 'sag':
+        warnings.warn("In Ridge, only 'sag' solver can currently fit the "
+                      "intercept. Solver has been "
+                      "automatically changed into 'sag'.")
         solver = 'sag'
 
     _dtype = [np.float64, np.float32]
@@ -404,14 +418,7 @@ def _ridge_regression(X, y, alpha, sample_weight=None, solver='auto',
         raise ValueError("Number of samples in X and y does not correspond:"
                          " %d != %d" % (n_samples, n_samples_))
 
-    has_sw = sample_weight is not None
 
-    if solver == 'auto':
-        # cholesky if it's a dense array and cg in any other case
-        if not sparse.issparse(X) or has_sw:
-            solver = 'cholesky'
-        else:
-            solver = 'sparse_cg'
 
     if has_sw:
         if np.atleast_1d(sample_weight).ndim > 1:
@@ -432,8 +439,6 @@ def _ridge_regression(X, y, alpha, sample_weight=None, solver='auto',
     if alpha.size == 1 and n_targets > 1:
         alpha = np.repeat(alpha, n_targets)
 
-    if solver not in ('sparse_cg', 'cholesky', 'svd', 'lsqr', 'sag', 'saga'):
-        raise ValueError('Solver %s not understood' % solver)
 
     n_iter = None
     if solver == 'sparse_cg':
@@ -555,7 +560,7 @@ def fit(self, X, y, sample_weight=None):
             # add the offset which was subtracted by _preprocess_data
             self.intercept_ += y_offset
         else:
-            if sparse.issparse(X):
+            if sparse.issparse(X) and self.solver == 'sparse_cg':
                 # required to fit intercept with sparse_cg solver
                 params = {'X_offset': X_offset, 'X_scale': X_scale}
             else:

sklearn/linear_model/tests/test_ridge.py

@@ -7,6 +7,7 @@
 
 from sklearn.utils.testing import assert_almost_equal
 from sklearn.utils.testing import assert_array_almost_equal
+from sklearn.utils.testing import assert_allclose
 from sklearn.utils.testing import assert_equal
 from sklearn.utils.testing import assert_array_equal
 from sklearn.utils.testing import assert_greater
@@ -778,7 +779,8 @@ def test_raises_value_error_if_solver_not_supported():
     wrong_solver = "This is not a solver (MagritteSolveCV QuantumBitcoin)"
 
     exception = ValueError
-    message = "Solver %s not understood" % wrong_solver
+    message = ("Known solvers are 'sparse_cg', 'cholesky', 'svd'"
+               " 'lsqr', 'sag' or 'saga'. Got %s." % wrong_solver)
 
     def func():
         X = np.eye(3)
@@ -837,6 +839,62 @@ def test_ridge_fit_intercept_sparse():
         assert_array_almost_equal(dense.coef_, sparse.coef_)
 
 
+@pytest.mark.parametrize('return_intercept', [False, True])
+@pytest.mark.parametrize('sample_weight', [None, np.ones(1000)])
+@pytest.mark.parametrize('arr_type', [np.array, sp.csr_matrix])
+@pytest.mark.parametrize('solver', ['auto', 'sparse_cg', 'cholesky', 'lsqr',
+                                    'sag', 'saga'])
+def test_ridge_regression_check_arguments_validity(return_intercept,
+                                                   sample_weight, arr_type,
+                                                   solver):
+    """check if all combinations of arguments give valid estimations"""
+
+    # test excludes 'svd' solver because it raises exception for sparse inputs
+
+    X = np.random.rand(1000, 3)
+    true_coefs = [1, 2, 0.1]
+    y = np.dot(X, true_coefs)
+    X_testing = arr_type(X)
+
+    target = ridge_regression(X_testing, y, 1,
+                              solver=solver,
+                              sample_weight=sample_weight,
+                              return_intercept=return_intercept
+                              )
+    if return_intercept:
+        coef, intercept = target
+        assert_allclose(coef, true_coefs, rtol=0, atol=0.03)
+        assert_allclose(intercept, 0, rtol=0, atol=0.03)
+    else:
+        assert_allclose(target, true_coefs, rtol=0, atol=0.03)
+
+
+def test_ridge_regression_warns_with_return_intercept():
+    # return_itercept is only supported by sag/saga
+
+    X, y = make_regression(n_samples=1000, n_features=2, n_informative=2,
+                           bias=10., random_state=42)
+
+    for solver in ['sparse_cg', 'cholesky', 'svd', 'lsqr', 'saga']:
+        with pytest.warns(UserWarning, match="In Ridge, only 'sag' solver"):
+            target = ridge_regression(X, y, 1,
+                                      solver=solver,
+                                      return_intercept=True
+                                      )
+            assert len(target) == 2
+            assert target[0].shape == (2,)
+
+    with pytest.warns(None) as record:
+        target = ridge_regression(X, y, 1,
+                                  solver="sag",
+                                  return_intercept=True
+                                  )
+        assert len(target) == 2
+        assert target[0].shape == (2,)
+
+    # no warning should be raised
+    assert len(record) == 0
+
 def test_errors_and_values_helper():
     ridgecv = _RidgeGCV()
     rng = check_random_state(42)