Merge pull request #221 from TeamHG-Memex/libsvm

libsvm-based sklearn estimators

docs/source/libraries/sklearn.rst

@@ -75,6 +75,11 @@ Linear SVMs from ``sklearn.svm`` are also supported:
 
 * LinearSVC_
 * LinearSVR_
+* SVC_ (only with ``kernel='linear'``, only for binary classification)
+* SVR_ (only with ``kernel='linear'``)
+* NuSVC_ (only with ``kernel='linear'``, only for binary classification)
+* NuSVR_ (only with ``kernel='linear'``)
+* OneClassSVM_ (only with ``kernel='linear'``)
 
 For linear scikit-learn classifiers :func:`eli5.explain_weights` supports
 one more keyword argument, in addition to common argument and extra arguments
@@ -122,7 +127,11 @@ for all scikit-learn estimators:
 .. _TheilSenRegressor: http://scikit-learn.org/stable/modules/generated/sklearn.linear_model.TheilSenRegressor.html#sklearn.linear_model.TheilSenRegressor
 .. _LinearSVC: http://scikit-learn.org/stable/modules/generated/sklearn.svm.LinearSVC.html#sklearn.svm.LinearSVC
 .. _LinearSVR: http://scikit-learn.org/stable/modules/generated/sklearn.svm.LinearSVR.html#sklearn.svm.LinearSVR
-
+.. _SVC: http://scikit-learn.org/stable/modules/generated/sklearn.svm.SVC.html#sklearn.svm.SVC
+.. _SVR: http://scikit-learn.org/stable/modules/generated/sklearn.svm.SVR.html#sklearn.svm.SVR
+.. _NuSVC: http://scikit-learn.org/stable/modules/generated/sklearn.svm.NuSVC.html#sklearn.svm.NuSVC
+.. _NuSVR: http://scikit-learn.org/stable/modules/generated/sklearn.svm.NuSVR.html#sklearn.svm.NuSVR
+.. _OneClassSVM: http://scikit-learn.org/stable/modules/generated/sklearn.svm.OneClassSVM.html#sklearn.svm.OneClassSVM
 
 Decision Trees, Ensembles
 -------------------------

eli5/sklearn/explain_prediction.py

@@ -35,7 +35,15 @@
     SGDRegressor,
     TheilSenRegressor,
 )
-from sklearn.svm import LinearSVC, LinearSVR  # type: ignore
+from sklearn.svm import (  # type: ignore
+    LinearSVC,
+    LinearSVR,
+    SVC,
+    SVR,
+    NuSVC,
+    NuSVR,
+    OneClassSVM,
+)
 from sklearn.multiclass import OneVsRestClassifier  # type: ignore
 from sklearn.tree import (   # type: ignore
     DecisionTreeClassifier,
@@ -175,7 +183,8 @@ def explain_prediction_linear_classifier(clf, doc,
     )
 
     _weights = _linear_weights(clf, x, top, feature_names, flt_indices)
-    display_names = get_target_display_names(clf.classes_, target_names,
+    classes = getattr(clf, "classes_", ["-1", "1"])  # OneClassSVM support
+    display_names = get_target_display_names(classes, target_names,
                                              targets, top_targets, score)
 
     if is_multiclass_classifier(clf):
@@ -201,6 +210,24 @@ def explain_prediction_linear_classifier(clf, doc,
     return res
 
 
+@register(NuSVC)
+@register(SVC)
+@register(OneClassSVM)
+def test_explain_prediction_libsvm_linear(clf, doc, *args, **kwargs):
+    if clf.kernel != 'linear':
+        return Explanation(
+            estimator=repr(clf),
+            error="only kernel='linear' is currently supported for "
+                  "libsvm-based classifiers",
+        )
+    if len(getattr(clf, 'classes_', [])) > 2:
+        return Explanation(
+            estimator=repr(clf),
+            error="only binary libsvm-based classifiers are supported",
+        )
+    return explain_prediction_linear_classifier(clf, doc, *args, **kwargs)
+
+
 @register(ElasticNet)
 @register(ElasticNetCV)
 @register(HuberRegressor)
@@ -215,6 +242,8 @@ def explain_prediction_linear_classifier(clf, doc,
 @register(RidgeCV)
 @register(SGDRegressor)
 @register(TheilSenRegressor)
+@register(SVR)
+@register(NuSVR)
 def explain_prediction_linear_regressor(reg, doc,
                                         vec=None,
                                         top=None,
@@ -242,6 +271,9 @@ def explain_prediction_linear_regressor(reg, doc,
     regressor ``reg``. Set it to True if you're passing ``vec``,
     but ``doc`` is already vectorized.
     """
+    if isinstance(reg, (SVR, NuSVR)) and reg.kernel != 'linear':
+        return explain_prediction_sklearn_not_supported(reg, doc)
+
     vec, feature_names = handle_vec(reg, doc, vec, vectorized, feature_names)
     X = get_X(doc, vec=vec, vectorized=vectorized, to_dense=True)
 

eli5/sklearn/explain_weights.py

@@ -29,7 +29,15 @@
     TheilSenRegressor,
 )
 from sklearn.multiclass import OneVsRestClassifier  # type: ignore
-from sklearn.svm import LinearSVC, LinearSVR  # type: ignore
+from sklearn.svm import (  # type: ignore
+    LinearSVC,
+    LinearSVR,
+    SVC,
+    SVR,
+    NuSVC,
+    NuSVR,
+    OneClassSVM,
+)
 # TODO: see https://github.com/scikit-learn/scikit-learn/pull/2250
 from sklearn.naive_bayes import BernoulliNB, MultinomialNB    # type: ignore
 from sklearn.ensemble import (  # type: ignore
@@ -214,7 +222,8 @@ def _features(label_id):
             coef = coef[flt_indices]
         return get_top_features(feature_names, coef, top)
 
-    display_names = get_target_display_names(clf.classes_, target_names, targets)
+    classes = getattr(clf, "classes_", ["-1", "1"])  # OneClassSVM support
+    display_names = get_target_display_names(classes, target_names, targets)
     if is_multiclass_classifier(clf):
         return Explanation(
             targets=[
@@ -244,6 +253,25 @@ def _features(label_id):
         )
 
 
+@register(SVC)
+@register(NuSVC)
+@register(OneClassSVM)
+def explain_libsvm_linear_classifier_weights(clf, *args, **kwargs):
+    if clf.kernel != 'linear':
+        return Explanation(
+            estimator=repr(clf),
+            error="only kernel='linear' is currently supported for "
+                  "libsvm-based classifiers",
+        )
+    if len(getattr(clf, 'classes_', [])) > 2:
+        return Explanation(
+            estimator=repr(clf),
+            error="only binary libsvm-based classifiers are supported",
+        )
+    return explain_linear_classifier_weights(clf, *args, **kwargs)
+
+
+
 @register(RandomForestClassifier)
 @register(RandomForestRegressor)
 @register(ExtraTreesClassifier)
@@ -355,6 +383,8 @@ def explain_decision_tree(estimator,
 @register(RidgeCV)
 @register(SGDRegressor)
 @register(TheilSenRegressor)
+@register(SVR)
+@register(NuSVR)
 def explain_linear_regressor_weights(reg,
                                      vec=None,
                                      top=_TOP,
@@ -381,6 +411,9 @@ def explain_linear_regressor_weights(reg,
     coef_scale[i] is not nan. Use it if you want to scale coefficients
     before displaying them, to take input feature sign or scale in account.
     """
+    if isinstance(reg, (SVR, NuSVR)) and reg.kernel != 'linear':
+        return explain_weights_sklearn_not_supported(reg)
+
     feature_names, coef_scale = handle_hashing_vec(vec, feature_names,
                                                    coef_scale)
     feature_names, flt_indices = get_feature_names_filtered(

eli5/sklearn/utils.py

@@ -153,13 +153,13 @@ def get_coef(clf, label_id, scale=None):
     """
     if len(clf.coef_.shape) == 2:
         # Most classifiers (even in binary case) and regressors
-        coef = clf.coef_[label_id]
+        coef = _dense_1d(clf.coef_[label_id])
     elif len(clf.coef_.shape) == 1:
         # SGDRegressor stores coefficients in a 1D array
         if label_id != 0:
             raise ValueError(
                 'Unexpected label_id %s for 1D coefficient' % label_id)
-        coef = clf.coef_
+        coef = _dense_1d(clf.coef_)
     elif len(clf.coef_.shape) == 0:
         # Lasso with one feature: 0D array
         coef = np.array([clf.coef_])
@@ -185,6 +185,12 @@ def get_coef(clf, label_id, scale=None):
     return np.hstack([coef, bias])
 
 
+def _dense_1d(arr):
+    if not sp.issparse(arr):
+        return arr
+    return arr.toarray().reshape(-1)
+
+
 def get_num_features(estimator):
     """ Return size of a feature vector estimator expects as an input. """
     if hasattr(estimator, 'coef_'):  # linear models

tests/test_sklearn_explain_prediction.py

@@ -5,6 +5,7 @@
 import re
 
 import pytest
+import numpy as np
 from sklearn.base import BaseEstimator
 from sklearn.datasets import make_regression
 from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
@@ -43,7 +44,15 @@
     SGDRegressor,
     TheilSenRegressor,
 )
-from sklearn.svm import LinearSVC, LinearSVR
+from sklearn.svm import (
+    LinearSVC,
+    LinearSVR,
+    SVC,
+    SVR,
+    NuSVC,
+    NuSVR,
+    OneClassSVM,
+)
 from sklearn.multiclass import OneVsRestClassifier
 from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
 
@@ -111,6 +120,23 @@ def assert_multiclass_linear_classifier_explained(newsgroups_train, clf,
     assert [t.target for t in top_neg_targets_res.targets] == sorted_targets[-1:]
 
 
+def assert_binary_linear_classifier_explained(newsgroups_train_binary, clf,
+                                              explain_prediction):
+    docs, y, target_names = newsgroups_train_binary
+    vec = TfidfVectorizer()
+
+    X = vec.fit_transform(docs)
+    clf.fit(X, y)
+
+    get_res = lambda **kwargs: explain_prediction(
+        clf, docs[2], vec=vec, target_names=target_names, top=20, **kwargs)
+    res = get_res()
+    pprint(res)
+    expl_text, expl_html = format_as_all(res, clf)
+    for expl in [expl_text, expl_html]:
+        assert 'software' in expl
+
+
 def assert_linear_regression_explained(boston_train, reg, explain_prediction,
                                        atol=1e-8, reg_has_intercept=None):
     X, y, feature_names = boston_train
@@ -260,6 +286,65 @@ def test_explain_linear(newsgroups_train, clf):
             newsgroups_train, clf, explain_prediction_sklearn)
 
 
+@pytest.mark.parametrize(['clf'], [
+    [LogisticRegression(random_state=42)],
+    [SGDClassifier(random_state=42)],
+    [SVC(kernel='linear', random_state=42)],
+    [SVC(kernel='linear', random_state=42, decision_function_shape='ovr')],
+    [SVC(kernel='linear', random_state=42, decision_function_shape='ovr',
+         probability=True)],
+    [SVC(kernel='linear', random_state=42, probability=True)],
+    [NuSVC(kernel='linear', random_state=42)],
+    [NuSVC(kernel='linear', random_state=42, decision_function_shape='ovr')],
+])
+def test_explain_linear_binary(newsgroups_train_binary, clf):
+    assert_binary_linear_classifier_explained(newsgroups_train_binary, clf,
+                                              explain_prediction)
+
+
+def test_explain_one_class_svm():
+    X = np.array([[0, 0], [0, 1], [5, 3], [93, 94], [90, 91]])
+    clf = OneClassSVM(kernel='linear', random_state=42).fit(X)
+    res = explain_prediction(clf, X[0])
+    assert res.targets[0].score < 0
+    for expl in format_as_all(res, clf):
+        assert 'BIAS' in expl
+        assert 'x0' not in expl
+        assert 'x1' not in expl
+
+    res = explain_prediction(clf, X[4])
+    assert res.targets[0].score > 0
+    for expl in format_as_all(res, clf):
+        assert 'BIAS' in expl
+        assert 'x0' in expl
+        assert 'x1' in expl
+
+
+@pytest.mark.parametrize(['clf'], [
+    [SVC()],
+    [NuSVC()],
+    [OneClassSVM()],
+])
+def test_explain_linear_classifiers_unsupported_kernels(clf, newsgroups_train_binary):
+    docs, y, target_names = newsgroups_train_binary
+    vec = TfidfVectorizer()
+    clf.fit(vec.fit_transform(docs), y)
+    res = explain_prediction(clf, docs[0], vec=vec)
+    assert 'supported' in res.error
+
+
+@pytest.mark.parametrize(['clf'], [
+    [SVC(kernel='linear')],
+    [NuSVC(kernel='linear')],
+])
+def test_explain_linear_unsupported_multiclass(clf, newsgroups_train):
+    docs, y, target_names = newsgroups_train
+    vec = TfidfVectorizer()
+    clf.fit(vec.fit_transform(docs), y)
+    expl = explain_prediction(clf, docs[0], vec=vec)
+    assert 'supported' in expl.error
+
+
 @pytest.mark.parametrize(['reg'], [
     [ElasticNet(random_state=42)],
     [ElasticNetCV(random_state=42)],
@@ -281,11 +366,24 @@ def test_explain_linear(newsgroups_train, clf):
     [RidgeCV()],
     [SGDRegressor(random_state=42)],
     [TheilSenRegressor()],
+    [SVR(kernel='linear')],
+    [NuSVR(kernel='linear')],
 ])
 def test_explain_linear_regression(boston_train, reg):
     assert_linear_regression_explained(boston_train, reg, explain_prediction)
 
 
+@pytest.mark.parametrize(['reg'], [
+    [SVR()],
+    [NuSVR()],
+])
+def test_explain_linear_regressors_unsupported_kernels(reg, boston_train):
+    X, y, feature_names = boston_train
+    reg.fit(X, y)
+    res = explain_prediction(reg, X[0], feature_names=feature_names)
+    assert 'supported' in res.error
+
+
 @pytest.mark.parametrize(['reg'], [
     [ElasticNet(random_state=42)],
     [Lars()],