Add decision function method for stacking classifiers

docs/sources/CHANGELOG.md

@@ -19,6 +19,7 @@ The CHANGELOG for the current development version is available at
 
 - The `SequentialFeatureSelector` now supports using pre-specified feature sets via the `fixed_features` parameter. ([#578](https://github.com/rasbt/mlxtend/pull/578))
 - Adds a new `accuracy_score` function to `mlxtend.evaluate` for computing basic classifcation accuracy, per-class accuracy, and average per-class accuracy. ([#624](https://github.com/rasbt/mlxtend/pull/624) via [Deepan Das](https://github.com/deepandas11))
+- `StackingClassifier` and `StackingCVClassifier`now have a `decision_function` method, which serves as a preferred choice over `predict_proba` in calculating roc_auc and average_precision scores when the meta estimator is a linear model or support vector classifier. ([#634](https://github.com/rasbt/mlxtend/pull/634) via [Qiang Gu](https://github.com/qiagu))
 
 ##### Changes
 

mlxtend/classifier/_base_classification.py

@@ -0,0 +1,80 @@
+import numpy as np
+from scipy import sparse
+from sklearn.base import ClassifierMixin
+from ..externals.estimator_checks import check_is_fitted
+
+
+class _BaseStackingClassifier(ClassifierMixin):
+    """Base class of stacking classifiers
+    """
+
+    def _do_predict(self, X, predict_fn):
+        meta_features = self.predict_meta_features(X)
+
+        if not self.use_features_in_secondary:
+            return predict_fn(meta_features)
+        elif sparse.issparse(X):
+            return predict_fn(sparse.hstack((X, meta_features)))
+        else:
+            return predict_fn(np.hstack((X, meta_features)))
+
+    def predict(self, X):
+        """ Predict target values for X.
+
+        Parameters
+        ----------
+        X : numpy array, shape = [n_samples, n_features]
+            Training vectors, where n_samples is the number of samples and
+            n_features is the number of features.
+
+        Returns
+        ----------
+        labels : array-like, shape = [n_samples]
+            Predicted class labels.
+
+        """
+        check_is_fitted(self, ['clfs_', 'meta_clf_'])
+
+        return self._do_predict(X, self.meta_clf_.predict)
+
+    def predict_proba(self, X):
+        """ Predict class probabilities for X.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
+            Training vectors, where n_samples is the number of samples and
+            n_features is the number of features.
+
+        Returns
+        ----------
+        proba : array-like, shape = [n_samples, n_classes] or a list of \
+                n_outputs of such arrays if n_outputs > 1.
+            Probability for each class per sample.
+
+        """
+        check_is_fitted(self, ['clfs_', 'meta_clf_'])
+
+        return self._do_predict(X, self.meta_clf_.predict_proba)
+
+    def decision_function(self, X):
+        """ Predict class confidence scores for X.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
+            Training vectors, where n_samples is the number of samples and
+            n_features is the number of features.
+
+        Returns
+        ----------
+        scores : shape=(n_samples,) if n_classes == 2 else \
+            (n_samples, n_classes).
+            Confidence scores per (sample, class) combination. In the binary
+            case, confidence score for self.classes_[1] where >0 means this
+            class would be predicted.
+
+        """
+        check_is_fitted(self, ['clfs_', 'meta_clf_'])
+
+        return self._do_predict(X, self.meta_clf_.decision_function)

mlxtend/classifier/ensemble_vote.py

@@ -173,7 +173,7 @@ def fit(self, X, y, sample_weight=None):
                 if sample_weight is None:
                     clf.fit(X, self.le_.transform(y))
                 else:
-                    clf.fit(X, self.le_.transform(y), 
+                    clf.fit(X, self.le_.transform(y),
                             sample_weight=sample_weight)
         return self
 

mlxtend/classifier/stacking_classification.py

@@ -11,14 +11,14 @@
 from ..externals.estimator_checks import check_is_fitted
 from ..externals.name_estimators import _name_estimators
 from ..utils.base_compostion import _BaseXComposition
+from ._base_classification import _BaseStackingClassifier
 from scipy import sparse
-from sklearn.base import ClassifierMixin
 from sklearn.base import TransformerMixin
 from sklearn.base import clone
 import numpy as np
 
 
-class StackingClassifier(_BaseXComposition, ClassifierMixin,
+class StackingClassifier(_BaseXComposition, _BaseStackingClassifier,
                          TransformerMixin):
 
     """A Stacking classifier for scikit-learn estimators for classification.
@@ -227,56 +227,3 @@ def predict_meta_features(self, X):
         else:
             vals = np.column_stack([clf.predict(X) for clf in self.clfs_])
         return vals
-
-    def predict(self, X):
-        """ Predict target values for X.
-
-        Parameters
-        ----------
-        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
-            Training vectors, where n_samples is the number of samples and
-            n_features is the number of features.
-
-        Returns
-        ----------
-        labels : array-like, shape = [n_samples] or [n_samples, n_outputs]
-            Predicted class labels.
-
-        """
-        check_is_fitted(self, 'clfs_')
-        meta_features = self.predict_meta_features(X)
-
-        if not self.use_features_in_secondary:
-            return self.meta_clf_.predict(meta_features)
-        elif sparse.issparse(X):
-            return self.meta_clf_.predict(sparse.hstack((X, meta_features)))
-        else:
-            return self.meta_clf_.predict(np.hstack((X, meta_features)))
-
-    def predict_proba(self, X):
-        """ Predict class probabilities for X.
-
-        Parameters
-        ----------
-        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
-            Training vectors, where n_samples is the number of samples and
-            n_features is the number of features.
-
-        Returns
-        ----------
-        proba : array-like, shape = [n_samples, n_classes] or a list of \
-                n_outputs of such arrays if n_outputs > 1.
-            Probability for each class per sample.
-
-        """
-        check_is_fitted(self, 'clfs_')
-        meta_features = self.predict_meta_features(X)
-
-        if not self.use_features_in_secondary:
-            return self.meta_clf_.predict_proba(meta_features)
-        elif sparse.issparse(X):
-            return self.meta_clf_.predict_proba(
-                sparse.hstack((X, meta_features))
-            )
-        else:
-            return self.meta_clf_.predict_proba(np.hstack((X, meta_features)))

mlxtend/classifier/stacking_cv_classification.py

@@ -12,17 +12,17 @@
 from ..externals.name_estimators import _name_estimators
 from ..externals.estimator_checks import check_is_fitted
 from ..utils.base_compostion import _BaseXComposition
+from ._base_classification import _BaseStackingClassifier
 import numpy as np
 from scipy import sparse
-from sklearn.base import ClassifierMixin
 from sklearn.base import TransformerMixin
 from sklearn.base import clone
 from sklearn.model_selection import cross_val_predict
 from sklearn.model_selection._split import check_cv
 # from sklearn.utils import check_X_y
 
 
-class StackingCVClassifier(_BaseXComposition, ClassifierMixin,
+class StackingCVClassifier(_BaseXComposition, _BaseStackingClassifier,
                            TransformerMixin):
 
     """A 'Stacking Cross-Validation' classifier for scikit-learn estimators.
@@ -331,49 +331,3 @@ def _stack_first_level_features(self, X, meta_features):
             stack_fn = np.hstack
 
         return stack_fn((X, meta_features))
-
-    def _do_predict(self, X, predict_fn):
-        meta_features = self.predict_meta_features(X)
-
-        if self.use_features_in_secondary:
-            meta_features = self._stack_first_level_features(X, meta_features)
-
-        return predict_fn(meta_features)
-
-    def predict(self, X):
-        """ Predict target values for X.
-
-        Parameters
-        ----------
-        X : numpy array, shape = [n_samples, n_features]
-            Training vectors, where n_samples is the number of samples and
-            n_features is the number of features.
-
-        Returns
-        ----------
-        labels : array-like, shape = [n_samples]
-            Predicted class labels.
-
-        """
-        check_is_fitted(self, ['clfs_', 'meta_clf_'])
-
-        return self._do_predict(X, self.meta_clf_.predict)
-
-    def predict_proba(self, X):
-        """ Predict class probabilities for X.
-
-        Parameters
-        ----------
-        X : numpy array, shape = [n_samples, n_features]
-            Training vectors, where n_samples is the number of samples and
-            n_features is the number of features.
-
-        Returns
-        ----------
-        proba : array-like, shape = [n_samples, n_classes]
-            Probability for each class per sample.
-
-        """
-        check_is_fitted(self, ['clfs_', 'meta_clf_'])
-
-        return self._do_predict(X, self.meta_clf_.predict_proba)

mlxtend/classifier/tests/test_stacking_classifier.py

@@ -10,6 +10,8 @@
 from mlxtend.externals.estimator_checks import NotFittedError
 from scipy import sparse
 from sklearn.linear_model import LogisticRegression
+from sklearn.linear_model import PassiveAggressiveClassifier
+from sklearn.svm import SVC
 from sklearn.naive_bayes import GaussianNB
 from sklearn.ensemble import RandomForestClassifier
 from sklearn.neighbors import KNeighborsClassifier
@@ -515,3 +517,50 @@ def test_clone():
                                meta_classifier=lr,
                                store_train_meta_features=True)
     clone(stclf)
+
+
+def test_decision_function():
+    np.random.seed(123)
+
+    # PassiveAggressiveClassifier has no predict_proba
+    meta = PassiveAggressiveClassifier(max_iter=1000,
+                                       tol=0.001,
+                                       random_state=42)
+    clf1 = RandomForestClassifier(n_estimators=10)
+    clf2 = GaussianNB()
+    sclf = StackingClassifier(classifiers=[clf1, clf2],
+                              use_probas=True,
+                              meta_classifier=meta)
+
+    # binarize target
+    y2 = y > 1
+    scores = cross_val_score(sclf,
+                             X,
+                             y2,
+                             cv=5,
+                             scoring='roc_auc')
+    scores_mean = (round(scores.mean(), 2))
+
+    if Version(sklearn_version) < Version("0.21"):
+        assert scores_mean == 0.96, scores_mean
+    else:
+        assert scores_mean == 0.93, scores_mean
+
+    # another test
+    meta = SVC(decision_function_shape='ovo')
+
+    sclf = StackingClassifier(classifiers=[clf1, clf2],
+                              use_probas=True,
+                              meta_classifier=meta)
+
+    scores = cross_val_score(sclf,
+                             X,
+                             y2,
+                             cv=5,
+                             scoring='roc_auc')
+    scores_mean = (round(scores.mean(), 2))
+
+    if Version(sklearn_version) < Version("0.21"):
+        assert scores_mean == 0.95, scores_mean
+    else:
+        assert scores_mean == 0.95, scores_mean

mlxtend/classifier/tests/test_stacking_cv_classifier.py

@@ -15,6 +15,8 @@
 from mlxtend.utils import assert_raises
 from mlxtend.data import iris_data
 from sklearn.linear_model import LogisticRegression
+from sklearn.linear_model import PassiveAggressiveClassifier
+from sklearn.svm import SVC
 from sklearn.naive_bayes import GaussianNB
 from sklearn.ensemble import RandomForestClassifier
 from sklearn.neighbors import KNeighborsClassifier
@@ -567,3 +569,42 @@ def test_works_with_df_if_fold_indexes_missing():
     stclf.fit(X_train, y_train)
     assert round(stclf.score(X_train, y_train), 2) == 0.99, \
         round(stclf.score(X_train, y_train), 2)
+
+
+def test_decision_function():
+    np.random.seed(123)
+
+    # PassiveAggressiveClassifier has no predict_proba
+    meta = PassiveAggressiveClassifier(random_state=42)
+    clf1 = RandomForestClassifier(n_estimators=10)
+    clf2 = GaussianNB()
+    sclf = StackingCVClassifier(classifiers=[clf1, clf2],
+                                use_probas=True,
+                                meta_classifier=meta)
+
+    scores = cross_val_score(sclf,
+                             X_breast,
+                             y_breast,
+                             cv=5,
+                             scoring='roc_auc')
+    scores_mean = (round(scores.mean(), 2))
+    assert scores_mean == 0.96, scores_mean
+
+    # another test
+    meta = SVC(decision_function_shape='ovo')
+
+    sclf = StackingCVClassifier(classifiers=[clf1, clf2],
+                                use_probas=True,
+                                meta_classifier=meta)
+
+    scores = cross_val_score(sclf,
+                             X_breast,
+                             y_breast,
+                             cv=5,
+                             scoring='roc_auc')
+    scores_mean = (round(scores.mean(), 2))
+
+    if Version(sklearn_version) < Version("0.21"):
+        assert scores_mean == 0.94, scores_mean
+    else:
+        assert scores_mean == 0.96, scores_mean