Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adds fit_params support for stacking classifiers #255

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/sources/CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -303,7 +303,7 @@ The documents containing code examples for the "User Guide" are generated from I
3. Convert the notebook to markdown using the `ipynb2markdown.py` converter

```python
~/github/mlxtend/docs$ python ipynb2markdown.py --ipynb_path ./sources/user_guide/subpackage/notebookname.ipynb
~/github/mlxtend/docs$ python ipynb2markdown.py --ipynb ./sources/user_guide/subpackage/notebookname.ipynb
```

**Note**
Expand Down
33 changes: 27 additions & 6 deletions mlxtend/classifier/stacking_classification.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,8 @@ def __init__(self, classifiers, meta_classifier,
self.verbose = verbose
self.use_features_in_secondary = use_features_in_secondary

def fit(self, X, y):
""" Fit ensemble classifers and the meta-classifier.
def fit(self, X, y, **fit_params):
"""Fit ensemble classifers and the meta-classifier.

Parameters
----------
Expand All @@ -87,18 +87,26 @@ def fit(self, X, y):
n_features is the number of features.
y : array-like, shape = [n_samples] or [n_samples, n_outputs]
Target values.
fit_params : dict, optional
Parameters to pass to the fit methods of `classifiers` and
`meta_classifier`.

Returns
-------
self : object

"""
self.clfs_ = [clone(clf) for clf in self.classifiers]
self.named_clfs_ = {key: value for key, value in
_name_estimators(self.clfs_)}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

dict(_name_estimators(self.clfs_)) might work, but no guarantee.

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hm yeah, I think it should be equivalent indeed

self.meta_clf_ = clone(self.meta_classifier)
self.named_meta_clf_ = {'meta-%s' % key: value for key, value in
_name_estimators([self.meta_clf_])}

if self.verbose > 0:
print("Fitting %d classifiers..." % (len(self.classifiers)))

for clf in self.clfs_:
for name, clf in six.iteritems(self.named_clfs_):

if self.verbose > 0:
i = self.clfs_.index(clf) + 1
Expand All @@ -112,14 +120,27 @@ def fit(self, X, y):
if self.verbose > 1:
print(_name_estimators((clf,))[0][1])

clf.fit(X, y)
# Extract fit_params for clf
clf_fit_params = {}
for key, value in six.iteritems(fit_params):
if name in key and 'meta-' not in key:
clf_fit_params[key.replace(name+'__', '')] = value
Copy link
Contributor

@qiagu qiagu Oct 2, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

for key in list(six.iterkeys(fit_params)):
    if name in key and 'meta-' not in key:
        clf_fit_params[key[len(name) + 2: ]] = fit_params.pop(key)

might be more efficient since the fit_params will be iterated again.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I suppose key.startswith(name) is True. Is that the case?


clf.fit(X, y, **clf_fit_params)

meta_features = self._predict_meta_features(X)
# Extract fit_params for meta_clf_
meta_fit_params = {}
meta_clf_name = list(self.named_meta_clf_.keys())[0]
for key, value in six.iteritems(fit_params):
if meta_clf_name in key and 'meta-' in meta_clf_name:
meta_fit_params[key.replace(meta_clf_name+'__', '')] = value

if not self.use_features_in_secondary:
self.meta_clf_.fit(meta_features, y)
self.meta_clf_.fit(meta_features, y, **meta_fit_params)
else:
self.meta_clf_.fit(np.hstack((X, meta_features)), y)
self.meta_clf_.fit(np.hstack((X, meta_features)), y,
**meta_fit_params)

return self

Expand Down
42 changes: 33 additions & 9 deletions mlxtend/classifier/stacking_cv_classification.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,29 +111,36 @@ def __init__(self, classifiers, meta_classifier,
self.stratify = stratify
self.shuffle = shuffle

def fit(self, X, y, groups=None):
def fit(self, X, y, groups=None, **fit_params):
""" Fit ensemble classifers and the meta-classifier.

Parameters
----------
X : numpy array, shape = [n_samples, n_features]
Training vectors, where n_samples is the number of samples and
n_features is the number of features.

y : numpy array, shape = [n_samples]
Target values.

groups : numpy array/None, shape = [n_samples]
The group that each sample belongs to. This is used by specific
folding strategies such as GroupKFold()
fit_params : dict, optional
Parameters to pass to the fit methods of `classifiers` and
`meta_classifier`. Note that only fit parameters for `classifiers`
that are the same for each cross-validation split are supported
(e.g. `sample_weight` is not currently supported).

Returns
-------
self : object

"""
self.clfs_ = [clone(clf) for clf in self.classifiers]
self.named_clfs_ = {key: value for key, value in
_name_estimators(self.clfs_)}
self.meta_clf_ = clone(self.meta_classifier)
self.named_meta_clf_ = {'meta-%s' % key: value for key, value in
_name_estimators([self.meta_clf_])}
if self.verbose > 0:
print("Fitting %d classifiers..." % (len(self.classifiers)))

Expand All @@ -144,8 +151,23 @@ def fit(self, X, y, groups=None):
final_cv.shuffle = self.shuffle
skf = list(final_cv.split(X, y, groups))

# Get fit_params for each classifier in self.named_clfs_
named_clfs_fit_params = {}
for name, clf in six.iteritems(self.named_clfs_):
clf_fit_params = {}
for key, value in six.iteritems(fit_params):
if name in key and 'meta-' not in key:
clf_fit_params[key.replace(name+'__', '')] = value
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

pop can help here as well.

named_clfs_fit_params[name] = clf_fit_params
# Get fit_params for self.named_meta_clf_
meta_fit_params = {}
meta_clf_name = list(self.named_meta_clf_.keys())[0]
for key, value in six.iteritems(fit_params):
if meta_clf_name in key and 'meta-' in meta_clf_name:
meta_fit_params[key.replace(meta_clf_name+'__', '')] = value

all_model_predictions = np.array([]).reshape(len(y), 0)
for model in self.clfs_:
for name, model in six.iteritems(self.named_clfs_):

if self.verbose > 0:
i = self.clfs_.index(model) + 1
Expand All @@ -172,7 +194,8 @@ def fit(self, X, y, groups=None):
((num + 1), final_cv.get_n_splits()))

try:
model.fit(X[train_index], y[train_index])
model.fit(X[train_index], y[train_index],
**named_clfs_fit_params[name])
except TypeError as e:
raise TypeError(str(e) + '\nPlease check that X and y'
'are NumPy arrays. If X and y are lists'
Expand Down Expand Up @@ -215,16 +238,17 @@ def fit(self, X, y, groups=None):
X[test_index]))

# Fit the base models correctly this time using ALL the training set
for model in self.clfs_:
model.fit(X, y)
for name, model in six.iteritems(self.named_clfs_):
model.fit(X, y, **named_clfs_fit_params[name])

# Fit the secondary model
if not self.use_features_in_secondary:
self.meta_clf_.fit(all_model_predictions, reordered_labels)
self.meta_clf_.fit(all_model_predictions, reordered_labels,
**meta_fit_params)
else:
self.meta_clf_.fit(np.hstack((reordered_features,
all_model_predictions)),
reordered_labels)
reordered_labels, **meta_fit_params)

return self

Expand Down
23 changes: 23 additions & 0 deletions mlxtend/classifier/tests/test_stacking_classifier.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,29 @@ def test_StackingClassifier_proba_concat_1():
assert scores_mean == 0.93, scores_mean


def test_StackingClassifier_fit_params():
np.random.seed(123)
meta = LogisticRegression()
clf1 = RandomForestClassifier()
clf2 = GaussianNB()
sclf = StackingClassifier(classifiers=[clf1, clf2],
meta_classifier=meta)
n_samples = X.shape[0]
fit_params = {
'randomforestclassifier__sample_weight': np.ones(n_samples),
'meta-logisticregression__sample_weight': np.arange(n_samples)
}

scores = cross_val_score(sclf,
X,
y,
cv=5,
scoring='accuracy',
fit_params=fit_params)
scores_mean = (round(scores.mean(), 2))
assert scores_mean == 0.95


def test_StackingClassifier_avg_vs_concat():
np.random.seed(123)
lr1 = LogisticRegression()
Expand Down
25 changes: 24 additions & 1 deletion mlxtend/classifier/tests/test_stacking_cv_classifier.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from mlxtend.classifier import StackingCVClassifier

import pandas as pd
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import LogisticRegression, SGDClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
Expand Down Expand Up @@ -61,6 +61,29 @@ def test_StackingClassifier_proba():
assert scores_mean == 0.93


def test_StackingClassifier_fit_params():
np.random.seed(123)
meta = LogisticRegression()
clf1 = RandomForestClassifier()
clf2 = SGDClassifier(random_state=2)
sclf = StackingCVClassifier(classifiers=[clf1, clf2],
meta_classifier=meta,
shuffle=False)
fit_params = {
'sgdclassifier__intercept_init': np.unique(y),
'meta-logisticregression__sample_weight': np.full(X.shape[0], 2)
}

scores = cross_val_score(sclf,
X,
y,
cv=5,
scoring='accuracy',
fit_params=fit_params)
scores_mean = (round(scores.mean(), 2))
assert scores_mean == 0.86


def test_gridsearch():
np.random.seed(123)
meta = LogisticRegression()
Expand Down
28 changes: 24 additions & 4 deletions mlxtend/regressor/stacking_regression.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ def __init__(self, regressors, meta_regressor, verbose=0):
_name_estimators([meta_regressor])}
self.verbose = verbose

def fit(self, X, y):
def fit(self, X, y, **fit_params):
"""Learn weight coefficients from training data for each regressor.

Parameters
Expand All @@ -75,18 +75,25 @@ def fit(self, X, y):
n_features is the number of features.
y : array-like, shape = [n_samples] or [n_samples, n_targets]
Target values.
fit_params : dict, optional
Parameters to pass to the fit methods of `regressors` and
`meta_regressor`.

Returns
-------
self : object

"""
self.regr_ = [clone(regr) for regr in self.regressors]
self.named_regr_ = {key: value for key, value in
_name_estimators(self.regr_)}
self.meta_regr_ = clone(self.meta_regressor)
self.named_meta_regr_ = {'meta-%s' % key: value for key, value in
_name_estimators([self.meta_regr_])}
if self.verbose > 0:
print("Fitting %d regressors..." % (len(self.regressors)))

for regr in self.regr_:
for name, regr in six.iteritems(self.named_regr_):

if self.verbose > 0:
i = self.regr_.index(regr) + 1
Expand All @@ -100,10 +107,23 @@ def fit(self, X, y):
if self.verbose > 1:
print(_name_estimators((regr,))[0][1])

regr.fit(X, y)
# Extract fit_params for regr
regr_fit_params = {}
for key, value in six.iteritems(fit_params):
if name in key and 'meta-' not in key:
regr_fit_params[key.replace(name+'__', '')] = value

regr.fit(X, y, **regr_fit_params)

meta_features = self._predict_meta_features(X)
self.meta_regr_.fit(meta_features, y)
# Extract fit_params for meta_regr_
meta_fit_params = {}
meta_regr_name = list(self.named_meta_regr_.keys())[0]
for key, value in six.iteritems(fit_params):
if meta_regr_name in key and 'meta-' in meta_regr_name:
meta_fit_params[key.replace(meta_regr_name+'__', '')] = value
self.meta_regr_.fit(meta_features, y, **meta_fit_params)

return self

@property
Expand Down
19 changes: 18 additions & 1 deletion mlxtend/regressor/tests/test_stacking_regression.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import numpy as np
from numpy.testing import assert_almost_equal
from nose.tools import raises
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import GridSearchCV, cross_val_score

# Generating a sample dataset
np.random.seed(1)
Expand Down Expand Up @@ -108,6 +108,23 @@ def test_gridsearch_numerate_regr():
assert best == got


def test_StackingRegressor_fit_params():
lr = LinearRegression()
svr_lin = SVR(kernel='linear')
ridge = Ridge(random_state=1)
svr_rbf = SVR(kernel='rbf')
stregr = StackingRegressor(regressors=[svr_lin, lr, ridge],
meta_regressor=svr_rbf)

fit_params = {'ridge__sample_weight': np.ones(X1.shape[0]),
'svr__sample_weight': np.ones(X1.shape[0]),
'meta-svr__sample_weight': np.ones(X1.shape[0])}

scores = cross_val_score(stregr, X1, y, cv=5, fit_params=fit_params)
scores_mean = (round(scores.mean(), 1))
assert scores_mean == 0.1
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just set up codacy today ... these msgs are annoying. Will see if I can disable those (at least for the asserts)



def test_get_coeff():
lr = LinearRegression()
svr_lin = SVR(kernel='linear')
Expand Down