From 61a027d21e00682a20ca26648b5da7240b88006f Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Thu, 21 Sep 2017 23:06:54 -0500 Subject: [PATCH 1/5] Adds fit_params support for StackingClassifier --- mlxtend/classifier/stacking_classification.py | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/mlxtend/classifier/stacking_classification.py b/mlxtend/classifier/stacking_classification.py index 232fc945a..880571e51 100644 --- a/mlxtend/classifier/stacking_classification.py +++ b/mlxtend/classifier/stacking_classification.py @@ -77,8 +77,8 @@ def __init__(self, classifiers, meta_classifier, self.verbose = verbose self.use_features_in_secondary = use_features_in_secondary - def fit(self, X, y): - """ Fit ensemble classifers and the meta-classifier. + def fit(self, X, y, **fit_params): + """Fit ensemble classifers and the meta-classifier. Parameters ---------- @@ -87,6 +87,9 @@ def fit(self, X, y): n_features is the number of features. y : array-like, shape = [n_samples] or [n_samples, n_outputs] Target values. + fit_params : dict, optional + Parameters to pass to the fit methods of the classifiers and + meta_classifier. Returns ------- @@ -94,11 +97,16 @@ def fit(self, X, y): """ self.clfs_ = [clone(clf) for clf in self.classifiers] + self.named_clfs_ = {key: value for key, value in + _name_estimators(self.clfs_)} self.meta_clf_ = clone(self.meta_classifier) + self.named_meta_clf_ = {'meta-%s' % key: value for key, value in + _name_estimators([self.meta_clf_])} + if self.verbose > 0: print("Fitting %d classifiers..." % (len(self.classifiers))) - for clf in self.clfs_: + for name, clf in six.iteritems(self.named_clfs_): if self.verbose > 0: i = self.clfs_.index(clf) + 1 @@ -112,14 +120,27 @@ def fit(self, X, y): if self.verbose > 1: print(_name_estimators((clf,))[0][1]) - clf.fit(X, y) + # Extract fit_params for clf + clf_fit_params = {} + for key, value in six.iteritems(fit_params): + if name in key and 'meta-' not in key: + clf_fit_params[key.replace(name+'__', '')] = value + + clf.fit(X, y, **clf_fit_params) meta_features = self._predict_meta_features(X) + # Extract fit_params for meta_clf_ + meta_clf_fit_params = {} + meta_clf_name = list(self.named_meta_clf_.keys())[0] + for key, value in six.iteritems(fit_params): + if meta_clf_name in key and 'meta-' in meta_clf_name: + meta_clf_fit_params[key.replace(meta_clf_name+'__', '')] = value if not self.use_features_in_secondary: - self.meta_clf_.fit(meta_features, y) + self.meta_clf_.fit(meta_features, y, **meta_clf_fit_params) else: - self.meta_clf_.fit(np.hstack((X, meta_features)), y) + self.meta_clf_.fit(np.hstack((X, meta_features)), y, + **meta_clf_fit_params) return self From 24473aca111186bc83213a670a35965f5fb4c6f3 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Thu, 21 Sep 2017 23:53:05 -0500 Subject: [PATCH 2/5] Adds fit_param test for StackingClassifier --- mlxtend/classifier/stacking_classification.py | 8 +++---- .../tests/test_stacking_classifier.py | 23 +++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/mlxtend/classifier/stacking_classification.py b/mlxtend/classifier/stacking_classification.py index 880571e51..d37551f70 100644 --- a/mlxtend/classifier/stacking_classification.py +++ b/mlxtend/classifier/stacking_classification.py @@ -130,17 +130,17 @@ def fit(self, X, y, **fit_params): meta_features = self._predict_meta_features(X) # Extract fit_params for meta_clf_ - meta_clf_fit_params = {} + meta_fit_params = {} meta_clf_name = list(self.named_meta_clf_.keys())[0] for key, value in six.iteritems(fit_params): if meta_clf_name in key and 'meta-' in meta_clf_name: - meta_clf_fit_params[key.replace(meta_clf_name+'__', '')] = value + meta_fit_params[key.replace(meta_clf_name+'__', '')] = value if not self.use_features_in_secondary: - self.meta_clf_.fit(meta_features, y, **meta_clf_fit_params) + self.meta_clf_.fit(meta_features, y, **meta_fit_params) else: self.meta_clf_.fit(np.hstack((X, meta_features)), y, - **meta_clf_fit_params) + **meta_fit_params) return self diff --git a/mlxtend/classifier/tests/test_stacking_classifier.py b/mlxtend/classifier/tests/test_stacking_classifier.py index 337e016b5..6cfdf672a 100644 --- a/mlxtend/classifier/tests/test_stacking_classifier.py +++ b/mlxtend/classifier/tests/test_stacking_classifier.py @@ -79,6 +79,29 @@ def test_StackingClassifier_proba_concat_1(): assert scores_mean == 0.93, scores_mean +def test_StackingClassifier_fit_params(): + np.random.seed(123) + meta = LogisticRegression() + clf1 = RandomForestClassifier() + clf2 = GaussianNB() + sclf = StackingClassifier(classifiers=[clf1, clf2], + meta_classifier=meta) + n_samples = X.shape[0] + fit_params = { + 'randomforestclassifier__sample_weight': np.ones(n_samples), + 'meta-logisticregression__sample_weight': np.arange(n_samples) + } + + scores = cross_val_score(sclf, + X, + y, + cv=5, + scoring='accuracy', + fit_params=fit_params) + scores_mean = (round(scores.mean(), 2)) + assert scores_mean == 0.95 + + def test_StackingClassifier_avg_vs_concat(): np.random.seed(123) lr1 = LogisticRegression() From 2b3668b607b2613fd7b0231c335de493a79be66a Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Wed, 27 Sep 2017 13:34:30 -0500 Subject: [PATCH 3/5] Adds fit_params to StackingRegressor fit method + adds test --- mlxtend/classifier/stacking_classification.py | 4 +-- mlxtend/regressor/stacking_regression.py | 28 ++++++++++++++++--- .../tests/test_stacking_regression.py | 19 ++++++++++++- 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/mlxtend/classifier/stacking_classification.py b/mlxtend/classifier/stacking_classification.py index d37551f70..15f575db0 100644 --- a/mlxtend/classifier/stacking_classification.py +++ b/mlxtend/classifier/stacking_classification.py @@ -88,8 +88,8 @@ def fit(self, X, y, **fit_params): y : array-like, shape = [n_samples] or [n_samples, n_outputs] Target values. fit_params : dict, optional - Parameters to pass to the fit methods of the classifiers and - meta_classifier. + Parameters to pass to the fit methods of `classifiers` and + `meta_classifier`. Returns ------- diff --git a/mlxtend/regressor/stacking_regression.py b/mlxtend/regressor/stacking_regression.py index cb5f58932..215f178c3 100644 --- a/mlxtend/regressor/stacking_regression.py +++ b/mlxtend/regressor/stacking_regression.py @@ -65,7 +65,7 @@ def __init__(self, regressors, meta_regressor, verbose=0): _name_estimators([meta_regressor])} self.verbose = verbose - def fit(self, X, y): + def fit(self, X, y, **fit_params): """Learn weight coefficients from training data for each regressor. Parameters @@ -75,6 +75,9 @@ def fit(self, X, y): n_features is the number of features. y : array-like, shape = [n_samples] or [n_samples, n_targets] Target values. + fit_params : dict, optional + Parameters to pass to the fit methods of `regressors` and + `meta_regressor`. Returns ------- @@ -82,11 +85,15 @@ def fit(self, X, y): """ self.regr_ = [clone(regr) for regr in self.regressors] + self.named_regr_ = {key: value for key, value in + _name_estimators(self.regr_)} self.meta_regr_ = clone(self.meta_regressor) + self.named_meta_regr_ = {'meta-%s' % key: value for key, value in + _name_estimators([self.meta_regr_])} if self.verbose > 0: print("Fitting %d regressors..." % (len(self.regressors))) - for regr in self.regr_: + for name, regr in six.iteritems(self.named_regr_): if self.verbose > 0: i = self.regr_.index(regr) + 1 @@ -100,10 +107,23 @@ def fit(self, X, y): if self.verbose > 1: print(_name_estimators((regr,))[0][1]) - regr.fit(X, y) + # Extract fit_params for regr + regr_fit_params = {} + for key, value in six.iteritems(fit_params): + if name in key and 'meta-' not in key: + regr_fit_params[key.replace(name+'__', '')] = value + + regr.fit(X, y, **regr_fit_params) meta_features = self._predict_meta_features(X) - self.meta_regr_.fit(meta_features, y) + # Extract fit_params for meta_regr_ + meta_fit_params = {} + meta_regr_name = list(self.named_meta_regr_.keys())[0] + for key, value in six.iteritems(fit_params): + if meta_regr_name in key and 'meta-' in meta_regr_name: + meta_fit_params[key.replace(meta_regr_name+'__', '')] = value + self.meta_regr_.fit(meta_features, y, **meta_fit_params) + return self @property diff --git a/mlxtend/regressor/tests/test_stacking_regression.py b/mlxtend/regressor/tests/test_stacking_regression.py index ee708e989..16b3a0443 100644 --- a/mlxtend/regressor/tests/test_stacking_regression.py +++ b/mlxtend/regressor/tests/test_stacking_regression.py @@ -11,7 +11,7 @@ import numpy as np from numpy.testing import assert_almost_equal from nose.tools import raises -from sklearn.model_selection import GridSearchCV +from sklearn.model_selection import GridSearchCV, cross_val_score # Generating a sample dataset np.random.seed(1) @@ -108,6 +108,23 @@ def test_gridsearch_numerate_regr(): assert best == got +def test_StackingRegressor_fit_params(): + lr = LinearRegression() + svr_lin = SVR(kernel='linear') + ridge = Ridge(random_state=1) + svr_rbf = SVR(kernel='rbf') + stregr = StackingRegressor(regressors=[svr_lin, lr, ridge], + meta_regressor=svr_rbf) + + fit_params = {'ridge__sample_weight': np.ones(X1.shape[0]), + 'svr__sample_weight': np.ones(X1.shape[0]), + 'meta-svr__sample_weight': np.ones(X1.shape[0])} + + scores = cross_val_score(stregr, X1, y, cv=5, fit_params=fit_params) + scores_mean = (round(scores.mean(), 1)) + assert scores_mean == 0.1 + + def test_get_coeff(): lr = LinearRegression() svr_lin = SVR(kernel='linear') From 7a67ce57ae9e16e95eee15e9549517ffaf82dc7f Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Sat, 14 Oct 2017 12:58:25 -0500 Subject: [PATCH 4/5] Adds fit_params to StackingCVClassifier fit method --- .../classifier/stacking_cv_classification.py | 42 +++++++++++++++---- .../tests/test_stacking_cv_classifier.py | 25 ++++++++++- 2 files changed, 57 insertions(+), 10 deletions(-) diff --git a/mlxtend/classifier/stacking_cv_classification.py b/mlxtend/classifier/stacking_cv_classification.py index 330a672a0..5138488a8 100644 --- a/mlxtend/classifier/stacking_cv_classification.py +++ b/mlxtend/classifier/stacking_cv_classification.py @@ -111,7 +111,7 @@ def __init__(self, classifiers, meta_classifier, self.stratify = stratify self.shuffle = shuffle - def fit(self, X, y, groups=None): + def fit(self, X, y, groups=None, **fit_params): """ Fit ensemble classifers and the meta-classifier. Parameters @@ -119,13 +119,16 @@ def fit(self, X, y, groups=None): X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. - y : numpy array, shape = [n_samples] Target values. - groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() + fit_params : dict, optional + Parameters to pass to the fit methods of `classifiers` and + `meta_classifier`. Note that only fit parameters for `classifiers` + that are the same for each cross-validation split are supported + (e.g. `sample_weight` is not currently supported). Returns ------- @@ -133,7 +136,11 @@ def fit(self, X, y, groups=None): """ self.clfs_ = [clone(clf) for clf in self.classifiers] + self.named_clfs_ = {key: value for key, value in + _name_estimators(self.clfs_)} self.meta_clf_ = clone(self.meta_classifier) + self.named_meta_clf_ = {'meta-%s' % key: value for key, value in + _name_estimators([self.meta_clf_])} if self.verbose > 0: print("Fitting %d classifiers..." % (len(self.classifiers))) @@ -144,8 +151,23 @@ def fit(self, X, y, groups=None): final_cv.shuffle = self.shuffle skf = list(final_cv.split(X, y, groups)) + # Get fit_params for each classifier in self.named_clfs_ + named_clfs_fit_params = {} + for name, clf in six.iteritems(self.named_clfs_): + clf_fit_params = {} + for key, value in six.iteritems(fit_params): + if name in key and 'meta-' not in key: + clf_fit_params[key.replace(name+'__', '')] = value + named_clfs_fit_params[name] = clf_fit_params + # Get fit_params for self.named_meta_clf_ + meta_fit_params = {} + meta_clf_name = list(self.named_meta_clf_.keys())[0] + for key, value in six.iteritems(fit_params): + if meta_clf_name in key and 'meta-' in meta_clf_name: + meta_fit_params[key.replace(meta_clf_name+'__', '')] = value + all_model_predictions = np.array([]).reshape(len(y), 0) - for model in self.clfs_: + for name, model in six.iteritems(self.named_clfs_): if self.verbose > 0: i = self.clfs_.index(model) + 1 @@ -172,7 +194,8 @@ def fit(self, X, y, groups=None): ((num + 1), final_cv.get_n_splits())) try: - model.fit(X[train_index], y[train_index]) + model.fit(X[train_index], y[train_index], + **named_clfs_fit_params[name]) except TypeError as e: raise TypeError(str(e) + '\nPlease check that X and y' 'are NumPy arrays. If X and y are lists' @@ -215,16 +238,17 @@ def fit(self, X, y, groups=None): X[test_index])) # Fit the base models correctly this time using ALL the training set - for model in self.clfs_: - model.fit(X, y) + for name, model in six.iteritems(self.named_clfs_): + model.fit(X, y, **named_clfs_fit_params[name]) # Fit the secondary model if not self.use_features_in_secondary: - self.meta_clf_.fit(all_model_predictions, reordered_labels) + self.meta_clf_.fit(all_model_predictions, reordered_labels, + **meta_fit_params) else: self.meta_clf_.fit(np.hstack((reordered_features, all_model_predictions)), - reordered_labels) + reordered_labels, **meta_fit_params) return self diff --git a/mlxtend/classifier/tests/test_stacking_cv_classifier.py b/mlxtend/classifier/tests/test_stacking_cv_classifier.py index a4f38acfb..e7e42af4d 100644 --- a/mlxtend/classifier/tests/test_stacking_cv_classifier.py +++ b/mlxtend/classifier/tests/test_stacking_cv_classifier.py @@ -8,7 +8,7 @@ from mlxtend.classifier import StackingCVClassifier import pandas as pd -from sklearn.linear_model import LogisticRegression +from sklearn.linear_model import LogisticRegression, SGDClassifier from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from sklearn.neighbors import KNeighborsClassifier @@ -61,6 +61,29 @@ def test_StackingClassifier_proba(): assert scores_mean == 0.93 +def test_StackingClassifier_fit_params(): + np.random.seed(123) + meta = LogisticRegression() + clf1 = RandomForestClassifier() + clf2 = SGDClassifier(random_state=2) + sclf = StackingCVClassifier(classifiers=[clf1, clf2], + meta_classifier=meta, + shuffle=False) + fit_params = { + 'sgdclassifier__intercept_init': np.unique(y), + 'meta-logisticregression__sample_weight': np.full(X.shape[0], 2) + } + + scores = cross_val_score(sclf, + X, + y, + cv=5, + scoring='accuracy', + fit_params=fit_params) + scores_mean = (round(scores.mean(), 2)) + assert scores_mean == 0.86 + + def test_gridsearch(): np.random.seed(123) meta = LogisticRegression() From ab389b10634e3411d920813c6e9a6e163b0c24bb Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Sun, 15 Oct 2017 10:21:18 -0500 Subject: [PATCH 5/5] Fixes typo in CONTRIBUTING.md --- docs/sources/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/CONTRIBUTING.md b/docs/sources/CONTRIBUTING.md index a9cda8e8c..c24929b71 100755 --- a/docs/sources/CONTRIBUTING.md +++ b/docs/sources/CONTRIBUTING.md @@ -303,7 +303,7 @@ The documents containing code examples for the "User Guide" are generated from I 3. Convert the notebook to markdown using the `ipynb2markdown.py` converter ```python -~/github/mlxtend/docs$ python ipynb2markdown.py --ipynb_path ./sources/user_guide/subpackage/notebookname.ipynb +~/github/mlxtend/docs$ python ipynb2markdown.py --ipynb ./sources/user_guide/subpackage/notebookname.ipynb ``` **Note**