/
bias_variance_decomp.py
162 lines (128 loc) · 5.43 KB
/
bias_variance_decomp.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
# Sebastian Raschka 2014-2024
# mlxtend Machine Learning Library Extensions
#
# Nonparametric Permutation Test
# Author: Sebastian Raschka <sebastianraschka.com>
#
# License: BSD 3 clause
import numpy as np
def _draw_bootstrap_sample(rng, X, y):
sample_indices = np.arange(X.shape[0])
bootstrap_indices = rng.choice(
sample_indices, size=sample_indices.shape[0], replace=True
)
return X[bootstrap_indices], y[bootstrap_indices]
def bias_variance_decomp(
estimator,
X_train,
y_train,
X_test,
y_test,
loss="0-1_loss",
num_rounds=200,
random_seed=None,
**fit_params
):
"""
estimator : object
A classifier or regressor object or class implementing both a
`fit` and `predict` method similar to the scikit-learn API.
X_train : array-like, shape=(num_examples, num_features)
A training dataset for drawing the bootstrap samples to carry
out the bias-variance decomposition.
y_train : array-like, shape=(num_examples)
Targets (class labels, continuous values in case of regression)
associated with the `X_train` examples.
X_test : array-like, shape=(num_examples, num_features)
The test dataset for computing the average loss, bias,
and variance.
y_test : array-like, shape=(num_examples)
Targets (class labels, continuous values in case of regression)
associated with the `X_test` examples.
loss : str (default='0-1_loss')
Loss function for performing the bias-variance decomposition.
Currently allowed values are '0-1_loss' and 'mse'.
num_rounds : int (default=200)
Number of bootstrap rounds (sampling from the training set)
for performing the bias-variance decomposition. Each bootstrap
sample has the same size as the original training set.
random_seed : int (default=None)
Random seed for the bootstrap sampling used for the
bias-variance decomposition.
fit_params : additional parameters
Additional parameters to be passed to the .fit() function of the
estimator when it is fit to the bootstrap samples.
Returns
----------
avg_expected_loss, avg_bias, avg_var : returns the average expected
average bias, and average bias (all floats), where the average
is computed over the data points in the test set.
Examples
-----------
For usage examples, please see
https://rasbt.github.io/mlxtend/user_guide/evaluate/bias_variance_decomp/
"""
supported = ["0-1_loss", "mse"]
if loss not in supported:
raise NotImplementedError("loss must be one of the following: %s" % supported)
for ary in (X_train, y_train, X_test, y_test):
if hasattr(ary, "loc"):
raise ValueError(
"The bias_variance_decomp does not "
"support pandas DataFrames yet. "
"Please check the inputs to "
"X_train, y_train, X_test, y_test. "
"If e.g., X_train is a pandas "
"DataFrame, try passing it as NumPy array via "
"X_train=X_train.values."
)
rng = np.random.RandomState(random_seed)
if loss == "0-1_loss":
dtype = np.int64
elif loss == "mse":
dtype = np.float64
all_pred = np.zeros((num_rounds, y_test.shape[0]), dtype=dtype)
for i in range(num_rounds):
X_boot, y_boot = _draw_bootstrap_sample(rng, X_train, y_train)
# Keras support
if estimator.__class__.__name__ in ["Sequential", "Functional"]:
# reset model
for ix, layer in enumerate(estimator.layers):
if hasattr(estimator.layers[ix], "kernel_initializer") and hasattr(
estimator.layers[ix], "bias_initializer"
):
weight_initializer = estimator.layers[ix].kernel_initializer
bias_initializer = estimator.layers[ix].bias_initializer
old_weights, old_biases = estimator.layers[ix].get_weights()
estimator.layers[ix].set_weights(
[
weight_initializer(shape=old_weights.shape),
bias_initializer(shape=len(old_biases)),
]
)
estimator.fit(X_boot, y_boot, **fit_params)
pred = estimator.predict(X_test).reshape(1, -1)
else:
pred = estimator.fit(X_boot, y_boot, **fit_params).predict(X_test)
all_pred[i] = pred
if loss == "0-1_loss":
main_predictions = np.apply_along_axis(
lambda x: np.argmax(np.bincount(x)), axis=0, arr=all_pred
)
avg_expected_loss = np.apply_along_axis(
lambda x: (x != y_test).mean(), axis=1, arr=all_pred
).mean()
avg_bias = np.sum(main_predictions != y_test) / y_test.size
var = np.zeros(pred.shape)
for pred in all_pred:
var += (pred != main_predictions).astype(np.int_)
var /= num_rounds
avg_var = var.sum() / y_test.shape[0]
else:
avg_expected_loss = np.apply_along_axis(
lambda x: ((x - y_test) ** 2).mean(), axis=1, arr=all_pred
).mean()
main_predictions = np.mean(all_pred, axis=0)
avg_bias = np.sum((main_predictions - y_test) ** 2) / y_test.size
avg_var = np.sum((main_predictions - all_pred) ** 2) / all_pred.size
return avg_expected_loss, avg_bias, avg_var