-
-
Notifications
You must be signed in to change notification settings - Fork 25.3k
/
_self_training.py
400 lines (327 loc) · 13.4 KB
/
_self_training.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
import warnings
import numpy as np
from ..base import MetaEstimatorMixin, clone, BaseEstimator
from ..utils.validation import check_is_fitted
from ..utils.metaestimators import available_if
from ..utils import safe_mask
__all__ = ["SelfTrainingClassifier"]
# Authors: Oliver Rausch <rauscho@ethz.ch>
# Patrice Becker <beckerp@ethz.ch>
# License: BSD 3 clause
def _estimator_has(attr):
"""Check if `self.base_estimator_ `or `self.base_estimator_` has `attr`."""
return lambda self: (
hasattr(self.base_estimator_, attr)
if hasattr(self, "base_estimator_")
else hasattr(self.base_estimator, attr)
)
class SelfTrainingClassifier(MetaEstimatorMixin, BaseEstimator):
"""Self-training classifier.
This class allows a given supervised classifier to function as a
semi-supervised classifier, allowing it to learn from unlabeled data. It
does this by iteratively predicting pseudo-labels for the unlabeled data
and adding them to the training set.
The classifier will continue iterating until either max_iter is reached, or
no pseudo-labels were added to the training set in the previous iteration.
Read more in the :ref:`User Guide <self_training>`.
Parameters
----------
base_estimator : estimator object
An estimator object implementing `fit` and `predict_proba`.
Invoking the `fit` method will fit a clone of the passed estimator,
which will be stored in the `base_estimator_` attribute.
threshold : float, default=0.75
The decision threshold for use with `criterion='threshold'`.
Should be in [0, 1). When using the `'threshold'` criterion, a
:ref:`well calibrated classifier <calibration>` should be used.
criterion : {'threshold', 'k_best'}, default='threshold'
The selection criterion used to select which labels to add to the
training set. If `'threshold'`, pseudo-labels with prediction
probabilities above `threshold` are added to the dataset. If `'k_best'`,
the `k_best` pseudo-labels with highest prediction probabilities are
added to the dataset. When using the 'threshold' criterion, a
:ref:`well calibrated classifier <calibration>` should be used.
k_best : int, default=10
The amount of samples to add in each iteration. Only used when
`criterion='k_best'`.
max_iter : int or None, default=10
Maximum number of iterations allowed. Should be greater than or equal
to 0. If it is `None`, the classifier will continue to predict labels
until no new pseudo-labels are added, or all unlabeled samples have
been labeled.
verbose : bool, default=False
Enable verbose output.
Attributes
----------
base_estimator_ : estimator object
The fitted estimator.
classes_ : ndarray or list of ndarray of shape (n_classes,)
Class labels for each output. (Taken from the trained
`base_estimator_`).
transduction_ : ndarray of shape (n_samples,)
The labels used for the final fit of the classifier, including
pseudo-labels added during fit.
labeled_iter_ : ndarray of shape (n_samples,)
The iteration in which each sample was labeled. When a sample has
iteration 0, the sample was already labeled in the original dataset.
When a sample has iteration -1, the sample was not labeled in any
iteration.
n_features_in_ : int
Number of features seen during :term:`fit`.
.. versionadded:: 0.24
feature_names_in_ : ndarray of shape (`n_features_in_`,)
Names of features seen during :term:`fit`. Defined only when `X`
has feature names that are all strings.
.. versionadded:: 1.0
n_iter_ : int
The number of rounds of self-training, that is the number of times the
base estimator is fitted on relabeled variants of the training set.
termination_condition_ : {'max_iter', 'no_change', 'all_labeled'}
The reason that fitting was stopped.
- `'max_iter'`: `n_iter_` reached `max_iter`.
- `'no_change'`: no new labels were predicted.
- `'all_labeled'`: all unlabeled samples were labeled before `max_iter`
was reached.
See Also
--------
LabelPropagation : Label propagation classifier.
LabelSpreading : Label spreading model for semi-supervised learning.
References
----------
:doi:`David Yarowsky. 1995. Unsupervised word sense disambiguation rivaling
supervised methods. In Proceedings of the 33rd annual meeting on
Association for Computational Linguistics (ACL '95). Association for
Computational Linguistics, Stroudsburg, PA, USA, 189-196.
<10.3115/981658.981684>`
Examples
--------
>>> import numpy as np
>>> from sklearn import datasets
>>> from sklearn.semi_supervised import SelfTrainingClassifier
>>> from sklearn.svm import SVC
>>> rng = np.random.RandomState(42)
>>> iris = datasets.load_iris()
>>> random_unlabeled_points = rng.rand(iris.target.shape[0]) < 0.3
>>> iris.target[random_unlabeled_points] = -1
>>> svc = SVC(probability=True, gamma="auto")
>>> self_training_model = SelfTrainingClassifier(svc)
>>> self_training_model.fit(iris.data, iris.target)
SelfTrainingClassifier(...)
"""
_estimator_type = "classifier"
def __init__(
self,
base_estimator,
threshold=0.75,
criterion="threshold",
k_best=10,
max_iter=10,
verbose=False,
):
self.base_estimator = base_estimator
self.threshold = threshold
self.criterion = criterion
self.k_best = k_best
self.max_iter = max_iter
self.verbose = verbose
def fit(self, X, y):
"""
Fit self-training classifier using `X`, `y` as training data.
Parameters
----------
X : {array-like, sparse matrix} of shape (n_samples, n_features)
Array representing the data.
y : {array-like, sparse matrix} of shape (n_samples,)
Array representing the labels. Unlabeled samples should have the
label -1.
Returns
-------
self : object
Fitted estimator.
"""
# we need row slicing support for sparce matrices, but costly finiteness check
# can be delegated to the base estimator.
X, y = self._validate_data(
X, y, accept_sparse=["csr", "csc", "lil", "dok"], force_all_finite=False
)
if self.base_estimator is None:
raise ValueError("base_estimator cannot be None!")
self.base_estimator_ = clone(self.base_estimator)
if self.max_iter is not None and self.max_iter < 0:
raise ValueError(f"max_iter must be >= 0 or None, got {self.max_iter}")
if not (0 <= self.threshold < 1):
raise ValueError(f"threshold must be in [0,1), got {self.threshold}")
if self.criterion not in ["threshold", "k_best"]:
raise ValueError(
"criterion must be either 'threshold' "
f"or 'k_best', got {self.criterion}."
)
if y.dtype.kind in ["U", "S"]:
raise ValueError(
"y has dtype string. If you wish to predict on "
"string targets, use dtype object, and use -1"
" as the label for unlabeled samples."
)
has_label = y != -1
if np.all(has_label):
warnings.warn("y contains no unlabeled samples", UserWarning)
if self.criterion == "k_best" and (
self.k_best > X.shape[0] - np.sum(has_label)
):
warnings.warn(
"k_best is larger than the amount of unlabeled "
"samples. All unlabeled samples will be labeled in "
"the first iteration",
UserWarning,
)
self.transduction_ = np.copy(y)
self.labeled_iter_ = np.full_like(y, -1)
self.labeled_iter_[has_label] = 0
self.n_iter_ = 0
while not np.all(has_label) and (
self.max_iter is None or self.n_iter_ < self.max_iter
):
self.n_iter_ += 1
self.base_estimator_.fit(
X[safe_mask(X, has_label)], self.transduction_[has_label]
)
# Predict on the unlabeled samples
prob = self.base_estimator_.predict_proba(X[safe_mask(X, ~has_label)])
pred = self.base_estimator_.classes_[np.argmax(prob, axis=1)]
max_proba = np.max(prob, axis=1)
# Select new labeled samples
if self.criterion == "threshold":
selected = max_proba > self.threshold
else:
n_to_select = min(self.k_best, max_proba.shape[0])
if n_to_select == max_proba.shape[0]:
selected = np.ones_like(max_proba, dtype=bool)
else:
# NB these are indices, not a mask
selected = np.argpartition(-max_proba, n_to_select)[:n_to_select]
# Map selected indices into original array
selected_full = np.nonzero(~has_label)[0][selected]
# Add newly labeled confident predictions to the dataset
self.transduction_[selected_full] = pred[selected]
has_label[selected_full] = True
self.labeled_iter_[selected_full] = self.n_iter_
if selected_full.shape[0] == 0:
# no changed labels
self.termination_condition_ = "no_change"
break
if self.verbose:
print(
f"End of iteration {self.n_iter_},"
f" added {selected_full.shape[0]} new labels."
)
if self.n_iter_ == self.max_iter:
self.termination_condition_ = "max_iter"
if np.all(has_label):
self.termination_condition_ = "all_labeled"
self.base_estimator_.fit(
X[safe_mask(X, has_label)], self.transduction_[has_label]
)
self.classes_ = self.base_estimator_.classes_
return self
@available_if(_estimator_has("predict"))
def predict(self, X):
"""Predict the classes of `X`.
Parameters
----------
X : {array-like, sparse matrix} of shape (n_samples, n_features)
Array representing the data.
Returns
-------
y : ndarray of shape (n_samples,)
Array with predicted labels.
"""
check_is_fitted(self)
X = self._validate_data(
X,
accept_sparse=True,
force_all_finite=False,
reset=False,
)
return self.base_estimator_.predict(X)
@available_if(_estimator_has("predict_proba"))
def predict_proba(self, X):
"""Predict probability for each possible outcome.
Parameters
----------
X : {array-like, sparse matrix} of shape (n_samples, n_features)
Array representing the data.
Returns
-------
y : ndarray of shape (n_samples, n_features)
Array with prediction probabilities.
"""
check_is_fitted(self)
X = self._validate_data(
X,
accept_sparse=True,
force_all_finite=False,
reset=False,
)
return self.base_estimator_.predict_proba(X)
@available_if(_estimator_has("decision_function"))
def decision_function(self, X):
"""Call decision function of the `base_estimator`.
Parameters
----------
X : {array-like, sparse matrix} of shape (n_samples, n_features)
Array representing the data.
Returns
-------
y : ndarray of shape (n_samples, n_features)
Result of the decision function of the `base_estimator`.
"""
check_is_fitted(self)
X = self._validate_data(
X,
accept_sparse=True,
force_all_finite=False,
reset=False,
)
return self.base_estimator_.decision_function(X)
@available_if(_estimator_has("predict_log_proba"))
def predict_log_proba(self, X):
"""Predict log probability for each possible outcome.
Parameters
----------
X : {array-like, sparse matrix} of shape (n_samples, n_features)
Array representing the data.
Returns
-------
y : ndarray of shape (n_samples, n_features)
Array with log prediction probabilities.
"""
check_is_fitted(self)
X = self._validate_data(
X,
accept_sparse=True,
force_all_finite=False,
reset=False,
)
return self.base_estimator_.predict_log_proba(X)
@available_if(_estimator_has("score"))
def score(self, X, y):
"""Call score on the `base_estimator`.
Parameters
----------
X : {array-like, sparse matrix} of shape (n_samples, n_features)
Array representing the data.
y : array-like of shape (n_samples,)
Array representing the labels.
Returns
-------
score : float
Result of calling score on the `base_estimator`.
"""
check_is_fitted(self)
X = self._validate_data(
X,
accept_sparse=True,
force_all_finite=False,
reset=False,
)
return self.base_estimator_.score(X, y)