/
_sparse_pca.py
465 lines (380 loc) · 14.9 KB
/
_sparse_pca.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
"""Matrix factorization with Sparse PCA."""
# Author: Vlad Niculae, Gael Varoquaux, Alexandre Gramfort
# License: BSD 3 clause
import warnings
import numpy as np
from ..utils import check_random_state
from ..utils.validation import check_is_fitted
from ..linear_model import ridge_regression
from ..base import BaseEstimator, TransformerMixin, _ClassNamePrefixFeaturesOutMixin
from ._dict_learning import dict_learning, dict_learning_online
class SparsePCA(_ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator):
"""Sparse Principal Components Analysis (SparsePCA).
Finds the set of sparse components that can optimally reconstruct
the data. The amount of sparseness is controllable by the coefficient
of the L1 penalty, given by the parameter alpha.
Read more in the :ref:`User Guide <SparsePCA>`.
Parameters
----------
n_components : int, default=None
Number of sparse atoms to extract. If None, then ``n_components``
is set to ``n_features``.
alpha : float, default=1
Sparsity controlling parameter. Higher values lead to sparser
components.
ridge_alpha : float, default=0.01
Amount of ridge shrinkage to apply in order to improve
conditioning when calling the transform method.
max_iter : int, default=1000
Maximum number of iterations to perform.
tol : float, default=1e-8
Tolerance for the stopping condition.
method : {'lars', 'cd'}, default='lars'
Method to be used for optimization.
lars: uses the least angle regression method to solve the lasso problem
(linear_model.lars_path)
cd: uses the coordinate descent method to compute the
Lasso solution (linear_model.Lasso). Lars will be faster if
the estimated components are sparse.
n_jobs : int, default=None
Number of parallel jobs to run.
``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
``-1`` means using all processors. See :term:`Glossary <n_jobs>`
for more details.
U_init : ndarray of shape (n_samples, n_components), default=None
Initial values for the loadings for warm restart scenarios. Only used
if `U_init` and `V_init` are not None.
V_init : ndarray of shape (n_components, n_features), default=None
Initial values for the components for warm restart scenarios. Only used
if `U_init` and `V_init` are not None.
verbose : int or bool, default=False
Controls the verbosity; the higher, the more messages. Defaults to 0.
random_state : int, RandomState instance or None, default=None
Used during dictionary learning. Pass an int for reproducible results
across multiple function calls.
See :term:`Glossary <random_state>`.
Attributes
----------
components_ : ndarray of shape (n_components, n_features)
Sparse components extracted from the data.
error_ : ndarray
Vector of errors at each iteration.
n_components_ : int
Estimated number of components.
.. versionadded:: 0.23
n_iter_ : int
Number of iterations run.
mean_ : ndarray of shape (n_features,)
Per-feature empirical mean, estimated from the training set.
Equal to ``X.mean(axis=0)``.
n_features_in_ : int
Number of features seen during :term:`fit`.
.. versionadded:: 0.24
feature_names_in_ : ndarray of shape (`n_features_in_`,)
Names of features seen during :term:`fit`. Defined only when `X`
has feature names that are all strings.
.. versionadded:: 1.0
See Also
--------
PCA : Principal Component Analysis implementation.
MiniBatchSparsePCA : Mini batch variant of `SparsePCA` that is faster but less
accurate.
DictionaryLearning : Generic dictionary learning problem using a sparse code.
Examples
--------
>>> import numpy as np
>>> from sklearn.datasets import make_friedman1
>>> from sklearn.decomposition import SparsePCA
>>> X, _ = make_friedman1(n_samples=200, n_features=30, random_state=0)
>>> transformer = SparsePCA(n_components=5, random_state=0)
>>> transformer.fit(X)
SparsePCA(...)
>>> X_transformed = transformer.transform(X)
>>> X_transformed.shape
(200, 5)
>>> # most values in the components_ are zero (sparsity)
>>> np.mean(transformer.components_ == 0)
0.9666...
"""
def __init__(
self,
n_components=None,
*,
alpha=1,
ridge_alpha=0.01,
max_iter=1000,
tol=1e-8,
method="lars",
n_jobs=None,
U_init=None,
V_init=None,
verbose=False,
random_state=None,
):
self.n_components = n_components
self.alpha = alpha
self.ridge_alpha = ridge_alpha
self.max_iter = max_iter
self.tol = tol
self.method = method
self.n_jobs = n_jobs
self.U_init = U_init
self.V_init = V_init
self.verbose = verbose
self.random_state = random_state
def fit(self, X, y=None):
"""Fit the model from data in X.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Training vector, where `n_samples` is the number of samples
and `n_features` is the number of features.
y : Ignored
Not used, present here for API consistency by convention.
Returns
-------
self : object
Returns the instance itself.
"""
random_state = check_random_state(self.random_state)
X = self._validate_data(X)
self.mean_ = X.mean(axis=0)
X = X - self.mean_
if self.n_components is None:
n_components = X.shape[1]
else:
n_components = self.n_components
code_init = self.V_init.T if self.V_init is not None else None
dict_init = self.U_init.T if self.U_init is not None else None
Vt, _, E, self.n_iter_ = dict_learning(
X.T,
n_components,
alpha=self.alpha,
tol=self.tol,
max_iter=self.max_iter,
method=self.method,
n_jobs=self.n_jobs,
verbose=self.verbose,
random_state=random_state,
code_init=code_init,
dict_init=dict_init,
return_n_iter=True,
)
self.components_ = Vt.T
components_norm = np.linalg.norm(self.components_, axis=1)[:, np.newaxis]
components_norm[components_norm == 0] = 1
self.components_ /= components_norm
self.n_components_ = len(self.components_)
self.error_ = E
return self
def transform(self, X):
"""Least Squares projection of the data onto the sparse components.
To avoid instability issues in case the system is under-determined,
regularization can be applied (Ridge regression) via the
`ridge_alpha` parameter.
Note that Sparse PCA components orthogonality is not enforced as in PCA
hence one cannot use a simple linear projection.
Parameters
----------
X : ndarray of shape (n_samples, n_features)
Test data to be transformed, must have the same number of
features as the data used to train the model.
Returns
-------
X_new : ndarray of shape (n_samples, n_components)
Transformed data.
"""
check_is_fitted(self)
X = self._validate_data(X, reset=False)
X = X - self.mean_
U = ridge_regression(
self.components_.T, X.T, self.ridge_alpha, solver="cholesky"
)
return U
@property
def _n_features_out(self):
"""Number of transformed output features."""
return self.components_.shape[0]
def _more_tags(self):
return {
"preserves_dtype": [np.float64, np.float32],
}
class MiniBatchSparsePCA(SparsePCA):
"""Mini-batch Sparse Principal Components Analysis.
Finds the set of sparse components that can optimally reconstruct
the data. The amount of sparseness is controllable by the coefficient
of the L1 penalty, given by the parameter alpha.
Read more in the :ref:`User Guide <SparsePCA>`.
Parameters
----------
n_components : int, default=None
Number of sparse atoms to extract. If None, then ``n_components``
is set to ``n_features``.
alpha : int, default=1
Sparsity controlling parameter. Higher values lead to sparser
components.
ridge_alpha : float, default=0.01
Amount of ridge shrinkage to apply in order to improve
conditioning when calling the transform method.
n_iter : int, default=100
Number of iterations to perform for each mini batch.
callback : callable, default=None
Callable that gets invoked every five iterations.
batch_size : int, default=3
The number of features to take in each mini batch.
verbose : int or bool, default=False
Controls the verbosity; the higher, the more messages. Defaults to 0.
shuffle : bool, default=True
Whether to shuffle the data before splitting it in batches.
n_jobs : int, default=None
Number of parallel jobs to run.
``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
``-1`` means using all processors. See :term:`Glossary <n_jobs>`
for more details.
method : {'lars', 'cd'}, default='lars'
Method to be used for optimization.
lars: uses the least angle regression method to solve the lasso problem
(linear_model.lars_path)
cd: uses the coordinate descent method to compute the
Lasso solution (linear_model.Lasso). Lars will be faster if
the estimated components are sparse.
random_state : int, RandomState instance or None, default=None
Used for random shuffling when ``shuffle`` is set to ``True``,
during online dictionary learning. Pass an int for reproducible results
across multiple function calls.
See :term:`Glossary <random_state>`.
Attributes
----------
components_ : ndarray of shape (n_components, n_features)
Sparse components extracted from the data.
n_components_ : int
Estimated number of components.
.. versionadded:: 0.23
n_iter_ : int
Number of iterations run.
mean_ : ndarray of shape (n_features,)
Per-feature empirical mean, estimated from the training set.
Equal to ``X.mean(axis=0)``.
n_features_in_ : int
Number of features seen during :term:`fit`.
.. versionadded:: 0.24
feature_names_in_ : ndarray of shape (`n_features_in_`,)
Names of features seen during :term:`fit`. Defined only when `X`
has feature names that are all strings.
.. versionadded:: 1.0
See Also
--------
DictionaryLearning : Find a dictionary that sparsely encodes data.
IncrementalPCA : Incremental principal components analysis.
PCA : Principal component analysis.
SparsePCA : Sparse Principal Components Analysis.
TruncatedSVD : Dimensionality reduction using truncated SVD.
Examples
--------
>>> import numpy as np
>>> from sklearn.datasets import make_friedman1
>>> from sklearn.decomposition import MiniBatchSparsePCA
>>> X, _ = make_friedman1(n_samples=200, n_features=30, random_state=0)
>>> transformer = MiniBatchSparsePCA(n_components=5, batch_size=50,
... random_state=0)
>>> transformer.fit(X)
MiniBatchSparsePCA(...)
>>> X_transformed = transformer.transform(X)
>>> X_transformed.shape
(200, 5)
>>> # most values in the components_ are zero (sparsity)
>>> np.mean(transformer.components_ == 0)
0.94
"""
def __init__(
self,
n_components=None,
*,
alpha=1,
ridge_alpha=0.01,
n_iter=100,
callback=None,
batch_size=3,
verbose=False,
shuffle=True,
n_jobs=None,
method="lars",
random_state=None,
):
super().__init__(
n_components=n_components,
alpha=alpha,
verbose=verbose,
ridge_alpha=ridge_alpha,
n_jobs=n_jobs,
method=method,
random_state=random_state,
)
self.n_iter = n_iter
self.callback = callback
self.batch_size = batch_size
self.shuffle = shuffle
def fit(self, X, y=None):
"""Fit the model from data in X.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Training vector, where `n_samples` is the number of samples
and `n_features` is the number of features.
y : Ignored
Not used, present for API consistency by convention.
Returns
-------
self : object
Returns the instance itself.
"""
random_state = check_random_state(self.random_state)
X = self._validate_data(X)
self.mean_ = X.mean(axis=0)
X = X - self.mean_
if self.n_components is None:
n_components = X.shape[1]
else:
n_components = self.n_components
with warnings.catch_warnings():
# return_n_iter and n_iter are deprecated. TODO Remove in 1.3
warnings.filterwarnings(
"ignore",
message=(
"'return_n_iter' is deprecated in version 1.1 and will be "
"removed in version 1.3. From 1.3 'n_iter' will never be "
"returned. Refer to the 'n_iter_' and 'n_steps_' attributes "
"of the MiniBatchDictionaryLearning object instead."
),
category=FutureWarning,
)
warnings.filterwarnings(
"ignore",
message=(
"'n_iter' is deprecated in version 1.1 and will be removed in "
"version 1.3. Use 'max_iter' instead."
),
category=FutureWarning,
)
Vt, _, self.n_iter_ = dict_learning_online(
X.T,
n_components,
alpha=self.alpha,
n_iter=self.n_iter,
return_code=True,
dict_init=None,
verbose=self.verbose,
callback=self.callback,
batch_size=self.batch_size,
shuffle=self.shuffle,
n_jobs=self.n_jobs,
method=self.method,
random_state=random_state,
return_n_iter=True,
)
self.components_ = Vt.T
components_norm = np.linalg.norm(self.components_, axis=1)[:, np.newaxis]
components_norm[components_norm == 0] = 1
self.components_ /= components_norm
self.n_components_ = len(self.components_)
return self