/
sparse_autoencoder.hpp
219 lines (198 loc) · 6.92 KB
/
sparse_autoencoder.hpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
/**
* @file methods/sparse_autoencoder/sparse_autoencoder.hpp
* @author Siddharth Agrawal
*
* An implementation of sparse autoencoders.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef MLPACK_METHODS_SPARSE_AUTOENCODER_SPARSE_AUTOENCODER_HPP
#define MLPACK_METHODS_SPARSE_AUTOENCODER_SPARSE_AUTOENCODER_HPP
#include <mlpack/core.hpp>
#include "maximal_inputs.hpp"
#include "sparse_autoencoder_function.hpp"
namespace mlpack {
/**
* A sparse autoencoder is a neural network whose aim to learn compressed
* representations of the data, typically for dimensionality reduction, with a
* constraint on the activity of the neurons in the network. Sparse autoencoders
* can be stacked together to learn a hierarchy of features, which provide a
* better representation of the data for classification. This is a method used
* in the recently developed field of deep learning. More technical details
* about the model can be found on the following webpage:
*
* http://deeplearning.stanford.edu/wiki/index.php/UFLDL_Tutorial
*
* An example of how to use the interface is shown below:
*
* @code
* arma::mat data; // Data matrix.
* const size_t vSize = 64; // Size of visible layer, depends on the data.
* const size_t hSize = 25; // Size of hidden layer, depends on requirements.
*
* // Train the model using default options.
* SparseAutoencoder encoder1(data, vSize, hSize);
*
* const size_t numBasis = 5; // Parameter required for L-BFGS algorithm.
* const size_t numIterations = 100; // Maximum number of iterations.
*
* // Use an instantiated optimizer for the training.
* SparseAutoencoderFunction saf(data, vSize, hSize);
* L_BFGS<SparseAutoencoderFunction> optimizer(saf, numBasis, numIterations);
* SparseAutoencoder<L_BFGS> encoder2(optimizer);
*
* arma::mat features1, features2; // Matrices for storing new representations.
*
* // Get new representations from the trained models.
* encoder1.GetNewFeatures(data, features1);
* encoder2.GetNewFeatures(data, features2);
* @endcode
*
* This implementation allows the use of arbitrary mlpack optimizers via the
* OptimizerType template parameter.
*
*/
class SparseAutoencoder
{
public:
/**
* Construct the sparse autoencoder model with the given training data. This
* will train the model. The parameters 'lambda', 'beta' and 'rho' can be set
* optionally. Changing these parameters will have an effect on regularization
* and sparsity of the model.
*
* @tparam OptimizerType The optimizer to use.
* @param data Input data with each column as one example.
* @param visibleSize Size of input vector expected at the visible layer.
* @param hiddenSize Size of input vector expected at the hidden layer.
* @param lambda L2-regularization parameter.
* @param beta KL divergence parameter.
* @param rho Sparsity parameter.
* @param optimizer Desired optimizer.
*/
template<typename OptimizerType = ens::L_BFGS>
SparseAutoencoder(const arma::mat& data,
const size_t visibleSize,
const size_t hiddenSize,
const double lambda = 0.0001,
const double beta = 3,
const double rho = 0.01,
OptimizerType optimizer = OptimizerType());
/**
* Construct the sparse autoencoder model with the given training data. This
* will train the model. The parameters 'lambda', 'beta' and 'rho' can be set
* optionally. Changing these parameters will have an effect on regularization
* and sparsity of the model.
*
* @tparam OptimizerType The optimizer to use.
* @tparam CallbackTypes Types of Callback Functions.
* @param data Input data with each column as one example.
* @param visibleSize Size of input vector expected at the visible layer.
* @param hiddenSize Size of input vector expected at the hidden layer.
* @param lambda L2-regularization parameter.
* @param beta KL divergence parameter.
* @param rho Sparsity parameter.
* @param optimizer Desired optimizer.
* @param callbacks Callback function for ensmallen optimizer `OptimizerType`.
* See https://www.ensmallen.org/docs.html#callback-documentation.
*/
template<typename OptimizerType, typename... CallbackTypes>
SparseAutoencoder(const arma::mat& data,
const size_t visibleSize,
const size_t hiddenSize,
const double lambda,
const double beta,
const double rho ,
OptimizerType optimizer,
CallbackTypes&&... callbacks);
/**
* Transforms the provided data into the representation learned by the sparse
* autoencoder. The function basically performs a feedforward computation
* using the learned weights, and returns the hidden layer activations.
*
* @param data Matrix of the provided data.
* @param features The hidden layer representation of the provided data.
*/
void GetNewFeatures(arma::mat& data, arma::mat& features);
/**
* Returns the elementwise sigmoid of the passed matrix, where the sigmoid
* function of a real number 'x' is [1 / (1 + exp(-x))].
*
* @param x Matrix of real values for which we require the sigmoid activation.
* @param output Output matrix.
*/
void Sigmoid(const arma::mat& x, arma::mat& output) const
{
output = (1.0 / (1 + exp(-x)));
}
//! Sets size of the visible layer.
void VisibleSize(const size_t visible)
{
this->visibleSize = visible;
}
//! Gets size of the visible layer.
size_t VisibleSize() const
{
return visibleSize;
}
//! Sets size of the hidden layer.
void HiddenSize(const size_t hidden)
{
this->hiddenSize = hidden;
}
//! Gets the size of the hidden layer.
size_t HiddenSize() const
{
return hiddenSize;
}
//! Sets the L2-regularization parameter.
void Lambda(const double l)
{
this->lambda = l;
}
//! Gets the L2-regularization parameter.
double Lambda() const
{
return lambda;
}
//! Sets the KL divergence parameter.
void Beta(const double b)
{
this->beta = b;
}
//! Gets the KL divergence parameter.
double Beta() const
{
return beta;
}
//! Sets the sparsity parameter.
void Rho(const double r)
{
this->rho = r;
}
//! Gets the sparsity parameter.
double Rho() const
{
return rho;
}
private:
//! Parameters after optimization.
arma::mat parameters;
//! Size of the visible layer.
size_t visibleSize;
//! Size of the hidden layer.
size_t hiddenSize;
//! L2-regularization parameter.
double lambda;
//! KL divergence parameter.
double beta;
//! Sparsity parameter.
double rho;
};
} // namespace mlpack
// Include implementation.
#include "sparse_autoencoder_impl.hpp"
#endif