New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP]Restricted Boltzmann Machine #1027

Closed
wants to merge 26 commits into
base: master
from
Commits
Jump to file or symbol
Failed to load files and symbols.
+936 −1
Diff settings

Always

Just for now

@@ -65,6 +65,17 @@ inline double Random(const double lo, const double hi)
return lo + (hi - lo) * randUniformDist(randGen);
}
/**
* Generates a 0/1 specified by the input.
*/
inline double RandBernoulli(const double input)
{
if (Random() < input)
return 1;
else
return 0;
}
/**
* Generates a uniform random integer.
*/
@@ -3,6 +3,8 @@
set(SOURCES
ffn.hpp
ffn_impl.hpp
rbm.hpp
rbm_impl.hpp
rnn.hpp
rnn_impl.hpp
)
@@ -6,6 +6,8 @@ set(SOURCES
add_merge.hpp
add_merge_impl.hpp
base_layer.hpp
binary_layer.hpp
binary_layer_impl.hpp
concat.hpp
concat_impl.hpp
concat_performance.hpp
@@ -0,0 +1,115 @@
/**
* @file binary_rbm.hpp
* @author Kris Singh
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef MLPACK_METHODS_ANN_LAYER_RBM_BINARY_LAYER_HPP
#define MLPACK_METHODS_ANN_LAYER_RBM_BINARY_LAYER_HPP
#include "layer_types.hpp"
#include <mlpack/prereqs.hpp>
#include <mlpack/methods/ann/activation_functions/logistic_function.hpp>
namespace mlpack{
namespace ann{
template <
typename InputDataType = arma::mat,
typename OutputDataType = arma::mat
>
class BinaryLayer
{
public:
/* The visible layer of the rbm
* network.
*
* @param: inSize: num of visible neurons
* @param: outSize: num of hidden neurons
*/
BinaryLayer(size_t inSize, size_t outSize,
bool typeVisible = true);
// Reset the variables
void Reset();
/**
* Calculate the acivations and send to the hidden layer.
* input if of the format datapoint + bias(other layer)
*
* @param input Input data used for evaluating the specified function.
* @param output Resulting output activation.
*/
void Forward(const InputDataType&& input, OutputDataType&& output);
/**
* Sample the output given the input parameters
* The sample are obtained from a distribution
* specified by the sampler.
*
* @param input the input parameters for
* @param output samples from the parameters
*/
void Sample(InputDataType&& input, OutputDataType&& output);
//! Get the parameters.
OutputDataType const& Parameters() const { return weights; }
//! Modify the parameters.
OutputDataType& Parameters() { return weights; }
//! Get the parameters.
OutputDataType const& Bias() const { return ownBias; }
//! Get the weights of the network
OutputDataType const& Weight() const { return weight; }
//! Modify the parameters.
OutputDataType& Bias() { return ownBias; }
/**
* Serialize the layer
*/
template<typename Archive>
void Serialize(Archive& ar, const unsigned int /* version */);
/**
* Calculate the pre-acivations and send to the hidden layer.
* input if of the format datapoint + bias(other layer)
*
* @param input Input data used for evaluating the specified function.
* @param output Resulting output from the linear function.
*/
void ForwardPreActivation(const InputDataType&& input,
OutputDataType&& output);
private:
//! Locally-stored number of input units.
size_t inSize;
//! Locally-stored number of output units.
size_t outSize;
//! Locally-stored type of layer
bool typeVisible;
//! Locally-stored weight object of the layer.
OutputDataType weights;
//! Locally-stored weight paramters of the layer.
OutputDataType weight;
//! Locally-stored bias paramters belonging to the layer
OutputDataType ownBias;
//! Locally-store bias paremeter belonging to the other layer.
OutputDataType otherBias;
}; // class BinaryLayer
} // namespace ann
} // namespace mlpack
// Include implementation.
#include "binary_layer_impl.hpp"
#endif
@@ -0,0 +1,96 @@
/**
* @file binary_rbm_impl.hpp
* @author Kris Singh
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef MLPACK_METHODS_ANN_LAYER_RBM_BINARY_LAYER_IMPL_HPP
#define MLPACK_METHODS_ANN_LAYER_RBM_BINARY_LAYER_IMPL_HPP
// In case it hasn't yet been included.
#include "binary_layer.hpp"
#include <mlpack/core/math/random.hpp>
#include <mlpack/methods/ann/activation_functions/logistic_function.hpp>
namespace mlpack {
namespace ann { /** Artificial Neural Network. */
template<typename InputDataType, typename OutputDataType>
BinaryLayer<InputDataType, OutputDataType>::BinaryLayer(
size_t inSize,
size_t outSize,
bool typeVisible):
inSize(inSize),
outSize(outSize),
typeVisible(typeVisible)
{
weights.set_size((outSize * inSize) + inSize + outSize, 1);
}
template<typename InputDataType, typename OutputDataType>
void BinaryLayer<InputDataType, OutputDataType>::Reset()
{
if (typeVisible)
{
weight = arma::mat(weights.memptr(), outSize, inSize, false, false);
ownBias = arma::mat(weights.memptr() + weight.n_elem + outSize,
inSize, 1, false, false);
otherBias = arma::mat(weights.memptr() + weight.n_elem , outSize,
1, false, false);
}
else
{
weight = arma::mat(weights.memptr(), inSize, outSize, false, false);
ownBias = arma::mat(weights.memptr() + weight.n_elem, inSize, 1,
false, false);
otherBias = arma::mat(weights.memptr() + weight.n_elem + inSize, outSize, 1,
false, false);
}
}
template<typename InputDataType, typename OutputDataType>
void BinaryLayer<InputDataType, OutputDataType>::Forward(
const InputDataType&& input, OutputDataType&& output)
{
ForwardPreActivation(std::move(input), std::move(output));
LogisticFunction::Fn(output, output);
}
template<typename InputDataType, typename OutputDataType>
void BinaryLayer<InputDataType, OutputDataType>::ForwardPreActivation(
const InputDataType&& input, OutputDataType&& output)
{
if (Parameters().empty())
Reset();
if (typeVisible)
output = weight * input + otherBias;
else
output = weight.t() * input + otherBias;
}
template<typename InputDataType, typename OutputDataType>
void BinaryLayer<InputDataType, OutputDataType>::Sample(InputDataType&& input,
OutputDataType&& output)
{
Forward(std::move(input), std::move(output));
for (size_t i = 0; i < output.n_elem; i++)
output(i) = math::RandBernoulli(output(i));

This comment has been minimized.

@zoq

zoq Jul 6, 2017

Member

I'm wondering if using arma::randi<arma::mat>(10, 10, arma::distr_param(0, 1)); is faster.

@zoq

zoq Jul 6, 2017

Member

I'm wondering if using arma::randi<arma::mat>(10, 10, arma::distr_param(0, 1)); is faster.

This comment has been minimized.

@kris-singh

kris-singh Jul 7, 2017

Contributor

I tested this and rand Bernoulli is faster

@kris-singh

kris-singh Jul 7, 2017

Contributor

I tested this and rand Bernoulli is faster

This comment has been minimized.

@zoq

zoq Jul 7, 2017

Member

Okay, thanks for checking.

@zoq

zoq Jul 7, 2017

Member

Okay, thanks for checking.

}
template<typename InputDataType, typename OutputDataType>
template<typename Archive>
void BinaryLayer<InputDataType, OutputDataType>::Serialize(
Archive& ar, const unsigned int /* version */)
{
ar & data::CreateNVP(weights, "weights");
ar & data::CreateNVP(inSize, "inSize");
ar & data::CreateNVP(outSize, "outSize");
ar & data::CreateNVP(typeVisible, "typeVisible");
}
} // namespace ann
} // namespace mlpack
#endif
@@ -26,5 +26,7 @@
#include "sequential.hpp"
#include "concat.hpp"
#include "vr_class_reward.hpp"
#include "binary_layer.hpp"
#endif
@@ -45,7 +45,7 @@ class Linear
* @param inSize The number of input units.
* @param outSize The number of output units.
*/
Linear(const size_t inSize, const size_t outSize);;
Linear(const size_t inSize, const size_t outSize);
/*
* Reset the layer parameter.
Oops, something went wrong.
ProTip! Use n and p to navigate between commits in a pull request.