Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implementation of Alias layer #1091

Closed
wants to merge 4 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
11 changes: 1 addition & 10 deletions src/mlpack/methods/ann/ffn.hpp
Expand Up @@ -15,15 +15,6 @@

#include <mlpack/prereqs.hpp>

#include "visitor/delete_visitor.hpp"
#include "visitor/delta_visitor.hpp"
#include "visitor/output_height_visitor.hpp"
#include "visitor/output_parameter_visitor.hpp"
#include "visitor/output_width_visitor.hpp"
#include "visitor/reset_visitor.hpp"
#include "visitor/weight_size_visitor.hpp"
#include "visitor/copy_visitor.hpp"

#include "init_rules/network_init.hpp"

#include <mlpack/methods/ann/layer/layer_types.hpp>
Expand Down Expand Up @@ -200,7 +191,7 @@ class FFN
arma::mat& Parameters() { return parameter; }

/**
* Reset the module infomration (weights/parameters).
* Reset the module information (weights/parameters).
*/
void ResetParameters();

Expand Down
16 changes: 8 additions & 8 deletions src/mlpack/methods/ann/ffn_impl.hpp
Expand Up @@ -15,14 +15,6 @@
// In case it hasn't been included yet.
#include "ffn.hpp"

#include "visitor/forward_visitor.hpp"
#include "visitor/backward_visitor.hpp"
#include "visitor/deterministic_set_visitor.hpp"
#include "visitor/gradient_set_visitor.hpp"
#include "visitor/gradient_visitor.hpp"
#include "visitor/set_input_height_visitor.hpp"
#include "visitor/set_input_width_visitor.hpp"

namespace mlpack {
namespace ann /** Artificial Neural Network. */ {

Expand Down Expand Up @@ -72,6 +64,8 @@ void FFN<OutputLayerType, InitializationRuleType>::ResetData(
this->predictors = std::move(predictors);
this->responses = std::move(responses);
this->deterministic = true;
// arma::mat tmp;
// boost::apply_visitor(ParametersVisitor(std::move(tmp)), network.front());
ResetDeterministic();

if (!reset)
Expand All @@ -87,6 +81,8 @@ void FFN<OutputLayerType, InitializationRuleType>::Train(
arma::mat responses,
OptimizerType& optimizer)
{
arma::mat tmp;
boost::apply_visitor(ParametersVisitor(std::move(tmp)), network.front());
ResetData(std::move(predictors), std::move(responses));

// Train the model.
Expand Down Expand Up @@ -256,7 +252,11 @@ void FFN<OutputLayerType, InitializationRuleType>::Gradient(
template<typename OutputLayerType, typename InitializationRuleType>
void FFN<OutputLayerType, InitializationRuleType>::ResetParameters()
{
arma::mat tmp;
boost::apply_visitor(ParametersVisitor(std::move(tmp)), network.front());
ResetDeterministic();
// arma::mat tmp;
// boost::apply_visitor(ParametersVisitor(std::move(tmp)), network.front());

// Reset the network parameter with the given initialization rule.
NetworkInitialization<InitializationRuleType> networkInit(initializeRule);
Expand Down
6 changes: 2 additions & 4 deletions src/mlpack/methods/ann/init_rules/network_init.hpp
Expand Up @@ -15,12 +15,10 @@

#include <mlpack/prereqs.hpp>

#include "../visitor/reset_visitor.hpp"
#include "../visitor/weight_size_visitor.hpp"
#include "../visitor/weight_set_visitor.hpp"
#include <mlpack/methods/ann/layer/layer.hpp>
#include <mlpack/methods/ann/layer/layer_types.hpp>
#include "init_rules_traits.hpp"

#include <mlpack/methods/ann/layer/layer_types.hpp>

namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
Expand Down
2 changes: 2 additions & 0 deletions src/mlpack/methods/ann/layer/CMakeLists.txt
Expand Up @@ -5,6 +5,8 @@ set(SOURCES
add_impl.hpp
add_merge.hpp
add_merge_impl.hpp
alias.hpp
alias_impl.hpp
base_layer.hpp
concat.hpp
concat_impl.hpp
Expand Down
130 changes: 130 additions & 0 deletions src/mlpack/methods/ann/layer/alias.hpp
@@ -0,0 +1,130 @@
/**
* @file alias_layer.hpp
* @author Ryan Curtin
* @author Shangtong Zhang
*
* This is an alias layer for another layer so that parameters can be shared
* between multiple networks. However, it is not threadsafe---so you cannot
* share parameters between networks in separate threads (although... it should
* be pretty easy to adapt this class, you just need to add locks).
*
* You can only alias a layer which implements Parameters() and you
* cannot alias a module.
*/
#ifndef MLPACK_METHODS_ANN_LAYER_ALIAS_LAYER_HPP
#define MLPACK_METHODS_ANN_LAYER_ALIAS_LAYER_HPP

#include <mlpack/prereqs.hpp>
#include <mlpack/methods/ann/layer/layer_types.hpp>
#include <mlpack/methods/ann/visitor/visitor.hpp>

namespace mlpack {
namespace ann {

//class WeightSizeVisitor;
//class ParametersVisitor;
//class ParametersSetVisitor;

class Alias
{
public:
/**
* Construct the AliasLayer as an alias of the given layer. When destructed,
* this class will not destruct the aliased layer.
*
* @param layer Layer to be aliased.
*/
template<typename LayerType>
Alias(LayerType& layer) : layer(&layer) {
parameter.set_size(boost::apply_visitor(WeightSizeVisitor(), this->layer), 1);
// arma::mat tmp;
// LayerTypes l(this);
// boost::apply_visitor(ParametersVisitor(std::move(tmp)), l);
};
template<typename LayerType>
Alias(LayerType* layer) : layer(layer) {
parameter.set_size(boost::apply_visitor(WeightSizeVisitor(), this->layer), 1);
};
Alias(LayerTypes& layer) : layer(layer) {
parameter.set_size(boost::apply_visitor(WeightSizeVisitor(), this->layer), 1);
};

/**
* Reset the parameters of the layer.
*/
void Reset() {}
// { boost::apply_visitor(ResetVisitor(), layer); }

/**
* Perform a forward pass of the aliased layer.
*/
template<typename eT>
void Forward(arma::Mat<eT>&& input, arma::Mat<eT>&& output);

/**
* Perform a backwards pass of the aliased layer.
*/
template<typename eT>
void Backward(arma::Mat<eT>&& input,
arma::Mat<eT>&& gy,
arma::Mat<eT>&& g);

/**
* Calculate the gradient of the aliased layer using the delta and the input
* activation.
*/
template<typename eT>
void Gradient(const arma::Mat<eT>&& input,
arma::Mat<eT>&& error,
arma::Mat<eT>&& gradient);

const arma::mat& OutputParameter() const {
return boost::apply_visitor(OutputParameterVisitor(), layer);
}
arma::mat& OutputParameter() {
return boost::apply_visitor(OutputParameterVisitor(), layer);
}

arma::mat& Parameters() { return parameter; }
const arma::mat& Parameters() const { return parameter; }

const arma::mat& Delta() const;
arma::mat& Delta();

template<typename LayerType>
void Add(LayerType* newLayer)
{ boost::apply_visitor(AddVisitor(newLayer), layer); }

size_t WeightSize() const
{ return boost::apply_visitor(WeightSizeVisitor(), layer); }

size_t SetWeight(arma::mat&& weight, const size_t offset)
{
parameter = arma::mat(weight.memptr() + offset, parameter.n_rows, parameter.n_cols, false, false);
return parameter.n_elem;
}

void SetParameters(arma::mat&& parameters)
{
parameter = static_cast<const arma::mat&>(parameters);
boost::apply_visitor(ParametersSetVisitor(std::move(parameters)), layer);
}

void GetParameters(arma::mat&& parameters)
{
boost::apply_visitor(ParametersVisitor(std::move(parameters)), layer);
parameter = static_cast<const arma::mat&>(parameters);
}

private:
LayerTypes layer;
arma::mat parameter;
};

} // namespace ann
} // namespace mlpack

#include "alias_impl.hpp"
#include <mlpack/methods/ann/visitor/visitor_impl.hpp>

#endif
77 changes: 77 additions & 0 deletions src/mlpack/methods/ann/layer/alias_impl.hpp
@@ -0,0 +1,77 @@
/**
* @file alias_layer_impl.hpp
* @author Ryan Curtin
* @author Shangtong Zhang
*
* This is an alias layer for another layer so that parameters can be shared
* between multiple networks. However, it is not threadsafe---so you cannot
* share parameters between networks in separate threads (although... it should
* be pretty easy to adapt this class, you just need to add locks).
*/
#ifndef MLPACK_METHODS_ANN_LAYER_ALIAS_LAYER_IMPL_HPP
#define MLPACK_METHODS_ANN_LAYER_ALIAS_LAYER_IMPL_HPP

#include "alias.hpp"

#include <mlpack/methods/ann/layer/layer_types.hpp>
#include <mlpack/methods/ann/visitor/forward_visitor.hpp>
#include <mlpack/methods/ann/visitor/backward_visitor.hpp>
#include <mlpack/methods/ann/visitor/gradient_visitor.hpp>
#include <mlpack/methods/ann/visitor/delta_visitor.hpp>

namespace mlpack {
namespace ann {

/**
* Perform a forward pass of the aliased layer.
*/
template<typename eT>
void Alias::Forward(arma::Mat<eT>&& input, arma::Mat<eT>&& output)
{
// Create a new visitor and call the layer's Forward() function.
// arma::mat tmp;
// boost::apply_visitor(ParametersVisitor(std::move(tmp)), layer);
boost::apply_visitor(ForwardVisitor(std::move(input), std::move(output)),
layer);
}

/**
* Perform a backwards pass of the aliased layer.
*/
template<typename eT>
void Alias::Backward(arma::Mat<eT>&& input,
arma::Mat<eT>&& gy,
arma::Mat<eT>&& g)
{
// Create a new visitor and call the layer's Backward() function.
boost::apply_visitor(BackwardVisitor(std::move(input), std::move(gy),
std::move(g)), layer);
}

/**
* Calculate the gradient of the aliased layer using the delta and the input
* activation.
*/
template<typename eT>
void Alias::Gradient(const arma::Mat<eT>&& input,
arma::Mat<eT>&& error,
arma::Mat<eT>&& gradient)
{
boost::apply_visitor(GradientVisitor(std::move(input), std::move(error),
std::move(gradient)), layer);
}

inline const arma::mat& Alias::Delta() const
{
return boost::apply_visitor(DeltaVisitor(), layer);
}

inline arma::mat& Alias::Delta()
{
return boost::apply_visitor(DeltaVisitor(), layer);
}

} // namespace ann
} // namespace mlpack

#endif
1 change: 1 addition & 0 deletions src/mlpack/methods/ann/layer/layer.hpp
Expand Up @@ -26,5 +26,6 @@
#include "sequential.hpp"
#include "concat.hpp"
#include "vr_class_reward.hpp"
#include "alias.hpp"

#endif
2 changes: 2 additions & 0 deletions src/mlpack/methods/ann/layer/layer_types.hpp
Expand Up @@ -44,6 +44,7 @@ namespace mlpack {
namespace ann {

template<typename InputDataType, typename OutputDataType> class AddMerge;
class Alias;
template<typename InputDataType, typename OutputDataType> class Concat;
template<typename InputDataType, typename OutputDataType> class DropConnect;
template<typename InputDataType, typename OutputDataType> class Glimpse;
Expand Down Expand Up @@ -79,6 +80,7 @@ class RecurrentAttention;
using LayerTypes = boost::variant<
Add<arma::mat, arma::mat>*,
AddMerge<arma::mat, arma::mat>*,
Alias*,
BaseLayer<LogisticFunction, arma::mat, arma::mat>*,
BaseLayer<IdentityFunction, arma::mat, arma::mat>*,
BaseLayer<TanhFunction, arma::mat, arma::mat>*,
Expand Down
5 changes: 5 additions & 0 deletions src/mlpack/methods/ann/layer/linear_impl.hpp
Expand Up @@ -48,6 +48,11 @@ template<typename eT>
void Linear<InputDataType, OutputDataType>::Forward(
const arma::Mat<eT>&& input, arma::Mat<eT>&& output)
{
arma::mat tmp0 = weights;
arma::mat tmp1 = weight;
// arma::mat tmp2 = input + 1;
// arma::mat tmp = weight * input;
// output = tmp;
output = weight * input;
output.each_col() += bias;
}
Expand Down
6 changes: 1 addition & 5 deletions src/mlpack/methods/ann/rnn.hpp
Expand Up @@ -14,14 +14,10 @@

#include <mlpack/prereqs.hpp>

#include "visitor/delete_visitor.hpp"
#include "visitor/delta_visitor.hpp"
#include "visitor/output_parameter_visitor.hpp"
#include "visitor/reset_visitor.hpp"

#include "init_rules/network_init.hpp"

#include <mlpack/methods/ann/layer/layer_types.hpp>
#include <mlpack/methods/ann/layer/layer.hpp>
#include <mlpack/methods/ann/init_rules/random_init.hpp>
#include <mlpack/core/optimizers/sgd/sgd.hpp>

Expand Down
10 changes: 0 additions & 10 deletions src/mlpack/methods/ann/rnn_impl.hpp
Expand Up @@ -15,16 +15,6 @@
// In case it hasn't been included yet.
#include "rnn.hpp"

#include "visitor/load_output_parameter_visitor.hpp"
#include "visitor/save_output_parameter_visitor.hpp"
#include "visitor/forward_visitor.hpp"
#include "visitor/backward_visitor.hpp"
#include "visitor/deterministic_set_visitor.hpp"
#include "visitor/gradient_set_visitor.hpp"
#include "visitor/gradient_visitor.hpp"
#include "visitor/weight_set_visitor.hpp"
#include "visitor/rho_set_visitor.hpp"

namespace mlpack {
namespace ann /** Artificial Neural Network. */ {

Expand Down
2 changes: 2 additions & 0 deletions src/mlpack/methods/ann/visitor/CMakeLists.txt
Expand Up @@ -47,6 +47,8 @@ set(SOURCES
set_input_height_visitor_impl.hpp
set_input_width_visitor.hpp
set_input_width_visitor_impl.hpp
visitor.hpp
visitor_impl.hpp
weight_set_visitor.hpp
weight_set_visitor_impl.hpp
weight_size_visitor.hpp
Expand Down
3 changes: 0 additions & 3 deletions src/mlpack/methods/ann/visitor/add_visitor.hpp
Expand Up @@ -55,7 +55,4 @@ class AddVisitor : public boost::static_visitor<void>
} // namespace ann
} // namespace mlpack

// Include implementation.
#include "add_visitor_impl.hpp"

#endif
3 changes: 0 additions & 3 deletions src/mlpack/methods/ann/visitor/add_visitor_impl.hpp
Expand Up @@ -12,9 +12,6 @@
#ifndef MLPACK_METHODS_ANN_VISITOR_ADD_VISITOR_IMPL_HPP
#define MLPACK_METHODS_ANN_VISITOR_ADD_VISITOR_IMPL_HPP

// In case it hasn't been included yet.
#include "add_visitor.hpp"

namespace mlpack {
namespace ann {

Expand Down