Rework the traits

wichtounet · Aug 10, 2016 · 0377837 · 0377837
1 parent 2dc2881
commit 0377837
Show file tree

Hide file tree

Showing 7 changed files with 63 additions and 45 deletions.
diff --git a/include/dll/conv_rbm.inl b/include/dll/conv_rbm.inl
@@ -5,6 +5,11 @@
 //  http://opensource.org/licenses/MIT)
 //=======================================================================
 
+/*!
+ * \file
+ * \brief Implementation of a Convolutional Restricted Boltzmann Machine
+ */
+
 #pragma once
 
 #include <cstddef>

diff --git a/include/dll/dbn_layers.hpp b/include/dll/dbn_layers.hpp
@@ -17,15 +17,13 @@ template <typename... Layers>
 struct is_dynamic : cpp::or_u<layer_traits<Layers>::is_dynamic()...> {};
 
 template <typename... Layers>
-struct is_convolutional : cpp::or_u<
-                          cpp::or_u<layer_traits<Layers>::is_convolutional_layer()...>::value,
-                          cpp::or_u<layer_traits<Layers>::is_convolutional_rbm_layer()...>::value> {};
+struct is_convolutional : cpp::or_u<layer_traits<Layers>::is_convolutional_layer()...> {};
 
 template <typename... Layers>
 struct is_multiplex : cpp::or_u<layer_traits<Layers>::is_multiplex_layer()...> {};
 
 template <typename... Layers>
-struct is_denoising : cpp::and_u<layer_traits<Layers>::is_standard_rbm_layer()...> {};
+struct is_denoising : cpp::and_u<layer_traits<Layers>::is_dense_rbm_layer()...> {};
 
 template <typename... Layers>
 struct has_shuffle_layer : cpp::or_u<layer_traits<Layers>::has_shuffle()...> {};

diff --git a/include/dll/dyn_dense_layer.inl b/include/dll/dyn_dense_layer.inl
@@ -38,9 +38,6 @@ struct dyn_dense_layer final : neural_base<dyn_dense_layer<Desc>> {
     template <std::size_t B>
     using input_batch_t = etl::fast_dyn_matrix<weight, B, 1>; //TODO CHECK THAT
 
-    template <std::size_t B>
-    using output_batch_t = etl::fast_dyn_matrix<weight, B, 1>; //TODO CHECK THAT
-
     using w_type = etl::dyn_matrix<weight, 2>;
     using b_type = etl::dyn_matrix<weight, 1>;
 

diff --git a/include/dll/layer_traits.hpp b/include/dll/layer_traits.hpp
@@ -21,41 +21,62 @@ namespace dll {
  */
 template <typename Layer>
 struct layer_traits {
-    using layer_t = Layer;
+    using layer_t = Layer; ///< The layer type being inspected
 
     /*!
-     * \brief Indicates if the layer is a standard layer.
+     * \brief Indicates if the layer is neural (dense or conv)
      */
-    static constexpr bool is_standard_layer() {
+    static constexpr bool is_neural_layer() {
         return is_dense_layer() || is_convolutional_layer();
     }
 
     /*!
-     * \brief Indicates if the layer is a standard dense layer.
+     * \brief Indicates if the layer is dense
      */
     static constexpr bool is_dense_layer() {
+        return is_standard_dense_layer() || is_dense_rbm_layer();
+    }
+
+    /*!
+     * \brief Indicates if the layer is convolutional
+     */
+    static constexpr bool is_convolutional_layer() {
+        return is_standard_convolutional_layer() || is_convolutional_rbm_layer();
+    }
+
+    /*!
+     * \brief Indicates if the layer is a standard (non-rbm) layer.
+     */
+    static constexpr bool is_standard_layer() {
+        return is_standard_dense_layer() || is_standard_convolutional_layer();
+    }
+
+    /*!
+     * \brief Indicates if the layer is a standard (non-rbm) dense layer.
+     */
+    static constexpr bool is_standard_dense_layer() {
         return cpp::is_specialization_of<dense_layer, layer_t>::value
             || cpp::is_specialization_of<dyn_dense_layer, layer_t>::value;
     }
 
     /*!
-     * \brief Indicates if the layer is a standard convolutionl layer.
+     * \brief Indicates if the layer is a standard (non-rbm) convolutionl layer.
      */
-    static constexpr bool is_convolutional_layer() {
+    static constexpr bool is_standard_convolutional_layer() {
         return cpp::is_specialization_of<conv_layer, layer_t>::value;
     }
 
     /*!
      * \brief Indicates if this layer is a RBM layer.
      */
     static constexpr bool is_rbm_layer() {
-        return is_standard_rbm_layer() || is_convolutional_rbm_layer();
+        return is_dense_rbm_layer() || is_convolutional_rbm_layer();
     }
 
     /*!
-     * \brief Indicates if this layer is a standard (non-convolutional) RBM layer.
+     * \brief Indicates if this layer is a dense RBM layer.
      */
-    static constexpr bool is_standard_rbm_layer() {
+    static constexpr bool is_dense_rbm_layer() {
         return cpp::is_specialization_of<dyn_rbm, layer_t>::value
             || cpp::is_specialization_of<rbm, layer_t>::value;
     }
@@ -355,15 +376,4 @@ constexpr std::size_t input_size(const RBM&) {
     return layer_traits<RBM>::input_size();
 }
 
-//TODO This should probably be moved into the traits class
-
-template <typename Layer>
-struct is_dense : cpp::bool_constant<decay_layer_traits<Layer>::is_dense_layer() || decay_layer_traits<Layer>::is_standard_rbm_layer()> {};
-
-template <typename Layer>
-struct is_conv : cpp::bool_constant<decay_layer_traits<Layer>::is_convolutional_layer() || decay_layer_traits<Layer>::is_convolutional_rbm_layer()> {};
-
-template <typename Layer>
-struct is_neural : cpp::or_c<is_dense<Layer>, is_conv<Layer>> {};
-
 } //end of dll namespace
diff --git a/include/dll/processor/processor.hpp b/include/dll/processor/processor.hpp
@@ -45,12 +45,12 @@ struct sgd_possible {
 };
 
 template <typename LastLayer>
-struct sgd_possible<LastLayer, std::enable_if_t<decay_layer_traits<LastLayer>::is_dense_layer()>> {
+struct sgd_possible<LastLayer, std::enable_if_t<decay_layer_traits<LastLayer>::is_standard_dense_layer()>> {
     static constexpr bool value = true;
 };
 
 template <typename LastLayer>
-struct sgd_possible<LastLayer, std::enable_if_t<decay_layer_traits<LastLayer>::is_standard_rbm_layer()>> {
+struct sgd_possible<LastLayer, std::enable_if_t<decay_layer_traits<LastLayer>::is_dense_rbm_layer()>> {
     static constexpr bool value = std::decay_t<LastLayer>::hidden_unit == unit_type::SOFTMAX;
 };
 

diff --git a/include/dll/trainer/sgd_context.hpp b/include/dll/trainer/sgd_context.hpp
@@ -29,7 +29,7 @@ struct sgd_context;
  * \copydoc sgd_context
  */
 template <typename DBN, typename Layer>
-struct sgd_context<DBN, Layer, std::enable_if_t<is_dense<Layer>::value && !layer_traits<Layer>::is_dynamic()>> {
+struct sgd_context<DBN, Layer, std::enable_if_t<layer_traits<Layer>::is_dense_layer() && !layer_traits<Layer>::is_dynamic()>> {
     using layer_t = Layer;
     using weight  = typename layer_t::weight;
 
@@ -55,7 +55,7 @@ struct sgd_context<DBN, Layer, std::enable_if_t<is_dense<Layer>::value && !layer
  * \copydoc sgd_context
  */
 template <typename DBN, typename Layer>
-struct sgd_context<DBN, Layer, std::enable_if_t<is_dense<Layer>::value && layer_traits<Layer>::is_dynamic()>> {
+struct sgd_context<DBN, Layer, std::enable_if_t<layer_traits<Layer>::is_dense_layer() && layer_traits<Layer>::is_dynamic()>> {
     using layer_t = Layer;
     using weight  = typename layer_t::weight;
 
@@ -80,7 +80,7 @@ struct sgd_context<DBN, Layer, std::enable_if_t<is_dense<Layer>::value && layer_
  * \copydoc sgd_context
  */
 template <typename DBN, typename Layer>
-struct sgd_context<DBN, Layer, std::enable_if_t<is_conv<Layer>::value>> {
+struct sgd_context<DBN, Layer, std::enable_if_t<layer_traits<Layer>::is_convolutional_layer()>> {
     using layer_t = Layer;
     using weight  = typename layer_t::weight;
 

diff --git a/include/dll/trainer/stochastic_gradient_descent.hpp b/include/dll/trainer/stochastic_gradient_descent.hpp
@@ -120,12 +120,14 @@ struct sgd_trainer {
         });
     }
 
-    template <typename Layer, typename Weight, typename Grad, typename Inputs, typename Errors, cpp_enable_if(is_dense<Layer>::value&& etl::decay_traits<Inputs>::dimensions() == 2)>
+    template <typename Layer, typename Weight, typename Grad, typename Inputs, typename Errors,
+             cpp_enable_if(decay_layer_traits<Layer>::is_dense_layer() && etl::decay_traits<Inputs>::dimensions() == 2)>
     static void compute_weight_gradients(Grad& grad, Inputs& inputs, Errors& errors) {
         dense_compute_weight_gradients<Weight>(grad, inputs, errors);
     }
 
-    template <typename Layer, typename Weight, typename Grad, typename Inputs, typename Errors, cpp_enable_if(is_dense<Layer>::value&& etl::decay_traits<Inputs>::dimensions() != 2)>
+    template <typename Layer, typename Weight, typename Grad, typename Inputs, typename Errors,
+             cpp_enable_if(decay_layer_traits<Layer>::is_dense_layer() && etl::decay_traits<Inputs>::dimensions() != 2)>
     static void compute_weight_gradients(Grad& grad, Inputs& inputs, Errors& errors) {
         dense_compute_weight_gradients<Weight>(grad, etl::reshape<batch_size, Layer::num_visible>(inputs), errors);
     }
@@ -153,7 +155,8 @@ struct sgd_trainer {
 
 #endif
 
-    template <typename Layer, typename Weight, typename Grad, typename Inputs, typename Errors, cpp_enable_if(is_conv<Layer>::value)>
+    template <typename Layer, typename Weight, typename Grad, typename Inputs, typename Errors,
+             cpp_enable_if(decay_layer_traits<Layer>::is_convolutional_layer())>
     static void compute_weight_gradients(Grad& grad, Inputs& inputs, Errors& errors) {
         constexpr const auto K   = Layer::K;
         constexpr const auto NC  = Layer::NC;
@@ -180,13 +183,14 @@ struct sgd_trainer {
         }
     }
 
-    template <typename Layer, typename Context, typename Inputs, cpp_enable_if((is_dense<Layer>::value || is_conv<Layer>::value))>
+    template <typename Layer, typename Context, typename Inputs,
+             cpp_enable_if((decay_layer_traits<Layer>::is_neural_layer()))>
     static void compute_gradients(Layer&, Context& ctx, Inputs& inputs) {
         ctx.w_grad = 0;
 
         compute_weight_gradients<Layer, weight>(ctx.w_grad, inputs, ctx.errors);
 
-        cpp::static_if<decay_layer_traits<Layer>::is_dense_layer() || decay_layer_traits<Layer>::is_standard_rbm_layer()>([&](auto f) {
+        cpp::static_if<decay_layer_traits<Layer>::is_dense_layer()>([&](auto f) {
             f(ctx.b_grad) = etl::sum_l(ctx.errors);
         }).else_([&](auto f) { f(ctx.b_grad) = etl::mean_r(etl::sum_l(f(ctx.errors))); });
 
@@ -251,29 +255,33 @@ struct sgd_trainer {
     }
 
     //Backpropagate errors from dense to (dense or conv)
-    template <typename Layer1, typename Context1, typename Layer2, typename Context2, cpp_enable_if(is_neural<Layer1>::value&& is_dense<Layer2>::value)>
+    template <typename Layer1, typename Context1, typename Layer2, typename Context2,
+             cpp_enable_if(decay_layer_traits<Layer1>::is_neural_layer() && decay_layer_traits<Layer2>::is_dense_layer())>
     static void compute_errors(Layer1& r1, Context1& ctx1, Layer2& r2, Context2& ctx2) {
         constexpr const auto a_f = extract_function<Layer1>::activation_function;
 
         compute_errors_from_dense(r1, ctx1, r2, ctx2, [&](std::size_t i) { return f_derivative<a_f>(ctx1.output(i)); });
     }
 
     //Backpropagate errors from conv to (dense or conv)
-    template <typename Layer1, typename Context1, typename Layer2, typename Context2, cpp_enable_if(is_neural<Layer1>::value&& is_conv<Layer2>::value)>
+    template <typename Layer1, typename Context1, typename Layer2, typename Context2,
+             cpp_enable_if(decay_layer_traits<Layer1>::is_neural_layer() && decay_layer_traits<Layer2>::is_convolutional_layer())>
     static void compute_errors(Layer1& r1, Context1& ctx1, Layer2& r2, Context2& ctx2) {
         constexpr const auto a_f = extract_function<Layer1>::activation_function;
 
         compute_errors_from_conv(r1, ctx1, r2, ctx2, [&](std::size_t i, std::size_t c) { return f_derivative<a_f>(ctx1.output(i)(c)); });
     }
 
     //Backpropagate errors from dense to pooling
-    template <typename Layer1, typename Context1, typename Layer2, typename Context2, cpp_enable_if(!is_neural<Layer1>::value && is_dense<Layer2>::value)>
+    template <typename Layer1, typename Context1, typename Layer2, typename Context2,
+             cpp_enable_if(!decay_layer_traits<Layer1>::is_neural_layer() && decay_layer_traits<Layer2>::is_dense_layer())>
     static void compute_errors(Layer1& r1, Context1& ctx1, Layer2& r2, Context2& ctx2) {
         compute_errors_from_dense(r1, ctx1, r2, ctx2, [](std::size_t) { return 1.0; });
     }
 
     //Backpropagate errors from conv to pooling
-    template <typename Layer1, typename Context1, typename Layer2, typename Context2, cpp_enable_if(!is_neural<Layer1>::value && is_conv<Layer2>::value)>
+    template <typename Layer1, typename Context1, typename Layer2, typename Context2,
+             cpp_enable_if(!decay_layer_traits<Layer1>::is_neural_layer() && decay_layer_traits<Layer2>::is_convolutional_layer())>
     static void compute_errors(Layer1& r1, Context1& ctx1, Layer2& r2, Context2& ctx2) {
         compute_errors_from_conv(r1, ctx1, r2, ctx2, [](std::size_t, std::size_t) { return 1.0; });
     }
@@ -300,7 +308,7 @@ struct sgd_trainer {
         }
     }
 
-    template <typename Layer, typename Context, typename Labels, cpp_enable_if(decay_layer_traits<Layer>::is_dense_layer())>
+    template <typename Layer, typename Context, typename Labels, cpp_enable_if(decay_layer_traits<Layer>::is_standard_dense_layer())>
     void compute_last_errors(Layer& /*layer*/, Context& context, Labels& labels) {
         constexpr const auto last_a_f = extract_function<Layer>::activation_function;
 
@@ -309,7 +317,7 @@ struct sgd_trainer {
         nan_check_deep(context.errors);
     }
 
-    template <typename Layer, typename Context, typename Labels, cpp_enable_if(decay_layer_traits<Layer>::is_standard_rbm_layer())>
+    template <typename Layer, typename Context, typename Labels, cpp_enable_if(decay_layer_traits<Layer>::is_dense_rbm_layer())>
     void compute_last_errors(Layer& /*layer*/, Context& context, Labels& labels) {
         constexpr const auto last_a_f = extract_function<Layer>::activation_function;
 
@@ -349,7 +357,7 @@ struct sgd_trainer {
         compute_outputs(inputs);
 
         static_assert(
-            decay_layer_traits<decltype(last_layer)>::is_dense_layer() || decay_layer_traits<decltype(last_layer)>::is_standard_rbm_layer(),
+            decay_layer_traits<decltype(last_layer)>::is_dense_layer(),
             "The last layer must be dense for SGD trainining");
 
         //Compute the errors of the last layer
@@ -376,7 +384,7 @@ struct sgd_trainer {
         });
     }
 
-    template <typename L, cpp_enable_if(is_dense<L>::value || is_conv<L>::value)>
+    template <typename L, cpp_enable_if(decay_layer_traits<L>::is_neural_layer())>
     void apply_gradients(L& layer, std::size_t n) {
         auto& context = layer.template get_sgd_context<dbn_t>();