Add support for residual blocks.

src/mlpack/methods/ann/layer/sequential.hpp

@@ -34,8 +34,15 @@ namespace ann /** Artificial Neural Network. */ {
  * feed-forward fully connected network container which plugs various layers
  * together.
  *
+ * This class can also be used as a container for a residual block. In that
+ * case, the sizes of the input and output matrices of this class should be
+ * equal.
+ *
  * Note: If this class is used as the first layer of a network, it should be
- * preceded by IdentityLayer<>.
+ *       preceded by IdentityLayer<>.
+ *
+ * Note: This class should at least have two layers for a call to its Gradient()
+ *       function.
  *
  * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
  *         arma::sp_mat or arma::cube).
@@ -54,8 +61,9 @@ class Sequential
    * Create the Sequential object using the specified parameters.
    *
    * @param model Expose the all network modules.
+   * @param residual Use the object as a residual block.
    */
-  Sequential(const bool model = true);
+  Sequential(const bool model = true, const bool residual = false);
 
   //! Destroy the Sequential object.
   ~Sequential();
@@ -111,6 +119,13 @@ class Sequential
    */
   void Add(LayerTypes<CustomLayers...> layer) { network.push_back(layer); }
 
+  /*
+   * Destroy all the modules added to the Sequential object.
+   *
+   * Note: Do not use when the model parameter is false to avoid double freeing.
+   */
+  void DeleteModules();
+
   //! Return the model modules.
   std::vector<LayerTypes<CustomLayers...> >& Model()
   {
@@ -137,7 +152,7 @@ class Sequential
   //! Modify the output parameter.
   arma::mat& OutputParameter() { return outputParameter; }
 
-  //! Get the delta.e
+  //! Get the delta.
   arma::mat const& Delta() const { return delta; }
   //! Modify the delta.
   arma::mat& Delta() { return delta; }
@@ -157,6 +172,9 @@ class Sequential
   //! Parameter which indicates if the modules should be exposed.
   bool model;
 
+  //! Parameter which indicates if the object is used as a residual block.
+  bool residual;
+
   //! Indicator if we already initialized the model.
   bool reset;
 

src/mlpack/methods/ann/layer/sequential_impl.hpp

@@ -28,7 +28,13 @@ namespace ann /** Artificial Neural Network. */ {
 template <typename InputDataType, typename OutputDataType,
           typename... CustomLayers>
 Sequential<InputDataType, OutputDataType, CustomLayers...>::Sequential(
-    const bool model) : model(model), reset(false), width(0), height(0)
+    const bool model,
+    const bool residual) :
+    model(model),
+    residual(residual),
+    reset(false),
+    width(0),
+    height(0)
 {
   // Nothing to do here.
 }
@@ -99,12 +105,23 @@ void Sequential<InputDataType, OutputDataType, CustomLayers...>::Forward(
     }
   }
 
-if (!reset)
-{
-  reset = true;
-}
+  if (!reset)
+  {
+    reset = true;
+  }
 
   output = boost::apply_visitor(outputParameterVisitor, network.back());
+
+  if (residual)
+  {
+    if (arma::size(output) != arma::size(input))
+    {
+      Log::Fatal << "The sizes of the output and input matrices of the residual"
+          << " block should be equal. Please examine the network architecture."
+          << std::endl;
+    }
+    output += input;
+  }
 }
 
 template<typename InputDataType, typename OutputDataType,
@@ -128,6 +145,11 @@ void Sequential<InputDataType, OutputDataType, CustomLayers...>::Backward(
   }
 
   g = boost::apply_visitor(deltaVisitor, network.front());
+
+  if (residual)
+  {
+    g += gy;
+  }
 }
 
 template<typename InputDataType, typename OutputDataType,
@@ -154,6 +176,14 @@ void Sequential<InputDataType, OutputDataType, CustomLayers...>::Gradient(
       boost::apply_visitor(deltaVisitor, network[1]))), network.front());
 }
 
+template <typename InputDataType, typename OutputDataType,
+          typename... CustomLayers>
+void Sequential<InputDataType, OutputDataType, CustomLayers...>::DeleteModules()
+{
+  for (LayerTypes<CustomLayers...>& layer : network)
+      boost::apply_visitor(deleteVisitor, layer);
+}
+
 template<typename InputDataType, typename OutputDataType,
          typename... CustomLayers>
 template<typename Archive>
@@ -168,6 +198,7 @@ void Sequential<InputDataType, OutputDataType, CustomLayers...>::serialize(
   }
 
   ar & BOOST_SERIALIZATION_NVP(model);
+  ar & BOOST_SERIALIZATION_NVP(residual);
   ar & BOOST_SERIALIZATION_NVP(network);
 }
 

src/mlpack/tests/ann_layer_test.cpp

@@ -2028,4 +2028,100 @@ BOOST_AUTO_TEST_CASE(GradientReparametrizationLayerBetaTest)
   BOOST_REQUIRE_LE(CheckGradient(function), 1e-4);
 }
 
+/**
+ * Simple sequential module residual block test.
+ */
+BOOST_AUTO_TEST_CASE(SimpleSequentialLayerResidualBlockTest)
+{
+  arma::mat outputA /* sequential */, outputB /* residual */, input,
+      deltaA /* sequential */, deltaB /* residual */;
+  Sequential<>* sequential = new Sequential<>(true, false);
+  Sequential<>* residual = new Sequential<>(true, true);
+
+  Linear<>* linearA = new Linear<>(10, 10);
+  linearA->Parameters().randu();
+  linearA->Reset();
+  Linear<>* linearB = new Linear<>(10, 10);
+  linearB->Parameters().randu();
+  linearB->Reset();
+
+  // Add the same layers(with the same parameters) to both normal Sequential and
+  // residual Sequential object.
+  sequential->Add(linearA);
+  sequential->Add(linearB);
+
+  residual->Add(linearA);
+  residual->Add(linearB);
+
+  // Test the Forward function(Pass the same input to both).
+  input = arma::randu(10, 1);
+  sequential->Forward(std::move(input), std::move(outputA));
+  residual->Forward(std::move(input), std::move(outputB));
+
+  CheckMatrices(outputA, outputB - input);
+
+  // Test the Backward function(Pass the same error to both).
+  sequential->Backward(std::move(input), std::move(input), std::move(deltaA));
+  residual->Backward(std::move(input), std::move(input), std::move(deltaB));
+
+  CheckMatrices(deltaA, deltaB - input);
+
+  delete sequential;
+  delete residual;
+  delete linearA;
+  delete linearB;
+}
+
+/**
+ * Sequential layer numerical gradient test.
+ */
+BOOST_AUTO_TEST_CASE(GradientSequentialLayerTest)
+{
+  // Linear function gradient instantiation.
+  struct GradientFunction
+  {
+    GradientFunction()
+    {
+      input = arma::randu(10, 1);
+      target = arma::mat("1");
+
+      model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
+      model->Predictors() = input;
+      model->Responses() = target;
+      model->Add<IdentityLayer<> >();
+
+      sequential = new Sequential<>();
+      sequential->Add<Linear<> >(10, 10);
+      sequential->Add<ReLULayer<> >();
+      sequential->Add<Linear<> >(10, 5);
+      sequential->Add<ReLULayer<> >();
+
+      model->Add(sequential);
+      model->Add<Linear<> >(5, 2);
+      model->Add<LogSoftMax<> >();
+    }
+
+    ~GradientFunction()
+    {
+      sequential->DeleteModules();
+      delete model;
+    }
+
+    double Gradient(arma::mat& gradient) const
+    {
+      double error = model->Evaluate(model->Parameters(), 0, 1);
+      model->Gradient(model->Parameters(), 0, gradient, 1);
+      return error;
+    }
+
+    arma::mat& Parameters() { return model->Parameters(); }
+
+    FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>* model;
+    Sequential<>* sequential;
+    arma::mat input, target;
+  } function;
+
+  BOOST_REQUIRE_LE(CheckGradient(function), 1e-4);
+}
+
 BOOST_AUTO_TEST_SUITE_END();