From 0a681856edd57ab2d8e84ca9d7704df92e475b26 Mon Sep 17 00:00:00 2001
From: Dhawal Arora <d.p.arora1@gmail.com>
Date: Fri, 4 Mar 2016 03:05:49 +0530
Subject: [PATCH 1/2] Added HardTanH Layer and its tests

---
 .../methods/ann/layer/hard_tanh_layer.hpp     | 254 ++++++++++++++++++
 .../tests/activation_functions_test.cpp       |  74 +++++
 2 files changed, 328 insertions(+)
 create mode 100644 src/mlpack/methods/ann/layer/hard_tanh_layer.hpp

diff --git a/src/mlpack/methods/ann/layer/hard_tanh_layer.hpp b/src/mlpack/methods/ann/layer/hard_tanh_layer.hpp
new file mode 100644
index 00000000000..7e71675c53d
--- /dev/null
+++ b/src/mlpack/methods/ann/layer/hard_tanh_layer.hpp
@@ -0,0 +1,254 @@
+/**
+ * @file hard_tanh_layer.hpp
+ * @author Dhawal Arora
+ *
+ * Implementation of hard_tanh activation function. The function is mentioned below.
+ */
+
+#ifndef __MLPACK_METHODS_ANN_LAYER_HARD_TANH_LAYER_HPP
+#define __MLPACK_METHODS_ANN_LAYER_HARD_TANH_LAYER_HPP
+
+#include <mlpack/core.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * The Hard Tanh activation function, defined by
+ *
+ * @f{eqnarray*}{
+ * f(x) &=& \left\{
+ *   \begin{array}{lr}
+ *     max & : x > maxValue \\
+ *     min & : x \le minValue \\
+ *     x   & : otherwise
+ *   \end{array}
+ * \right.
+ * f'(x) &=& \left\{
+ *   \begin{array}{lr}
+ *     0 & : x > maxValue \\ 
+ *     0 & : x \le minValue \\
+ *     1 & : otherwise
+ *   \end{array}
+ * \right.
+ * @f}
+ */
+
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+
+class HardTanHLayer
+{
+ public:
+
+  /**
+   * Constructor. Default maxValue is set to 1 and default minValue is set to -1.
+   *
+   */
+
+  HardTanHLayer(const double maxValue = 1.00, const double minValue = -1.00) : maxValue(maxValue), minValue(minValue)
+  {
+     // Nothing to do here.
+  }
+
+  /**
+   * Computes the HardTanH function.
+   *
+   * @param x Input data.
+   * @return f(x).
+   */
+  double fn(const double x)
+  {
+    double val;
+    val = x;
+    if (x > maxValue)
+      val = maxValue;
+    else if (x < minValue)
+      val = minValue;
+    return val;
+  }
+
+  /**
+   * Computes the HardTanH function using a dense matrix as input.
+   *
+   * @param x Input data.
+   * @param y The resulting output activation.
+   */
+   
+  template<typename eT>
+  void fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
+  {
+    arma::Mat<eT> t;
+    t = x;
+    y = t.transform( [&](eT val) { return std::min( std::max( val, minValue ), maxValue ); } );
+  }
+
+  /**
+   * Computes the HardTanH function using a 3rd-order tensor as input.
+   *
+   * @param x Input data.
+   * @param y The resulting output activation.
+   */
+  template<typename eT>
+  void fn(const arma::Cube<eT>& x, arma::Cube<eT>& y)
+  {
+    y = x;
+    for (size_t s = 0; s < x.n_slices; s++)
+      fn(x.slice(s), y.slice(s));
+  }
+
+  /**
+   * Computes the first derivative of the HardTanH function.
+   *
+   * @param x Input data.
+   * @return f'(x)
+   */
+  double deriv(const double x)
+  {
+    return (x > maxValue || x < minValue) ? 0 : 1;
+  }
+
+  /**
+   * Computes the first derivative of the HardTanH function.
+   *
+   * @param y Input activations.
+   * @param x The resulting derivatives.
+   */
+  template<typename InputType, typename OutputType>
+  void deriv(const InputType& x, OutputType& y)
+  {
+    y = x;
+
+    for (size_t i = 0; i < x.n_elem; i++)
+      y(i) = deriv(x(i));
+  }
+
+  /**
+   * Ordinary feed forward pass of a neural network, evaluating the function
+   * f(x) by propagating the activity forward through f.
+   *
+   * @param input Input data used for evaluating the specified function.
+   * @param output Resulting output activation.
+   */
+  
+  template<typename InputType, typename OutputType>
+  void Forward(const InputType& input, OutputType& output)
+  {
+    fn(input, output);
+  }
+
+  /**
+   * Ordinary feed backward pass of a neural network, calculating the function
+   * f(x) by propagating x backwards through f. Using the results from the feed
+   * forward pass.
+   *
+   * @param input The propagated input activation.
+   * @param gy The backpropagated error.
+   * @param g The calculated gradient.
+   */
+  template<typename DataType>
+  void Backward(const DataType& input,
+                const DataType& gy,
+                DataType& g)
+  {
+    DataType derivative;
+    deriv(input, derivative);
+    g = gy % derivative;
+  }
+
+  /**
+   * Ordinary feed backward pass of a neural network, calculating the function
+   * f(x) by propagating x backwards through f. Using the results from the feed
+   * forward pass.
+   *
+   * @param input The propagated input activation.
+   * @param gy The backpropagated error.
+   * @param g The calculated gradient.
+   */
+  template<typename eT>
+  void Backward(const arma::Cube<eT>& input,
+                const arma::Mat<eT>& gy,
+                arma::Cube<eT>& g)
+  {
+    // Generate a cube using the backpropagated error matrix.
+    arma::Cube<eT> mappedError = arma::zeros<arma::cube>(input.n_rows,
+        input.n_cols, input.n_slices);
+
+    for (size_t s = 0, j = 0; s < mappedError.n_slices; s+= gy.n_cols, j++)
+    {
+      for (size_t i = 0; i < gy.n_cols; i++)
+      {
+        arma::Col<eT> temp = gy.col(i).subvec(
+            j * input.n_rows * input.n_cols,
+            (j + 1) * input.n_rows * input.n_cols - 1);
+
+        mappedError.slice(s + i) = arma::Mat<eT>(temp.memptr(),
+            input.n_rows, input.n_cols);
+      }
+    }
+
+    arma::Cube<eT> derivative;
+    deriv(input, derivative);
+    g = mappedError % derivative;
+  }
+
+  //! Get the input parameter.
+  InputDataType const& InputParameter() const { return inputParameter; }
+  //! Modify the input parameter.
+  InputDataType& InputParameter() { return inputParameter; }
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
+
+  //! Get the Maximum value.
+  double const& getmaxValue() const { return maxValue; }
+  //! Modify the Maximum value.
+  double& setmaxValue() { return maxValue; }
+
+  //! Get the Minimum value.
+  double const& getminValue() const { return minValue; }
+  //! Modify the Minimum value.
+  double& setminValue() { return minValue; }
+
+  
+  /**
+   * Serialize the layer.
+   */
+  template<typename Archive>
+  void Serialize(Archive& /* ar */, const unsigned int /* version */)
+  {
+    /* Nothing to do here */
+  }
+
+ private:
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored input parameter object.
+  InputDataType inputParameter;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+
+  //! Maximum value for the HardTanH function.
+  double maxValue;
+
+  //! Minimum value for the HardTanH function.
+  double minValue;
+
+
+}; // class HardTanHLayer
+
+} // namespace ann
+} // namespace mlpack
+
+#endif
diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp
index 5c40e2ee66c..26bedb5edbd 100644
--- a/src/mlpack/tests/activation_functions_test.cpp
+++ b/src/mlpack/tests/activation_functions_test.cpp
@@ -21,6 +21,7 @@
 #include <mlpack/methods/ann/layer/linear_layer.hpp>
 #include <mlpack/methods/ann/layer/base_layer.hpp>
 #include <mlpack/methods/ann/layer/binary_classification_layer.hpp>
+#include <mlpack/methods/ann/layer/hard_tanh_layer.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "old_boost_test_definitions.hpp"
@@ -120,6 +121,62 @@ void CheckInverseCorrect(const arma::colvec input)
   }
 }
 
+
+/*
+ * Implementation of the HardTanH activation function test. The function is implemented as a HardTanH Layer
+ * in file hard_tanh_layer.hpp
+ * @param input Input data used for evaluating the HardTanH activation function.
+ * @param target Target data used to evaluate the HardTanH activation.
+ *
+ */
+void CheckHardTanHActivationCorrect(const arma::colvec input, const arma::colvec target)
+{
+  HardTanHLayer<> htf;
+  // Test the activation function using a single value as input.
+  for (size_t i = 0; i < target.n_elem; i++)
+  {
+    BOOST_REQUIRE_CLOSE(htf.fn(input.at(i)),
+        target.at(i), 1e-3);
+  }
+
+  // Test the activation function using the entire vector as input.
+  arma::colvec activations;
+  htf.fn(input, activations);
+  for (size_t i = 0; i < activations.n_elem; i++)
+  {
+    BOOST_REQUIRE_CLOSE(activations.at(i), target.at(i), 1e-3);
+  }
+}
+
+/*
+ * Implementation of the HardTanH activation function derivative test. The derivative is implemented in HardTanH Layer
+ * in file hard_tanh_layer.hpp
+ * @param input Input data used for evaluating the HardTanH activation function.
+ * @param target Target data used to evaluate the HardTanH activation.
+ *
+ */
+
+void CheckHardTanHDerivativeCorrect(const arma::colvec input, const arma::colvec target)
+{
+  HardTanHLayer<> htf;
+  // Test the calculation of the derivatives using a single value as input.
+  for (size_t i = 0; i < target.n_elem; i++)
+  {
+    BOOST_REQUIRE_CLOSE(htf.deriv(input.at(i)),
+        target.at(i), 1e-3);
+  }
+
+  // Test the calculation of the derivatives using the entire vector as input.
+  arma::colvec derivatives;
+  htf.deriv(input, derivatives);
+  for (size_t i = 0; i < derivatives.n_elem; i++)
+  {
+    BOOST_REQUIRE_CLOSE(derivatives.at(i), target.at(i), 1e-3);
+  }
+}
+
+
+
 /**
  * Basic test of the tanh function.
  */
@@ -199,4 +256,21 @@ BOOST_AUTO_TEST_CASE(RectifierFunctionTest)
       desiredDerivatives);
 }
 
+/**
+ * Basic test of the HardTanH function.
+ */
+BOOST_AUTO_TEST_CASE(HardTanHFunctionTest)
+{
+  const arma::colvec desiredActivations("-1 1 1 -1 \
+                                         1 -1 1 0");
+
+  const arma::colvec desiredDerivatives("0 0 0 0 \
+                                         1 1 0 1");
+
+  CheckHardTanHActivationCorrect(activationData, desiredActivations);
+  CheckHardTanHDerivativeCorrect(activationData, desiredDerivatives);
+}
+
+
+
 BOOST_AUTO_TEST_SUITE_END();

From 7de16454b3b97268ce7402f433f3371376696a40 Mon Sep 17 00:00:00 2001
From: Dhawal Arora <d.p.arora1@gmail.com>
Date: Mon, 7 Mar 2016 16:05:49 +0530
Subject: [PATCH 2/2] Changed scope of some functions to private in HardTanH
 Layer and its tests; some minor style changes

---
 .../methods/ann/layer/hard_tanh_layer.hpp     | 168 ++++++++++--------
 .../tests/activation_functions_test.cpp       |  10 +-
 2 files changed, 104 insertions(+), 74 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/hard_tanh_layer.hpp b/src/mlpack/methods/ann/layer/hard_tanh_layer.hpp
index 7e71675c53d..89114b0e8dc 100644
--- a/src/mlpack/methods/ann/layer/hard_tanh_layer.hpp
+++ b/src/mlpack/methods/ann/layer/hard_tanh_layer.hpp
@@ -54,75 +54,16 @@ class HardTanHLayer
   }
 
   /**
-   * Computes the HardTanH function.
-   *
-   * @param x Input data.
-   * @return f(x).
-   */
-  double fn(const double x)
-  {
-    double val;
-    val = x;
-    if (x > maxValue)
-      val = maxValue;
-    else if (x < minValue)
-      val = minValue;
-    return val;
-  }
-
-  /**
-   * Computes the HardTanH function using a dense matrix as input.
-   *
-   * @param x Input data.
-   * @param y The resulting output activation.
-   */
-   
-  template<typename eT>
-  void fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
-  {
-    arma::Mat<eT> t;
-    t = x;
-    y = t.transform( [&](eT val) { return std::min( std::max( val, minValue ), maxValue ); } );
-  }
-
-  /**
-   * Computes the HardTanH function using a 3rd-order tensor as input.
-   *
-   * @param x Input data.
-   * @param y The resulting output activation.
-   */
-  template<typename eT>
-  void fn(const arma::Cube<eT>& x, arma::Cube<eT>& y)
-  {
-    y = x;
-    for (size_t s = 0; s < x.n_slices; s++)
-      fn(x.slice(s), y.slice(s));
-  }
-
-  /**
-   * Computes the first derivative of the HardTanH function.
-   *
-   * @param x Input data.
-   * @return f'(x)
+   * Ordinary feed forward pass of a neural network, evaluating the function
+   * f(x) by propagating the activity forward through f.
+   * 
+   * @param x Input data used for evaluating the specified function. This is for just one input value. 
+   * @return f(x) The activation value for the input. 
    */
-  double deriv(const double x)
-  {
-    return (x > maxValue || x < minValue) ? 0 : 1;
-  }
 
-  /**
-   * Computes the first derivative of the HardTanH function.
-   *
-   * @param y Input activations.
-   * @param x The resulting derivatives.
-   */
-  template<typename InputType, typename OutputType>
-  void deriv(const InputType& x, OutputType& y)
+  double Forward(const double x)
   {
-    y = x;
-
-    for (size_t i = 0; i < x.n_elem; i++)
-      y(i) = deriv(x(i));
+    return fn(x);
   }
 
   /**
@@ -139,6 +80,22 @@ class HardTanHLayer
     fn(input, output);
   }
 
+  /**
+   * Ordinary feed backward pass of a neural network, calculating the function
+   * f(x) by propagating x backwards through f. Using the results from the feed
+   * forward pass.
+   *
+   * @param input The propagated input activation. This function is for just a single input. 
+   * @param gy The backpropagated error.
+   * @return The calculated gradient.
+   */
+
+  double Backward(const double input,
+                  const double gy)
+  {
+    return gy * deriv(input);
+  }
+
   /**
    * Ordinary feed backward pass of a neural network, calculating the function
    * f(x) by propagating x backwards through f. Using the results from the feed
@@ -210,14 +167,14 @@ class HardTanHLayer
   OutputDataType& Delta() { return delta; }
 
   //! Get the Maximum value.
-  double const& getmaxValue() const { return maxValue; }
+  double const& MaxValue() const { return maxValue; }
   //! Modify the Maximum value.
-  double& setmaxValue() { return maxValue; }
+  double& MaxValue() { return maxValue; }
 
   //! Get the Minimum value.
-  double const& getminValue() const { return minValue; }
+  double const& MinValue() const { return minValue; }
   //! Modify the Minimum value.
-  double& setminValue() { return minValue; }
+  double& MinValue() { return minValue; }
 
   
   /**
@@ -230,6 +187,77 @@ class HardTanHLayer
   }
 
  private:
+
+
+  /**
+   * Computes the HardTanH function.
+   *
+   * @param x Input data.
+   * @return f(x).
+   */
+  double fn(const double x)
+  {
+    if (x > maxValue)
+      return maxValue;
+    else if (x < minValue)
+      return minValue;
+    return x;
+  }
+
+  /**
+   * Computes the HardTanH function using a dense matrix as input.
+   *
+   * @param x Input data.
+   * @param y The resulting output activation.
+   */
+   
+  template<typename eT>
+  void fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
+  {
+    y = x;
+    y = y.transform( [&](eT val) { return std::min( std::max( val, minValue ), maxValue ); } );
+  }
+
+  /**
+   * Computes the HardTanH function using a 3rd-order tensor as input.
+   *
+   * @param x Input data.
+   * @param y The resulting output activation.
+   */
+  template<typename eT>
+  void fn(const arma::Cube<eT>& x, arma::Cube<eT>& y)
+  {
+    y = x;
+    for (size_t s = 0; s < x.n_slices; s++)
+      fn(x.slice(s), y.slice(s));
+  }
+
+  /**
+   * Computes the first derivative of the HardTanH function.
+   *
+   * @param x Input data.
+   * @return f'(x)
+   */
+  double deriv(const double x)
+  {
+    return (x > maxValue || x < minValue) ? 0 : 1;
+  }
+
+  /**
+   * Computes the first derivative of the HardTanH function.
+   *
+   * @param y Input activations.
+   * @param x The resulting derivatives.
+   */
+  template<typename InputType, typename OutputType>
+  void deriv(const InputType& x, OutputType& y)
+  {
+    y = x;
+
+    for (size_t i = 0; i < x.n_elem; i++)
+      y(i) = deriv(x(i));
+  }
+
   //! Locally-stored delta object.
   OutputDataType delta;
 
diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp
index 26bedb5edbd..986e89a8536 100644
--- a/src/mlpack/tests/activation_functions_test.cpp
+++ b/src/mlpack/tests/activation_functions_test.cpp
@@ -135,13 +135,13 @@ void CheckHardTanHActivationCorrect(const arma::colvec input, const arma::colvec
   // Test the activation function using a single value as input.
   for (size_t i = 0; i < target.n_elem; i++)
   {
-    BOOST_REQUIRE_CLOSE(htf.fn(input.at(i)),
+    BOOST_REQUIRE_CLOSE(htf.Forward(input.at(i)),
         target.at(i), 1e-3);
   }
 
   // Test the activation function using the entire vector as input.
   arma::colvec activations;
-  htf.fn(input, activations);
+  htf.Forward(input, activations);
   for (size_t i = 0; i < activations.n_elem; i++)
   {
     BOOST_REQUIRE_CLOSE(activations.at(i), target.at(i), 1e-3);
@@ -162,13 +162,15 @@ void CheckHardTanHDerivativeCorrect(const arma::colvec input, const arma::colvec
   // Test the calculation of the derivatives using a single value as input.
   for (size_t i = 0; i < target.n_elem; i++)
   {
-    BOOST_REQUIRE_CLOSE(htf.deriv(input.at(i)),
+    BOOST_REQUIRE_CLOSE(htf.Backward(input.at(i), 1),
         target.at(i), 1e-3);
   }
 
   // Test the calculation of the derivatives using the entire vector as input.
   arma::colvec derivatives;
-  htf.deriv(input, derivatives);
+  // This error vector will be set to 1 to get the derivatives.
+  arma::colvec error(input.n_elem);
+  htf.Backward(input, (arma::colvec)error.ones(), derivatives);
   for (size_t i = 0; i < derivatives.n_elem; i++)
   {
     BOOST_REQUIRE_CLOSE(derivatives.at(i), target.at(i), 1e-3);