Adapt C_ReLU, ReLU6, FlexibleReLU

src/mlpack/methods/ann/layer/not_adapted/c_relu.hpp → src/mlpack/methods/ann/layer/c_relu.hpp

@@ -39,14 +39,12 @@ namespace mlpack {
  * }
  * @endcode
  *
- * @tparam InputType The type of the layer's inputs. The layer automatically
- *     cast inputs to this type (Default: arma::mat).
- * @tparam OutputType The type of the computation which also causes the output
- *     to also be in this type. The type also allows the computation and weight
- *     type to differ from the input type (Default: arma::mat).
+ * @tparam MatType Matrix representation to accept as input and allows the
+ *         computation and weight type to differ from the input type
+ *         (Default: arma::mat).
  */
-template<typename InputType = arma::mat, typename OutputType = arma::mat>
-class CReLUType : public Layer<InputType, OutputType>
+template<typename MatType = arma::mat>
+class CReLUType : public Layer<MatType>
 {
  public:
   //! Create the CReLU object.
@@ -55,6 +53,18 @@ class CReLUType : public Layer<InputType, OutputType>
   //! Clone the CReLUType object. This handles polymorphism correctly.
   CReLUType* Clone() const { return new CReLUType(*this); }
 
+  // Virtual destructor.
+  virtual ~CReLUType() { }
+
+  //! Copy the given CReLUType.
+  CReLUType(const CReLUType& other);
+  //! Take ownership of the given CReLUType.
+  CReLUType(CReLUType&& other);
+  //! Copy the given CReLUType.
+  CReLUType& operator=(const CReLUType& other);
+  //! Take ownership of the given CReLUType.
+  CReLUType& operator=(CReLUType&& other);
+
   /**
    * Ordinary feed forward pass of a neural network, evaluating the function
    * f(x) by propagating the activity forward through f.
@@ -63,7 +73,7 @@ class CReLUType : public Layer<InputType, OutputType>
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
-  void Forward(const InputType& input, OutputType& output);
+  void Forward(const MatType& input, MatType& output);
 
   /**
    * Ordinary feed backward pass of a neural network, calculating the function
@@ -74,7 +84,7 @@ class CReLUType : public Layer<InputType, OutputType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const InputType& input, const OutputType& gy, OutputType& g);
+  void Backward(const MatType& input, const MatType& gy, MatType& g);
 
   //! Serialize the layer.
   template<typename Archive>
@@ -84,7 +94,7 @@ class CReLUType : public Layer<InputType, OutputType>
 // Convenience typedefs.
 
 // Standard CReLU layer.
-typedef CReLUType<arma::mat, arma::mat> CReLU;
+typedef CReLUType<arma::mat> CReLU;
 
 } // namespace mlpack
 

src/mlpack/methods/ann/layer/c_relu_impl.hpp

@@ -0,0 +1,95 @@
+/**
+ * @file methods/ann/layer/c_relu_impl.hpp
+ * @author Jeffin Sam
+ *
+ * Implementation of CReLU layer.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LAYER_C_RELU_IMPL_HPP
+#define MLPACK_METHODS_ANN_LAYER_C_RELU_IMPL_HPP
+
+// In case it hasn't yet been included.
+#include "c_relu.hpp"
+
+namespace mlpack {
+
+template<typename MatType>
+CReLUType<MatType>::CReLUType() :
+    Layer<MatType>()
+{
+  // Nothing to do here.
+}
+
+template<typename MatType>
+CReLUType<MatType>::CReLUType(
+    const CReLUType& other) :
+    Layer<MatType>(other)
+{
+  // Nothing to do here.
+}
+
+template<typename MatType>
+CReLUType<MatType>::CReLUType(
+    CReLUType&& other) :
+    Layer<MatType>(std::move(other))
+{
+  // Nothing to do here.
+}
+
+template<typename MatType>
+CReLUType<MatType>&
+CReLUType<MatType>::operator=(const CReLUType& other)
+{
+  if (&other != this)
+  {
+    Layer<MatType>::operator=(other);
+  }
+
+  return *this;
+}
+
+template<typename MatType>
+CReLUType<MatType>&
+CReLUType<MatType>::operator=(CReLUType&& other)
+{
+  if (&other != this)
+  {
+    Layer<MatType>::operator=(std::move(other));
+  }
+
+  return *this;
+}
+
+template<typename MatType>
+void CReLUType<MatType>::Forward(
+    const MatType& input, MatType& output)
+{
+  output = arma::join_cols(arma::max(input, 0.0 * input), arma::max(
+      (-1 * input), 0.0 * input));
+}
+
+template<typename MatType>
+void CReLUType<MatType>::Backward(
+    const MatType& input, const MatType& gy, MatType& g)
+{
+  MatType temp = gy % (input >= 0.0);
+  g = temp.rows(0, (input.n_rows / 2 - 1)) - temp.rows(input.n_rows / 2,
+      (input.n_rows - 1));
+}
+
+template<typename MatType>
+template<typename Archive>
+void CReLUType<MatType>::serialize(
+    Archive& ar,
+    const uint32_t /* version */)
+{
+  ar(cereal::base_class<Layer<MatType>>(this));
+}
+
+} // namespace mlpack
+
+#endif

src/mlpack/methods/ann/layer/not_adapted/flexible_relu.hpp → src/mlpack/methods/ann/layer/flexible_relu.hpp

@@ -12,8 +12,8 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef MLPACK_METHODS_ANN_LAYER_FLEXIBLERELU_HPP
-#define MLPACK_METHODS_ANN_LAYER_FLEXIBLERELU_HPP
+#ifndef MLPACK_METHODS_ANN_LAYER_FLEXIBLE_RELU_HPP
+#define MLPACK_METHODS_ANN_LAYER_FLEXIBLE_RELU_HPP
 
 #include <mlpack/prereqs.hpp>
 
@@ -47,14 +47,12 @@ namespace mlpack {
  * }
  * @endcode
  *
- * @tparam InputType The type of the layer's inputs. The layer automatically
- *     cast inputs to this type (Default: arma::mat).
- * @tparam OutputType The type of the computation which also causes the output
- *     to also be in this type. The type also allows the computation and weight
- *     type to differ from the input type (Default: arma::mat).
+ * @tparam MatType Matrix representation to accept as input and allows the
+ *         computation and weight type to differ from the input type
+ *         (Default: arma::mat).
  */
-template<typename InputType = arma::mat, typename OutputType = arma::mat>
-class FlexibleReLUType : public Layer<InputType, OutputType>
+template<typename MatType = arma::mat>
+class FlexibleReLUType : public Layer<MatType>
 {
  public:
   /**
@@ -69,11 +67,33 @@ class FlexibleReLUType : public Layer<InputType, OutputType>
   //! Clone the FlexibleReLUType object. This handles polymorphism correctly.
   FlexibleReLUType* Clone() const { return new FlexibleReLUType(*this); }
 
+  // Virtual destructor.
+  virtual ~FlexibleReLUType() { }
+
+  //! Copy the given FlexibleReLUType.
+  FlexibleReLUType(const FlexibleReLUType& other);
+  //! Take ownership of the given FlexibleReLUType.
+  FlexibleReLUType(FlexibleReLUType&& other);
+  //! Copy the given FlexibleReLUType.
+  FlexibleReLUType& operator=(const FlexibleReLUType& other);
+  //! Take ownership of the given FlexibleReLUType.
+  FlexibleReLUType& operator=(FlexibleReLUType&& other);
+
   /**
    * Reset the layer parameter (alpha). The method is called to
    * assign the allocated memory to the learnable layer parameter.
    */
-  void SetWeights(typename OutputType::elem_type* weightsPtr);
+  void SetWeights(typename MatType::elem_type* weightsPtr);
+
+  /**
+   * Initialize the weight matrix of the layer.
+   *
+   * @param W Weight matrix to initialize.
+   * @param elements Number of elements.
+   */
+  void CustomInitialize(
+      MatType& W,
+      const size_t elements);
 
   /**
    * Ordinary feed forward pass of a neural network, evaluating the function
@@ -82,7 +102,7 @@ class FlexibleReLUType : public Layer<InputType, OutputType>
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
-  void Forward(const InputType& input, OutputType& output);
+  void Forward(const MatType& input, MatType& output);
 
   /**
    * Ordinary feed backward pass of a neural network, calculating the function
@@ -93,7 +113,7 @@ class FlexibleReLUType : public Layer<InputType, OutputType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const InputType& input, const OutputType& gy, OutputType& g);
+  void Backward(const MatType& input, const MatType& gy, MatType& g);
 
   /**
    * Calculate the gradient using the output delta and the input activation.
@@ -102,21 +122,22 @@ class FlexibleReLUType : public Layer<InputType, OutputType>
    * @param error The calculated error.
    * @param gradient The calculated gradient.
    */
-  void Gradient(const InputType& input,
-                const OutputType& error,
-                OutputType& gradient);
+  void Gradient(const MatType& input,
+                const MatType& error,
+                MatType& gradient);
 
   //! Get the parameters.
-  OutputType const& Parameters() const { return alpha; }
+  MatType const& Parameters() const { return alpha; }
   //! Modify the parameters.
-  OutputType& Parameters() { return alpha; }
+  MatType& Parameters() { return alpha; }
 
   //! Get the parameter controlling the range of the ReLU function.
   const double& Alpha() const { return alpha; }
   //! Modify the parameter controlling the range of the ReLU function.
   double& Alpha() { return alpha; }
 
-  const size_t WeightSize() const { return 1; }
+  //! Get size of weights.
+  size_t WeightSize() const { return 1; }
 
   /**
    * Serialize the layer.
@@ -126,19 +147,16 @@ class FlexibleReLUType : public Layer<InputType, OutputType>
 
  private:
   //! Parameter object.
-  OutputType alpha;
+  MatType alpha;
 
   //! Parameter controlling the range of the ReLU function.
   double userAlpha;
-
-  //! Whether or not a forward pass has ever been performed.
-  bool initialized;
 }; // class FlexibleReLUType
 
 // Convenience typedefs.
 
 // Standard flexible ReLU layer.
-typedef FlexibleReLUType<arma::mat, arma::mat> FlexibleReLU;
+typedef FlexibleReLUType<arma::mat> FlexibleReLU;
 
 } // namespace mlpack