Added Hyper-Sinh activation function

HISTORY.md

@@ -6,6 +6,10 @@
 
   * Add `serialize` method to `GaussianInitialization`, `KathirvalavakumarSubavathiInitialization`, `KathirvalavakumarSubavathiInitialization`, `NguyenWidrowInitialization`, and `OrthogonalInitialization` (#3483).
 
+  * Adapt FTSwish Activation Function (#3485)
+
+  * Adapt Hyper-Sinh activation Function (#3491)
+
 ### mlpack 4.1.0
 ###### 2023-04-26
 

src/mlpack/methods/ann/activation_functions/activation_functions.hpp

@@ -35,5 +35,6 @@
 #include "swish_function.hpp"
 #include "tanh_exponential_function.hpp"
 #include "tanh_function.hpp"
+#include "hyper_sinh_function.hpp"
 
 #endif

src/mlpack/methods/ann/activation_functions/hyper_sinh_function.hpp

@@ -0,0 +1,113 @@
+/**
+ * @file methods/ann/activation_functions/hyper_sinh_function.hpp
+ * @author Mayank Raj
+ *
+ * Definition and implementation of the Hyper-sinh function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_HYPER_SINH_FUNCTION_HPP
+#define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_HYPER_SINH_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+
+/**
+ * The Hyper-sinh function, defined by
+ *
+ * @f{eqnarray*}{
+ * f(x) &=& \begin{cases}
+ * \sinh(x) / 3, & \text{if } x > 0 \\
+ * (x^3) / 4, & \text{if } x \leq 0
+ * \end{cases} \\
+ * f'(x) &=& \begin{cases}
+ * \cosh(x) / 3, & \text{if } x > 0 \\
+ * (3/4) x^2, & \text{if } x \leq 0
+ * \end{cases} \\
+ * @f}
+ */
+class HyperSinhFunction
+{
+ public:
+  /**
+   * Computes the Hyper-sinh function.
+   *
+   * @param x Input data.
+   * @return f(x).
+   */
+  static double Fn(const double x)
+  {
+    if (x > 0)
+      return (std::sinh(x) / 3.0);
+    else
+      return (std::pow(x, 3.0) / 4.0);
+  }
+
+  /**
+   * Computes the Hyper-sinh function.
+   *
+   * @param x Input data.
+   * @param y The resulting output activation.
+   */
+  template<typename InputVecType, typename OutputVecType>
+  static void Fn(const InputVecType& x, OutputVecType& y)
+  {
+    y.set_size(x.size());
+    for(size_t i = 0; i < x.n_elem; ++i)
+    {
+      if (x(i) > 0)
+      {
+        y(i) = std::sinh(x(i)) / 3;
+      }
+      else
+      {
+        y(i) = std::pow(x(i), 3) / 4;
+      }
+    }
+  }
+
+
+  /**
+   * Computes the first derivative of the Hyper-sinh function.
+   *
+   * @param y Input activation.
+   * @return f'(x)
+   */
+  static double Deriv(const double y)
+  {
+    if (y > 0)
+      return (std::cosh(y) / 3);
+    else
+      return ((3.0 / 4.0) * std::pow(y, 2));
+  }
+
+  /**
+   * Computes the first derivatives of the Hyper-sinh function.
+   *
+   * @param y Input activations.
+   * @param x The resulting derivatives.
+   */
+  template<typename InputVecType, typename OutputVecType>
+  static void Deriv(const InputVecType& y, OutputVecType& x)
+  {
+    x.set_size(y.size());
+    for(size_t i = 0; i < y.n_elem; ++i)
+    {
+      if (y(i) > 0)
+      {
+        x(i) = std::cosh(y(i)) / 3;
+      }
+      else
+      {
+        x(i) = (3.0 / 4.0) * std::pow(y(i), 2);
+      }
+    }
+  }
+}; // class HyperSinhFunction
+} // namespace mlpack
+
+#endif

src/mlpack/methods/ann/layer/base_layer.hpp

@@ -30,6 +30,7 @@
 #include <mlpack/methods/ann/activation_functions/hard_swish_function.hpp>
 #include <mlpack/methods/ann/activation_functions/tanh_exponential_function.hpp>
 #include <mlpack/methods/ann/activation_functions/silu_function.hpp>
+#include <mlpack/methods/ann/activation_functions/hyper_sinh_function.hpp>
 #include "layer.hpp"
 
 namespace mlpack {

src/mlpack/tests/ann/activation_functions_test.cpp

@@ -786,3 +786,23 @@ TEST_CASE("SILUFunctionTest", "[ActivationFunctionsTest]")
   CheckActivationCorrect<SILUFunction>(activationData, desiredActivation);
   CheckDerivativeCorrect<SILUFunction>(desiredActivation, desiredDerivate);
 }
+
+/**
+ * Basic test of the Hyper-Sinh Function
+ */
+TEST_CASE("HyperSinhFunctionTest", "[ActivationFunctionsTest]")
+{
+  const arma::colvec activationData("-2.1 3.2 0.3 0 -1.7 2.8 0.11 -1.2 -0.4");
+
+  // Hand-calculated values.
+  arma::colvec desiredActivation(
+      "-2.31525 4.08196 0.101507 0 -1.22825 2.73064 0.0367407 \
+      -0.432 -0.016");
+
+  // Hand-calculated values.
+  arma::colvec desiredDerivate(
+      "4.02029 9.87973 0.335052 0 1.13145 2.56798 0.333558 0.139968 0.000192");
+
+  CheckActivationCorrect<HyperSinhFunction>(activationData, desiredActivation);
+  CheckDerivativeCorrect<HyperSinhFunction>(desiredActivation, desiredDerivate);
+}

src/mlpack/tests/ann/not_adapted/activation_functions_test.cpp

@@ -35,6 +35,7 @@
 #include <mlpack/methods/ann/activation_functions/hard_swish_function.hpp>
 #include <mlpack/methods/ann/activation_functions/tanh_exponential_function.hpp>
 #include <mlpack/methods/ann/activation_functions/silu_function.hpp>
+#include <mlpack/methods/ann/activation_functions/hyper_sinh_function.hpp>
 
 #include "../catch.hpp"