Add mish and softplus nonlinear functions

josephjaspers · Nov 22, 2019 · 87fa2f4 · 87fa2f4
1 parent ac9f986
commit 87fa2f4
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 3 deletions.
diff --git a/include/neural_networks/layers/Nonlinear.h b/include/neural_networks/layers/Nonlinear.h
@@ -31,6 +31,8 @@ auto funcName(SystemTag system, BC::size_t inputs) {\
 BC_NONLINEAR_DEF(Tanh, tanh)
 BC_NONLINEAR_DEF(Logistic, logistic)
 BC_NONLINEAR_DEF(Relu, relu)
+BC_NONLINEAR_DEF(SoftPlus, softplus)
+BC_NONLINEAR_DEF(Mish, mish)
 
 #undef BC_NONLINEAR_DEF
 }

diff --git a/include/operations/CMath.h b/include/operations/CMath.h
@@ -157,9 +157,13 @@ BLACKCAT_FUNCTOR_DEF(Logistic, logistic, (1 / (1 + std::exp(-x))),
 
 BLACKCAT_FUNCTOR_DEF(Relu, relu, BC::traits::max(0, x), DERIVATIVE_DEF(x > 0 ? 1 : 0));
 BLACKCAT_FUNCTOR_DEF(Logical, logical, x != 0 ? 1 : 0);
-
-
-
+BLACKCAT_FUNCTOR_DEF(SoftPlus, softplus, std::log(1 + std::exp(x)), DERIVATIVE_DEF(Logistic::apply(x)))
+BLACKCAT_FUNCTOR_DEF(Mish, mish,
+		x * std::tanh(SoftPlus::apply(x)),
+	DERIVATIVE_DEF(
+		std::exp(x)*
+			(4*(x+1) + 4*(std::exp(2*x)) + std::exp(3*x) + std::exp(x)*(4*x+6)) /
+			std::pow((2*std::exp(x) + std::exp(2*x) + 2),2)))
 
 #undef BLACKCAT_FUNCTOR_DEF
 #undef BLACKCAT_MATH_DEF