Implementing MISH activation; Fixes issue #191

src/enums/ActivationType.ts

@@ -14,5 +14,6 @@ export enum ActivationType {
     HardTanhActivation,
     AbsoluteActivation,
     InverseActivation,
-    SELUActivation
+    SELUActivation,
+    MISHActivation,
 }

src/methods/Activation.ts

@@ -53,6 +53,8 @@ abstract class Activation {
                 return new InverseActivation();
             case ActivationType.SELUActivation:
                 return new SELUActivation();
+            case ActivationType.MISHActivation:
+                return new MISHActivation();
         }
         throw new ReferenceError(activationType + " is not the name of any available activations! These are all available activations: " + ALL_ACTIVATIONS);
     }
@@ -454,6 +456,37 @@ class SELUActivation implements Activation {
     }
 }
 
+/**
+ * MISH: Self Regularized Non-Monotonic Activation Function
+ *
+ * @see {@link https://github.com/digantamisra98/Mish Neural Networks}
+ *
+ * @param x Input value to activation function
+ * @param derivative Flag to select derivative function
+ *
+ * @example
+ * let { methods, Node } = require("@liquid-carrot/carrot");
+ *
+ * // Changing a neuron's activation function
+ * let A = new Node();
+ * A.squash = new MISHActivation();
+ */
+class MISHActivation implements Activation {
+    public readonly type: ActivationType = ActivationType.MISHActivation;
+
+    public calc(x: number, derivative: boolean = false): number {
+        const ex: number = Math.exp(x);
+
+        if (derivative) {
+            const w: number = ex * ex * ex + 4 * (ex * ex + x * ex + x + 1) + 6 * ex;
+            const d: number = 2 * ex + ex * ex + 2;
+            return ex * w / (d * d);
+        } else {
+            return x * Math.tanh(Math.log(1 + ex));
+        }
+    }
+}
+
 const ALL_ACTIVATIONS: ActivationType[] = [
     ActivationType.LogisticActivation,
     ActivationType.TanhActivation,
@@ -469,7 +502,8 @@ const ALL_ACTIVATIONS: ActivationType[] = [
     ActivationType.HardTanhActivation,
     ActivationType.AbsoluteActivation,
     ActivationType.InverseActivation,
-    ActivationType.SELUActivation
+    ActivationType.SELUActivation,
+    ActivationType.MISHActivation
 ];
 
 export {
@@ -489,5 +523,6 @@ export {
     HardTanhActivation,
     AbsoluteActivation,
     InverseActivation,
-    SELUActivation
+    SELUActivation,
+    MISHActivation
 };

test/units/methods/ActivationTest.ts

@@ -1,3 +1,4 @@
+import {expect} from "chai";
 import {
     AbsoluteActivation,
     BentIdentityActivation,
@@ -8,6 +9,7 @@ import {
     IdentityActivation,
     InverseActivation,
     LogisticActivation,
+    MISHActivation,
     RELUActivation,
     SELUActivation,
     SinusoidActivation,
@@ -16,7 +18,6 @@ import {
     TanhActivation
 } from "../../../src/methods/Activation";
 import {randDouble} from "../../../src/methods/Utils";
-import {expect} from "chai";
 
 describe("Activation", () => {
     describe("activation.LOGISTIC()", () => {
@@ -179,4 +180,21 @@ describe("Activation", () => {
             expect(new SELUActivation().calc(x, true)).to.be.closeTo(z, 0.01);
         });
     });
+    describe("activation.MISH()", () => {
+        it("activation.MISH(number, derivate=false) => {number}", () => {
+            const x: number = randDouble(-50, 50);
+            const z: number = x * Math.tanh(Math.log(1 + Math.exp(x)));
+            expect(new MISHActivation().calc(x, false)).to.be.closeTo(z, 0.01);
+        });
+        it("activation.MISH(number, derivate=true) => {number}", () => {
+            const x: number = randDouble(-50, 50);
+
+            const ex: number = Math.exp(x);
+            const w: number = ex * ex * ex + 4 * (ex * ex + x * ex + x + 1) + 6 * ex;
+            const d: number = 2 * ex + ex * ex + 2;
+            const z: number = ex * w / (d * d);
+
+            expect(new MISHActivation().calc(x, true)).to.be.closeTo(z, 0.01);
+        });
+    });
 });