Added support for Swish activation function.

mlfromscratch/deep_learning/activation_functions.py

@@ -73,3 +73,19 @@ def __call__(self, x):
     def gradient(self, x):
         return 1 / (1 + np.exp(-x))
 
+class Swish():
+    # Reference : https://arxiv.org/abs/1710.05941v1
+    def __init__(self):
+        self.beta = 1
+
+    def __call__(self, x):
+        return x * self.__sigmoid(self.beta * x)
+
+    def __sigmoid(self, x):
+        return 1 / (1 + np.exp(-x))
+
+    def gradient(self, x):
+        betaX = self.beta * x
+        sig = self.__sigmoid(betaX)
+        return betaX * (sig * (1 - sig)) + sig
+

mlfromscratch/deep_learning/layers.py

@@ -3,7 +3,7 @@
 import math
 import numpy as np
 import copy
-from mlfromscratch.deep_learning.activation_functions import Sigmoid, ReLU, SoftPlus, LeakyReLU
+from mlfromscratch.deep_learning.activation_functions import Sigmoid, ReLU, SoftPlus, LeakyReLU, Swish
 from mlfromscratch.deep_learning.activation_functions import TanH, ELU, SELU, Softmax
 
 
@@ -610,7 +610,8 @@ def output_shape(self):
     'softmax': Softmax,
     'leaky_relu': LeakyReLU,
     'tanh': TanH,
-    'softplus': SoftPlus
+    'softplus': SoftPlus,
+    'swish': Swish
 }
 
 class Activation(Layer):

mlfromscratch/supervised_learning/perceptron.py

@@ -4,7 +4,7 @@
 
 # Import helper functions
 from mlfromscratch.utils import train_test_split, to_categorical, normalize, accuracy_score
-from mlfromscratch.deep_learning.activation_functions import Sigmoid, ReLU, SoftPlus, LeakyReLU, TanH, ELU
+from mlfromscratch.deep_learning.activation_functions import Sigmoid, ReLU, SoftPlus, Swish, LeakyReLU, TanH, ELU
 from mlfromscratch.deep_learning.loss_functions import CrossEntropy, SquareLoss
 from mlfromscratch.utils import Plot
 from mlfromscratch.utils.misc import bar_widgets
@@ -19,7 +19,7 @@ class Perceptron():
         The number of training iterations the algorithm will tune the weights for.
     activation_function: class
         The activation that shall be used for each neuron.
-        Possible choices: Sigmoid, ExpLU, ReLU, LeakyReLU, SoftPlus, TanH
+        Possible choices: Sigmoid, ExpLU, ReLU, LeakyReLU, SoftPlus, TanH, Swish
     loss: class
         The loss function used to assess the model's performance.
         Possible choices: SquareLoss, CrossEntropy