SINGA-425 Add 3 operators

python/singa/autograd.py

@@ -1336,3 +1336,55 @@ def step_forward(self, x, h, c, Wx, Wh, Bx, Bh):
         hout = tanh(cout)
         hout = mul(o, hout)
         return hout, cout
+
+class Abs(Operation):
+
+    def forward(self, a):
+        if training:
+            self.input = a
+        return singa.Abs(a)
+
+    def backward(self, dy):
+        dx = singa.Sign(self.input)
+        return singa.__mul__(dy, dx)
+
+def abs(a):
+    return Abs()(a)[0]
+
+class Exp(Operation):
+
+    def forward(self, a):
+        if training:
+            self.input = a
+        return singa.Exp(a)
+
+    def backward(self, dy):
+        dx = singa.Exp(self.input)
+        return singa.__mul__(dy, dx)
+
+def exp(a):
+    return Exp()(a)[0]
+
+class LeakyRelu(Operation):
+
+    def forward(self, x, a):
+        if training:
+            self.input = x
+        x1 = singa.LTFloat(x, 0.0)
+        x1 = singa.__mul__(x, x1)
+        x1 = singa.MultFloat(x1, a)  
+        x2 = singa.ReLU(x)
+        x1 = singa.__add__(x1, x2)
+        return x1
+
+    def backward(self, dy, a):
+
+        dx1 = singa.GTFloat(self.input, 0.0)
+        dx2 = singa.LTFloat(self.input, 0.0)
+        dx2 = singa.MultFloat(x1, a) 
+        dx =  singa.__add__(x1, x2) 
+        return singa.__mul__(dy, dx)
+
+
+def leakyrelu(x,a=0.01):
+    return LeakyRelu()(x,a)[0]

test/python/test_operation.py

@@ -264,6 +264,48 @@ def test_MeanSquareError(self):
         loss_np=tensor.to_numpy(loss)
         self.assertAlmostEqual(loss_np, 0.0366666, places=4)
         self.check_shape(dx.shape(), (3, 2))
+
+    def test_Abs(self):
+        X=np.array([0.8,-1.2,3.3,-3.6,-0.5,0.5]).reshape(3,2).astype(np.float32)
+        XT=np.array([0.8,1.2,3.3,3.6,0.5,0.5]).reshape(3,2).astype(np.float32)
+        x=tensor.from_numpy(X)
+        x.to_device(gpu_dev)
+
+        result=autograd.abs(x)
+        Err=XT-result
+        dx=result.creator.backward()[0]
+
+        for ii in Err.flatten():
+            self.assertAlmostEquals(ii,0., places=3)
+        self.check_shape(dx.shape(), (3, 2))
+
+    def test_Exp(self):
+        X=np.array([0.8,-1.2,3.3,-3.6,-0.5,0.5]).reshape(3,2).astype(np.float32)
+        XT=np.array([2.2255409,0.22313017,27.112638,0.02732372,0.60653067,1.6487212]).reshape(3,2).astype(np.float32)
+        x=tensor.from_numpy(X)
+        x.to_device(gpu_dev)
+
+        result=autograd.exp(x)
+        Err=XT-result
+        dx=result.creator.backward()[0]
+
+        for ii in Err.flatten():
+            self.assertAlmostEquals(ii,0., places=3)
+        self.check_shape(dx.shape(), (3, 2))
+
+    def test_LeakyRelu(self):
+        X=np.array([0.8,-1.2,3.3,-3.6,-0.5,0.5]).reshape(3,2).astype(np.float32)
+        XT=np.array([0.8,-0.012,3.3,-0.036,-0.005,0.5]).reshape(3,2).astype(np.float32)
+        x=tensor.from_numpy(X)
+        x.to_device(gpu_dev)
+
+        result=autograd.LeakyRelu(x)
+        Err=XT-result
+        dx=result.creator.backward()[0]
+
+        for ii in Err.flatten():
+            self.assertAlmostEquals(ii,0., places=3)
+        self.check_shape(dx.shape(), (3, 2))
 
 
 if __name__ == '__main__':