From c1b8ea747a93496366a6c5c5d446774ecbcf6afc Mon Sep 17 00:00:00 2001 From: pinpom Date: Mon, 29 Jul 2019 09:55:20 +0800 Subject: [PATCH 1/4] add Sqrt operator --- python/singa/autograd.py | 20 ++++++++++++++++++++ test/python/test_operation.py | 11 +++++++++++ 2 files changed, 31 insertions(+) diff --git a/python/singa/autograd.py b/python/singa/autograd.py index 605a7b60e..069ca58dc 100644 --- a/python/singa/autograd.py +++ b/python/singa/autograd.py @@ -1828,3 +1828,23 @@ def backward(self, dy): def leakyrelu(x, a=0.01): return LeakyRelu(a)(x)[0] + + +class Sqrt(Operation): + def __init__(self): + super(Sqrt, self).__init__() + + def forward(self, x): + if training: + self.input = x + return singa.Sqrt(x) +​ + def backward(self, dy): + dx = singa.PowFloat(self.input,-0.5) + dx = singa.MultFloat(dx,0.5) + dx = singa.__mul__(dy, dx) + return dx +​ + +def sqrt(x): + return Sqrt()(x)[0] \ No newline at end of file diff --git a/test/python/test_operation.py b/test/python/test_operation.py index 82af71910..32261c409 100755 --- a/test/python/test_operation.py +++ b/test/python/test_operation.py @@ -610,6 +610,17 @@ def test_Atanh_gpu(self): np.testing.assert_array_almost_equal(tensor.to_numpy(result), XT, decimal=5) self.check_shape(dx.shape(), (3, 2)) + def test_Sqrt(self): + X=np.array([0.1,1.0,0.4,4.0,0.9,9.0]).reshape(3,2).astype(np.float32) + XT=numpy.sqrt(X) + x=tensor.from_numpy(X) + x.to_device(gpu_dev) + + result=autograd.sqrt(x) + dx=result.creator.backward(x.data) + + np.testing.assert_array_almost_equal(tensor.to_numpy(result), XT) + self.check_shape(dx.shape(), (3, 2)) if __name__ == '__main__': unittest.main() From 60be2257837f1d8621232c2aa8ad8bcf6775ebaf Mon Sep 17 00:00:00 2001 From: pinpom Date: Thu, 1 Aug 2019 16:29:57 +0800 Subject: [PATCH 2/4] fix backward test --- test/python/test_operation.py | 38 ++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/test/python/test_operation.py b/test/python/test_operation.py index 32261c409..2447a3abe 100755 --- a/test/python/test_operation.py +++ b/test/python/test_operation.py @@ -34,6 +34,23 @@ singa.Gaussian(0.0, 1.0, dy) +def eval_numerical_gradient(f, x): + h = 0.0001 + fx = f(x) # evaluate function value at original point + grad = np.zeros( x.shape) + it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite']) + while not it.finished: + ix = it.multi_index + old_value = x[ix] + x[ix] = old_value + h # increment by h + fxh = f(x) # evalute f(x + h) + x[ix] = old_value + grad[ix] = (fxh - fx) / h # the slope + it.iternext() + + return grad + + def _tuple_to_string(t): lt = [str(x) for x in t] return '(' + ', '.join(lt) + ')' @@ -611,16 +628,23 @@ def test_Atanh_gpu(self): self.check_shape(dx.shape(), (3, 2)) def test_Sqrt(self): - X=np.array([0.1,1.0,0.4,4.0,0.9,9.0]).reshape(3,2).astype(np.float32) - XT=numpy.sqrt(X) - x=tensor.from_numpy(X) - x.to_device(gpu_dev) + x=np.array([0.1,1.0,0.4,4.0,0.9,9.0]).reshape(3,2).astype(np.float32) + + y=np.sqrt(x) + f = lambda x : np.sum(np.sqrt(x)) + grad = eval_numerical_gradient(f,x) + x=tensor.from_numpy(x) + x.to_device(gpu_dev) result=autograd.sqrt(x) - dx=result.creator.backward(x.data) - np.testing.assert_array_almost_equal(tensor.to_numpy(result), XT) - self.check_shape(dx.shape(), (3, 2)) + dy=tensor.from_numpy(np.ones((3,2)).astype(np.float32)) + dy.to_device(gpu_dev) + dx=result.creator.backward(dy.data) + + np.testing.assert_array_almost_equal(tensor.to_numpy(result), y) + np.testing.assert_array_almost_equal(tensor.to_numpy(tensor.from_raw_tensor(dx)), grad, decimal=2) + if __name__ == '__main__': unittest.main() From 7a39adee2d4bfb022d15d30bb651ffd77756748f Mon Sep 17 00:00:00 2001 From: pinpom Date: Wed, 7 Aug 2019 17:45:26 +0800 Subject: [PATCH 3/4] change backward test function --- test/python/test_operation.py | 44 +++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/test/python/test_operation.py b/test/python/test_operation.py index 2447a3abe..99aea4356 100755 --- a/test/python/test_operation.py +++ b/test/python/test_operation.py @@ -627,24 +627,44 @@ def test_Atanh_gpu(self): np.testing.assert_array_almost_equal(tensor.to_numpy(result), XT, decimal=5) self.check_shape(dx.shape(), (3, 2)) - def test_Sqrt(self): - x=np.array([0.1,1.0,0.4,4.0,0.9,9.0]).reshape(3,2).astype(np.float32) + def test_Sqrt_cpu(self): + X = np.array([0.1,1.0,0.4,4.0,0.9,9.0]).reshape(3,2).astype(np.float32) + XT = np.sqrt(X) + DY = np.ones((3, 2), dtype = np.float32) - y=np.sqrt(x) - f = lambda x : np.sum(np.sqrt(x)) - grad = eval_numerical_gradient(f,x) + x = tensor.from_numpy(X) + dy = tensor.from_numpy(DY) + x.to_device(cpu_dev) + dy.to_device(cpu_dev) - x=tensor.from_numpy(x) - x.to_device(gpu_dev) - result=autograd.sqrt(x) + result = autograd.sqrt(x) + dx = result.creator.backward(dy.data) + + G = 0.5 * np.power(X, -0.5) + DX = np.multiply(G, DY) + + np.testing.assert_array_almost_equal(tensor.to_numpy(result), XT, decimal=5) + np.testing.assert_array_almost_equal(tensor.to_numpy(tensor.from_raw_tensor(dx)), DX, decimal=5) + + def test_Sqrt_gpu(self): + X = np.array([0.1,1.0,0.4,4.0,0.9,9.0]).reshape(3,2).astype(np.float32) + XT = np.sqrt(X) + DY = np.ones((3, 2), dtype = np.float32) - dy=tensor.from_numpy(np.ones((3,2)).astype(np.float32)) + x = tensor.from_numpy(X) + dy = tensor.from_numpy(DY) + x.to_device(gpu_dev) dy.to_device(gpu_dev) - dx=result.creator.backward(dy.data) - np.testing.assert_array_almost_equal(tensor.to_numpy(result), y) - np.testing.assert_array_almost_equal(tensor.to_numpy(tensor.from_raw_tensor(dx)), grad, decimal=2) + result = autograd.sqrt(x) + dx = result.creator.backward(dy.data) + G = 0.5 * np.power(X, -0.5) + DX = np.multiply(G, DY) + np.testing.assert_array_almost_equal(tensor.to_numpy(result), XT, decimal=5) + np.testing.assert_array_almost_equal(tensor.to_numpy(tensor.from_raw_tensor(dx)), DX, decimal=5) + + if __name__ == '__main__': unittest.main() From 6719ce9985c6771ff892cab0db0a37b06cd712ad Mon Sep 17 00:00:00 2001 From: pinpom Date: Wed, 7 Aug 2019 17:47:24 +0800 Subject: [PATCH 4/4] change backward test function --- test/python/test_operation.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/test/python/test_operation.py b/test/python/test_operation.py index 99aea4356..88978e775 100755 --- a/test/python/test_operation.py +++ b/test/python/test_operation.py @@ -34,23 +34,6 @@ singa.Gaussian(0.0, 1.0, dy) -def eval_numerical_gradient(f, x): - h = 0.0001 - fx = f(x) # evaluate function value at original point - grad = np.zeros( x.shape) - it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite']) - while not it.finished: - ix = it.multi_index - old_value = x[ix] - x[ix] = old_value + h # increment by h - fxh = f(x) # evalute f(x + h) - x[ix] = old_value - grad[ix] = (fxh - fx) / h # the slope - it.iternext() - - return grad - - def _tuple_to_string(t): lt = [str(x) for x in t] return '(' + ', '.join(lt) + ')'