From ecf2594c504f3a54a6c71055a3c5389c5d6f35c4 Mon Sep 17 00:00:00 2001 From: niboshi Date: Fri, 18 Aug 2017 15:33:01 +0900 Subject: [PATCH 1/3] New-style LeakyReLU --- chainer/functions/activation/leaky_relu.py | 79 +++++++++++++------ .../activation_tests/test_leaky_relu.py | 27 ++++++- 2 files changed, 80 insertions(+), 26 deletions(-) diff --git a/chainer/functions/activation/leaky_relu.py b/chainer/functions/activation/leaky_relu.py index 5b981bb12578..81282cfd4ecb 100644 --- a/chainer/functions/activation/leaky_relu.py +++ b/chainer/functions/activation/leaky_relu.py @@ -1,15 +1,21 @@ from chainer import cuda -from chainer import function +from chainer import function_node from chainer.utils import type_check -def _kern(): - return cuda.elementwise( - 'T cond, T x, T slope', 'T y', - 'y = cond >= 0 ? x : (T)(slope * x)', 'lrelu') +_kern = None -class LeakyReLU(function.Function): +def _get_kern(): + global _kern + if _kern is None: + _kern = cuda.elementwise( + 'T cond, T x, T slope', 'T y', + 'y = cond >= 0 ? x : (T)(slope * x)', 'lrelu') + return _kern + + +class LeakyReLU(function_node.FunctionNode): """Leaky rectifier unit.""" @@ -21,37 +27,60 @@ def check_type_forward(self, in_types): x_type, = in_types type_check.expect(x_type.dtype.kind == 'f') - def forward_cpu(self, x): - y = x[0].copy() - y[x[0] < 0] *= self.slope + def forward_cpu(self, inputs): + x, = inputs + y = x.copy() + y[x < 0] *= self.slope if self.slope >= 0: - self.retain_inputs(()) self.retain_outputs((0,)) + else: + self.retain_inputs((0,)) return y, - def forward_gpu(self, x): - y = _kern()(x[0], x[0], self.slope) + def forward_gpu(self, inputs): + x, = inputs + y = _get_kern()(x, x, self.slope) if self.slope >= 0: - self.retain_inputs(()) self.retain_outputs((0,)) + else: + self.retain_inputs((0,)) return y, - def backward_cpu(self, x, gy): - gx = gy[0].copy() + def backward(self, indexes, grad_outputs): + if self.slope >= 0: + x = None + y = self.get_retained_outputs()[0].data + else: + x = self.get_retained_inputs()[0].data + y = None + return _LeakyReLUGrad(x, y, self.slope).apply(grad_outputs) + + +class _LeakyReLUGrad(function_node.FunctionNode): + + def __init__(self, x, y, slope): + self.slope = slope + self.x = x + self.y = y + + def forward_cpu(self, inputs): + gy, = inputs if self.slope >= 0: - y = self.output_data - gx[y[0] < 0] *= self.slope + gy[self.y < 0] *= self.slope else: - gx[x[0] < 0] *= self.slope - return gx, + gy[self.x < 0] *= self.slope + return gy, - def backward_gpu(self, x, gy): + def forward_gpu(self, inputs): + gy, = inputs if self.slope >= 0: - y = self.output_data - gx = _kern()(y[0], gy[0], self.slope) + gy = _get_kern()(self.y, gy, self.slope) else: - gx = _kern()(x[0], gy[0], self.slope) - return gx, + gy = _get_kern()(self.x, gy, self.slope) + return gy, + + def backward(self, indexes, grad_outputs): + return _LeakyReLUGrad(self.x, self.y, self.slope).apply(grad_outputs) def leaky_relu(x, slope=0.2): @@ -86,4 +115,4 @@ def leaky_relu(x, slope=0.2): [-0.40000001, 1. ]], dtype=float32) """ - return LeakyReLU(slope)(x) + return LeakyReLU(slope).apply((x,))[0] diff --git a/tests/chainer_tests/functions_tests/activation_tests/test_leaky_relu.py b/tests/chainer_tests/functions_tests/activation_tests/test_leaky_relu.py index 6c2965d5f695..71b57f2b6647 100644 --- a/tests/chainer_tests/functions_tests/activation_tests/test_leaky_relu.py +++ b/tests/chainer_tests/functions_tests/activation_tests/test_leaky_relu.py @@ -25,6 +25,7 @@ def setUp(self): if -0.05 < self.x[i] < 0.05: self.x[i] = 0.5 self.gy = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype) + self.ggx = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype) self.slope = random.random() self.check_forward_options = {} self.check_backward_options = {'dtype': numpy.float64} @@ -56,8 +57,11 @@ def test_forward_gpu(self): self.check_forward(cuda.to_gpu(self.x)) def check_backward(self, x_data, y_grad): + def f(x): + return functions.leaky_relu(x, self.slope) + gradient_check.check_backward( - functions.LeakyReLU(self.slope), x_data, y_grad, + f, x_data, y_grad, **self.check_backward_options) @condition.retry(10) @@ -69,5 +73,26 @@ def test_backward_cpu(self): def test_backward_gpu(self): self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy)) + def check_double_backward(self, x_data, y_grad, x_grad_grad): + def f(x): + y = functions.leaky_relu(x, self.slope) + return y * y + + gradient_check.check_double_backward( + f, x_data, y_grad, x_grad_grad, + **self.check_backward_options) + + @condition.retry(10) + def test_double_backward_cpu(self): + self.check_double_backward(self.x, self.gy, self.ggx) + + @attr.gpu + @condition.retry(10) + def test_double_backward_gpu(self): + self.check_double_backward( + cuda.to_gpu(self.x), + cuda.to_gpu(self.gy), + cuda.to_gpu(self.ggx)) + testing.run_module(__name__, __file__) From ff462a5c6e8a71d9e724086f37022e9aef2ba32d Mon Sep 17 00:00:00 2001 From: niboshi Date: Tue, 22 Aug 2017 15:25:21 +0900 Subject: [PATCH 2/3] Relax numerial tolerance --- .../functions_tests/activation_tests/test_leaky_relu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/chainer_tests/functions_tests/activation_tests/test_leaky_relu.py b/tests/chainer_tests/functions_tests/activation_tests/test_leaky_relu.py index 71b57f2b6647..0bd4d2df8f12 100644 --- a/tests/chainer_tests/functions_tests/activation_tests/test_leaky_relu.py +++ b/tests/chainer_tests/functions_tests/activation_tests/test_leaky_relu.py @@ -32,7 +32,7 @@ def setUp(self): if self.dtype == numpy.float16: self.check_forward_options = {'atol': 1e-4, 'rtol': 1e-3} self.check_backward_options = { - 'dtype': numpy.float64, 'atol': 5e-4, 'rtol': 5e-3} + 'dtype': numpy.float64, 'atol': 5e-3, 'rtol': 5e-2} def check_forward(self, x_data): x = chainer.Variable(x_data) From bca94b57e2236280bc87be624274e284932d600b Mon Sep 17 00:00:00 2001 From: niboshi Date: Tue, 22 Aug 2017 17:31:34 +0900 Subject: [PATCH 3/3] Copy gy --- chainer/functions/activation/leaky_relu.py | 1 + 1 file changed, 1 insertion(+) diff --git a/chainer/functions/activation/leaky_relu.py b/chainer/functions/activation/leaky_relu.py index 81282cfd4ecb..86a043a0329f 100644 --- a/chainer/functions/activation/leaky_relu.py +++ b/chainer/functions/activation/leaky_relu.py @@ -65,6 +65,7 @@ def __init__(self, x, y, slope): def forward_cpu(self, inputs): gy, = inputs + gy = gy.copy() if self.slope >= 0: gy[self.y < 0] *= self.slope else: