Skip to content

Commit

Permalink
Merge 360260e into a48476d
Browse files Browse the repository at this point in the history
  • Loading branch information
@toslunar
toslunar committed Apr 3, 2018
2 parents a48476d + 360260e commit 22f7de1
Showing 1 changed file with 44 additions and 163 deletions.
207 changes: 44 additions & 163 deletions tests/chainer_tests/functions_tests/normalization_tests/test_batch_normalization.py
View file
Expand Up @@ -25,18 +25,28 @@ def _batch_normalization(args):
return y_expect


@testing.parameterize(*(testing.product({
'param_shape': [(3,), (3, 4), (3, 2, 3)],
'ndim': [0, 1, 2],
'eps': [2e-5, 5e-1],
'dtype': [numpy.float32],
'c_contiguous': [True, False],
}) + testing.product({
@testing.parameterize(*(testing.product_dict(
testing.product({
'param_shape': [(3,), (3, 4), (3, 2, 3)],
'ndim': [0, 1, 2],
'axis': [None],
}) + [
{'shape': (5, 4, 3, 2), 'axis': (0, 2, 3)},
{'shape': (5, 4), 'axis': 0},
{'shape': (5, 4, 3), 'axis': (0, 1)},
],
testing.product({
'dtype': [numpy.float32],
'eps': [2e-5, 5e-1],
'c_contiguous': [True, False],
}),
) + testing.product({
'param_shape': [(3,)],
'ndim': [1],
'eps': [2e-5, 5e-1],
'dtype': [numpy.float16, numpy.float32, numpy.float64],
'c_contiguous': [True, False],
'axis': [None],
})))
@backend.inject_backend_tests(
['test_forward', 'test_backward', 'test_double_backward'],
Expand All @@ -53,176 +63,47 @@ def _batch_normalization(args):
class TestBatchNormalization(unittest.TestCase):

def setUp(self):
param_shape = self.param_shape
dtype = self.dtype
ndim = self.ndim
aggr_axes = self.axis

if aggr_axes is None:
param_shape = self.param_shape
ndim = self.ndim
shape = (5,) + param_shape + (2,) * ndim

else:
if isinstance(self.axis, int):
aggr_axes = self.axis,
param_shape = tuple(
s
for i, s in enumerate(self.shape)
if i not in aggr_axes
)
shape = self.shape

gamma = numpy.random.uniform(.5, 1, param_shape).astype(dtype)
beta = numpy.random.uniform(-1, 1, param_shape).astype(dtype)
head_ndim = gamma.ndim + 1
shape = (5,) + param_shape + (2,) * ndim
x = numpy.random.uniform(-1, 1, shape).astype(dtype)
gy = numpy.random.uniform(-1, 1, shape).astype(dtype)
ggx = numpy.random.uniform(-1, 1, shape).astype(dtype)
gggamma = numpy.random.uniform(-1, 1, param_shape).astype(dtype)
ggbeta = numpy.random.uniform(-1, 1, param_shape).astype(dtype)

aggr_axes = (0,) + tuple(six.moves.range(head_ndim, x.ndim))
mean = x.mean(axis=aggr_axes)
var = x.var(axis=aggr_axes)
if aggr_axes is None:
head_ndim = gamma.ndim + 1
aggr_axes = (0,) + tuple(six.moves.range(head_ndim, x.ndim))

self.decay = 0.9
self.expander = (None, Ellipsis) + (None,) * ndim
self.mean = mean
self.var = var

self.inputs = [x, gamma, beta]
self.grad_outputs = [gy]
self.grad_grad_inputs = [ggx, gggamma, ggbeta]

self.check_forward_options = {'atol': 1e-4, 'rtol': 1e-3}
self.check_backward_options = {'dtype': numpy.float64}
self.check_double_backward_options = {
'dtype': numpy.float64, 'atol': 1e-3, 'rtol': 1e-2}
if self.dtype == numpy.float16:
self.check_forward_options = {'atol': 1e-2, 'rtol': 1e-2}
self.check_backward_options = {
'dtype': numpy.float64, 'atol': 1e-2, 'rtol': 1e-2}
self.check_double_backward_options = {
'dtype': numpy.float64, 'atol': 1e-2, 'rtol': 1e-2}

def forward_cpu(self, inputs):
y_expect = _batch_normalization(
inputs + [self.mean, self.var, self.eps, self.expander])
return y_expect,

def check_forward(self, inputs, backend_config):
y_expected, = self.forward_cpu(inputs)

if backend_config.use_cuda:
inputs = cuda.to_gpu(inputs)
if not self.c_contiguous:
inputs = _to_fcontiguous(inputs)

with backend_config:
y = functions.batch_normalization(
*inputs, running_mean=None,
running_var=None, decay=self.decay, eps=self.eps)
assert y.data.dtype == self.dtype

testing.assert_allclose(
y_expected, y.data, **self.check_forward_options)

def test_forward(self, backend_config):
self.check_forward(self.inputs, backend_config)

def check_backward(self, inputs, grad_outputs, backend_config):
if backend_config.use_cuda:
inputs = cuda.to_gpu(inputs)
grad_outputs = cuda.to_gpu(grad_outputs)
if not self.c_contiguous:
inputs = _to_fcontiguous(inputs)
grad_outputs = _to_fcontiguous(grad_outputs)
self.expander = (None, Ellipsis) + (None,) * ndim
else:
self.expander = tuple(
None if i in aggr_axes else slice(None)
for i in range(x.ndim)
)

def f(*inputs):
y = functions.batch_normalization(
*inputs, decay=self.decay, eps=self.eps)
return y,

with backend_config:
gradient_check.check_backward(
f, inputs, grad_outputs,
**self.check_backward_options)

def test_backward(self, backend_config):
self.check_backward(self.inputs, self.grad_outputs, backend_config)

def check_double_backward(
self, inputs, grad_outputs, grad_grad_inputs, backend_config):
if backend_config.use_cuda:
inputs = cuda.to_gpu(inputs)
grad_outputs = cuda.to_gpu(grad_outputs)
grad_grad_inputs = cuda.to_gpu(grad_grad_inputs)
if not self.c_contiguous:
inputs = _to_fcontiguous(inputs)
grad_outputs = _to_fcontiguous(grad_outputs)
grad_grad_inputs = _to_fcontiguous(grad_grad_inputs)

def f(*inputs):
y = functions.batch_normalization(
*inputs, decay=self.decay, eps=self.eps)
return y * y, # make nonlinear against beta

with backend_config:
gradient_check.check_double_backward(
f, inputs, grad_outputs, grad_grad_inputs,
**self.check_double_backward_options)

def test_double_backward(self, backend_config):
self.check_double_backward(
self.inputs, self.grad_outputs, self.grad_grad_inputs,
backend_config)


@testing.parameterize(*testing.product_dict(
[
{'shape': (5, 4, 3, 2), 'axis': (0, 2, 3)},
{'shape': (5, 4), 'axis': 0},
{'shape': (5, 4, 3), 'axis': (0, 1)},
],
[
{'dtype': numpy.float32},
],
[
{'eps': 2e-5}, {'eps': 5e-1},
],
[
{'c_contiguous': True}, {'c_contiguous': False},
],
))
@backend.inject_backend_tests(
['test_forward', 'test_backward', 'test_double_backward'],
# CPU tests
testing.product({
'use_cuda': [False],
})
# GPU tests
+ testing.product({
'use_cuda': [True],
'use_cudnn': ['never', 'always'],
}))
class TestBatchNormalizationAxis(unittest.TestCase):

def setUp(self):
dtype = self.dtype

_axis = self.axis
if isinstance(self.axis, int):
_axis = self.axis,
param_shape = []
for i, s in enumerate(self.shape):
if i not in _axis:
param_shape.append(s)

gamma = numpy.random.uniform(.5, 1, param_shape).astype(dtype)
beta = numpy.random.uniform(-1, 1, param_shape).astype(dtype)
x = numpy.random.uniform(-1, 1, self.shape).astype(dtype)
gy = numpy.random.uniform(-1, 1, self.shape).astype(dtype)
ggx = numpy.random.uniform(-1, 1, self.shape).astype(dtype)
gggamma = numpy.random.uniform(-1, 1, param_shape).astype(dtype)
ggbeta = numpy.random.uniform(-1, 1, param_shape).astype(dtype)

mean = x.mean(axis=self.axis)
var = x.var(axis=self.axis)
mean = x.mean(axis=aggr_axes)
var = x.var(axis=aggr_axes)

self.decay = 0.9
expander = [None for _ in range(x.ndim)]
j = 0
for i in range(x.ndim):
if i not in _axis:
expander[i] = slice(gamma.shape[j])
j += 1
self.expander = expander
self.mean = mean
self.var = var

Expand Down

0 comments on commit 22f7de1

Please sign in to comment.