From 7a51d6a2d792f1105050160b5e5ff2b3a7062251 Mon Sep 17 00:00:00 2001 From: Siju Samuel Date: Thu, 5 Jul 2018 10:15:02 +0530 Subject: [PATCH 1/3] Batchnorm added as part of Dense op for running rnn model for next word prediction --- nnvm/python/nnvm/frontend/darknet.py | 20 ++++++++++++------- .../python/frontend/darknet/test_forward.py | 14 +++++++++++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/nnvm/python/nnvm/frontend/darknet.py b/nnvm/python/nnvm/frontend/darknet.py index 112e5294396a..2fa9a108cd16 100644 --- a/nnvm/python/nnvm/frontend/darknet.py +++ b/nnvm/python/nnvm/frontend/darknet.py @@ -122,7 +122,7 @@ def _darknet_avgpooling(inputs, attrs): def _darknet_batch_norm(inputs, attrs): """Process the batchnormalization operation.""" - op_name, new_attrs = 'darknet_batch_norm', {} + op_name, new_attrs = 'batch_norm', {} new_attrs['axis'] = attrs.get('axis', 1) new_attrs['epsilon'] = attrs.get('eps', 0.000001) new_attrs['center'] = True @@ -226,13 +226,16 @@ def _darknet_dense(inputs, attrs): """Process the dense operation.""" op_name, new_attrs = 'dense', {} new_attrs['units'] = _darknet_required_attr(attrs, 'num_hidden') - + out_name = {} if attrs.get('use_bias', False) is True: new_attrs['use_bias'] = True if attrs.get('use_flatten', False) is True: inputs[0] = _sym.flatten(inputs[0]) sym = _darknet_get_nnvm_op(op_name)(*inputs, **new_attrs) - out_name = sym.list_output_names()[0].replace('_output', '') + out_name[0] = sym.list_output_names()[0].replace('_output', '') + if 'use_batchNorm' in attrs: + sym, _ = _darknet_batch_norm(*sym, attrs) + out_name[1] = sym.list_output_names()[0].replace('_output', '') if 'activation' in attrs: new_attrs = {} new_attrs['activation'] = attrs['activation'] @@ -430,13 +433,16 @@ def _get_connected_weights(layer, opname, params, dtype): weights = _read_memory_buffer((layer.outputs, layer.inputs), layer.weights, dtype) biases = _read_memory_buffer((layer.outputs, ), layer.biases, dtype) - k = _get_tvm_params_name(opname, 'weight') + k = _get_tvm_params_name(opname[0], 'weight') params[k] = tvm.nd.array(weights) - k = _get_tvm_params_name(opname, 'bias') - params[k] = tvm.nd.array(biases) if layer.batch_normalize == 1 and layer.dontloadscales != 1: - _get_batchnorm_weights(layer, opname, params, layer.outputs, dtype) + _get_batchnorm_weights(layer, opname[1], params, layer.outputs, dtype) + k = _get_tvm_params_name(opname[1], 'beta') + params[k] = tvm.nd.array(biases) + else: + k = _get_tvm_params_name(opname[0], 'bias') + params[k] = tvm.nd.array(biases) def _get_batchnorm_weights(layer, opname, params, size, dtype): """Parse the weights for batchnorm, which includes, scales, moving mean diff --git a/nnvm/tests/python/frontend/darknet/test_forward.py b/nnvm/tests/python/frontend/darknet/test_forward.py index a61a8d5a9738..e1ca2c6d5262 100644 --- a/nnvm/tests/python/frontend/darknet/test_forward.py +++ b/nnvm/tests/python/frontend/darknet/test_forward.py @@ -169,6 +169,19 @@ def test_forward_dense(): test_forward(net) LIB.free_network(net) +def test_forward_dense_batchnorm(): + '''test fully connected layer with batchnorm''' + net = LIB.make_network(1) + layer = LIB.make_connected_layer(1, 75, 5, 1, 1, 0) + for i in range(5): + layer.rolling_mean[i] = np.random.rand(1) + layer.rolling_variance[i] = np.random.rand(1) + net.layers[0] = layer + net.w = net.h = 5 + LIB.resize_network(net, 5, 5) + test_forward(net) + LIB.free_network(net) + def test_forward_maxpooling(): '''test maxpooling layer''' net = LIB.make_network(1) @@ -264,6 +277,7 @@ def test_forward_elu(): test_forward_batch_norm() test_forward_shortcut() test_forward_dense() + test_forward_dense_batchnorm() test_forward_reorg() test_forward_region() test_forward_elu() From a4a40b0ed394b7cd6650c9cc35256a1e2caf5146 Mon Sep 17 00:00:00 2001 From: Siju Samuel Date: Thu, 5 Jul 2018 13:31:29 +0530 Subject: [PATCH 2/3] CI issue resolved --- nnvm/python/nnvm/frontend/darknet.py | 6 ++++-- nnvm/tests/python/frontend/darknet/test_forward.py | 7 ++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/nnvm/python/nnvm/frontend/darknet.py b/nnvm/python/nnvm/frontend/darknet.py index 2fa9a108cd16..4bb5100ece92 100644 --- a/nnvm/python/nnvm/frontend/darknet.py +++ b/nnvm/python/nnvm/frontend/darknet.py @@ -122,7 +122,7 @@ def _darknet_avgpooling(inputs, attrs): def _darknet_batch_norm(inputs, attrs): """Process the batchnormalization operation.""" - op_name, new_attrs = 'batch_norm', {} + op_name, new_attrs = '_darknet_batch_norm', {} new_attrs['axis'] = attrs.get('axis', 1) new_attrs['epsilon'] = attrs.get('eps', 0.000001) new_attrs['center'] = True @@ -234,7 +234,9 @@ def _darknet_dense(inputs, attrs): sym = _darknet_get_nnvm_op(op_name)(*inputs, **new_attrs) out_name[0] = sym.list_output_names()[0].replace('_output', '') if 'use_batchNorm' in attrs: - sym, _ = _darknet_batch_norm(*sym, attrs) + op_name, new_attrs = 'batch_norm', {} + new_attrs['epsilon'] = 0.000001 + sym = _darknet_get_nnvm_op(op_name)(*sym, **new_attrs) out_name[1] = sym.list_output_names()[0].replace('_output', '') if 'activation' in attrs: new_attrs = {} diff --git a/nnvm/tests/python/frontend/darknet/test_forward.py b/nnvm/tests/python/frontend/darknet/test_forward.py index e1ca2c6d5262..6077d6b9d127 100644 --- a/nnvm/tests/python/frontend/darknet/test_forward.py +++ b/nnvm/tests/python/frontend/darknet/test_forward.py @@ -172,13 +172,14 @@ def test_forward_dense(): def test_forward_dense_batchnorm(): '''test fully connected layer with batchnorm''' net = LIB.make_network(1) - layer = LIB.make_connected_layer(1, 75, 5, 1, 1, 0) + layer = LIB.make_connected_layer(1, 12, 2, 1, 1, 0) for i in range(5): layer.rolling_mean[i] = np.random.rand(1) layer.rolling_variance[i] = np.random.rand(1) + layer.scales[i] = np.random.rand(1) net.layers[0] = layer - net.w = net.h = 5 - LIB.resize_network(net, 5, 5) + net.w = net.h = 2 + LIB.resize_network(net, 2, 2) test_forward(net) LIB.free_network(net) From 9b25e39fae887d5b7cf23f60f72ef820972c6cd9 Mon Sep 17 00:00:00 2001 From: Siju Date: Thu, 5 Jul 2018 14:05:41 +0530 Subject: [PATCH 3/3] Update darknet.py --- nnvm/python/nnvm/frontend/darknet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nnvm/python/nnvm/frontend/darknet.py b/nnvm/python/nnvm/frontend/darknet.py index 4bb5100ece92..f2c744bea375 100644 --- a/nnvm/python/nnvm/frontend/darknet.py +++ b/nnvm/python/nnvm/frontend/darknet.py @@ -122,7 +122,7 @@ def _darknet_avgpooling(inputs, attrs): def _darknet_batch_norm(inputs, attrs): """Process the batchnormalization operation.""" - op_name, new_attrs = '_darknet_batch_norm', {} + op_name, new_attrs = 'darknet_batch_norm', {} new_attrs['axis'] = attrs.get('axis', 1) new_attrs['epsilon'] = attrs.get('eps', 0.000001) new_attrs['center'] = True