Skip to content

Enable skip biases in Conv3dLayer, the same as beta and gamma in BN Layer #421

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Mar 17, 2018
Merged

Enable skip biases in Conv3dLayer, the same as beta and gamma in BN Layer #421

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ec90a03
Merge pull request #1 from tensorlayer/master
lllcho Mar 14, 2018
427b550
update by lllcho on March 15
lllcho Mar 15, 2018
0d7ac7c
update logging
lllcho Mar 15, 2018
da41eea
Merge pull request #2 from tensorlayer/master
lllcho Mar 15, 2018
7542a43
b_init in c3d can None and gamma/beta in BN layer can skip
lllcho Mar 16, 2018
e54ab3d
fix some comments
lllcho Mar 16, 2018
597da8e
Merge branch 'master' into master
lllcho Mar 16, 2018
8afebe5
add comments in bn layer
lllcho Mar 17, 2018
2470ccc
Merge remote-tracking branch 'origin/master'
lllcho Mar 17, 2018
87591b4
Merge branch 'master' into master
lllcho Mar 17, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 24 additions & 13 deletions tensorlayer/layers/convolution.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,9 +123,9 @@ class Conv2dLayer(Layer):
padding : str
The padding algorithm type: "SAME" or "VALID".
W_init : initializer
The initializer for the the weight matrix.
The initializer for the weight matrix.
b_init : initializer or None
The initializer for the the bias vector. If None, skip biases.
The initializer for the bias vector. If None, skip biases.
W_init_args : dictionary
The arguments for the weight matrix initializer.
b_init_args : dictionary
Expand Down Expand Up @@ -358,8 +358,8 @@ class Conv3dLayer(Layer):
The padding algorithm type: "SAME" or "VALID".
W_init : initializer
The initializer for the weight matrix.
b_init : initializer
The initializer for the bias vector.
b_init : initializer or None
The initializer for the bias vector. If None, skip biases.
W_init_args : dictionary
The arguments for the weight matrix initializer.
b_init_args : dictionary
Expand Down Expand Up @@ -403,16 +403,22 @@ def __init__(
# W = tf.Variable(W_init(shape=shape, **W_init_args), name='W_conv')
# b = tf.Variable(b_init(shape=[shape[-1]], **b_init_args), name='b_conv')
W = tf.get_variable(name='W_conv3d', shape=shape, initializer=W_init, dtype=LayersConfig.tf_dtype, **W_init_args)
b = tf.get_variable(name='b_conv3d', shape=(shape[-1]), initializer=b_init, dtype=LayersConfig.tf_dtype, **b_init_args)
self.outputs = act(tf.nn.conv3d(self.inputs, W, strides=strides, padding=padding, name=None) + b)
if b_init:
b = tf.get_variable(name='b_conv3d', shape=(shape[-1]), initializer=b_init, dtype=LayersConfig.tf_dtype, **b_init_args)
self.outputs = act(tf.nn.conv3d(self.inputs, W, strides=strides, padding=padding, name=None) + b)
else:
self.outputs = act(tf.nn.conv3d(self.inputs, W, strides=strides, padding=padding, name=None))

# self.outputs = act( tf.nn.conv3d(self.inputs, W, strides=strides, padding=padding, name=None) + b )

# self.all_layers = list(layer.all_layers)
# self.all_params = list(layer.all_params)
# self.all_drop = dict(layer.all_drop)
self.all_layers.append(self.outputs)
self.all_params.extend([W, b])
if b_init:
self.all_params.extend([W, b])
else:
self.all_params.extend([W])


class DeConv3dLayer(Layer):
Expand All @@ -435,8 +441,8 @@ class DeConv3dLayer(Layer):
The padding algorithm type: "SAME" or "VALID".
W_init : initializer
The initializer for the weight matrix.
b_init : initializer
The initializer for the bias vector.
b_init : initializer or None
The initializer for the bias vector. If None, skip biases.
W_init_args : dictionary
The arguments for the weight matrix initializer.
b_init_args : dictionary
Expand Down Expand Up @@ -474,15 +480,20 @@ def __init__(

with tf.variable_scope(name):
W = tf.get_variable(name='W_deconv3d', shape=shape, initializer=W_init, dtype=LayersConfig.tf_dtype, **W_init_args)
b = tf.get_variable(name='b_deconv3d', shape=(shape[-2]), initializer=b_init, dtype=LayersConfig.tf_dtype, **b_init_args)

self.outputs = act(tf.nn.conv3d_transpose(self.inputs, W, output_shape=output_shape, strides=strides, padding=padding) + b)
if b_init:
b = tf.get_variable(name='b_deconv3d', shape=(shape[-2]), initializer=b_init, dtype=LayersConfig.tf_dtype, **b_init_args)
self.outputs = act(tf.nn.conv3d_transpose(self.inputs, W, output_shape=output_shape, strides=strides, padding=padding) + b)
else:
self.outputs = act(tf.nn.conv3d_transpose(self.inputs, W, output_shape=output_shape, strides=strides, padding=padding))

# self.all_layers = list(layer.all_layers)
# self.all_params = list(layer.all_params)
# self.all_drop = dict(layer.all_drop)
self.all_layers.append(self.outputs)
self.all_params.extend([W, b])
if b_init:
self.all_params.extend([W, b])
else:
self.all_params.extend([W])


class UpSampling2dLayer(Layer):
Expand Down
45 changes: 28 additions & 17 deletions tensorlayer/layers/normalization.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,13 @@ class BatchNormLayer(Layer):
The activation function of this layer.
is_train : boolean
Is being used for training or inference.
beta_init : initializer
The initializer for initializing beta.
gamma_init : initializer
The initializer for initializing gamma.
beta_init : initializer or None
The initializer for initializing beta, if None, skip beta.
Usually you should not skip beta unless you know what happened.
gamma_init : initializer or None
The initializer for initializing gamma, if None, skip gamma.
When the batch normalization layer is use instead of 'biases', or the next layer is linear, this can be
disabled since the scaling can be done by the next layer. see `Inception-ResNet-v2 <https://github.com/tensorflow/models/blob/master/research/slim/nets/inception_resnet_v2.py>`__
dtype : TensorFlow dtype
tf.float32 (default) or tf.float16.
name : str
Expand Down Expand Up @@ -112,19 +115,27 @@ def __init__(

with tf.variable_scope(name):
axis = list(range(len(x_shape) - 1))

# 1. beta, gamma
if tf.__version__ > '0.12.1' and beta_init == tf.zeros_initializer:
beta_init = beta_init()
beta = tf.get_variable('beta', shape=params_shape, initializer=beta_init, dtype=LayersConfig.tf_dtype, trainable=is_train)

gamma = tf.get_variable(
'gamma',
shape=params_shape,
initializer=gamma_init,
dtype=LayersConfig.tf_dtype,
trainable=is_train,
)
variables = []
if beta_init:
if tf.__version__ > '0.12.1' and beta_init == tf.zeros_initializer:
beta_init = beta_init()
beta = tf.get_variable('beta', shape=params_shape, initializer=beta_init, dtype=LayersConfig.tf_dtype, trainable=is_train)
variables.append(beta)
else:
beta = None

if gamma_init:
gamma = tf.get_variable(
'gamma',
shape=params_shape,
initializer=gamma_init,
dtype=LayersConfig.tf_dtype,
trainable=is_train,
)
variables.append(gamma)
else:
gamma = None

# 2.
if tf.__version__ > '0.12.1':
Expand Down Expand Up @@ -163,7 +174,7 @@ def mean_var_with_update():
else:
self.outputs = act(tf.nn.batch_normalization(self.inputs, moving_mean, moving_variance, beta, gamma, epsilon))

variables = [beta, gamma, moving_mean, moving_variance]
variables.extend([moving_mean, moving_variance])

# logging.info(len(variables))
# for idx, v in enumerate(variables):
Expand Down