From 06488402ca855fd4492e85608ced4cc8e839ce01 Mon Sep 17 00:00:00 2001 From: zsdonghao Date: Tue, 17 Apr 2018 01:30:32 +0100 Subject: [PATCH 1/5] update deconv2d function-->class, support dilation --- README.md | 2 +- docs/modules/layers.rst | 2 +- example/tutorial_mnist_simple.py | 3 +- tensorlayer/layers/convolution.py | 161 +++++++++++++++--------------- tests/test_layers_convolution.py | 10 +- 5 files changed, 91 insertions(+), 87 deletions(-) diff --git a/README.md b/README.md index 944bb7f8a..b74c62eee 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ - + [![Codacy Badge](https://api.codacy.com/project/badge/Grade/ca2a29ddcf7445588beff50bee5406d9)](https://app.codacy.com/app/tensorlayer/tensorlayer?utm_source=github.com&utm_medium=referral&utm_content=tensorlayer/tensorlayer&utm_campaign=badger) [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/tensorlayer/Lobby#?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Build Status](https://travis-ci.org/tensorlayer/tensorlayer.svg?branch=master)](https://travis-ci.org/tensorlayer/tensorlayer) diff --git a/docs/modules/layers.rst b/docs/modules/layers.rst index 81e43c607..9cc603af9 100644 --- a/docs/modules/layers.rst +++ b/docs/modules/layers.rst @@ -492,7 +492,7 @@ APIs may better for you. 2D Deconvolution ^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. autofunction:: DeConv2d +.. autoclass:: DeConv2d 3D Deconvolution ^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/example/tutorial_mnist_simple.py b/example/tutorial_mnist_simple.py index 0e18abed9..4bc8d6a1b 100644 --- a/example/tutorial_mnist_simple.py +++ b/example/tutorial_mnist_simple.py @@ -2,14 +2,13 @@ # -*- coding: utf-8 -*- import tensorflow as tf - import tensorlayer as tl sess = tf.InteractiveSession() # prepare data X_train, y_train, X_val, y_val, X_test, y_test = \ - tl.files.load_mnist_dataset(shape=(-1,784)) + tl.files.load_mnist_dataset(shape=(-1, 784)) # define placeholder x = tf.placeholder(tf.float32, shape=[None, 784], name='x') y_ = tf.placeholder(tf.int64, shape=[None], name='y_') diff --git a/tensorlayer/layers/convolution.py b/tensorlayer/layers/convolution.py index 162a2c22b..c2aa67ad8 100644 --- a/tensorlayer/layers/convolution.py +++ b/tensorlayer/layers/convolution.py @@ -1415,6 +1415,7 @@ def __init__( strides=(1, 1), act=tf.identity, padding='SAME', + dilation_rate=(1, 1), W_init=tf.truncated_normal_initializer(stddev=0.02), b_init=tf.constant_initializer(value=0.0), W_init_args=None, @@ -1472,7 +1473,7 @@ def __init__( strides=strides, padding=padding, data_format='channels_last', - dilation_rate=(1, 1), + dilation_rate=dilation_rate, activation=act, use_bias=(False if b_init is None else True), kernel_initializer=W_init, #None, @@ -1520,20 +1521,8 @@ def __init__( self.all_params.append(W) -@deprecated_alias(layer='prev_layer', n_out_channel='n_filter', end_support_version=1.9) # TODO remove this line for the 1.9 release -def deconv2d(prev_layer, - n_filter, - filter_size=(3, 3), - out_size=(30, 30), - strides=(2, 2), - padding='SAME', - batch_size=None, - act=tf.identity, - W_init=tf.truncated_normal_initializer(stddev=0.02), - b_init=tf.constant_initializer(value=0.0), - W_init_args=None, - b_init_args=None, - name='decnn2d'): +# @deprecated_alias(layer='prev_layer', end_support_version=1.9) # TODO remove this line for the 1.9 release +class DeConv2d(Layer): """Simplified version of :class:`DeConv2dLayer`. Parameters @@ -1550,8 +1539,8 @@ def deconv2d(prev_layer, The stride step (height, width). padding : str The padding algorithm type: "SAME" or "VALID". - batch_size : int - Require if TF version < 1.3, int or None. + batch_size : int or None + Require if TF < 1.3, int or None. If None, try to find the `batch_size` from the first dim of net.outputs (you should define the `batch_size` in the input placeholder). act : activation function The activation function of this layer. @@ -1560,79 +1549,87 @@ def deconv2d(prev_layer, b_init : initializer or None The initializer for the bias vector. If None, skip biases. W_init_args : dictionary - The arguments for the weight matrix initializer. + The arguments for the weight matrix initializer (For TF < 1.3). b_init_args : dictionary - The arguments for the bias vector initializer. + The arguments for the bias vector initializer (For TF < 1.3). name : str A unique layer name. - Returns - ------- - :class:`Layer` - A :class:`DeConv2dLayer` object. - """ - logging.info("DeConv2d %s: n_filters:%s strides:%s pad:%s act:%s" % (name, str(n_filter), str(strides), padding, act.__name__)) + def __init__( + self, + prev_layer, + n_filter=32, + filter_size=(3, 3), + out_size=(30, 30), # remove + strides=(2, 2), + padding='SAME', + batch_size=None, # remove + act=tf.identity, + W_init=tf.truncated_normal_initializer(stddev=0.02), + b_init=tf.constant_initializer(value=0.0), + W_init_args=None, # remove + b_init_args=None, # remove + name='decnn2d'): + super(DeConv2d, self).__init__(prev_layer=prev_layer, name=name) + logging.info("DeConv2d %s: n_filters:%s strides:%s pad:%s act:%s" % (name, str(n_filter), str(strides), padding, act.__name__)) - if W_init_args is None: - W_init_args = {} - if b_init_args is None: - b_init_args = {} - if act is None: - act = tf.identity - - if len(strides) != 2: - raise ValueError("len(strides) should be 2, DeConv2d and DeConv2dLayer are different.") - - if tf.__version__ > '1.3': - inputs = prev_layer.outputs - scope_name = tf.get_variable_scope().name - # if scope_name: - # whole_name = scope_name + '/' + name - # else: - # whole_name = name - net_new = Layer(prev_layer=None, name=name) - # with tf.name_scope(name): - with tf.variable_scope(name) as vs: - net_new.outputs = tf.contrib.layers.conv2d_transpose( - inputs=inputs, - num_outputs=n_filter, + if W_init_args is None: + W_init_args = {} + if b_init_args is None: + b_init_args = {} + if act is None: + act = tf.identity + + if len(strides) != 2: + raise ValueError("len(strides) should be 2, DeConv2d and DeConv2dLayer are different.") + + if tf.__version__ > '1.3': + self.inputs = prev_layer.outputs + # scope_name = tf.get_variable_scope().name + # net_new = Layer(prev_layer=layer, name=name) + + conv2d_transpose = tf.layers.Conv2DTranspose( + filters=n_filter, kernel_size=filter_size, - stride=strides, + strides=strides, padding=padding, - activation_fn=act, - weights_initializer=W_init, - biases_initializer=b_init, - scope=name) - new_variables = tf.get_collection(TF_GRAPHKEYS_VARIABLES, scope=vs.name) - net_new.all_layers = list(prev_layer.all_layers) - net_new.all_params = list(prev_layer.all_params) - net_new.all_drop = dict(prev_layer.all_drop) - net_new.all_layers.extend([net_new.outputs]) - net_new.all_params.extend(new_variables) - return net_new - else: - if batch_size is None: - # batch_size = tf.shape(net.outputs)[0] - fixed_batch_size = prev_layer.outputs.get_shape().with_rank_at_least(1)[0] - if fixed_batch_size.value: - batch_size = fixed_batch_size.value - else: - from tensorflow.python.ops import array_ops - batch_size = array_ops.shape(prev_layer.outputs)[0] - return DeConv2dLayer( - prev_layer=prev_layer, - act=act, - shape=(filter_size[0], filter_size[1], n_filter, int(prev_layer.outputs.get_shape()[-1])), - output_shape=(batch_size, int(out_size[0]), int(out_size[1]), n_filter), - strides=(1, strides[0], strides[1], 1), - padding=padding, - W_init=W_init, - b_init=b_init, - W_init_args=W_init_args, - b_init_args=b_init_args, - name=name) + activation=act, + kernel_initializer=W_init, + bias_initializer=b_init, + name=name) + # new_variables = tf.get_collection(TF_GRAPHKEYS_VARIABLES, scope=vs.name) + self.outputs = conv2d_transpose(self.inputs) + new_variables = conv2d_transpose.weights + # net_new.all_layers = list(prev_layer.all_layers) + # net_new.all_params = list(prev_layer.all_params) + # net_new.all_drop = dict(prev_layer.all_drop) + self.all_layers.append(self.outputs) + self.all_params.extend(new_variables) + # return net_new + else: + raise Exception("please update TF > 1.3 or downgrade TL < 1.8.4") + # if batch_size is None: + # # batch_size = tf.shape(net.outputs)[0] + # fixed_batch_size = prev_layer.outputs.get_shape().with_rank_at_least(1)[0] + # if fixed_batch_size.value: + # batch_size = fixed_batch_size.value + # else: + # from tensorflow.python.ops import array_ops + # batch_size = array_ops.shape(prev_layer.outputs)[0] + # return DeConv2dLayer( + # prev_layer=prev_layer, + # act=act, + # shape=(filter_size[0], filter_size[1], n_filter, int(prev_layer.outputs.get_shape()[-1])), + # output_shape=(batch_size, int(out_size[0]), int(out_size[1]), n_filter), + # strides=(1, strides[0], strides[1], 1), + # padding=padding, + # W_init=W_init, + # b_init=b_init, + # W_init_args=W_init_args, + # b_init_args=b_init_args, + # name=name) class DeConv3d(Layer): @@ -2022,4 +2019,4 @@ def __init__( AtrousConv1dLayer = atrous_conv1d Conv1d = conv1d # Conv2d = conv2d -DeConv2d = deconv2d +# DeConv2d = deconv2d diff --git a/tests/test_layers_convolution.py b/tests/test_layers_convolution.py index b5560352d..57a151d17 100644 --- a/tests/test_layers_convolution.py +++ b/tests/test_layers_convolution.py @@ -61,9 +61,17 @@ n = tl.layers.DeConv2d(nin, n_filter=32, filter_size=(3, 3), strides=(2, 2), name='DeConv2d') print(n) shape = n.outputs.get_shape().as_list() +print(shape[1:]) # if (shape[1] != 200) or (shape[2] != 200) or (shape[3] != 32): # TODO: why [None None None 32] ? -if (shape[3] != 32): +if (shape[1:] != [200, 200, 32]): raise Exception("shape do not match") +n = tl.layers.DeConv2d(n, n_filter=16, filter_size=(3, 3), strides=(2, 2), name='DeConv2d_2') +print(n) +shape = n.outputs.get_shape().as_list() +if (shape[1:] != [400, 400, 16]): + raise Exception("shape do not match") +if len(n.all_params) != 4: + raise Exception("params do not match") n = tl.layers.DepthwiseConv2d(nin, shape=(3, 3), strides=(2, 2), act=tf.nn.relu, depth_multiplier=2, name='depthwise') print(n) From 42464531a68941c4893b868a1c9c004a3af3036c Mon Sep 17 00:00:00 2001 From: zsdonghao Date: Tue, 17 Apr 2018 11:19:35 +0100 Subject: [PATCH 2/5] fix review comment --- tensorlayer/layers/convolution.py | 13 +++---------- tests/test_layers_convolution.py | 17 ++--------------- 2 files changed, 5 insertions(+), 25 deletions(-) diff --git a/tensorlayer/layers/convolution.py b/tensorlayer/layers/convolution.py index c2aa67ad8..74decd955 100644 --- a/tensorlayer/layers/convolution.py +++ b/tensorlayer/layers/convolution.py @@ -1521,7 +1521,7 @@ def __init__( self.all_params.append(W) -# @deprecated_alias(layer='prev_layer', end_support_version=1.9) # TODO remove this line for the 1.9 release +@deprecated_alias(layer='prev_layer', end_support_version=1.9) # TODO remove this line for the 1.9 release class DeConv2d(Layer): """Simplified version of :class:`DeConv2dLayer`. @@ -1588,8 +1588,6 @@ def __init__( if tf.__version__ > '1.3': self.inputs = prev_layer.outputs # scope_name = tf.get_variable_scope().name - # net_new = Layer(prev_layer=layer, name=name) - conv2d_transpose = tf.layers.Conv2DTranspose( filters=n_filter, kernel_size=filter_size, @@ -1599,17 +1597,12 @@ def __init__( kernel_initializer=W_init, bias_initializer=b_init, name=name) - # new_variables = tf.get_collection(TF_GRAPHKEYS_VARIABLES, scope=vs.name) self.outputs = conv2d_transpose(self.inputs) - new_variables = conv2d_transpose.weights - # net_new.all_layers = list(prev_layer.all_layers) - # net_new.all_params = list(prev_layer.all_params) - # net_new.all_drop = dict(prev_layer.all_drop) + new_variables = conv2d_transpose.weights # new_variables = tf.get_collection(TF_GRAPHKEYS_VARIABLES, scope=vs.name) self.all_layers.append(self.outputs) self.all_params.extend(new_variables) - # return net_new else: - raise Exception("please update TF > 1.3 or downgrade TL < 1.8.4") + raise RuntimeError("please update TF > 1.3 or downgrade TL < 1.8.4") # if batch_size is None: # # batch_size = tf.shape(net.outputs)[0] # fixed_batch_size = prev_layer.outputs.get_shape().with_rank_at_least(1)[0] diff --git a/tests/test_layers_convolution.py b/tests/test_layers_convolution.py index 57a151d17..695e0a1ab 100644 --- a/tests/test_layers_convolution.py +++ b/tests/test_layers_convolution.py @@ -11,7 +11,6 @@ raise Exception("shape do not match") n = tl.layers.Conv1d(nin, n_filter=32, filter_size=5, stride=2) -print(n) shape = n.outputs.get_shape().as_list() if (shape[1] != 50) or (shape[2] != 32): raise Exception("shape do not match") @@ -30,7 +29,6 @@ W_init=tf.truncated_normal_initializer(stddev=5e-2), b_init=tf.constant_initializer(value=0.0), name='conv2dlayer') -print(n) shape = n.outputs.get_shape().as_list() if (shape[1] != 50) or (shape[2] != 50) or (shape[3] != 32): raise Exception("shape do not match") @@ -44,7 +42,6 @@ raise Exception("params do not match") n = tl.layers.Conv2d(nin, n_filter=32, filter_size=(3, 3), strides=(2, 2), act=tf.nn.relu, b_init=None, name='conv2d_no_bias') -print(n) shape = n.outputs.get_shape().as_list() if (shape[1] != 50) or (shape[2] != 50) or (shape[3] != 32): raise Exception("shape do not match") @@ -52,21 +49,16 @@ raise Exception("params do not match") n = tl.layers.DeConv2dLayer(nin, shape=(5, 5, 32, 3), output_shape=(100, 200, 200, 32), strides=(1, 2, 2, 1), name='deconv2dlayer') -print(n) shape = n.outputs.get_shape().as_list() if (shape[1] != 200) or (shape[2] != 200) or (shape[3] != 32): raise Exception("shape do not match") -print(nin.outputs) n = tl.layers.DeConv2d(nin, n_filter=32, filter_size=(3, 3), strides=(2, 2), name='DeConv2d') -print(n) shape = n.outputs.get_shape().as_list() -print(shape[1:]) # if (shape[1] != 200) or (shape[2] != 200) or (shape[3] != 32): # TODO: why [None None None 32] ? if (shape[1:] != [200, 200, 32]): raise Exception("shape do not match") n = tl.layers.DeConv2d(n, n_filter=16, filter_size=(3, 3), strides=(2, 2), name='DeConv2d_2') -print(n) shape = n.outputs.get_shape().as_list() if (shape[1:] != [400, 400, 16]): raise Exception("shape do not match") @@ -74,14 +66,12 @@ raise Exception("params do not match") n = tl.layers.DepthwiseConv2d(nin, shape=(3, 3), strides=(2, 2), act=tf.nn.relu, depth_multiplier=2, name='depthwise') -print(n) shape = n.outputs.get_shape().as_list() if (shape[1] != 50) or (shape[2] != 50) or (shape[3] != 6): raise Exception("shape do not match") n = tl.layers.Conv2d(nin, n_filter=32, filter_size=(3, 3), strides=(2, 2), act=tf.nn.relu, name='conv2d2') n = tl.layers.GroupConv2d(n, n_filter=32, filter_size=(3, 3), strides=(2, 2), name='group') -print(n) shape = n.outputs.get_shape().as_list() if (shape[1] != 25) or (shape[2] != 25) or (shape[3] != 32): raise Exception("shape do not match") @@ -99,8 +89,8 @@ # AtrousConv2dLayer n = tl.layers.SeparableConv2d(nin, n_filter=32, filter_size=(3, 3), strides=(1, 1), act=tf.nn.relu, name='seperable1') -n.print_layers() -n.print_params(False) +# n.print_layers() +# n.print_params(False) shape = n.outputs.get_shape().as_list() if shape[1:] != [98, 98, 32]: @@ -114,14 +104,12 @@ if n.count_params() != 155: raise Exception("params do not match") -# exit() ## 3D x = tf.placeholder(tf.float32, (None, 100, 100, 100, 3)) nin = tl.layers.InputLayer(x, name='in3') n = tl.layers.Conv3dLayer(nin, shape=(2, 2, 2, 3, 32), strides=(1, 2, 2, 2, 1)) -print(n) shape = n.outputs.get_shape().as_list() if (shape[1] != 50) or (shape[2] != 50) or (shape[3] != 50) or (shape[4] != 32): raise Exception("shape do not match") @@ -132,6 +120,5 @@ n = tl.layers.DeConv3d(nin, n_filter=32, filter_size=(3, 3, 3), strides=(2, 2, 2)) shape = n.outputs.get_shape().as_list() -print(shape) if (shape[1] != 200) or (shape[2] != 200) or (shape[3] != 200) or (shape[4] != 32): raise Exception("shape do not match") From 2b1bd51e33577ab8ec2b3c65bd44dbe0c1d7cbd9 Mon Sep 17 00:00:00 2001 From: zsdonghao Date: Tue, 17 Apr 2018 11:23:37 +0100 Subject: [PATCH 3/5] fix review comment --- tensorlayer/layers/convolution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorlayer/layers/convolution.py b/tensorlayer/layers/convolution.py index 74decd955..f5d92baa1 100644 --- a/tensorlayer/layers/convolution.py +++ b/tensorlayer/layers/convolution.py @@ -1521,7 +1521,6 @@ def __init__( self.all_params.append(W) -@deprecated_alias(layer='prev_layer', end_support_version=1.9) # TODO remove this line for the 1.9 release class DeConv2d(Layer): """Simplified version of :class:`DeConv2dLayer`. @@ -1557,6 +1556,7 @@ class DeConv2d(Layer): """ + @deprecated_alias(layer='prev_layer', end_support_version=1.9) # TODO remove this line for the 1.9 release def __init__( self, prev_layer, From 92a0e2ef81d5a866daf197584352e680db114e20 Mon Sep 17 00:00:00 2001 From: zsdonghao Date: Tue, 17 Apr 2018 12:16:18 +0100 Subject: [PATCH 4/5] fix yapf --- tests/test_layers_convolution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_layers_convolution.py b/tests/test_layers_convolution.py index e797278b8..d408ec101 100644 --- a/tests/test_layers_convolution.py +++ b/tests/test_layers_convolution.py @@ -5,6 +5,7 @@ import tensorflow as tf import tensorlayer as tl + class Layer_Convolution_Test(unittest.TestCase): @classmethod def setUpClass(cls): @@ -164,4 +165,3 @@ def test_layers_n10(self): tf.logging.set_verbosity(tf.logging.DEBUG) unittest.main() - From e7cbc54118c1e638ee2a1bb556ce6dba5eb06026 Mon Sep 17 00:00:00 2001 From: zsdonghao Date: Tue, 17 Apr 2018 13:12:18 +0100 Subject: [PATCH 5/5] fix review commend --- tensorlayer/layers/convolution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorlayer/layers/convolution.py b/tensorlayer/layers/convolution.py index f5d92baa1..9a9382fe6 100644 --- a/tensorlayer/layers/convolution.py +++ b/tensorlayer/layers/convolution.py @@ -1556,7 +1556,7 @@ class DeConv2d(Layer): """ - @deprecated_alias(layer='prev_layer', end_support_version=1.9) # TODO remove this line for the 1.9 release + @deprecated_alias(layer='prev_layer', n_out_channel='n_filter', end_support_version=1.9) # TODO remove this line for the 1.9 release def __init__( self, prev_layer,