From bc787af9e84b06dad8c74dc178e46ed8cc89c57f Mon Sep 17 00:00:00 2001 From: wanglong03 Date: Fri, 16 Mar 2018 14:48:42 +0800 Subject: [PATCH 1/8] add LRN layer for fluid --- python/paddle/fluid/layers/nn.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index bf161d6618b10..5960304c25128 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -73,6 +73,7 @@ 'smooth_l1', 'one_hot', 'autoincreased_step_counter', + 'lrn', ] @@ -3292,3 +3293,28 @@ def autoincreased_step_counter(counter_name=None, begin=1, step=1): counter.stop_gradient = True return counter + + +def lrn(input, n=5, k=2.0, alpha=1e-4, beta=0.75, name=None): + """ + This function helps create an operator to implement + the LRN layer using the configurations from the input parameters. + """ + helper = LayerHelper('lrn', **locals()) + dtype = helper.input_dtype() + + mid_out = helper.create_tmp_variable(dtype=dtype, stop_gradient=True) + lrn_out = helper.create_tmp_variable(dtype) + helper.append_op( + type="lrn", + inputs={"X": input}, + outputs={ + "Out": lrn_out, + "MidOut": mid_out, + }, + attrs={"n": n, + "k": k, + "alpha": alpha, + "beta": beta}) + + return lrn_out From 1d36397838d9bd3b6c41abb5feb095ee20f74cf9 Mon Sep 17 00:00:00 2001 From: wanglong03 Date: Fri, 16 Mar 2018 14:59:54 +0800 Subject: [PATCH 2/8] add LRN layer for fluid --- python/paddle/fluid/layers/nn.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 5960304c25128..0568494c8bdf2 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -3299,6 +3299,13 @@ def lrn(input, n=5, k=2.0, alpha=1e-4, beta=0.75, name=None): """ This function helps create an operator to implement the LRN layer using the configurations from the input parameters. + + Args: + n(int): The number of channels to sum over + k(float): An offset (usually positive to avoid dividing by 0) + alpha(float): The scaling parameter + beta(float): The exponent + name(str): A name for this operation """ helper = LayerHelper('lrn', **locals()) dtype = helper.input_dtype() From bb9c711f632e8edec58697d7af3e23bab3d8748a Mon Sep 17 00:00:00 2001 From: wanglong03 Date: Mon, 19 Mar 2018 15:24:55 +0800 Subject: [PATCH 3/8] add documentation for LRN layer --- doc/fluid/dev/src/fc.py | 42 ++++++++++++++++ python/paddle/fluid/layers/nn.py | 49 ++++++++++++++++--- .../fluid/tests/unittests/test_layers.py | 7 +++ 3 files changed, 91 insertions(+), 7 deletions(-) diff --git a/doc/fluid/dev/src/fc.py b/doc/fluid/dev/src/fc.py index 3b074821cc227..ff3780c742831 100644 --- a/doc/fluid/dev/src/fc.py +++ b/doc/fluid/dev/src/fc.py @@ -79,3 +79,45 @@ def fc(input, data = fluid.layers.data(name="data", shape=[32, 32], dtype="float32") fc = fluid.layers.fc(input=data, size=1000, act="tanh") """ + + +def lrn(input, n=5, k=2.0, alpha=1e-4, beta=0.75, name=None): + """ + **Local Response Normalization Operator** + + This operator comes from the paper: + <>. + + .. math:: + + Output(i, x, y) = Input(i, x, y) / \left( + k + \alpha \sum\limits^{\min(C, c + n/2)}_{j = \max(0, c - n/2)} + (Input(j, x, y))^2 \right)^{\beta} + + In the above equation: + + * :math:`n`: The number of channels to sum over. + * :math:`k`: The offset (usually positive to avoid dividing by 0). + * :math:`alpha`: The scaling parameter. + * :math:`beta`: The exponent. + + Args: + input(Variable): The input tensor of this layer. The dims of the input tensor must be 4 and it's order should be 'NCHW'. + n(int, default 5): The number of channels to sum over. + k(float, default 2.0): An offset (usually positive to avoid dividing by 0). + alpha(float, default 1e-4): The scaling parameter. + beta(float, default 0.75): The exponent. + name(str, default None): A name for this operation. + + Raises: + ValueError: If rank of the input tensor is not 4. + + Returns: + A tensor variable storing the transformation result. + + Examples: + .. code-block:: python + + data = fluid.layers.data(name="data", shape=[3, 112, 112], dtype="float32") + lrn = fluid.layers.lrn(input=data) + """ diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 0568494c8bdf2..71ae1f630aa75 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -3297,18 +3297,53 @@ def autoincreased_step_counter(counter_name=None, begin=1, step=1): def lrn(input, n=5, k=2.0, alpha=1e-4, beta=0.75, name=None): """ - This function helps create an operator to implement - the LRN layer using the configurations from the input parameters. + **Local Response Normalization Operator** + + This operator comes from the paper: + <>. + + .. math:: + + Output(i, x, y) = Input(i, x, y) / \left( + k + \alpha \sum\limits^{\min(C, c + n/2)}_{j = \max(0, c - n/2)} + (Input(j, x, y))^2 \right)^{\beta} + + In the above equation: + + * :math:`n`: The number of channels to sum over. + * :math:`k`: The offset (usually positive to avoid dividing by 0). + * :math:`alpha`: The scaling parameter. + * :math:`beta`: The exponent parameter. Args: - n(int): The number of channels to sum over - k(float): An offset (usually positive to avoid dividing by 0) - alpha(float): The scaling parameter - beta(float): The exponent - name(str): A name for this operation + input(Variable): The input tensor of this layer, and the dims of the input tensor must be 4. + n(int, default 5): The number of channels to sum over. + k(float, default 2.0): An offset (usually positive to avoid dividing by 0). + alpha(float, default 1e-4): The scaling parameter. + beta(float, default 0.75): The exponent. + name(str, default None): A name for this operation. + + Raises: + ValueError: If rank of the input tensor is not 4. + + Returns: + A tensor variable storing the transformation result. + + Examples: + .. code-block:: python + + data = fluid.layers.data(name="data", shape=[3, 112, 112], dtype="float32") + lrn = fluid.layers.lrn(input=data) """ helper = LayerHelper('lrn', **locals()) dtype = helper.input_dtype() + input_shape = input.shape + dims = len(input_shape) + + if dims != 4: + raise ValueError( + "dims of input must be 4(not %d), and it's order must be NCHW" % + (dims)) mid_out = helper.create_tmp_variable(dtype=dtype, stop_gradient=True) lrn_out = helper.create_tmp_variable(dtype) diff --git a/python/paddle/fluid/tests/unittests/test_layers.py b/python/paddle/fluid/tests/unittests/test_layers.py index 90d70aa39fdc4..5ac7e514cdb34 100644 --- a/python/paddle/fluid/tests/unittests/test_layers.py +++ b/python/paddle/fluid/tests/unittests/test_layers.py @@ -231,6 +231,13 @@ def test_softmax(self): self.assertIsNotNone(layers.softmax(hid)) print(str(program)) + def test_lrn(self): + program = Program() + with program_guard(program): + data = layers.data(name='data', shape=[6, 2, 2], dtype='float32') + self.assertIsNotNone(layers.lrn(data)) + print(str(program)) + def test_get_places(self): program = Program() with program_guard(program): From bb668280383d1aede06c5602efdec32d3894efe5 Mon Sep 17 00:00:00 2001 From: wanglong03 Date: Mon, 19 Mar 2018 16:45:26 +0800 Subject: [PATCH 4/8] add paper reference for LRN layer --- doc/fluid/dev/src/fc.py | 19 ++++++++++--------- python/paddle/fluid/layers/nn.py | 20 +++++++++++--------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/doc/fluid/dev/src/fc.py b/doc/fluid/dev/src/fc.py index ff3780c742831..75a6b55650b51 100644 --- a/doc/fluid/dev/src/fc.py +++ b/doc/fluid/dev/src/fc.py @@ -81,12 +81,13 @@ def fc(input, """ -def lrn(input, n=5, k=2.0, alpha=1e-4, beta=0.75, name=None): +def lrn(input, n=5, k=1.0, alpha=1e-4, beta=0.75, name=None): """ **Local Response Normalization Operator** - This operator comes from the paper: - <>. + Refer to `ImageNet Classification with Deep Convolutional Neural Networks `_. + + The formula is as follows: .. math:: @@ -102,12 +103,12 @@ def lrn(input, n=5, k=2.0, alpha=1e-4, beta=0.75, name=None): * :math:`beta`: The exponent. Args: - input(Variable): The input tensor of this layer. The dims of the input tensor must be 4 and it's order should be 'NCHW'. - n(int, default 5): The number of channels to sum over. - k(float, default 2.0): An offset (usually positive to avoid dividing by 0). - alpha(float, default 1e-4): The scaling parameter. - beta(float, default 0.75): The exponent. - name(str, default None): A name for this operation. + input (Variable): The input tensor of this layer. The dimension of input tensor must be 4 and it's order should be 'NCHW'. + n (int, default 5): The number of channels to sum over. + k (float, default 1.0): An offset (usually positive to avoid dividing by 0). + alpha (float, default 1e-4): The scaling parameter. + beta (float, default 0.75): The exponent. + name (str, default None): A name for this operation. Raises: ValueError: If rank of the input tensor is not 4. diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 71ae1f630aa75..1e12091bf89db 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -3295,12 +3295,14 @@ def autoincreased_step_counter(counter_name=None, begin=1, step=1): return counter -def lrn(input, n=5, k=2.0, alpha=1e-4, beta=0.75, name=None): +def lrn(input, n=5, k=1.0, alpha=1e-4, beta=0.75, name=None): """ **Local Response Normalization Operator** - This operator comes from the paper: - <>. + Refer to `ImageNet Classification with Deep Convolutional Neural Networks + `_ + + The formula is as follows: .. math:: @@ -3316,12 +3318,12 @@ def lrn(input, n=5, k=2.0, alpha=1e-4, beta=0.75, name=None): * :math:`beta`: The exponent parameter. Args: - input(Variable): The input tensor of this layer, and the dims of the input tensor must be 4. - n(int, default 5): The number of channels to sum over. - k(float, default 2.0): An offset (usually positive to avoid dividing by 0). - alpha(float, default 1e-4): The scaling parameter. - beta(float, default 0.75): The exponent. - name(str, default None): A name for this operation. + input (Variable): The input tensor of this layer, and the dimension of input tensor must be 4. + n (int, default 5): The number of channels to sum over. + k (float, default 1.0): An offset (usually positive to avoid dividing by 0). + alpha (float, default 1e-4): The scaling parameter. + beta (float, default 0.75): The exponent. + name (str, default None): A name for this operation. Raises: ValueError: If rank of the input tensor is not 4. From 308b3c22247709f5f7fad566ac094588020e8002 Mon Sep 17 00:00:00 2001 From: wanglong03 Date: Mon, 19 Mar 2018 16:51:02 +0800 Subject: [PATCH 5/8] add seperate documentation for LRN layer --- doc/fluid/dev/src/fc.py | 43 ------------------------------ doc/fluid/dev/src/lrn.py | 57 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 43 deletions(-) create mode 100644 doc/fluid/dev/src/lrn.py diff --git a/doc/fluid/dev/src/fc.py b/doc/fluid/dev/src/fc.py index 75a6b55650b51..3b074821cc227 100644 --- a/doc/fluid/dev/src/fc.py +++ b/doc/fluid/dev/src/fc.py @@ -79,46 +79,3 @@ def fc(input, data = fluid.layers.data(name="data", shape=[32, 32], dtype="float32") fc = fluid.layers.fc(input=data, size=1000, act="tanh") """ - - -def lrn(input, n=5, k=1.0, alpha=1e-4, beta=0.75, name=None): - """ - **Local Response Normalization Operator** - - Refer to `ImageNet Classification with Deep Convolutional Neural Networks `_. - - The formula is as follows: - - .. math:: - - Output(i, x, y) = Input(i, x, y) / \left( - k + \alpha \sum\limits^{\min(C, c + n/2)}_{j = \max(0, c - n/2)} - (Input(j, x, y))^2 \right)^{\beta} - - In the above equation: - - * :math:`n`: The number of channels to sum over. - * :math:`k`: The offset (usually positive to avoid dividing by 0). - * :math:`alpha`: The scaling parameter. - * :math:`beta`: The exponent. - - Args: - input (Variable): The input tensor of this layer. The dimension of input tensor must be 4 and it's order should be 'NCHW'. - n (int, default 5): The number of channels to sum over. - k (float, default 1.0): An offset (usually positive to avoid dividing by 0). - alpha (float, default 1e-4): The scaling parameter. - beta (float, default 0.75): The exponent. - name (str, default None): A name for this operation. - - Raises: - ValueError: If rank of the input tensor is not 4. - - Returns: - A tensor variable storing the transformation result. - - Examples: - .. code-block:: python - - data = fluid.layers.data(name="data", shape=[3, 112, 112], dtype="float32") - lrn = fluid.layers.lrn(input=data) - """ diff --git a/doc/fluid/dev/src/lrn.py b/doc/fluid/dev/src/lrn.py new file mode 100644 index 0000000000000..36762f04738d6 --- /dev/null +++ b/doc/fluid/dev/src/lrn.py @@ -0,0 +1,57 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def lrn(input, n=5, k=1.0, alpha=1e-4, beta=0.75, name=None): + """ + **Local Response Normalization Operator** + + Refer to `ImageNet Classification with Deep Convolutional Neural Networks + `_ + + The formula is as follows: + + .. math:: + + Output(i, x, y) = Input(i, x, y) / \left( + k + \alpha \sum\limits^{\min(C, c + n/2)}_{j = \max(0, c - n/2)} + (Input(j, x, y))^2 \right)^{\beta} + + In the above equation: + + * :math:`n`: The number of channels to sum over. + * :math:`k`: The offset (usually positive to avoid dividing by 0). + * :math:`alpha`: The scaling parameter. + * :math:`beta`: The exponent. + + Args: + input (Variable): The input tensor of this layer. The dimension of input tensor must be 4 and it's order should be 'NCHW'. + n (int, default 5): The number of channels to sum over. + k (float, default 1.0): An offset (usually positive to avoid dividing by 0). + alpha (float, default 1e-4): The scaling parameter. + beta (float, default 0.75): The exponent. + name (str, default None): A name for this operation. + + Raises: + ValueError: If rank of the input tensor is not 4. + + Returns: + A tensor variable storing the transformation result. + + Examples: + .. code-block:: python + + data = fluid.layers.data(name="data", shape=[3, 112, 112], dtype="float32") + lrn = fluid.layers.lrn(input=data) + """ From 0caf2f2fac39a7728608d1327072545ae863f832 Mon Sep 17 00:00:00 2001 From: wanglong03 Date: Mon, 19 Mar 2018 17:26:12 +0800 Subject: [PATCH 6/8] rm lrn.py in doc/fluid/dev/src --- doc/fluid/dev/src/lrn.py | 57 ---------------------------------------- 1 file changed, 57 deletions(-) delete mode 100644 doc/fluid/dev/src/lrn.py diff --git a/doc/fluid/dev/src/lrn.py b/doc/fluid/dev/src/lrn.py deleted file mode 100644 index 36762f04738d6..0000000000000 --- a/doc/fluid/dev/src/lrn.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -def lrn(input, n=5, k=1.0, alpha=1e-4, beta=0.75, name=None): - """ - **Local Response Normalization Operator** - - Refer to `ImageNet Classification with Deep Convolutional Neural Networks - `_ - - The formula is as follows: - - .. math:: - - Output(i, x, y) = Input(i, x, y) / \left( - k + \alpha \sum\limits^{\min(C, c + n/2)}_{j = \max(0, c - n/2)} - (Input(j, x, y))^2 \right)^{\beta} - - In the above equation: - - * :math:`n`: The number of channels to sum over. - * :math:`k`: The offset (usually positive to avoid dividing by 0). - * :math:`alpha`: The scaling parameter. - * :math:`beta`: The exponent. - - Args: - input (Variable): The input tensor of this layer. The dimension of input tensor must be 4 and it's order should be 'NCHW'. - n (int, default 5): The number of channels to sum over. - k (float, default 1.0): An offset (usually positive to avoid dividing by 0). - alpha (float, default 1e-4): The scaling parameter. - beta (float, default 0.75): The exponent. - name (str, default None): A name for this operation. - - Raises: - ValueError: If rank of the input tensor is not 4. - - Returns: - A tensor variable storing the transformation result. - - Examples: - .. code-block:: python - - data = fluid.layers.data(name="data", shape=[3, 112, 112], dtype="float32") - lrn = fluid.layers.lrn(input=data) - """ From 4316a6711f436368a848afeb89f73bde87fc7a0d Mon Sep 17 00:00:00 2001 From: wanglong03 Date: Tue, 20 Mar 2018 10:24:53 +0800 Subject: [PATCH 7/8] change code style in lrn --- python/paddle/fluid/layers/nn.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 5a00340731864..329123033806b 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -3393,10 +3393,8 @@ def lod_reset(x, y=None, target_lod=None): def lrn(input, n=5, k=1.0, alpha=1e-4, beta=0.75, name=None): """ - **Local Response Normalization Operator** - - Refer to `ImageNet Classification with Deep Convolutional Neural Networks - `_ + Local Response Normalization Layer. This layer performs a kind of + "lateral inhibition" by normalizing over local input regions. The formula is as follows: @@ -3413,6 +3411,9 @@ def lrn(input, n=5, k=1.0, alpha=1e-4, beta=0.75, name=None): * :math:`alpha`: The scaling parameter. * :math:`beta`: The exponent parameter. + Refer to `ImageNet Classification with Deep Convolutional Neural Networks + `_ + Args: input (Variable): The input tensor of this layer, and the dimension of input tensor must be 4. n (int, default 5): The number of channels to sum over. From ce8e013309a976fa7132730e4dcbe1b2e6ddac42 Mon Sep 17 00:00:00 2001 From: wanglong03 Date: Wed, 21 Mar 2018 14:14:03 +0800 Subject: [PATCH 8/8] fix style of comments in lrn --- python/paddle/fluid/layers/nn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 329123033806b..aa21f915fb60a 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -3393,7 +3393,7 @@ def lod_reset(x, y=None, target_lod=None): def lrn(input, n=5, k=1.0, alpha=1e-4, beta=0.75, name=None): """ - Local Response Normalization Layer. This layer performs a kind of + Local Response Normalization Layer. This layer performs a type of "lateral inhibition" by normalizing over local input regions. The formula is as follows: @@ -3407,7 +3407,7 @@ def lrn(input, n=5, k=1.0, alpha=1e-4, beta=0.75, name=None): In the above equation: * :math:`n`: The number of channels to sum over. - * :math:`k`: The offset (usually positive to avoid dividing by 0). + * :math:`k`: The offset (avoid being divided by 0). * :math:`alpha`: The scaling parameter. * :math:`beta`: The exponent parameter.