add norm 2.0 api, test=develop #26465
Conversation
|
Thanks for your contribution! |
|
✅ This PR's description meets the template requirements! |
python/paddle/nn/functional/norm.py
Outdated
|
|
||
| helper = LayerHelper('batch_norm', **locals()) | ||
| saved_mean = helper.create_variable_for_type_inference( | ||
| dtype=x.dtype, stop_gradient=True) |
There was a problem hiding this comment.
make sure this is the correct behavior for fp16?
There was a problem hiding this comment.
done . get_default_type
| } | ||
|
|
||
| helper = LayerHelper('batch_norm', **locals()) | ||
| saved_mean = helper.create_variable_for_type_inference( |
There was a problem hiding this comment.
this is not resolved, also, please just reply to the comments and don't mark them resolved, they should be marked resolved by the reviewer
There was a problem hiding this comment.
mean and var will still be in fp16 if x is fp16?
| data_format, "use_mkldnn", False, "fuse_with_relu", False, | ||
| "use_global_stats", use_global_stats) | ||
| batch_norm_out, _, _, _, _, _ = core.ops.batch_norm( | ||
| x, weight, bias, running_mean, running_var, mean_out, variance_out, |
There was a problem hiding this comment.
what happens when weight|bias is None?
There was a problem hiding this comment.
weight and bias need not be None.
python/paddle/nn/functional/norm.py
Outdated
| weight=None, | ||
| bias=None, | ||
| use_input_stats=True, | ||
| momentum=0.1, |
| use_input_stats=True, | ||
| momentum=0.1, | ||
| eps=1e-05, | ||
| data_format="NCHW", |
| weight_attr=None, | ||
| bias_attr=None, | ||
| track_running_stats=False, | ||
| data_format="NCHW", |
python/paddle/nn/functional/norm.py
Outdated
|
|
||
| assert len(x.shape) >= 2, "input dim must be larger than 1" | ||
|
|
||
| assert weight is not None, "the weight must not be None, please use nn.BatchNorm1d, nn.BatchNorm2d or nn.BatchNorm3d" |
There was a problem hiding this comment.
simply make them required arguments?
| assert weight is not None, "the weight must not be None, please use nn.BatchNorm1d, nn.BatchNorm2d or nn.BatchNorm3d" | ||
| assert bias is not None, "the bias must not be None, please use nn.BatchNorm1d, nn.BatchNorm2d or nn.BatchNorm3d" | ||
| # we use not training means use_global_status, more details see nn._BatchNormBase | ||
| use_global_stats = not training |
There was a problem hiding this comment.
when training and track_status, this is true.
| check_variable_and_dtype(x, 'input', ['float16', 'float32', 'float64'], | ||
| 'BatchNorm') | ||
|
|
||
| attrs = { |
There was a problem hiding this comment.
move attr up, no need to set twice
| } | ||
|
|
||
| helper = LayerHelper('batch_norm', **locals()) | ||
| saved_mean = helper.create_variable_for_type_inference( |
There was a problem hiding this comment.
mean and var will still be in fp16 if x is fp16?
| epsilon(float, optional): A value added to the denominator for | ||
| numerical stability. Default is 1e-5. | ||
| momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9. | ||
| track_running_stats(bool, optional): Whether to use global mean and |
There was a problem hiding this comment.
this is not the same as use_global_stats, looks like it is not implemented
There was a problem hiding this comment.
fixed fp16 problem
python/paddle/nn/layer/norm.py
Outdated
| class GroupNorm(layers.Layer): | ||
| """ | ||
| :alias_main: paddle.nn.GroupNorm | ||
| :alias: paddle.nn.GroupNorm,paddle.nn.layer.GroupNorm,paddle.nn.layer.norm.GroupNorm |
|
|
||
| class LayerNorm(layers.Layer): | ||
| """ | ||
| :alias_main: paddle.nn.LayerNorm |
python/paddle/nn/layer/norm.py
Outdated
| - :math:`b`: the trainable bias parameter. | ||
|
|
||
| Parameters: | ||
| normalized_shape(int or list or tuple): Input shape from an expected input of |
There was a problem hiding this comment.
may be necessary to support tensor as input
There was a problem hiding this comment.
can solve this problem next version
python/paddle/nn/layer/norm.py
Outdated
| self._variance, | ||
| weight=self.weight, | ||
| bias=self.bias, | ||
| training=self.training or not self._track_running_stats, |
There was a problem hiding this comment.
track can be false even for train mode
There was a problem hiding this comment.
add note for users
python/paddle/nn/layer/norm.py
Outdated
| will create ParamAttr as bias_attr. If it is set to Fasle, the weight is not learnable. | ||
| If the Initializer of the bias_attr is not set, the bias is initialized zero. Default: None. | ||
| data_format(str, optional): Specify the input data format, the data format can be "NCHW" or "NHWC". Default: NCHW. | ||
| track_running_stats(bool, optional): Whether to use global mean and |
There was a problem hiding this comment.
this is not the same as use_global_stats, looks like it is not implemented
There was a problem hiding this comment.
will solve next version
python/paddle/nn/layer/norm.py
Outdated
| If the Initializer of the bias_attr is not set, the bias is initialized zero. Default: None. | ||
| data_format(str, optional): Specify the input data format, the data format can be "NCHW" or "NHWC". Default: NCHW. | ||
| track_running_stats(bool, optional): Whether to use global mean and | ||
| variance. In train mode, when setting track_running_stats True, the global mean |
There was a problem hiding this comment.
update description for track_running_stats
python/paddle/nn/layer/norm.py
Outdated
| self._variance, | ||
| weight=self.weight, | ||
| bias=self.bias, | ||
| training=self.training or not self._track_running_stats, |
There was a problem hiding this comment.
if both training and track are false, the bn op will run in train mode.
also add check and warning
|
|
||
| self._check_input_dim(input) | ||
|
|
||
| if not self.training and not self._track_running_stats: |
There was a problem hiding this comment.
should error out, also change training=self.training below
There was a problem hiding this comment.
on top of that, add warning when track_running_stats is set to false, this is not supported properly right now
| # from .norm import data_norm #DEFINE_ALIAS | ||
| # from .norm import group_norm #DEFINE_ALIAS | ||
| # from .norm import instance_norm #DEFINE_ALIAS | ||
| from .norm import l2_normalize #DEFINE_ALIAS |
python/paddle/nn/functional/norm.py
Outdated
| epsilon(float, optional): The small value added to the variance to prevent division by zero. Default: 1e-5. | ||
| momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9. | ||
| training(bool, optional): True means train mode which compute by batch data and track global mean and var during train period. False means inference mode which compute by global mean and var which calculated by train period. Defalut False. | ||
| data_format(str, optional): Specify the input data format. Defalut "NCHW". |
python/paddle/nn/functional/norm.py
Outdated
| momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9. | ||
| training(bool, optional): True means train mode which compute by batch data and track global mean and var during train period. False means inference mode which compute by global mean and var which calculated by train period. Defalut False. | ||
| data_format(str, optional): Specify the input data format. Defalut "NCHW". | ||
| name(str, optional): Default: None. |
There was a problem hiding this comment.
python/paddle/nn/functional/norm.py
Outdated
| momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9. | ||
| use_input_stats(bool): Default True. | ||
| data_format(str, optional): Specify the input data format. Default: NCHW. | ||
| name(str, optional): Default None. |
There was a problem hiding this comment.
data_format and name
see above
python/paddle/nn/layer/norm.py
Outdated
| If the Initializer of the bias_attr is not set, the bias is initialized zero. | ||
| If it is set to False, will not create bias_attr. Default: None. | ||
| data_format(str, optional): Specify the input data format. Default: NCL. | ||
| name(str, optional): Default None. |
python/paddle/nn/layer/norm.py
Outdated
| bias :math:`b`. If it is set to False, no bias will be added to the output units. | ||
| If it is set to None, the bias is initialized zero. Default: None. | ||
| data_format(str, optional): Specify the input data format. Only NCHW is supported. Default: NCHW. | ||
| name(str, optional): Default None. |
python/paddle/nn/layer/norm.py
Outdated
| bias_attr(ParamAttr|bool, optional): The parameter attribute for the learnable | ||
| bias :math:`b`. If is False, bias is None. If is None, a default :code:`ParamAttr` would be added as bias. The | ||
| :attr:`bias_attr` is initialized as 0 if it is added. Default: None. | ||
| name(str, optional): parameter name. Default None. |
There was a problem hiding this comment.
python/paddle/nn/layer/norm.py
Outdated
| track_running_stats(bool, optional): Whether to use global mean and variance. In train period, | ||
| True will track global mean and variance used for inference. When inference, track_running_stats must be | ||
| True. Default: True. | ||
| name(str, optional): Default: None. |
There was a problem hiding this comment.
python/paddle/nn/layer/norm.py
Outdated
| track_running_stats(bool, optional): Whether to use global mean and variance. In train period, | ||
| True will track global mean and variance used for inference. When inference, track_running_stats must be | ||
| True. Default: True. | ||
| name(str, optional): Default: None. |
There was a problem hiding this comment.
python/paddle/nn/layer/norm.py
Outdated
| track_running_stats(bool, optional): Whether to use global mean and variance. In train period, | ||
| True will track global mean and variance used for inference. When inference, track_running_stats must be | ||
| True. Default: True. | ||
| name(str, optional): Default: None. |
There was a problem hiding this comment.
frankwhzhang
dismissed stale reviews from lanxianghit, WuHaobo, and willthefrog
via
5fa037c
python/paddle/nn/layer/norm.py
Outdated
| x = np.array([[[[0.3, 0.4], [0.3, 0.07]], [[0.83, 0.37], [0.18, 0.93]]]]).astype('float32') | ||
| paddle.disable_static() | ||
| x = paddle.to_tensor(x) | ||
| if paddle.fluid.is_compiled_with_cuda(): |
There was a problem hiding this comment.
probably we need to make is_compiled_with_cuda visible to users via :paddle.is_compiled_with_cuda
PR types
New features
PR changes
OPs, APIs, DOCs
Describe
add BatchNorm1d , BatchNorm2d, BatchNorm3d and so on