-
Notifications
You must be signed in to change notification settings - Fork 5.5k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Update the epsilon in batch_norm_layer to a variable in v2. #5692
Changes from 4 commits
8a49f7f
ac46018
27d7b2c
9580c45
5502abb
90e05a4
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -21,7 +21,7 @@ namespace paddle { | |
|
||
REGISTER_LAYER(cudnn_batch_norm, CudnnBatchNormLayer); | ||
|
||
const double CudnnBatchNormLayer::EPS = 1E-5; | ||
const double CudnnBatchNormLayer::MIN_EPS = 1E-5; | ||
|
||
bool CudnnBatchNormLayer::init(const LayerMap& layerMap, | ||
const ParameterMap& parameterMap) { | ||
|
@@ -61,6 +61,15 @@ void CudnnBatchNormLayer::forward(PassType passType) { | |
real* movingMean = movingMean_->getW()->getData(); | ||
real* movingVar = movingVar_->getW()->getData(); | ||
|
||
/** | ||
* If epsilon_ equals to 1e-5 and eps_ is assigned the value of | ||
* static_cast<double>(epsilon_), The CUDNN_STATUS_BAD_PARAM error | ||
* will occur due to eps_ value is less than | ||
* CUDNN_BN_MIN_EPSILON. | ||
* The following code is to ensure that the eps_ meets requirement. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. comment 信息太冗余。 cuDNN does not allow an epsilon value less than CUDNN_BN_MIN_EPSILON. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done |
||
*/ | ||
eps_ = std::max(MIN_EPS, static_cast<double>(epsilon_)); | ||
|
||
if (!useGlobalStats_) { | ||
REGISTER_TIMER_INFO("CudnnBatchFwTimer", getName().c_str()); | ||
real* savedMean = savedMean_->getData(); | ||
|
@@ -75,7 +84,7 @@ void CudnnBatchNormLayer::forward(PassType passType) { | |
1.0 - movingAvgFraction_, | ||
movingMean, | ||
movingVar, | ||
EPS, | ||
eps_, | ||
savedMean, | ||
savedInvVar); | ||
} else { | ||
|
@@ -90,7 +99,7 @@ void CudnnBatchNormLayer::forward(PassType passType) { | |
beta, | ||
movingMean, | ||
movingVar, | ||
EPS); | ||
eps_); | ||
} else { | ||
// There is a limitation in cudnn library. | ||
// When the batch size is larger than 1024 in cuDNN v5.1, | ||
|
@@ -101,7 +110,7 @@ void CudnnBatchNormLayer::forward(PassType passType) { | |
beta, | ||
movingMean, | ||
movingVar, | ||
EPS, | ||
eps_, | ||
batchSize, | ||
channels_, | ||
imageH_ * imageD_, | ||
|
@@ -128,6 +137,15 @@ void CudnnBatchNormLayer::backward(const UpdateCallback& callback) { | |
real* savedMean = savedMean_->getData(); | ||
real* savedInvVar = savedInvVar_->getData(); | ||
|
||
/** | ||
* If epsilon_ equals to 1e-5 and eps_ is assigned the value of | ||
* static_cast<double>(epsilon_), The CUDNN_STATUS_BAD_PARAM error | ||
* will occur due to eps_ value is less than | ||
* CUDNN_BN_MIN_EPSILON. | ||
* The following code is to ensure that the eps_ meets requirement. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. comment 信息太冗余,不要解释这个赋值的逻辑,没有啥用处。 cuDNN does not allow an epsilon value less than CUDNN_BN_MIN_EPSILON. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done |
||
*/ | ||
eps_ = std::max(MIN_EPS, static_cast<double>(epsilon_)); | ||
|
||
auto create = [](MatrixPtr& m, size_t h, size_t w, real** p) { | ||
Matrix::resizeOrCreate(m, h, w, false, true); | ||
m->zeroMem(); | ||
|
@@ -157,7 +175,7 @@ void CudnnBatchNormLayer::backward(const UpdateCallback& callback) { | |
gamma, | ||
gammaGrad, | ||
betaGrad, | ||
EPS, | ||
eps_, | ||
savedMean, | ||
savedInvVar); | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -540,6 +540,10 @@ message LayerConfig { | |
|
||
// for switch order layer | ||
optional ReshapeConfig reshape_conf = 59; | ||
|
||
// for batch normalization layer | ||
// small constant added to the variance to avoid numerical problems. | ||
optional double epsilon = 60 [ default = 0.00001 ]; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done |
||
} | ||
|
||
message EvaluatorConfig { | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2434,6 +2434,7 @@ def __init__(self, | |
bias=True, | ||
img3D=False, | ||
use_global_stats=True, | ||
epsilon=1e-5, | ||
moving_average_fraction=0.9, | ||
batch_norm_type=None, | ||
mean_var_names=None, | ||
|
@@ -2483,6 +2484,8 @@ def __init__(self, | |
if moving_average_fraction is not None: | ||
self.config.moving_average_fraction = moving_average_fraction | ||
|
||
self.config.epsilon = epsilon | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 为了确保老接口使用bn层一致,这里统一将epsilon大小的判断放在了config_parser.py中,所有类型的batch norm layer都需要保证输入的epsilon大小必须大于或等于1e-5,和cuDNN的约束一致。 |
||
|
||
input_layer = self.get_input_layer(0) | ||
image_conf = self.config.inputs[0].image_conf | ||
if img3D: | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3036,6 +3036,7 @@ def batch_norm_layer(input, | |
param_attr=None, | ||
layer_attr=None, | ||
batch_norm_type=None, | ||
epsilon=1e-5, | ||
moving_average_fraction=0.9, | ||
use_global_stats=None, | ||
mean_var_names=None): | ||
|
@@ -3106,6 +3107,8 @@ def batch_norm_layer(input, | |
will use the mean and variance of the current batch | ||
of test data. | ||
:type use_global_stats: bool | None. | ||
:param epsilon: Small constant added to the variance to avoid numerical problems. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done |
||
:type epsilon: float. | ||
:param moving_average_fraction: Factor used in the moving average computation. | ||
:math:`runningMean = newMean*(1-factor) + runningMean*factor` | ||
:type moving_average_fraction: float. | ||
|
@@ -3123,6 +3126,9 @@ def batch_norm_layer(input, | |
assert (batch_norm_type is None) or (batch_norm_type == "batch_norm") or \ | ||
(batch_norm_type == "mkldnn_batch_norm") or \ | ||
(batch_norm_type == "cudnn_batch_norm") | ||
|
||
assert epsilon >= 1e-5, "epsilon must be no less than 1e-5." | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 所有类型的bn层都需要统一满足限制,已改为在config_parse.py中统一判断。 |
||
|
||
l = Layer( | ||
name=name, | ||
img3D=img3D, | ||
|
@@ -3132,6 +3138,7 @@ def batch_norm_layer(input, | |
type=LayerType.BATCH_NORM_LAYER, | ||
batch_norm_type=batch_norm_type, | ||
bias=ParamAttr.to_bias(bias_attr), | ||
epsilon=epsilon, | ||
moving_average_fraction=moving_average_fraction, | ||
use_global_stats=use_global_stats, | ||
mean_var_names=mean_var_names, | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -65,6 +65,7 @@ layers { | |
height: 227 | ||
width: 227 | ||
depth: 1 | ||
epsilon: 1e-05 | ||
} | ||
layers { | ||
name: "__crmnorm_0__" | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -65,6 +65,7 @@ layers { | |
height: 256 | ||
width: 256 | ||
depth: 1 | ||
epsilon: 1e-05 | ||
} | ||
layers { | ||
name: "__crmnorm_0__" | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -36,6 +36,7 @@ layers { | |
height: 6 | ||
width: 20 | ||
depth: 3 | ||
epsilon: 1e-05 | ||
} | ||
parameters { | ||
name: "___batch_norm_0__.w0" | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
不要用这个 magic number ,直接用 CUDNN_BN_MIN_EPSILON。
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Done