-
Notifications
You must be signed in to change notification settings - Fork 659
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Fix upsample sbp infer bug and add global test #7884
Changes from all commits
e6e6973
a0abdd5
00522df
9a40937
8ff6ec6
d38ba32
303c5f9
cd5964d
c90a291
54f71f9
68e0e08
f154da2
f76f499
0b90f9b
da30b1a
21741e6
a5f4921
468e775
81faf24
fb812b7
f08840a
9f6551c
2ab15d4
84cf459
1c7cfa7
b65823f
c89fe8b
86db50d
20ceb35
d587929
23e4199
d0023c7
bccad55
01890db
600a508
4c13987
593278a
ddd5a37
e6ccea4
81dee25
d35ce15
6db23bb
382dd5d
75e34a3
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -46,11 +46,11 @@ __global__ void UpsampleBicubic2dForward(const int64_t elem_cnt, const T* in_dpt | |
T* out = out_dptr; | ||
|
||
const T real_x = GetAreaPixel(scale_width, output_x, align_corners, /*cubic=*/true); | ||
int64_t input_x = std::floor(1.0 * real_x); | ||
int64_t input_x = floor(1.0 * real_x); | ||
const T t_x = real_x - input_x; | ||
|
||
const T real_y = GetAreaPixel(scale_height, output_y, align_corners, /*cubic=*/true); | ||
int64_t input_y = std::floor(1.0 * real_y); | ||
int64_t input_y = floor(1.0 * real_y); | ||
const T t_y = real_y - input_y; | ||
|
||
for (int64_t c = 0; c < channels * nbatch; c++) { | ||
|
@@ -92,11 +92,11 @@ __global__ void UpsampleBicubic2dBackward(const int64_t elem_cnt, const T* dy_dp | |
const T* out = dy_dptr; | ||
|
||
T real_x = GetAreaPixel(scale_width, output_x, align_corners, true); | ||
int64_t input_x = std::floor(1.0 * real_x); | ||
int64_t input_x = floor(1.0 * real_x); | ||
T t_x = real_x - input_x; | ||
|
||
T real_y = GetAreaPixel(scale_height, output_y, align_corners, true); | ||
int64_t input_y = std::floor(1.0 * real_y); | ||
int64_t input_y = floor(1.0 * real_y); | ||
T t_y = real_y - input_y; | ||
|
||
T x_coeffs[4]; | ||
|
@@ -105,7 +105,7 @@ __global__ void UpsampleBicubic2dBackward(const int64_t elem_cnt, const T* dy_dp | |
get_cubic_upsample_coefficients<T>(x_coeffs, t_x); | ||
get_cubic_upsample_coefficients<T>(y_coeffs, t_y); | ||
|
||
for (int64_t c = 0; c < channels; c++) { | ||
for (int64_t c = 0; c < channels * nbatch; c++) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 这里导致了后向计算只有第一个batch生效的bug |
||
T out_value = out[output_y * out_width + output_x]; | ||
|
||
for (int64_t i = 0; i < 4; i++) { | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,113 @@ | ||
""" | ||
Copyright 2020 The OneFlow Authors. All rights reserved. | ||
|
||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
|
||
http://www.apache.org/licenses/LICENSE-2.0 | ||
|
||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
""" | ||
|
||
import unittest | ||
from collections import OrderedDict | ||
|
||
import numpy as np | ||
from oneflow.test_utils.test_util import GenArgList | ||
from oneflow.test_utils.automated_test_util import * | ||
|
||
import oneflow as flow | ||
import oneflow.unittest | ||
|
||
|
||
@autotest(n=1, auto_backward=True, check_graph=False) | ||
def _test_global_upsample2d_nearest(test_case, placement, sbp): | ||
x = random_tensor(ndim=3, dim0=8, dim1=16).to_global(placement, sbp) | ||
print(x) | ||
m = torch.nn.Upsample(scale_factor=random().to(int), mode="nearest",) | ||
y = m(x) | ||
return y | ||
|
||
|
||
@autotest(n=1, auto_backward=True, check_graph=False) | ||
def _test_global_upsample2d_linear(test_case, placement, sbp): | ||
x = random_tensor(ndim=3, dim0=8, dim1=16).to_global(placement, sbp) | ||
m = torch.nn.Upsample( | ||
scale_factor=random().to(int), mode="linear", align_corners=random_bool(), | ||
) | ||
y = m(x) | ||
return y | ||
|
||
|
||
@autotest(n=1, auto_backward=True, check_graph=False) | ||
def _test_global_upsample2d_bilinear(test_case, placement, sbp): | ||
x = random_tensor(ndim=4, dim0=8, dim1=16).to_global(placement, sbp) | ||
m = torch.nn.Upsample( | ||
scale_factor=random().to(int), mode="bilinear", align_corners=random_bool(), | ||
) | ||
y = m(x) | ||
return y | ||
|
||
|
||
@autotest(n=1, auto_backward=True, check_graph=False) | ||
def _test_global_upsample2d_bicubic(test_case, placement, sbp): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. bicubic模式下,oneflow 后向计算结果与pytorch对不上,不知道是否是在实现上有差异 复现命令:设置```auto_backward=True````,
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 这里需要确定是否真的对不上,我们要以Pytorch1.11为准。如果确实对不上,那么就更新到:https://github.com/Oneflow-Inc/OneTeam/issues/1207#issuecomment-1073432125 ,我来debug。 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
升级pytorch到1.11后(原本是在1.10下侧的),后向计算结果仍然不一样,已更新在Oneflow-Inc/OneTeam#1207 (comment) 中 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
合并 pr7916 后,打开后向测试,直接 Abroted 的了,关闭后向没有问题,可以正常跑,报错信息如下: python test_consistent_upsample.py --verbose --failfast
test_global_upsample2d_bicubic (__main__.TestGlobalUpsample2d) ... Environment has been initialized, this env init will do nothing.
/home/hanbinbin/anaconda3/envs/oneflow/lib/python3.8/site-packages/torch/_tensor.py:1104: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at aten/src/ATen/core/TensorBody.h:475.)
return self._grad
free(): corrupted unsorted chunks
Aborted (core dumped) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 神奇,我再看看 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
好的 |
||
x = random_tensor(ndim=4, dim0=8, dim1=16).to_global(placement, sbp) | ||
m = torch.nn.Upsample( | ||
scale_factor=random().to(int), mode="bicubic", align_corners=random_bool(), | ||
) | ||
y = m(x) | ||
return y | ||
|
||
|
||
@autotest(n=1, auto_backward=True, check_graph=False) | ||
def _test_global_upsample2d_trilinear(test_case, placement, sbp): | ||
x = random_tensor(ndim=5, dim0=8, dim1=16).to_global(placement, sbp) | ||
m = torch.nn.Upsample( | ||
scale_factor=random().to(int), mode="trilinear", align_corners=random_bool(), | ||
) | ||
y = m(x) | ||
return y | ||
|
||
|
||
class TestGlobalUpsample2d(flow.unittest.TestCase): | ||
@unittest.skip( | ||
"The nearest interpolate operation in pytorch has bug, https://github.com/pytorch/pytorch/issues/65200" | ||
) | ||
@globaltest | ||
def test_global_upsample2d_nearest(test_case): | ||
Comment on lines
+78
to
+82
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 这个里pytroch中的issue已经close了,但是这测结果还是和pytorch对不上,不知道只oneflow的问题还是pytorch的问题 复现命令:注释
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 可能和我们CI环境下的PyTorch版本比较旧有关,这里暂时也和Local一样skip吧 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @caishenghang There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 之前我和盛航在升级的过程中碰到了很多问题,还没有来得及一一解决。 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 我这边重复测试了一下,发现原因是因为pytorch的cpu和gpu结果在缩放系数不是整数情况下跑出的结果对不上。这个bug我之前确实反馈了,但pytorch不修就直接把我issue关了,这个问题先不管吧。 |
||
for placement in all_placement(): | ||
for sbp in all_sbp(placement, max_dim=1): | ||
_test_global_upsample2d_nearest(test_case, placement, sbp) | ||
|
||
@globaltest | ||
def test_global_upsample2d_linear(test_case): | ||
for placement in all_placement(): | ||
for sbp in all_sbp(placement, max_dim=1): | ||
_test_global_upsample2d_linear(test_case, placement, sbp) | ||
|
||
@globaltest | ||
def test_global_upsample2d_bilinear(test_case): | ||
for placement in all_placement(): | ||
for sbp in all_sbp(placement, max_dim=1): | ||
_test_global_upsample2d_bilinear(test_case, placement, sbp) | ||
|
||
@globaltest | ||
def test_global_upsample2d_bicubic(test_case): | ||
for placement in all_placement(): | ||
for sbp in all_sbp(placement, max_dim=1): | ||
_test_global_upsample2d_bicubic(test_case, placement, sbp) | ||
|
||
@globaltest | ||
def test_global_upsample2d_trilinear(test_case): | ||
for placement in all_placement(): | ||
for sbp in all_sbp(placement, max_dim=1): | ||
_test_global_upsample2d_trilinear(test_case, placement, sbp) | ||
|
||
|
||
if __name__ == "__main__": | ||
unittest.main() |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
这里channels已经是原始channel和batch的乘积了,这里导致了segment fault的bug