Skip to content

Commit

Permalink
[Cherry-Pick] Modify the bf16 accuracy checking framework in OpTest (#…
Browse files Browse the repository at this point in the history 
…54658)

* modify the bf16 accuracy checking framework in OpTest

* modify the bf16 accuracy checking framework in OpTest

* modify the bf16 accuracy checking framework in OpTest

* modify the bf16 accuracy checking framework in OpTest

* modify the bf16 accuracy checking framework in OpTest

* modify the bf16 accuracy checking framework in OpTest
  • Loading branch information
@Vvsmile
Vvsmile committed Jun 16, 2023
1 parent 0abd9ff commit 8077d79
Show file tree
Hide file tree
Showing 3 changed files with 100 additions and 32 deletions.
122 changes: 91 additions & 31 deletions test/legacy_test/eager_op_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -550,8 +550,17 @@ def is_fp16_compared_with_fp32(self):
not in op_accuracy_white_list.NO_FP16_COMPARED_WITH_FP32_OP_LIST
)

def is_bf16_compared_with_fp32(self):
return self.is_bfloat16_op() and (
self.op_type
not in op_accuracy_white_list.NO_BF16_COMPARED_WITH_FP32_OP_LIST
)

def enable_cal_ref_output(self):
self.is_calc_ref = self.is_fp16_compared_with_fp32()
self.is_calc_ref = (
self.is_fp16_compared_with_fp32()
or self.is_bf16_compared_with_fp32()
)

def disable_cal_ref_output(self):
self.is_calc_ref = False
Expand Down Expand Up @@ -652,46 +661,86 @@ def feed_var(self, input_vars, place):
if isinstance(np_value, tuple):
tensor.set(np_value[0], place)
dtype = np.array(np_value[1]).dtype
if self.is_calc_ref and dtype == np.float16:
if isinstance(np_value[1], list):
tensor.set_recursive_sequence_lengths(
np.array(np_value[1]).astype(np.float32)
)

if self.is_calc_ref:
# convert the float16 to float by numpy.astype
if dtype == np.float16:
if isinstance(np_value[1], list):
tensor.set_recursive_sequence_lengths(
np.array(np_value[1]).astype(np.float32)
)
else:
tensor.set_recursive_sequence_lengths(
np_value[1].astype(np.float32)
)
# convert the bfloat16 to float by convert_uint16_to_float
# provided in this file
elif dtype == np.uint16:
if isinstance(np_value[1], list):
tensor.set_recursive_sequence_lengths(
convert_uint16_to_float(
np.array(np_value[1])
)
)
else:
tensor.set_recursive_sequence_lengths(
convert_uint16_to_float(np_value[1])
)
else:
tensor.set_recursive_sequence_lengths(
np_value[1].astype(np.float32)
np_value[1]
)
else:
tensor.set_recursive_sequence_lengths(np_value[1])
else:
if self.is_calc_ref and np_value.dtype == np.float16:
tensor.set(np_value.astype(np.float32), place)
if self.is_calc_ref:
if np_value.dtype == np.float16:
tensor.set(np_value.astype(np.float32), place)
elif np_value.dtype == np.uint16:
tensor.set(
convert_uint16_to_float(np_value), place
)
else:
tensor.set(np_value, place)
else:
tensor.set(np_value, place)
feed_map[name] = tensor
else:
tensor = core.LoDTensor()
if isinstance(self.inputs[var_name], tuple):
tensor.set(self.inputs[var_name][0], place)
if (
self.is_calc_ref
and self.inputs[var_name][1].dtype == np.float16
):
tensor.set_recursive_sequence_lengths(
self.inputs[var_name][1].astype(np.float32)
)
if self.is_calc_ref:
if self.inputs[var_name][1].dtype == np.float16:
tensor.set_recursive_sequence_lengths(
self.inputs[var_name][1].astype(np.float32)
)
elif self.inputs[var_name][1].dtype == np.uint16:
tensor.set_recursive_sequence_lengths(
convert_uint16_to_float(
self.inputs[var_name][1]
)
)
else:
tensor.set_recursive_sequence_lengths(
self.inputs[var_name][1]
)
else:
tensor.set_recursive_sequence_lengths(
self.inputs[var_name][1]
)
else:
if (
self.is_calc_ref
and self.inputs[var_name].dtype == np.float16
):
tensor.set(
self.inputs[var_name].astype(np.float32), place
)
if self.is_calc_ref:
if self.inputs[var_name].dtype == np.float16:
tensor.set(
self.inputs[var_name].astype(np.float32), place
)
elif self.inputs[var_name].dtype == np.uint16:
tensor.set(
convert_uint16_to_float(self.inputs[var_name]),
place,
)
else:
tensor.set(self.inputs[var_name], place)
else:
tensor.set(self.inputs[var_name], place)
feed_map[var_name] = tensor
Expand Down Expand Up @@ -1761,7 +1810,10 @@ def _compare_list(self, name, actual, expect):
def compare_single_output_with_expect(self, name, expect):
actual, actual_np = self.find_actual_value(name)
# expect_np = expect[0] if isinstance(expect, tuple) else expect
if self.op_test.is_fp16_compared_with_fp32():
if (
self.op_test.is_fp16_compared_with_fp32()
or self.op_test.is_bf16_compared_with_fp32()
):
expect, expect_np = self.find_expect_value(name)
else:
expect_np = (
Expand Down Expand Up @@ -1816,7 +1868,10 @@ def calculate_output(self):
)
self.outputs = outs
self.fetch_list = fetch_list
if self.op_test.is_fp16_compared_with_fp32():
if (
self.op_test.is_fp16_compared_with_fp32()
or self.op_test.is_bf16_compared_with_fp32()
):
self.op_test.enable_cal_ref_output()
ref_outs, ref_fetch_list = self.op_test._calc_output(
place, no_check_set=no_check_set
Expand Down Expand Up @@ -1883,7 +1938,10 @@ def calculate_output(self):
place, no_check_set=no_check_set
)
self.outputs = dygraph_outs
if self.op_test.is_fp16_compared_with_fp32():
if (
self.op_test.is_fp16_compared_with_fp32()
or self.op_test.is_bf16_compared_with_fp32()
):
self.op_test.enable_cal_ref_output()
self.is_python_api_test = True
self.ref_outputs = self.op_test._calc_python_api_output(
Expand Down Expand Up @@ -2228,9 +2286,8 @@ def _assert_is_close(
atol=atol,
equal_nan=False,
err_msg=(
"Operator %s error, %s variable %s (shape: %s, dtype: %s) max gradient diff over limit"
)
% (
"Operator {} error, {} variable {} (shape: {}, dtype: {}) max gradient diff over limit"
).format(
self.op_type,
msg_prefix,
name,
Expand Down Expand Up @@ -2486,7 +2543,10 @@ def check_grad_with_place(
if numeric_place is None:
numeric_place = place

if user_defined_grads is None and self.is_fp16_compared_with_fp32():
if user_defined_grads is None and (
self.is_fp16_compared_with_fp32()
or self.is_bf16_compared_with_fp32()
):
self.enable_cal_ref_output()
numeric_grads = self._get_gradient(
inputs_to_check,
Expand Down Expand Up @@ -2769,7 +2829,7 @@ def _get_gradient(
feed_dict = self.feed_var(inputs, place)

if user_defined_grad_outputs is None:
if self.dtype == np.uint16:
if self.dtype == np.uint16 and not self.is_calc_ref:
cast_inputs = list(map(block.var, output_names))
if self.op_type in ["broadcast_tensors", "meshgrid"]:
output_names = self.cast_bf16_output(block, cast_inputs)
Expand Down
2 changes: 1 addition & 1 deletion test/legacy_test/testsuite.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ def create_var(block, name, np_list, var_proto, is_calc_ref=False):
if is_input:
shape = list(np_value.shape)
lod_level = 0
if is_calc_ref and dtype == np.float16:
if is_calc_ref and (dtype == np.float16 or dtype == np.uint16):
dtype = np.float32
return block.create_var(
dtype=dtype, shape=shape, lod_level=lod_level, name=name
Expand Down
8 changes: 8 additions & 0 deletions test/white_list/op_accuracy_white_list.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,3 +94,11 @@
'fake_quantize_moving_average_abs_max',
'p_norm',
]


NO_BF16_COMPARED_WITH_FP32_OP_LIST = [
'unique',
'fusion_gru',
'fusion_lstm',
'dequantize',
]

0 comments on commit 8077d79

Please sign in to comment.