Skip to content

Enable UFMT on test/cpp_api_parity, test/cpp_extensions, test/create_dummy_torchscript_model.py, test/custom_backend, test/custom_operator #123518

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 5 commits into from
Closed

Enable UFMT on test/cpp_api_parity, test/cpp_extensions, test/create_dummy_torchscript_model.py, test/custom_backend, test/custom_operator #123518

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ef233b2
style: UFMT formatting on test/cpp_api_parity
shink Apr 7, 2024
1db5b7c
style: UFMT formatting on test/cpp_extensions
shink Apr 7, 2024
405e4d6
style: UFMT formatting on test/create_dummy_torchscript_model.py
shink Apr 7, 2024
7af6e92
style: UFMT formatting on test/custom_backend
shink Apr 7, 2024
a9d4cb3
style: UFMT formatting on test/custom_operator
shink Apr 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 0 additions & 15 deletions .lintrunner.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1014,21 +1014,6 @@ exclude_patterns = [
'test/_nvfuser/test_dynamo.py',
'test/_nvfuser/test_python_frontend.py',
'test/_nvfuser/test_torchscript.py',
'test/cpp_api_parity/__init__.py',
'test/cpp_api_parity/functional_impl_check.py',
'test/cpp_api_parity/module_impl_check.py',
'test/cpp_api_parity/parity_table_parser.py',
'test/cpp_api_parity/sample_functional.py',
'test/cpp_api_parity/sample_module.py',
'test/cpp_api_parity/utils.py',
'test/cpp_extensions/no_python_abi_suffix_test/setup.py',
'test/cpp_extensions/setup.py',
'test/cpp_extensions/torch_test_cpp_extension/__init__.py',
'test/create_dummy_torchscript_model.py',
'test/custom_backend/backend.py',
'test/custom_backend/test_custom_backend.py',
'test/custom_operator/model.py',
'test/custom_operator/test_custom_ops.py',
'test/delete.py',
'test/distributed/_shard/sharded_optim/test_sharded_optim.py',
'test/distributed/_shard/sharded_tensor/ops/test_binary_cmp.py',
Expand Down
202 changes: 141 additions & 61 deletions test/cpp_api_parity/functional_impl_check.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,26 +14,39 @@
# 4. Compare Python/C++ functional's forward output. If they are the same, then we
# have implementation parity between Python/C++ module.

import os
import pprint
import re
import tempfile
from string import Template
import re
import pprint
import os

import torch
from cpp_api_parity.utils import TorchNNFunctionalTestParams, TORCH_NN_COMMON_TEST_HARNESS, \
compile_cpp_code_inline, set_python_tensors_requires_grad, move_python_tensors_to_device, \
add_test, compute_cpp_args_construction_stmts_and_forward_arg_symbols, serialize_arg_dict_as_script_module, \
compute_arg_dict, decorate_test_fn, compute_temp_file_path, generate_error_msg, is_torch_nn_functional_test, \
try_remove_folder

from cpp_api_parity.sample_functional import SAMPLE_FUNCTIONAL_CPP_SOURCE
from cpp_api_parity.utils import (
add_test,
compile_cpp_code_inline,
compute_arg_dict,
compute_cpp_args_construction_stmts_and_forward_arg_symbols,
compute_temp_file_path,
decorate_test_fn,
generate_error_msg,
is_torch_nn_functional_test,
move_python_tensors_to_device,
serialize_arg_dict_as_script_module,
set_python_tensors_requires_grad,
TORCH_NN_COMMON_TEST_HARNESS,
TorchNNFunctionalTestParams,
try_remove_folder,
)

# Expected substitutions:
#
# ${functional_variant_name} (e.g. `BCELoss_no_reduce`)
# ${cpp_args_construction_stmts}
# ${cpp_function_call}
TORCH_NN_FUNCTIONAL_TEST_FORWARD = Template("""
TORCH_NN_FUNCTIONAL_TEST_FORWARD = Template(
"""
void ${functional_variant_name}_test_forward(
const std::string& arg_dict_file_path,
const std::string& forward_output_file_path) {
Expand All @@ -56,17 +69,24 @@
// Save the output into a file to be compared in Python later
write_ivalue_to_file(torch::IValue(cpp_output), forward_output_file_path);
}
""")
"""
)


def run_forward(unit_test_class, test_params):
device = test_params.device

inputs = set_python_tensors_requires_grad(move_python_tensors_to_device(
[arg_value for _, arg_value in test_params.arg_dict['input']], device))
inputs = set_python_tensors_requires_grad(
move_python_tensors_to_device(
[arg_value for _, arg_value in test_params.arg_dict["input"]], device
)
)
inputs += move_python_tensors_to_device(
[arg_value for _, arg_value in test_params.arg_dict['target']], device)
[arg_value for _, arg_value in test_params.arg_dict["target"]], device
)
inputs += move_python_tensors_to_device(
[arg_value for _, arg_value in test_params.arg_dict['extra_args']], device)
[arg_value for _, arg_value in test_params.arg_dict["extra_args"]], device
)

# Some functionals (such as `F.rrelu`) create random tensors in their call path.
# To make sure the random tensors created are the same in Python/C++, we need
Expand All @@ -76,6 +96,7 @@ def run_forward(unit_test_class, test_params):

return python_output


def test_forward(unit_test_class, test_params):
functional_variant_name = test_params.functional_variant_name
cpp_tmp_folder = test_params.cpp_tmp_folder
Expand All @@ -87,105 +108,149 @@ def test_forward(unit_test_class, test_params):
python_output = run_forward(unit_test_class, test_params)

# Save Python arguments to be used from C++ function
arg_dict_file_path = compute_temp_file_path(cpp_tmp_folder, functional_variant_name, 'arg_dict')
arg_dict_file_path = compute_temp_file_path(
cpp_tmp_folder, functional_variant_name, "arg_dict"
)
serialize_arg_dict_as_script_module(test_params.arg_dict).save(arg_dict_file_path)

cpp_test_name = f'{test_params.functional_variant_name}_test_forward'
cpp_test_fn = getattr(unit_test_class.functional_impl_check_cpp_module, cpp_test_name)
cpp_test_name = f"{test_params.functional_variant_name}_test_forward"
cpp_test_fn = getattr(
unit_test_class.functional_impl_check_cpp_module, cpp_test_name
)

def run_cpp_test_fn_and_check_output():
forward_output_file_path = compute_temp_file_path(cpp_tmp_folder, functional_variant_name, 'forward_output')
forward_output_file_path = compute_temp_file_path(
cpp_tmp_folder, functional_variant_name, "forward_output"
)

cpp_test_fn(arg_dict_file_path, forward_output_file_path)
cpp_output = torch.load(forward_output_file_path)

# Check that forward outputs are equal
unit_test_class.assertEqual(
python_output, cpp_output,
msg=generate_error_msg("forward output", cpp_output, python_output))
python_output,
cpp_output,
msg=generate_error_msg("forward output", cpp_output, python_output),
)

run_cpp_test_fn_and_check_output()

# Remove temporary folder that stores C++ outputs
try_remove_folder(cpp_tmp_folder)


def compute_functional_name(test_params_dict):
def camel_case_to_snake_case(camel_case_str):
return re.sub(r'(?<!^)(?=[A-Z])', '_', camel_case_str).lower()
return re.sub(r"(?<!^)(?=[A-Z])", "_", camel_case_str).lower()

if 'cpp_options_args' in test_params_dict:
if "cpp_options_args" in test_params_dict:
# Expected format for `cpp_options_args`: `F::FunctionalFuncOptions(...)`
# Example output: `binary_cross_entropy`
return camel_case_to_snake_case(
test_params_dict['cpp_options_args'].split('(')[0].replace('F::', '').replace('FuncOptions', ''))
elif 'cpp_function_call' in test_params_dict:
test_params_dict["cpp_options_args"]
.split("(")[0]
.replace("F::", "")
.replace("FuncOptions", "")
)
elif "cpp_function_call" in test_params_dict:
# Expected format for `cpp_function_call`: `F::functional_name(...)`
# Example output: `binary_cross_entropy`
return test_params_dict['cpp_function_call'].split('(')[0].replace('F::', '')
return test_params_dict["cpp_function_call"].split("(")[0].replace("F::", "")
else:
raise RuntimeError(
"`cpp_options_args` or `cpp_function_call` entry must be present in test params dict:\n{}".format(
pprint.pformat(test_params_dict)))
pprint.pformat(test_params_dict)
)
)


def compute_cpp_function_call(test_params_dict, arg_dict, functional_name):
if 'cpp_function_call' in test_params_dict:
return test_params_dict['cpp_function_call']
elif 'cpp_options_args' in test_params_dict:
cpp_forward_args_symbols = [arg_name for arg_name, _ in
arg_dict['input'] + arg_dict['target'] + arg_dict['extra_args']]
return 'F::{}({}, {})'.format(
functional_name, ", ".join(cpp_forward_args_symbols), test_params_dict['cpp_options_args'])
if "cpp_function_call" in test_params_dict:
return test_params_dict["cpp_function_call"]
elif "cpp_options_args" in test_params_dict:
cpp_forward_args_symbols = [
arg_name
for arg_name, _ in arg_dict["input"]
+ arg_dict["target"]
+ arg_dict["extra_args"]
]
return "F::{}({}, {})".format(
functional_name,
", ".join(cpp_forward_args_symbols),
test_params_dict["cpp_options_args"],
)
else:
raise RuntimeError(
"`cpp_options_args` or `cpp_function_call` entry must be present in test params dict:\n{}".format(
pprint.pformat(test_params_dict)))
pprint.pformat(test_params_dict)
)
)


def process_test_params_for_functional(test_params_dict, device, test_instance_class):
test_instance = test_instance_class(**test_params_dict)
functional_name = compute_functional_name(test_params_dict)
assert test_instance.get_name().startswith('test_')
assert test_instance.get_name().startswith("test_")
# Example output: `BCELoss_no_reduce_cuda`
functional_variant_name = test_instance.get_name()[5:] + (('_' + device) if device != 'cpu' else '')
functional_variant_name = test_instance.get_name()[5:] + (
("_" + device) if device != "cpu" else ""
)
arg_dict = compute_arg_dict(test_params_dict, test_instance)

return TorchNNFunctionalTestParams(
functional_name=functional_name,
functional_variant_name=functional_variant_name,
test_instance=test_instance,
cpp_function_call=compute_cpp_function_call(test_params_dict, arg_dict, functional_name),
cpp_function_call=compute_cpp_function_call(
test_params_dict, arg_dict, functional_name
),
arg_dict=arg_dict,
has_parity=test_params_dict.get('has_parity', True),
has_parity=test_params_dict.get("has_parity", True),
device=device,
cpp_tmp_folder=tempfile.mkdtemp(),
)


def write_test_to_test_class(
unit_test_class, test_params_dict, test_instance_class, parity_table, devices):
unit_test_class, test_params_dict, test_instance_class, parity_table, devices
):
assert is_torch_nn_functional_test(test_params_dict)

assert 'cpp_options_args' in test_params_dict or 'cpp_function_call' in test_params_dict, (
assert (
"cpp_options_args" in test_params_dict
or "cpp_function_call" in test_params_dict
), (
"To enable C++ API parity test, "
"`cpp_options_args` or `cpp_function_call` entry must be present in test params dict:\n{}. \n"
"If you are interested in adding the C++ API parity test, please see:\n"
"NOTE [How to check NN module / functional API parity between Python and C++ frontends]. \n"
"If not, please add `test_cpp_api_parity=False` to the test params dict and file an issue about this."
).format(pprint.pformat(test_params_dict))
).format(
pprint.pformat(test_params_dict)
)

assert not ('cpp_options_args' in test_params_dict and 'cpp_function_call' in test_params_dict), (
assert not (
"cpp_options_args" in test_params_dict
and "cpp_function_call" in test_params_dict
), (
"Only one of `cpp_options_args` and `cpp_function_call` entries "
f"should be present in test params dict:\n{pprint.pformat(test_params_dict)}")
f"should be present in test params dict:\n{pprint.pformat(test_params_dict)}"
)

functional_name = compute_functional_name(test_params_dict)

assert hasattr(torch.nn.functional, functional_name), \
"`torch.nn.functional` doesn't have function `{}`. (Discovered while processing\n{}.)".format(
functional_name, pprint.pformat(test_params_dict))
assert hasattr(
torch.nn.functional, functional_name
), "`torch.nn.functional` doesn't have function `{}`. (Discovered while processing\n{}.)".format(
functional_name, pprint.pformat(test_params_dict)
)

functional_full_name = 'F::' + functional_name
functional_full_name = "F::" + functional_name

assert functional_full_name in parity_table['torch::nn::functional'], (
assert functional_full_name in parity_table["torch::nn::functional"], (
"Please add `{}` entry to `torch::nn::functional` section of `test/cpp_api_parity/parity-tracker.md`. "
"(Discovered while processing\n{}.)").format(functional_full_name, pprint.pformat(test_params_dict))
"(Discovered while processing\n{}.)"
).format(functional_full_name, pprint.pformat(test_params_dict))

for device in devices:
test_params = process_test_params_for_functional(
Expand All @@ -194,24 +259,37 @@ def write_test_to_test_class(
test_instance_class=test_instance_class,
)
try_remove_folder(test_params.cpp_tmp_folder)
unit_test_name = f'test_torch_nn_functional_{test_params.functional_variant_name}'
unit_test_name = (
f"test_torch_nn_functional_{test_params.functional_variant_name}"
)
unit_test_class.functional_test_params_map[unit_test_name] = test_params

def test_fn(self):
test_forward(
unit_test_class=self, test_params=unit_test_class.functional_test_params_map[self._testMethodName])
unit_test_class=self,
test_params=unit_test_class.functional_test_params_map[
self._testMethodName
],
)

test_fn = decorate_test_fn(
test_fn=test_fn,
test_cuda=test_params_dict.get('test_cuda', True),
has_impl_parity=parity_table['torch::nn::functional'][functional_full_name][0] and
test_params_dict.get('has_parity', True),
device=device)
test_cuda=test_params_dict.get("test_cuda", True),
has_impl_parity=parity_table["torch::nn::functional"][functional_full_name][
0
]
and test_params_dict.get("has_parity", True),
device=device,
)

add_test(unit_test_class, unit_test_name, test_fn)


def generate_test_cpp_sources(test_params, template):
cpp_args_construction_stmts, _ = compute_cpp_args_construction_stmts_and_forward_arg_symbols(test_params)
(
cpp_args_construction_stmts,
_,
) = compute_cpp_args_construction_stmts_and_forward_arg_symbols(test_params)

test_cpp_sources = template.substitute(
functional_variant_name=test_params.functional_variant_name,
Expand All @@ -220,19 +298,21 @@ def generate_test_cpp_sources(test_params, template):
)
return test_cpp_sources


# Build all C++ tests together, instead of once per test.
def build_cpp_tests(unit_test_class, print_cpp_source=False):
assert len(unit_test_class.functional_test_params_map) > 0
cpp_sources = TORCH_NN_COMMON_TEST_HARNESS + SAMPLE_FUNCTIONAL_CPP_SOURCE
functions = []
for test_params in unit_test_class.functional_test_params_map.values():
cpp_sources += generate_test_cpp_sources(test_params=test_params, template=TORCH_NN_FUNCTIONAL_TEST_FORWARD)
functions.append(f'{test_params.functional_variant_name}_test_forward')
cpp_sources += generate_test_cpp_sources(
test_params=test_params, template=TORCH_NN_FUNCTIONAL_TEST_FORWARD
)
functions.append(f"{test_params.functional_variant_name}_test_forward")
if print_cpp_source:
print(cpp_sources)

cpp_module = compile_cpp_code_inline(
name='functional_impl_check',
cpp_sources=cpp_sources,
functions=functions)
name="functional_impl_check", cpp_sources=cpp_sources, functions=functions
)
unit_test_class.functional_impl_check_cpp_module = cpp_module
Loading