diff --git a/.azure-pipelines/Linux-CI.yml b/.azure-pipelines/Linux-CI.yml index bef654b72dc..20d35967a4e 100644 --- a/.azure-pipelines/Linux-CI.yml +++ b/.azure-pipelines/Linux-CI.yml @@ -119,7 +119,7 @@ jobs: - script: | source venv/bin/activate - python onnx/backend/test/cmd_tools.py generate-data + python onnx/backend/test/cmd_tools.py generate-data --clean git status # Skip *output_*.pb because NumPy functions might behave differently on different platforms # Skip test_log's input.pb because it uses np.random, which might behave differently on different platforms @@ -128,6 +128,11 @@ jobs: echo "git diff for test generation returned failures. Please check updated node test files" exit 1 fi + git diff --exit-code --diff-filter=ADR -- . ':!onnx/onnx-data.proto' ':!onnx/onnx-data.proto3' + if [ $? -ne 0 ]; then + echo "Test generation returned failures. Please check the number of node test files (input_*.pb or output_*.pb)" + exit 1 + fi displayName: Test backend test data diff --git a/.azure-pipelines/MacOS-CI.yml b/.azure-pipelines/MacOS-CI.yml index 63211481dd3..9248bdca940 100644 --- a/.azure-pipelines/MacOS-CI.yml +++ b/.azure-pipelines/MacOS-CI.yml @@ -96,12 +96,17 @@ jobs: exit 1 fi - python onnx/backend/test/cmd_tools.py generate-data + python onnx/backend/test/cmd_tools.py generate-data --clean git status git diff --exit-code -- . ':!onnx/onnx-data.proto' ':!onnx/onnx-data.proto3' ':!*output_*.pb' ':!*input_*.pb' if [ $? -ne 0 ]; then echo "git diff for test generation returned failures. Please check updated node test files" exit 1 fi + git diff --exit-code --diff-filter=ADR -- . ':!onnx/onnx-data.proto' ':!onnx/onnx-data.proto3' + if [ $? -ne 0 ]; then + echo "Test generation returned failures. Please check the number of node test files (input_*.pb or output_*.pb)" + exit 1 + fi displayName: 'Run ONNX Tests' diff --git a/.azure-pipelines/Windows-CI.yml b/.azure-pipelines/Windows-CI.yml index 4aa60eda77d..6f838e70512 100644 --- a/.azure-pipelines/Windows-CI.yml +++ b/.azure-pipelines/Windows-CI.yml @@ -65,19 +65,24 @@ jobs: python onnx/gen_proto.py -l python onnx/gen_proto.py -l --ml - git diff --exit-code -- . :(exclude)onnx/onnx-data.proto :(exclude)onnx/onnx-data.proto3 + git diff --exit-code -- . :(exclude)onnx/onnx-data.proto :(exclude)onnx/onnx-data.proto3 IF NOT %ERRORLEVEL% EQU 0 ( @echo "git diff returned failures" EXIT 1 ) - python onnx/backend/test/cmd_tools.py generate-data + python onnx/backend/test/cmd_tools.py generate-data --clean git status - git diff --exit-code -- . :!onnx/onnx-data.proto :!onnx/onnx-data.proto3 :!*output_*.pb :!*input_*.pb + git diff --exit-code -- . :!onnx/onnx-data.proto :!onnx/onnx-data.proto3 :!*output_*.pb :!*input_*.pb IF NOT %ERRORLEVEL% EQU 0 ( @echo "git diff for test generation returned failures. Please check updated node test files" EXIT 1 ) + git diff --exit-code --diff-filter=ADR -- . :!onnx/onnx-data.proto :!onnx/onnx-data.proto3 + IF NOT %ERRORLEVEL% EQU 0 ( + @echo "Test generation returned failures. Please check the number of node test files (input_*.pb or output_*.pb)." + EXIT 1 + ) rm -rf .setuptools-cmake-build diff --git a/docs/AddNewOp.md b/docs/AddNewOp.md index 870f028adbc..8e3bc1f863b 100644 --- a/docs/AddNewOp.md +++ b/docs/AddNewOp.md @@ -56,7 +56,7 @@ Once the criteria of proposing new operator/function has been satisfied, you wil 3. Example: [onnx/backend/test/case/node/abs.py](/onnx/backend/test/case/node/abs.py) 5. Add at least one automatic upgrade test for your operator in [onnx/test/automatic_upgrade_test.py](/onnx/test/automatic_upgrade_test.py) using `_test_op_upgrade`. These tests create a given operator at a given opset version (usually the version the operator was introduced in) and test that the version converter is able to convert them to the highest available version. So for a new operator `_test_op_upgrade` will not test anything, but as soon as the operator gets updated in a future opset the test will automatically become nontrivial. 6. Update the documentation and generate the test data. - 1. Running [the script](/tools/update_doc.sh) + 1. Running [the script](/tools/update_doc.sh). If you have files under `onnx/backend/test/data/node` which cannot be generated by the scripts from `onnx/backend/test/case/node`, please further use `python onnx/backend/test/cmd_tools.py generate-data --clean` to cleanup the directory and only preserve needed test data. to update the doc and generate the test data. 7. Shape Inference function 1. Please provide a shape inference function in cases where it is meaningful and applicable. diff --git a/onnx/backend/test/cmd_tools.py b/onnx/backend/test/cmd_tools.py index e506a8deede..e6c0d52dc98 100644 --- a/onnx/backend/test/cmd_tools.py +++ b/onnx/backend/test/cmd_tools.py @@ -4,6 +4,7 @@ import json import os import shutil +import warnings import onnx.backend.test.case.model as model_test import onnx.backend.test.case.node as node_test @@ -19,14 +20,27 @@ def prepare_dir(path: str) -> None: shutil.rmtree(path) os.makedirs(path) + # Clean the output directory before generating data for node testcases + # It is used to check new generated data is correct in CIs + node_root = os.path.join(args.output, "node") + original_dir_number = len( + [name for name in os.listdir(node_root) if os.path.isfile(name)] + ) + if args.clean and os.path.exists(node_root): + shutil.rmtree(node_root) + os.makedirs(node_root) + cases = model_test.collect_testcases() # If op_type is specified, only include those testcases including the given operator # Otherwise, include all of the testcases cases += node_test.collect_testcases(args.op_type) + node_number = 0 for case in cases: output_dir = os.path.join(args.output, case.kind, case.name) prepare_dir(output_dir) + if case.kind == "node": + node_number += 1 if case.kind == "real": with open(os.path.join(output_dir, "data.json"), "w") as fi: json.dump( @@ -105,6 +119,15 @@ def prepare_dir(path: str) -> None: output, case.model.graph.output[j].name ).SerializeToString() ) + if not args.clean and node_number != original_dir_number: + warnings.warn( + "There are some models under 'onnx/backend/test/data/node' which cannot not" + + " be generated by the script from 'onnx/backend/test/case/node'. Please add" + + " '--clean' option for 'python onnx/backend/test/cmd_tools.py generate-data'" + + " to cleanup the existing directories and regenerate them.", + Warning, + stacklevel=2, + ) def parse_args() -> argparse.Namespace: @@ -112,7 +135,14 @@ def parse_args() -> argparse.Namespace: subparsers = parser.add_subparsers() subparser = subparsers.add_parser( - "generate-data", help="convert testcases to test data" + "generate-data", help="convert testcases to test data." + ) + subparser.add_argument( + "-c", + "--clean", + default=False, + action="store_true", + help="Clean the output directory before generating data for node testcases.", ) subparser.add_argument( "-o", diff --git a/onnx/backend/test/data/node/test_basic_convinteger/model.onnx b/onnx/backend/test/data/node/test_basic_convinteger/model.onnx deleted file mode 100644 index 023bd4a9f21..00000000000 Binary files a/onnx/backend/test/data/node/test_basic_convinteger/model.onnx and /dev/null differ diff --git a/onnx/backend/test/data/node/test_basic_convinteger/test_data_set_0/input_0.pb b/onnx/backend/test/data/node/test_basic_convinteger/test_data_set_0/input_0.pb deleted file mode 100644 index 33cea026337..00000000000 --- a/onnx/backend/test/data/node/test_basic_convinteger/test_data_set_0/input_0.pb +++ /dev/null @@ -1 +0,0 @@ -BxJ  diff --git a/onnx/backend/test/data/node/test_basic_convinteger/test_data_set_0/input_1.pb b/onnx/backend/test/data/node/test_basic_convinteger/test_data_set_0/input_1.pb deleted file mode 100644 index 53980d95daf..00000000000 --- a/onnx/backend/test/data/node/test_basic_convinteger/test_data_set_0/input_1.pb +++ /dev/null @@ -1 +0,0 @@ -BwJ \ No newline at end of file diff --git a/onnx/backend/test/data/node/test_basic_convinteger/test_data_set_0/input_2.pb b/onnx/backend/test/data/node/test_basic_convinteger/test_data_set_0/input_2.pb deleted file mode 100644 index 6741b098107..00000000000 --- a/onnx/backend/test/data/node/test_basic_convinteger/test_data_set_0/input_2.pb +++ /dev/null @@ -1 +0,0 @@ -B x_zero_pointJ \ No newline at end of file diff --git a/onnx/backend/test/data/node/test_basic_convinteger/test_data_set_0/output_0.pb b/onnx/backend/test/data/node/test_basic_convinteger/test_data_set_0/output_0.pb deleted file mode 100644 index f81e5d710a9..00000000000 Binary files a/onnx/backend/test/data/node/test_basic_convinteger/test_data_set_0/output_0.pb and /dev/null differ diff --git a/onnx/backend/test/data/node/test_convtranspose_with_kernel/model.onnx b/onnx/backend/test/data/node/test_convtranspose_with_kernel/model.onnx deleted file mode 100644 index 3fdea1f6be2..00000000000 --- a/onnx/backend/test/data/node/test_convtranspose_with_kernel/model.onnx +++ /dev/null @@ -1,28 +0,0 @@ - backend-test: -w -x -wytest" ConvTranspose* - kernel_shape@@* -output_padding@@* - output_shape@ -@* -strides@@test_convtranspose_with_kernelZ -x - - - - -Z -w - - - - -b -y - - - - - -B \ No newline at end of file diff --git a/onnx/backend/test/data/node/test_convtranspose_with_kernel/test_data_set_0/input_0.pb b/onnx/backend/test/data/node/test_convtranspose_with_kernel/test_data_set_0/input_0.pb deleted file mode 100644 index dd4c0c66ceb..00000000000 Binary files a/onnx/backend/test/data/node/test_convtranspose_with_kernel/test_data_set_0/input_0.pb and /dev/null differ diff --git a/onnx/backend/test/data/node/test_convtranspose_with_kernel/test_data_set_0/input_1.pb b/onnx/backend/test/data/node/test_convtranspose_with_kernel/test_data_set_0/input_1.pb deleted file mode 100644 index 45d0d5dd6fe..00000000000 Binary files a/onnx/backend/test/data/node/test_convtranspose_with_kernel/test_data_set_0/input_1.pb and /dev/null differ diff --git a/onnx/backend/test/data/node/test_convtranspose_with_kernel/test_data_set_0/output_0.pb b/onnx/backend/test/data/node/test_convtranspose_with_kernel/test_data_set_0/output_0.pb deleted file mode 100644 index 9fd2487d548..00000000000 Binary files a/onnx/backend/test/data/node/test_convtranspose_with_kernel/test_data_set_0/output_0.pb and /dev/null differ diff --git a/onnx/backend/test/data/node/test_depthtospace_crd_mode/model.onnx b/onnx/backend/test/data/node/test_depthtospace_crd_mode/model.onnx deleted file mode 100644 index 72060b1c14d..00000000000 --- a/onnx/backend/test/data/node/test_depthtospace_crd_mode/model.onnx +++ /dev/null @@ -1,17 +0,0 @@ - backend-test: -6 -xy" DepthToSpace* - blocksize* -mode"CRDtest_depthtospace_crd_modeZ -x - - - - -b -y - - - - -B \ No newline at end of file diff --git a/onnx/backend/test/data/node/test_depthtospace_crd_mode/test_data_set_0/input_0.pb b/onnx/backend/test/data/node/test_depthtospace_crd_mode/test_data_set_0/input_0.pb deleted file mode 100644 index 660603548a0..00000000000 Binary files a/onnx/backend/test/data/node/test_depthtospace_crd_mode/test_data_set_0/input_0.pb and /dev/null differ diff --git a/onnx/backend/test/data/node/test_depthtospace_crd_mode/test_data_set_0/output_0.pb b/onnx/backend/test/data/node/test_depthtospace_crd_mode/test_data_set_0/output_0.pb deleted file mode 100644 index f1a31fb5bc4..00000000000 Binary files a/onnx/backend/test/data/node/test_depthtospace_crd_mode/test_data_set_0/output_0.pb and /dev/null differ diff --git a/onnx/backend/test/data/node/test_depthtospace_dcr_mode/model.onnx b/onnx/backend/test/data/node/test_depthtospace_dcr_mode/model.onnx deleted file mode 100644 index 483c41297bc..00000000000 --- a/onnx/backend/test/data/node/test_depthtospace_dcr_mode/model.onnx +++ /dev/null @@ -1,17 +0,0 @@ - backend-test: -6 -xy" DepthToSpace* - blocksize* -mode"DCRtest_depthtospace_dcr_modeZ -x - - - - -b -y - - - - -B \ No newline at end of file diff --git a/onnx/backend/test/data/node/test_depthtospace_dcr_mode/test_data_set_0/input_0.pb b/onnx/backend/test/data/node/test_depthtospace_dcr_mode/test_data_set_0/input_0.pb deleted file mode 100644 index 660603548a0..00000000000 Binary files a/onnx/backend/test/data/node/test_depthtospace_dcr_mode/test_data_set_0/input_0.pb and /dev/null differ diff --git a/onnx/backend/test/data/node/test_depthtospace_dcr_mode/test_data_set_0/output_0.pb b/onnx/backend/test/data/node/test_depthtospace_dcr_mode/test_data_set_0/output_0.pb deleted file mode 100644 index 8721c7484d2..00000000000 Binary files a/onnx/backend/test/data/node/test_depthtospace_dcr_mode/test_data_set_0/output_0.pb and /dev/null differ diff --git a/onnx/backend/test/data/node/test_pow_types_float/model.onnx b/onnx/backend/test/data/node/test_pow_types_float/model.onnx deleted file mode 100644 index c0fd50393cd..00000000000 --- a/onnx/backend/test/data/node/test_pow_types_float/model.onnx +++ /dev/null @@ -1,16 +0,0 @@ - backend-test:Y - -x -yz"Powtest_pow_types_floatZ -x - - -Z -y - - -b -z - - -B \ No newline at end of file diff --git a/onnx/backend/test/data/node/test_pow_types_float/test_data_set_0/input_0.pb b/onnx/backend/test/data/node/test_pow_types_float/test_data_set_0/input_0.pb deleted file mode 100644 index d91963f15c8..00000000000 Binary files a/onnx/backend/test/data/node/test_pow_types_float/test_data_set_0/input_0.pb and /dev/null differ diff --git a/onnx/backend/test/data/node/test_pow_types_float/test_data_set_0/input_1.pb b/onnx/backend/test/data/node/test_pow_types_float/test_data_set_0/input_1.pb deleted file mode 100644 index a760307aaa9..00000000000 Binary files a/onnx/backend/test/data/node/test_pow_types_float/test_data_set_0/input_1.pb and /dev/null differ diff --git a/onnx/backend/test/data/node/test_pow_types_float/test_data_set_0/output_0.pb b/onnx/backend/test/data/node/test_pow_types_float/test_data_set_0/output_0.pb deleted file mode 100644 index c77d2103a13..00000000000 Binary files a/onnx/backend/test/data/node/test_pow_types_float/test_data_set_0/output_0.pb and /dev/null differ diff --git a/onnx/backend/test/data/node/test_pow_types_int/model.onnx b/onnx/backend/test/data/node/test_pow_types_int/model.onnx deleted file mode 100644 index 2e62ba5ad4f..00000000000 --- a/onnx/backend/test/data/node/test_pow_types_int/model.onnx +++ /dev/null @@ -1,16 +0,0 @@ - backend-test:W - -x -yz"Powtest_pow_types_intZ -x - - -Z -y - - -b -z - - -B \ No newline at end of file diff --git a/onnx/backend/test/data/node/test_pow_types_int/test_data_set_0/input_0.pb b/onnx/backend/test/data/node/test_pow_types_int/test_data_set_0/input_0.pb deleted file mode 100644 index 62e4e87e30c..00000000000 Binary files a/onnx/backend/test/data/node/test_pow_types_int/test_data_set_0/input_0.pb and /dev/null differ diff --git a/onnx/backend/test/data/node/test_pow_types_int/test_data_set_0/input_1.pb b/onnx/backend/test/data/node/test_pow_types_int/test_data_set_0/input_1.pb deleted file mode 100644 index 2a776165494..00000000000 Binary files a/onnx/backend/test/data/node/test_pow_types_int/test_data_set_0/input_1.pb and /dev/null differ diff --git a/onnx/backend/test/data/node/test_pow_types_int/test_data_set_0/output_0.pb b/onnx/backend/test/data/node/test_pow_types_int/test_data_set_0/output_0.pb deleted file mode 100644 index 0cc39708ca5..00000000000 Binary files a/onnx/backend/test/data/node/test_pow_types_int/test_data_set_0/output_0.pb and /dev/null differ diff --git a/onnx/backend/test/data/node/test_reduce_log_sum/model.onnx b/onnx/backend/test/data/node/test_reduce_log_sum/model.onnx deleted file mode 100644 index 2d4f60fce29..00000000000 Binary files a/onnx/backend/test/data/node/test_reduce_log_sum/model.onnx and /dev/null differ diff --git a/onnx/backend/test/data/node/test_reduce_log_sum/test_data_set_0/input_0.pb b/onnx/backend/test/data/node/test_reduce_log_sum/test_data_set_0/input_0.pb deleted file mode 100644 index 913f023fdcb..00000000000 --- a/onnx/backend/test/data/node/test_reduce_log_sum/test_data_set_0/input_0.pb +++ /dev/null @@ -1 +0,0 @@ -BdataJ  ?7?N?w} ?H>QY%?n >~J?e?^k?l?Z{=p= <&U?H5G?^?z?L?G>G?9=#?4>q?ڗ?N>s>.4F?>?<\?N?c?yq?Ƌ.?k>>2?v=9*?+?nW>A>>L8>j?a>}?=U>R.%>2'?p>I>Jz> \ No newline at end of file diff --git a/onnx/backend/test/data/node/test_reduce_log_sum/test_data_set_0/output_0.pb b/onnx/backend/test/data/node/test_reduce_log_sum/test_data_set_0/output_0.pb deleted file mode 100644 index b1470001e95..00000000000 --- a/onnx/backend/test/data/node/test_reduce_log_sum/test_data_set_0/output_0.pb +++ /dev/null @@ -1 +0,0 @@ -BreducedJ ! @F@@ \ No newline at end of file diff --git a/onnx/backend/test/data/node/test_reduce_sum_empty_axes_input_noop_random/model.onnx b/onnx/backend/test/data/node/test_reduce_sum_empty_axes_input_noop_random/model.onnx deleted file mode 100644 index e69caaee69d..00000000000 Binary files a/onnx/backend/test/data/node/test_reduce_sum_empty_axes_input_noop_random/model.onnx and /dev/null differ diff --git a/onnx/backend/test/data/node/test_reduce_sum_empty_axes_input_noop_random/test_data_set_0/input_0.pb b/onnx/backend/test/data/node/test_reduce_sum_empty_axes_input_noop_random/test_data_set_0/input_0.pb deleted file mode 100644 index 71549335933..00000000000 --- a/onnx/backend/test/data/node/test_reduce_sum_empty_axes_input_noop_random/test_data_set_0/input_0.pb +++ /dev/null @@ -1 +0,0 @@ -BdataJ0y?@@Ie?qÿ:@Ɵ@A_A1;@&? \ No newline at end of file diff --git a/onnx/backend/test/data/node/test_reduce_sum_empty_axes_input_noop_random/test_data_set_0/input_1.pb b/onnx/backend/test/data/node/test_reduce_sum_empty_axes_input_noop_random/test_data_set_0/input_1.pb deleted file mode 100644 index 2f3f037e956..00000000000 Binary files a/onnx/backend/test/data/node/test_reduce_sum_empty_axes_input_noop_random/test_data_set_0/input_1.pb and /dev/null differ diff --git a/onnx/backend/test/data/node/test_reduce_sum_empty_axes_input_noop_random/test_data_set_0/output_0.pb b/onnx/backend/test/data/node/test_reduce_sum_empty_axes_input_noop_random/test_data_set_0/output_0.pb deleted file mode 100644 index 63d96ff6956..00000000000 --- a/onnx/backend/test/data/node/test_reduce_sum_empty_axes_input_noop_random/test_data_set_0/output_0.pb +++ /dev/null @@ -1 +0,0 @@ -BreducedJ0y?@@Ie?qÿ:@Ɵ@A_A1;@&? \ No newline at end of file diff --git a/onnx/backend/test/data/node/test_unsqueeze_axis_3/model.onnx b/onnx/backend/test/data/node/test_unsqueeze_axis_3/model.onnx deleted file mode 100644 index 13437cae7c8..00000000000 --- a/onnx/backend/test/data/node/test_unsqueeze_axis_3/model.onnx +++ /dev/null @@ -1,15 +0,0 @@ - backend-test:m - -xy" Unsqueeze* -axes@test_unsqueeze_axis_3Z -x - - - -b -y - - - - -B \ No newline at end of file diff --git a/onnx/backend/test/data/node/test_unsqueeze_axis_3/test_data_set_0/input_0.pb b/onnx/backend/test/data/node/test_unsqueeze_axis_3/test_data_set_0/input_0.pb deleted file mode 100644 index bae0ffd6324..00000000000 --- a/onnx/backend/test/data/node/test_unsqueeze_axis_3/test_data_set_0/input_0.pb +++ /dev/null @@ -1 +0,0 @@ -BxJx?h>z?j@$ ?.z8s?bhdӽ9>(>%?^B?0= B>]ת>=?RiJ>Z/d#S'?K]?=C@(Hm;= ?2??>>Ec! >*z??Oƾmǚ6&õgڿ?xFKྙ[ G?4οYL=e> kQN>.:=ݚ>b"6 \ No newline at end of file diff --git a/onnx/backend/test/data/node/test_unsqueeze_axis_3/test_data_set_0/output_0.pb b/onnx/backend/test/data/node/test_unsqueeze_axis_3/test_data_set_0/output_0.pb deleted file mode 100644 index 5b5348d4f0f..00000000000 --- a/onnx/backend/test/data/node/test_unsqueeze_axis_3/test_data_set_0/output_0.pb +++ /dev/null @@ -1 +0,0 @@ -ByJx?h>z?j@$ ?.z8s?bhdӽ9>(>%?^B?0= B>]ת>=?RiJ>Z/d#S'?K]?=C@(Hm;= ?2??>>Ec! >*z??Oƾmǚ6&õgڿ?xFKྙ[ G?4οYL=e> kQN>.:=ݚ>b"6 \ No newline at end of file