diff --git a/iree_tests/configs/onnx_cpu_llvm_sync.json b/iree_tests/configs/onnx_cpu_llvm_sync.json index 95544c8e1..957bb8470 100644 --- a/iree_tests/configs/onnx_cpu_llvm_sync.json +++ b/iree_tests/configs/onnx_cpu_llvm_sync.json @@ -8,6 +8,7 @@ ], "skip_compile_tests": [ "onnx/node/generated/test_dequantizelinear", + "onnx/node/generated/test_einsum_inner_prod", "onnx/node/generated/test_slice_default_axes" ], "skip_run_tests": [], @@ -29,15 +30,9 @@ "onnx/node/generated/test_ai_onnx_ml_label_encoder_string_int", "onnx/node/generated/test_ai_onnx_ml_label_encoder_string_int_no_default", "onnx/node/generated/test_ai_onnx_ml_label_encoder_tensor_value_only_mapping", - "onnx/node/generated/test_averagepool_2d_dilations", "onnx/node/generated/test_averagepool_2d_precomputed_same_upper", "onnx/node/generated/test_averagepool_2d_same_lower", "onnx/node/generated/test_averagepool_2d_same_upper", - "onnx/node/generated/test_averagepool_3d_dilations_large_count_include_pad_is_0_ceil_mode_is_False", - "onnx/node/generated/test_averagepool_3d_dilations_large_count_include_pad_is_0_ceil_mode_is_True", - "onnx/node/generated/test_averagepool_3d_dilations_large_count_include_pad_is_1_ceil_mode_is_False", - "onnx/node/generated/test_averagepool_3d_dilations_large_count_include_pad_is_1_ceil_mode_is_True", - "onnx/node/generated/test_averagepool_3d_dilations_small", "onnx/node/generated/test_basic_deform_conv_with_padding", "onnx/node/generated/test_basic_deform_conv_without_padding", "onnx/node/generated/test_batchnorm_epsilon_training_mode", @@ -143,7 +138,6 @@ "onnx/node/generated/test_edge_pad", "onnx/node/generated/test_einsum_batch_diagonal", "onnx/node/generated/test_einsum_batch_matmul", - "onnx/node/generated/test_einsum_inner_prod", "onnx/node/generated/test_einsum_sum", "onnx/node/generated/test_equal_string", "onnx/node/generated/test_equal_string_broadcast", @@ -183,21 +177,14 @@ "onnx/node/generated/test_image_decoder_decode_tiff_rgb", "onnx/node/generated/test_image_decoder_decode_webp_rgb", "onnx/node/generated/test_loop11", - "onnx/node/generated/test_lppool_1d_default", - "onnx/node/generated/test_lppool_2d_default", "onnx/node/generated/test_lppool_2d_dilations", - "onnx/node/generated/test_lppool_2d_pads", "onnx/node/generated/test_lppool_2d_same_lower", "onnx/node/generated/test_lppool_2d_same_upper", - "onnx/node/generated/test_lppool_2d_strides", - "onnx/node/generated/test_lppool_3d_default", - "onnx/node/generated/test_lrn", - "onnx/node/generated/test_lrn_default", "onnx/node/generated/test_lstm_batchwise", "onnx/node/generated/test_lstm_defaults", "onnx/node/generated/test_lstm_with_initial_bias", "onnx/node/generated/test_lstm_with_peepholes", - "onnx/node/generated/test_maxpool_1d_default", + "onnx/node/generated/test_maxpool_2d_ceil_output_size_reduce_by_one", "onnx/node/generated/test_maxpool_2d_precomputed_same_upper", "onnx/node/generated/test_maxpool_2d_same_lower", "onnx/node/generated/test_maxpool_2d_same_upper", @@ -213,7 +200,6 @@ "onnx/node/generated/test_momentum_multiple", "onnx/node/generated/test_mvn", "onnx/node/generated/test_nesterov_momentum", - "onnx/node/generated/test_nllloss_NC", "onnx/node/generated/test_nllloss_NCd1", "onnx/node/generated/test_nllloss_NCd1_ii", "onnx/node/generated/test_nllloss_NCd1_mean_weight_negative_ii", @@ -243,11 +229,6 @@ "onnx/node/generated/test_nonzero_example", "onnx/node/generated/test_onehot_with_axis", "onnx/node/generated/test_onehot_with_negative_axis", - "onnx/node/generated/test_optional_get_element_tensor", - "onnx/node/generated/test_optional_has_element_empty_no_input_name_optional_input", - "onnx/node/generated/test_optional_has_element_empty_no_input_name_tensor_input", - "onnx/node/generated/test_optional_has_element_empty_no_input_optional_input", - "onnx/node/generated/test_optional_has_element_empty_no_input_tensor_input", "onnx/node/generated/test_pow_types_int32_float32", "onnx/node/generated/test_pow_types_int32_int32", "onnx/node/generated/test_pow_types_int64_float32", @@ -387,8 +368,6 @@ "onnx/node/generated/test_resize_upsample_sizes_nearest_axes_3_2", "onnx/node/generated/test_resize_upsample_sizes_nearest_floor_align_corners", "onnx/node/generated/test_resize_upsample_sizes_nearest_not_larger", - "onnx/node/generated/test_reversesequence_batch", - "onnx/node/generated/test_reversesequence_time", "onnx/node/generated/test_rnn_seq_length", "onnx/node/generated/test_roialign_aligned_false", "onnx/node/generated/test_roialign_aligned_true", @@ -427,8 +406,6 @@ "onnx/node/generated/test_sce_mean_3d_expanded", "onnx/node/generated/test_sce_mean_3d_log_prob", "onnx/node/generated/test_sce_mean_3d_log_prob_expanded", - "onnx/node/generated/test_sce_mean_expanded", - "onnx/node/generated/test_sce_mean_log_prob_expanded", "onnx/node/generated/test_sce_mean_no_weight_ii_3d", "onnx/node/generated/test_sce_mean_no_weight_ii_3d_expanded", "onnx/node/generated/test_sce_mean_no_weight_ii_3d_log_prob", @@ -437,8 +414,6 @@ "onnx/node/generated/test_sce_mean_no_weight_ii_4d_expanded", "onnx/node/generated/test_sce_mean_no_weight_ii_4d_log_prob", "onnx/node/generated/test_sce_mean_no_weight_ii_4d_log_prob_expanded", - "onnx/node/generated/test_sce_mean_no_weight_ii_expanded", - "onnx/node/generated/test_sce_mean_no_weight_ii_log_prob_expanded", "onnx/node/generated/test_sce_mean_weight", "onnx/node/generated/test_sce_mean_weight_expanded", "onnx/node/generated/test_sce_mean_weight_ii", @@ -455,14 +430,10 @@ "onnx/node/generated/test_sce_mean_weight_ii_log_prob_expanded", "onnx/node/generated/test_sce_mean_weight_log_prob", "onnx/node/generated/test_sce_mean_weight_log_prob_expanded", - "onnx/node/generated/test_sce_none_expanded", - "onnx/node/generated/test_sce_none_log_prob_expanded", "onnx/node/generated/test_sce_none_weights", "onnx/node/generated/test_sce_none_weights_expanded", "onnx/node/generated/test_sce_none_weights_log_prob", "onnx/node/generated/test_sce_none_weights_log_prob_expanded", - "onnx/node/generated/test_sce_sum_expanded", - "onnx/node/generated/test_sce_sum_log_prob_expanded", "onnx/node/generated/test_shrink_hard", "onnx/node/generated/test_shrink_soft", "onnx/node/generated/test_simple_rnn_batchwise", @@ -529,6 +500,7 @@ ], "expected_run_failures": [ "onnx/node/generated/test_asin", + "onnx/node/generated/test_averagepool_3d_dilations_large_count_include_pad_is_0_ceil_mode_is_True", "onnx/node/generated/test_bernoulli", "onnx/node/generated/test_bernoulli_double", "onnx/node/generated/test_bernoulli_double_expanded", @@ -565,12 +537,7 @@ "onnx/node/generated/test_gridsample_nearest", "onnx/node/generated/test_gridsample_nearest_align_corners_0_additional_1", "onnx/node/generated/test_gridsample_nearest_align_corners_1_additional_1", - "onnx/node/generated/test_hardsigmoid", - "onnx/node/generated/test_hardsigmoid_default", - "onnx/node/generated/test_hardsigmoid_example", - "onnx/node/generated/test_hardswish_expanded", "onnx/node/generated/test_max_float64", - "onnx/node/generated/test_maxpool_2d_ceil_output_size_reduce_by_one", "onnx/node/generated/test_min_float64", "onnx/node/generated/test_mod_mixed_sign_int16", "onnx/node/generated/test_mod_mixed_sign_int32", @@ -596,7 +563,9 @@ "onnx/node/generated/test_resize_downsample_scales_linear_align_corners", "onnx/node/generated/test_scatter_elements_with_negative_indices", "onnx/node/generated/test_sce_mean_no_weight_ii", + "onnx/node/generated/test_sce_mean_no_weight_ii_expanded", "onnx/node/generated/test_sce_mean_no_weight_ii_log_prob", + "onnx/node/generated/test_sce_mean_no_weight_ii_log_prob_expanded", "onnx/node/generated/test_shape_end_1", "onnx/node/generated/test_shape_start_1", "onnx/node/generated/test_shape_start_1_end_2", diff --git a/iree_tests/conftest.py b/iree_tests/conftest.py index c42c25f6a..2561553d4 100644 --- a/iree_tests/conftest.py +++ b/iree_tests/conftest.py @@ -303,8 +303,11 @@ def __init__(self, spec, **kwargs): super().__init__(**kwargs) self.spec = spec + relative_test_directory = self.spec.test_directory.relative_to( + IREE_TESTS_ROOT + ).as_posix() self.user_properties.append( - ("test_directory_name", self.spec.test_directory.name) + ("relative_test_directory_name", relative_test_directory) ) self.user_properties.append(("input_mlir_name", self.spec.input_mlir_name)) self.user_properties.append(("test_name", self.spec.test_name)) @@ -389,7 +392,9 @@ def test_compile(self): def test_run(self): mlir_name = self.spec.input_mlir_name vmfb_name = f"{self.spec.input_mlir_stem}_{self.spec.test_name}.vmfb" - compile_cmd = get_compile_cmd(mlir_name, vmfb_name, self.spec.iree_compile_flags) + compile_cmd = get_compile_cmd( + mlir_name, vmfb_name, self.spec.iree_compile_flags + ) iree_run_module(vmfb_name, self.run_args, self.test_cwd, compile_cmd) def repr_failure(self, excinfo): @@ -411,5 +416,6 @@ def reportinfo(self): def _initrequest(self): pass + class IreeXFailCompileRunException(Exception): pass diff --git a/iree_tests/update_config_xfails.py b/iree_tests/update_config_xfails.py index ca9dd89f8..8263266bb 100644 --- a/iree_tests/update_config_xfails.py +++ b/iree_tests/update_config_xfails.py @@ -85,14 +85,16 @@ def check_field(field_name): # TODO(scotttodd): handle multiple config files writing to one log file? - # Find the test directory name, since that is what our config.json uses - # to label tests for skipping or XFAIL. - test_directory_name = "" + # Find the test directory path relative to the iree_tests root, since + # that is what our config.json uses to label tests for skipping or XFAIL. + relative_test_directory_name = "" for user_property in user_properties: - if user_property[0] == "test_directory_name": - test_directory_name = user_property[1] - if not test_directory_name: - logger.warning("Missing 'test_directory_name' property, ignoring log line") + if user_property[0] == "relative_test_directory_name": + relative_test_directory_name = user_property[1] + if not relative_test_directory_name: + logger.warning( + "Missing 'relative_test_directory_name' property, ignoring log line" + ) continue # If the test failed, it should have a "repr" from repr_failure(). @@ -102,20 +104,26 @@ def check_field(field_name): continue if "Error invoking iree-compile" in repr: - logger.debug(f"test {test_directory_name} failed to compile") - compile_failures.append(test_directory_name) + logger.debug(f"test {relative_test_directory_name} failed to compile") + compile_failures.append(relative_test_directory_name) elif "Error invoking iree-run-module" in repr: - logger.debug(f"test {test_directory_name} failed to run") - run_failures.append(test_directory_name) + logger.debug(f"test {relative_test_directory_name} failed to run") + run_failures.append(relative_test_directory_name) elif "remove from 'expected_compile_failures'" in repr: - logger.debug(f"test {test_directory_name} compiled and ran successfully") + logger.debug( + f"test {relative_test_directory_name} compiled and ran successfully" + ) elif "remove from 'expected_run_failures'" in repr: - logger.debug(f"test {test_directory_name} ran successfully") + logger.debug(f"test {relative_test_directory_name} ran successfully") elif "move to 'expected_run_failures'" in repr: - logger.debug(f"test {test_directory_name} compile xfail -> run fail") - run_failures.append(test_directory_name) + logger.debug( + f"test {relative_test_directory_name} compile xfail -> run fail" + ) + run_failures.append(relative_test_directory_name) else: - logger.warning(f"Unhandled error for {test_directory_name}: '{repr}'") + logger.warning( + f"Unhandled error for {relative_test_directory_name}: '{repr}'" + ) logger.info(f"Updating config") # Remove duplicates and sort.