diff --git a/backends/vulkan/test/test_vulkan_delegate.py b/backends/vulkan/test/test_vulkan_delegate.py index 7c9f31b720c..ff709618259 100644 --- a/backends/vulkan/test/test_vulkan_delegate.py +++ b/backends/vulkan/test/test_vulkan_delegate.py @@ -7,6 +7,7 @@ # pyre-unsafe import ctypes +import functools import unittest from typing import Tuple @@ -42,6 +43,24 @@ pass +def disable_test(reason): + """Disable a test while still reporting it as executed. + + Some test runners do not handle skipped results consistently, so this keeps + disabled tests visible in logs without using unittest.skip. + """ + + def decorator(fn): + @functools.wraps(fn) + def wrapper(*args, **kwargs): + print(f"DISABLED_TEST: {fn.__qualname__}: {reason}") + return None + + return wrapper + + return decorator + + def lower_module( model: torch.nn.Module, sample_inputs: Tuple[torch.Tensor], dynamic_shapes=None ) -> EdgeProgramManager: @@ -743,7 +762,7 @@ def forward(self, x): self.lower_module_and_test_output(model, sample_inputs) - @unittest.skip( + @disable_test( "Currently this test is failing due to weird partitioning because the eq scalar" "operator is not supported yet. Re-enable when the operator is supported." ) @@ -810,7 +829,7 @@ def forward(self, x): self.lower_module_and_test_output(module, sample_inputs) - @unittest.skip( + @disable_test( "Reduce shader does not support multiple reduction axes at the moment" ) def test_vulkan_backend_sum_dim_list(self): @@ -831,7 +850,7 @@ def forward(self, x): sample_inputs, ) - @unittest.skip( + @disable_test( "Reduce shader does not support multiple reduction axes at the moment" ) def test_vulkan_backend_sum(self): @@ -1028,7 +1047,7 @@ def forward(self, x): sample_inputs, ) - @unittest.skip("layer norm compute shader not working with swiftshader") + @disable_test("layer norm compute shader not working with swiftshader") def test_vulkan_backend_native_layer_norm(self): class NativeLayerNormModule(torch.nn.Module): def __init__(self): @@ -1459,7 +1478,7 @@ def forward(self, x): sample_inputs, ) - @unittest.skip( + @disable_test( "Softmax shader with shared memory does not work with swiftshader due to potential swiftshader bug" ) def test_vulkan_backend_softmax(self): @@ -1480,7 +1499,7 @@ def forward(self, x): sample_inputs, ) - @unittest.skip( + @disable_test( "Softmax shader with shared memory does not work with swiftshader due to potential swiftshader bug" ) def test_vulkan_backend_logsoftmax(self): @@ -1512,7 +1531,7 @@ def forward(self, x): self.lower_unary_module_and_test_output(GeluModule()) - @unittest.skip( + @disable_test( "Reduce shader does not support multiple reduction axes at the moment" ) def test_vulkan_backend_mean(self): @@ -2364,7 +2383,7 @@ def apply_quantization(self): quantized_linear_module_gemm, sample_inputs_gemm, atol=1e-2, rtol=1e-2 ) - @unittest.skip("Cannot run on swiftshader due to no integer dot product support") + @disable_test("Cannot run on swiftshader due to no integer dot product support") def test_vulkan_backend_xnnpack_pt2e_quantized_linear_sequence(self): """ Test a sequence of linear layers quantized with XNNPACK quantization config. @@ -2439,7 +2458,7 @@ def forward(self, x): rtol=1e-1, ) - @unittest.skip("Cannot run on swiftshader due to no integer dot product support") + @disable_test("Cannot run on swiftshader due to no integer dot product support") def test_vulkan_backend_xnnpack_pt2e_quantized_conv_sequence(self): """ Test a sequence of convolution layers quantized with PT2E quantization. @@ -2530,7 +2549,7 @@ def forward(self, x): rtol=1e-1, ) - @unittest.skip("Cannot run on swiftshader due to no integer dot product support") + @disable_test("Cannot run on swiftshader due to no integer dot product support") def test_vulkan_backend_xnnpack_pt2e_quantized_conv_sequence_all_reduced(self): """ Test a sequence of convolution layers quantized with PT2E quantization. @@ -2610,7 +2629,7 @@ def forward(self, x): rtol=1e-1, ) - @unittest.skip("Cannot run on swiftshader due to no 8-bit int support") + @disable_test("Cannot run on swiftshader due to no 8-bit int support") def test_vulkan_backend_torchao_8da4w_quantized_linear(self): """ Test TorchAO 8da4w quantization (int8 dynamic activation + int4 weight) with Vulkan backend. diff --git a/devtools/inspector/tests/inspector_test.py b/devtools/inspector/tests/inspector_test.py index b33c5b37164..4c59190650c 100644 --- a/devtools/inspector/tests/inspector_test.py +++ b/devtools/inspector/tests/inspector_test.py @@ -7,6 +7,7 @@ # pyre-unsafe import copy +import functools import os import random import statistics @@ -90,6 +91,28 @@ def forward(self, indices: torch.Tensor, values: torch.Tensor) -> torch.Tensor: ETRECORD_PATH = "unittest_etrecord_path" +def disable_if(condition, reason): + """Disable a test when condition is true, still reporting it as executed. + + Conditional analogue of unittest.skipIf that keeps disabled tests visible in + logs instead of producing a skipped result, which some test runners handle + inconsistently. + """ + + def decorator(fn): + if not condition: + return fn + + @functools.wraps(fn) + def wrapper(*args, **kwargs): + print(f"DISABLED_TEST: {fn.__qualname__}: {reason}") + return None + + return wrapper + + return decorator + + # TODO: write an E2E test: create an inspector instance, mock just the file reads, and then verify the external correctness class TestInspector(unittest.TestCase): def test_perf_data(self) -> None: @@ -1504,7 +1527,7 @@ def test_calculate_numeric_gap_with_edge_dialect_exported_program_name(self): self.assertIsInstance(df, pd.DataFrame) self.assertEqual(len(df), 1) - @unittest.skipIf(sys.platform.startswith("win"), "Skipping on Windows") + @disable_if(sys.platform.startswith("win"), "Skipping on Windows") def test_transformer_block_xnnpack_numeric_gap_within_tolerance(self): """ Test that the numeric gap between AOT and runtime intermediate outputs @@ -1693,7 +1716,7 @@ def forward( f"Stack trace for {op_name} doesn't contain file info", ) - @unittest.skipIf(sys.platform.startswith("win"), "Skipping on Windows") + @disable_if(sys.platform.startswith("win"), "Skipping on Windows") def test_intermediate_tensor_comparison_with_torch_export(self): """Test intermediate tensor comparison using torch.export.export and to_edge_transform_and_lower. @@ -1840,7 +1863,7 @@ def _gen_random_runtime_output( ) -> List[Union[None, List[torch.Tensor], bool, float, int, str, torch.Tensor]]: return [torch.randn(RAW_DATA_SIZE)] - @unittest.skipIf(sys.platform.startswith("win"), "Skipping on Windows") + @disable_if(sys.platform.startswith("win"), "Skipping on Windows") def test_disable_debug_handle_validation_with_symbolic_shapes(self): """ Test that demonstrates the issue with symbolic shape related nodes losing from_node info