Disable unstable tests

onnxscript/backend/onnx_export_test.py

@@ -99,6 +99,8 @@ def skip(pattern: str | Pattern, reason: str, *, condition: bool = True):
             "^test_resize_upsample_scales_linear_half_pixel_symmetric",
             "cannot import module, import_module does not work",
         ),
+        # tests are too unstable on Windows, not always the same ones are failing.
+        skip("test_", "cannot import module"),
     )
 
 

tests/function_libs/torch_lib/ops_test_data.py

@@ -39,6 +39,7 @@
 import copy
 import dataclasses
 import functools
+import sys
 from typing import Any, Callable, Collection, Optional
 
 import numpy as np
@@ -726,7 +727,10 @@ def _where_input_wrangler(
     # TorchLibOpInfo("copy", core_ops.aten_copy),  # copy is not in OPS_DB
     TorchLibOpInfo("cos", core_ops.aten_cos),
     TorchLibOpInfo("cosh", core_ops.aten_cosh),
-    TorchLibOpInfo("cross", core_ops.aten_cross, tolerance={torch.float16: (6e-3, 3e-3)}),
+    TorchLibOpInfo("cross", core_ops.aten_cross, tolerance={torch.float16: (6e-2, 2e-1)}).skip(
+        dtypes=(torch.float16 if sys.platform != "linux" else torch.complex64,),
+        reason="fixme: test is failing on windows and torch nightly",
+    ),
     TorchLibOpInfo("deg2rad", core_ops.aten_deg2rad),
     # TorchLibOpInfo("detach", core_ops.aten_detach),  # detach is not in OP-TEST-DB
     TorchLibOpInfo("diagonal", core_ops.aten_diagonal),
@@ -797,6 +801,9 @@ def _where_input_wrangler(
     TorchLibOpInfo(
         "full_like",
         core_ops.aten_full_like,
+    ).skip(
+        enabled_if=ops_test_common.IS_MACOS,
+        reason="fixme: memory allocation issue on CI",
     ),
     TorchLibOpInfo("gather", core_ops.aten_gather).skip(
         matcher=lambda sample: sample.input.numel() == 0 or sample.args[1].numel() == 0,
@@ -1026,8 +1033,11 @@ def _where_input_wrangler(
     TorchLibOpInfo(
         "ops.aten.embedding_bag",
         core_ops.aten_embedding_bag,
-        tolerance={torch.float16: (1e-2, 5e-2)},
+        tolerance={torch.float32: (1e-4, 5e-4)},
         compare_shape_only_for_output=(1, 2, 3),
+    ).skip(
+        dtypes=(torch.float16,),
+        reason="fixme: results mismatch in torch nightly.",
     ),
     TorchLibOpInfo(
         "ops.aten.embedding_bag.padding_idx",
@@ -1584,9 +1594,18 @@ def _where_input_wrangler(
         "ops.aten.layer_norm",
         core_ops.aten_layer_norm,
         tolerance={torch.float32: (3.7e-5, 1.8e-4)},
-    ).xfail(
+    )
+    .xfail(
         dtypes=(torch.int64,),
         reason="fixme: ORT `LayerNormKernelImpl` not implemented for int64",
+    )
+    .skip(
+        matcher=lambda sample: sample.input.shape[-1] <= 1,
+        reason="fixme: onnxruntime fail when no reduction is needed",
+    )
+    .skip(
+        dtypes=(torch.float32 if sys.platform != "linux" else torch.complex64,),
+        reason="fixme: test is unstable on macosx, windows",
     ),
     TorchLibOpInfo("logit", core_ops.aten_logit, tolerance={torch.float16: (1e-1, 7e-4)}),
     TorchLibOpInfo("max_dim", core_ops.aten_max_dim)
@@ -1694,10 +1713,10 @@ def _where_input_wrangler(
         core_ops.aten_native_layer_norm,
         tolerance={torch.float32: (3.7e-5, 1.8e-4), torch.float16: (1e-1, 7e-4)},
     )
-    .xfail(
+    .skip(
         dtypes=(torch.float32,),
-        matcher=lambda sample: len(sample.input.shape) == 1,
-        enabled_if=ops_test_common.IS_MACOS,
+        matcher=lambda sample: sample.input.shape[-1] <= 1,
+        # enabled_if=ops_test_common.IS_MACOS,
         reason="fixme: result mismatch. https://github.com/microsoft/onnxruntime/issues/20676",
     )
     .skip(