From a9d68795209cdbbe7b06c89cf991fa57202bc385 Mon Sep 17 00:00:00 2001 From: Scott Todd Date: Fri, 12 Jul 2024 16:05:07 -0700 Subject: [PATCH] Bump test suite to include punet tests and fix xfail script. --- .github/workflows/pkgci_regression_test.yml | 6 +++--- .../pkgci/external_test_suite/models_gpu_rocm_gfx90a.json | 8 +++++--- .../pkgci/external_test_suite/models_gpu_rocm_gfx942.json | 6 ++++-- .../pkgci/external_test_suite/models_gpu_vulkan.json | 4 ++-- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pkgci_regression_test.yml b/.github/workflows/pkgci_regression_test.yml index 1a61c7677371..d2d7c0fe5075 100644 --- a/.github/workflows/pkgci_regression_test.yml +++ b/.github/workflows/pkgci_regression_test.yml @@ -90,7 +90,7 @@ jobs: uses: actions/checkout@v4.1.7 with: repository: nod-ai/SHARK-TestSuite - ref: a06e730ce325c12db40bb89b43e8e6e897052e96 + ref: 1da89a12a8b00dc77506d702f61300803b6240b7 path: SHARK-TestSuite submodules: false lfs: false @@ -198,7 +198,7 @@ jobs: uses: actions/checkout@v4.1.7 with: repository: nod-ai/SHARK-TestSuite - ref: a06e730ce325c12db40bb89b43e8e6e897052e96 + ref: 1da89a12a8b00dc77506d702f61300803b6240b7 path: SHARK-TestSuite submodules: false lfs: true @@ -220,7 +220,7 @@ jobs: pytest \ SHARK-TestSuite/iree_tests/pytorch/models \ SHARK-TestSuite/iree_tests/sharktank \ - -rpfE \ + -rA \ -k real_weights \ --no-skip-tests-missing-files \ --capture=no \ diff --git a/build_tools/pkgci/external_test_suite/models_gpu_rocm_gfx90a.json b/build_tools/pkgci/external_test_suite/models_gpu_rocm_gfx90a.json index 8b7f59c85926..86f9a7a80b27 100644 --- a/build_tools/pkgci/external_test_suite/models_gpu_rocm_gfx90a.json +++ b/build_tools/pkgci/external_test_suite/models_gpu_rocm_gfx90a.json @@ -20,9 +20,11 @@ "pytorch/models/resnet50", "pytorch/models/sdxl-vae-decode-tank", - // error: 'builtin.module' op failed to run transform dialect passes - // (transform spec file is specific to SDXL?) - "sharktank/llama/open-llama-3b-v2-f16" + // TODO(#17874): error: a handle passed as operand #0 and consumed by this operation points to a payload entity more than once + "sharktank/llama/open-llama-3b-v2-f16", + + "sharktank/punet/fp16", + "sharktank/punet/int8", ], "expected_run_failures": [] } diff --git a/build_tools/pkgci/external_test_suite/models_gpu_rocm_gfx942.json b/build_tools/pkgci/external_test_suite/models_gpu_rocm_gfx942.json index 5d451f0b6f20..4a0e50713845 100644 --- a/build_tools/pkgci/external_test_suite/models_gpu_rocm_gfx942.json +++ b/build_tools/pkgci/external_test_suite/models_gpu_rocm_gfx942.json @@ -20,9 +20,11 @@ "pytorch/models/resnet50", "pytorch/models/sdxl-vae-decode-tank", - // error: 'builtin.module' op failed to run transform dialect passes - // (transform spec file is specific to SDXL?) + // TODO(#17874): error: a handle passed as operand #0 and consumed by this operation points to a payload entity more than once "sharktank/llama/open-llama-3b-v2-f16" + + "sharktank/punet/fp16", + "sharktank/punet/int8", ], "expected_run_failures": [] } diff --git a/build_tools/pkgci/external_test_suite/models_gpu_vulkan.json b/build_tools/pkgci/external_test_suite/models_gpu_vulkan.json index d8af23a7b7ab..b5f1731a9243 100644 --- a/build_tools/pkgci/external_test_suite/models_gpu_vulkan.json +++ b/build_tools/pkgci/external_test_suite/models_gpu_vulkan.json @@ -12,12 +12,12 @@ // TODO(#17344): need to regenerate .mlirbc "pytorch/models/opt-125M", "pytorch/models/resnet50", + "pytorch/models/sdxl-prompt-encoder-tank", "pytorch/models/sdxl-scheduled-unet-3-tank", "pytorch/models/sdxl-vae-decode-tank", - // error: 'builtin.module' op failed to run transform dialect passes - // (transform spec file is specific to SDXL?) + // TODO(#17874): error: a handle passed as operand #0 and consumed by this operation points to a payload entity more than once "sharktank/llama/open-llama-3b-v2-f16" ], "expected_run_failures": []