From ee30e8c80c06348f504487cb637d4d856b6a5564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5ns=20Nilsson?= Date: Mon, 17 Nov 2025 10:23:52 +0100 Subject: [PATCH 1/2] Arm backend: Enable VGF backend testing in github Reapply "Arm backend: Enable pytest VKML op unit testing in ciflow/trunk" (#15841) This reverts commit 0c0cee5fd5adfff71fdf896181c51ba984427f48. Reason why the first commit was reverted was because the all op test timed out. Therefore on top of the "reapply" test_pytest_ops_vkml is split out into its own job, test-arm-backend-vkml. This means the original all ops test will run like before. Change-Id: Ic4316bd617625dd6e839bfcd8cdf1721023ab4f3 --- .ci/scripts/test_backend.sh | 12 ++++++++- .github/workflows/test-backend-arm.yml | 2 +- .github/workflows/trunk.yml | 34 ++++++++++++++++++++++++++ backends/test/suite/flow.py | 4 +++ backends/test/suite/flows/arm.py | 12 +++++++++ 5 files changed, 62 insertions(+), 2 deletions(-) diff --git a/.ci/scripts/test_backend.sh b/.ci/scripts/test_backend.sh index a48cc9ec41a..e959a2f074a 100755 --- a/.ci/scripts/test_backend.sh +++ b/.ci/scripts/test_backend.sh @@ -57,8 +57,13 @@ if [[ "$FLOW" == *vulkan* ]]; then fi if [[ "$FLOW" == *arm* ]]; then + # Setup ARM deps. - .ci/scripts/setup-arm-baremetal-tools.sh + if [[ "$FLOW" == *vgf* ]]; then + .ci/scripts/setup-arm-baremetal-tools.sh --enable-mlsdk-deps --install-mlsdk-deps-with-pip + else + .ci/scripts/setup-arm-baremetal-tools.sh + fi source examples/arm/ethos-u-scratch/setup_path.sh if [[ "$FLOW" == *ethos_u* ]]; then @@ -66,6 +71,11 @@ if [[ "$FLOW" == *arm* ]]; then backends/arm/scripts/build_executorch.sh backends/arm/test/setup_testing.sh fi + + if [[ "$FLOW" == *vgf* ]]; then + # Prepare a test runner binary for VKML runtime + backends/arm/test/setup_testing_vkml.sh + fi fi if [[ $IS_MACOS -eq 1 ]]; then diff --git a/.github/workflows/test-backend-arm.yml b/.github/workflows/test-backend-arm.yml index 22e3d524f6b..638d5a2079f 100644 --- a/.github/workflows/test-backend-arm.yml +++ b/.github/workflows/test-backend-arm.yml @@ -26,7 +26,7 @@ jobs: uses: ./.github/workflows/_test_backend.yml with: backend: arm - flows: '["arm_tosa_fp", "arm_tosa_int", "arm_ethos_u55", "arm_ethos_u85"]' + flows: '["arm_tosa_fp", "arm_tosa_int", "arm_ethos_u55", "arm_ethos_u85", "arm_vgf_fp", "arm_vgf_int"]' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 120 run-linux: true diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index f428766ba64..cc918034988 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -317,6 +317,40 @@ jobs: # Test test_arm_baremetal.sh with test backends/arm/test/test_arm_baremetal.sh "${ARM_TEST}" + test-arm-backend-vkml: + name: test-arm-backend-vkml + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + permissions: + id-token: write + contents: read + strategy: + matrix: + include: + - test_arm_baremetal: test_pytest_ops_vkml + fail-fast: false + with: + runner: linux.2xlarge.memory + docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk + submodules: 'recursive' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 120 + script: | + # The generic Linux job chooses to use base env, not the one setup by the image + CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") + conda activate "${CONDA_ENV}" + source .ci/scripts/utils.sh + install_executorch "--use-pt-pinned-commit" + + .ci/scripts/setup-arm-baremetal-tools.sh --disable-ethos-u-deps --enable-mlsdk-deps --install-mlsdk-deps-with-pip + + # Increase number of files user can monitor to bypass buck failures. + # Hopefully this is high enough for this setup. + sudo sysctl fs.inotify.max_user_watches=1048576 # 1024 * 1024 + + ARM_TEST=${{ matrix.test_arm_baremetal }} + + backends/arm/test/test_arm_baremetal.sh "${ARM_TEST}" + test-arm-cortex-m-size-test: name: test-arm-cortex-m-size-test uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main diff --git a/backends/test/suite/flow.py b/backends/test/suite/flow.py index 29394951bd7..f3c9ee75083 100644 --- a/backends/test/suite/flow.py +++ b/backends/test/suite/flow.py @@ -132,6 +132,8 @@ def all_flows() -> dict[str, TestFlow]: ARM_ETHOS_U85_FLOW, ARM_TOSA_FP_FLOW, ARM_TOSA_INT_FLOW, + ARM_VGF_FP_FLOW, + ARM_VGF_INT_FLOW, ) flows += [ @@ -139,6 +141,8 @@ def all_flows() -> dict[str, TestFlow]: ARM_TOSA_INT_FLOW, ARM_ETHOS_U55_FLOW, ARM_ETHOS_U85_FLOW, + ARM_VGF_FP_FLOW, + ARM_VGF_INT_FLOW, ] except Exception as e: logger.info(f"Skipping ARM flow registration: {e}") diff --git a/backends/test/suite/flows/arm.py b/backends/test/suite/flows/arm.py index db3bb2cfd9e..29ef504d50c 100644 --- a/backends/test/suite/flows/arm.py +++ b/backends/test/suite/flows/arm.py @@ -78,3 +78,15 @@ def create_quantize_stage() -> Quantize: lambda: common.get_u85_compile_spec(), quantize=True, ) +ARM_VGF_FP_FLOW = _create_arm_flow( + "arm_vgf_fp", + lambda: common.get_vgf_compile_spec(tosa_spec="TOSA-1.0+FP"), + quantize=False, + use_portable_ops=False, +) +ARM_VGF_INT_FLOW = _create_arm_flow( + "arm_vgf_int", + lambda: common.get_vgf_compile_spec(tosa_spec="TOSA-1.0+INT"), + quantize=True, + use_portable_ops=False, +) From f95e71910f42924784969b562cb2bee483c28213 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5ns=20Nilsson?= Date: Mon, 17 Nov 2025 14:17:18 +0100 Subject: [PATCH 2/2] Do not call build_executorch.sh for test_pytest_ops_vkml --- backends/arm/test/test_arm_baremetal.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/backends/arm/test/test_arm_baremetal.sh b/backends/arm/test/test_arm_baremetal.sh index 4bc4fe0f06d..de8768873c4 100755 --- a/backends/arm/test/test_arm_baremetal.sh +++ b/backends/arm/test/test_arm_baremetal.sh @@ -157,7 +157,6 @@ test_pytest_ethosu_fvp() { # Same as test_pytest but also sometime verify using test_pytest_ops_vkml() { # Same as test_pytest but also sometime verify using VKML runtime echo "${TEST_SUITE_NAME}: Run pytest operator tests with VKML runtime" - backends/arm/scripts/build_executorch.sh backends/arm/test/setup_testing_vkml.sh pytest --verbose --color=yes --numprocesses=auto --durations=10 backends/arm/test/ \