Arm backend: Add NGP unit test scripts

backends/arm/CMakeLists.txt

@@ -73,9 +73,10 @@ if(EXECUTORCH_BUILD_VGF)
   # vgf backend
   list(TRANSFORM _vgf_backend_sources PREPEND "${EXECUTORCH_ROOT}/")
   add_library(vgf_backend ${_vgf_backend_sources})
+  install(TARGETS vgf_backend EXPORT ExecuTorchTargets)
   target_include_directories(
-    vgf_backend PUBLIC ${_common_include_directories} ${VULKAN_HEADERS_PATH}
-                       ${VOLK_HEADERS_PATH}
+    vgf_backend PRIVATE ${_common_include_directories} ${VULKAN_HEADERS_PATH}
+                        ${VOLK_HEADERS_PATH}
   )
   target_compile_options(
     vgf_backend PRIVATE -DUSE_VULKAN_WRAPPER -DUSE_VULKAN_VOLK

backends/arm/README.md

@@ -88,6 +88,19 @@ You can test to run some models with the full fvp test flow
 backends/arm/test/test_arm_baremetal.sh test_full_ethosu_fvp
 ```
 
+To run the unit test suite with VKML use the following. Note Vulkan SDK need to be installed.
+Have a look at install_vulkan_sdk() in .ci/scripts/setup-vulkan-linux-deps.sh on how to install Vulkan SDK.
+
+```
+backends/arm/test/test_arm_baremetal.sh test_pytest_vkml
+```
+
+You can test to run some models with the full VKML flow
+
+```
+backends/arm/test/test_arm_baremetal.sh test_full_vkml
+```
+
 ## Unit tests
 
 This is the structure of the test directory
@@ -102,6 +115,7 @@ test                            #  Root test folder
 ├── tosautil                    #  Utility functions for TOSA artifacts
 ├ common.py                     #  Common functions and definitions used by many tests
 ├ setup_testing.sh              #  Script to prepare testing for using the Corstone 3x0 FVP
+├ setup_testing_vkml.sh         #  Script to prepare testing for using the VKML
 ├ test_arm_baremetal.sh         #  Help script to trigger testing
 ```
 
@@ -123,7 +137,7 @@ first you need to build and prepare some used target libs
 
 ```
 examples/arm/run.sh --model_name=add --build_only
-backends/arm/test/setup_testing.sh
+backends/arm/test/setup_testing.sh and/or backends/arm/test/setup_testing_vkml.sh
 ```
 
 The you can run the tests with

backends/arm/scripts/build_executor_runner_vkml.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# Copyright 2025 Arm Limited and/or its affiliates.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -eu
+
+script_dir=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
+et_root_dir=$(cd ${script_dir}/../../.. && pwd)
+et_root_dir=$(realpath ${et_root_dir})
+setup_path_script=${et_root_dir}/examples/arm/ethos-u-scratch/setup_path.sh
+_setup_msg="please refer to ${et_root_dir}/examples/arm/setup.sh to properly install necessary tools."
+
+build_type="Release"
+build_with_etdump=false
+extra_build_flags=""
+output_folder="cmake-out-vkml"
+
+build_with_etdump_flags=" -DEXECUTORCH_ENABLE_EVENT_TRACER=OFF "
+
+help() {
+    echo "Usage: $(basename $0) [options]"
+    echo "Options:"
+    echo "  --build_type=<TYPE>             Build with Release, Debug or RelWithDebInfo, default is ${build_type}"
+    echo "  --etdump                        Adds Devtools etdump support to track timing, etdump area will be base64 encoded in the log"
+    echo "  --extra_build_flags=<FLAGS>     Extra flags to pass to cmake. Default: none "
+    echo "  --output=<FOLDER>               Output folder Default: $(output_folder)"
+    exit 0
+}
+
+for arg in "$@"; do
+    case $arg in
+      -h|--help) help ;;
+      --build_type=*) build_type="${arg#*=}";;
+      --etdump) build_with_etdump=true ;;
+      --extra_build_flags=*) extra_build_flags="${arg#*=}";;
+      --output=*) output_folder="${arg#*=}";;
+      --select_ops_list=*) select_ops_list="${arg#*=}";;
+      *)
+      ;;
+    esac
+done
+
+# Source the tools
+# This should be prepared by the setup.sh
+[[ -f ${setup_path_script} ]] \
+    || { echo "Missing ${setup_path_script}. ${_setup_msg}"; exit 1; }
+
+source ${setup_path_script}
+
+mkdir -p "${output_folder}"
+output_folder=$(realpath ${output_folder})
+
+echo "--------------------------------------------------------------------------------"
+echo "Build Arm VKML executor runner: '${output_folder}' with extra build flags: ${extra_build_flags}"
+echo "--------------------------------------------------------------------------------"
+
+cd ${et_root_dir}/examples/arm/executor_runner
+
+if [ "$build_with_etdump" = true ] ; then
+    build_with_etdump_flags=" -DEXECUTORCH_ENABLE_EVENT_TRACER=ON "
+fi
+
+echo "Building with extra flags: ${build_with_etdump_flags} ${extra_build_flags}"
+cmake \
+    -DCMAKE_BUILD_TYPE=${build_type}            \
+    -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
+    -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
+    -DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
+    -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
+    -DEXECUTORCH_BUILD_XNNPACK=OFF \
+    -DEXECUTORCH_BUILD_VULKAN=ON \
+    -DEXECUTORCH_BUILD_VGF=ON \
+    -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
+    -DEXECUTORCH_BUILD_KERNELS_QUANTIZED_AOT=ON \
+    -DEXECUTORCH_ENABLE_LOGGING=ON \
+    -DPYTHON_EXECUTABLE=$(which python3)        \
+    ${extra_build_flags}                        \
+    -B ${output_folder} ${et_root_dir}
+
+echo "[${BASH_SOURCE[0]}] Configured CMAKE"
+
+cmake --build ${output_folder} -j$(nproc)
+
+echo "[${BASH_SOURCE[0]}] Built VKML runner: "
+find ${output_folder} -name "executor_runner"

backends/arm/scripts/run_vkml.sh

@@ -19,14 +19,14 @@ _setup_msg="please refer to ${et_root_dir}/examples/arm/setup.sh to properly ins
 
 
 model=""
-build_path="cmake-out"
+build_path="cmake-out-vkml"
 converter="model-converter"
 
 help() {
     echo "Usage: $(basename $0) [options]"
     echo "Options:"
     echo "  --model=<MODEL_FILE>    .pte model file to run"
-    echo "  --build=<BUILD_PATH>    Target to build and run for Default: ${build_path}"
+    echo "  --build_path=<BUILD_PATH>  Path to executor_runner build. for Default: ${build_path}"
     exit 0
 }
 

backends/arm/test/ops/test_add.py

@@ -7,7 +7,6 @@
 
 from typing import Tuple
 
-import pytest
 import torch
 from executorch.backends.arm.quantizer import arm_quantizer
 from executorch.backends.arm.test import common, conftest
@@ -186,13 +185,13 @@ def test_add_tensor_u85_INT_2(test_data: input_t2):
     pipeline.run()
 
 
-@common.parametrize("test_data", Add.test_data)
+# TODO/MLETORCH-1282: remove once inputs are not hard coded to ones
+skip_keys = {"5d_float", "1d_ones", "1d_randn"}
+filtered_test_data = {k: v for k, v in Add.test_data.items() if k not in skip_keys}
+
+
+@common.parametrize("test_data", filtered_test_data)
 @common.SkipIfNoModelConverter
-@common.XfailfNoVKMLEmulationLayer
-@pytest.mark.xfail(
-    reason="VGF runtime is not yet fully supported for FP pipeline (MLETORCH-1234)",
-    strict=True,
-)
 def test_add_tensor_vgf_FP(test_data: input_t1):
     pipeline = VgfPipeline[input_t1](
         Add(),
@@ -205,7 +204,7 @@ def test_add_tensor_vgf_FP(test_data: input_t1):
     pipeline.run()
 
 
-@common.parametrize("test_data", Add.test_data)
+@common.parametrize("test_data", filtered_test_data)
 @common.SkipIfNoModelConverter
 def test_add_tensor_vgf_INT(test_data: input_t1):
     pipeline = VgfPipeline[input_t1](
@@ -214,5 +213,6 @@ def test_add_tensor_vgf_INT(test_data: input_t1):
         aten_op,
         exir_op,
         tosa_version="TOSA-1.0+INT",
+        run_on_vulkan_runtime=True,
     )
     pipeline.run()

backends/arm/test/runner_utils.py

@@ -257,11 +257,21 @@ def run_vkml_emulation_layer(
     result_stdout = result.stdout.decode()  # noqa: F841
     # TODO: MLETORCH-1234: Support VGF e2e tests in VgfPipeline
     # TODO: Add regex to check for error or fault messages in stdout from Emulation Layer
-    # TODO: Retrieve and return the output tensors once VGF runtime is able to dump them.
-    raise NotImplementedError(
-        "Output parsing from VKML Emulation Layer is not yet implemented. "
+    # Regex to extract tensor values from stdout
+    output_np = []
+    matches = re.findall(
+        r"Output\s+\d+:\s+tensor\(sizes=\[(.*?)\],\s+\[(.*?)\]\)",
+        result_stdout,
+        re.DOTALL,
     )
 
+    for shape_str, values_str in matches:
+        shape = list(map(int, shape_str.split(",")))
+        values = list(map(float, re.findall(r"[-+]?\d*\.\d+|\d+", values_str)))
+        output_np.append(torch.tensor(values).reshape(shape))
+
+    return tuple(output_np)
+
 
 def run_corstone(
     executorch_program_manager: ExecutorchProgramManager,
@@ -626,7 +636,8 @@ def vkml_emulation_layer_installed() -> bool:
 def assert_elf_path_exists(elf_path):
     if not os.path.exists(elf_path):
         raise FileNotFoundError(
-            f"Did not find build arm_executor_runner or executor_runner in path {elf_path}, run setup_testing.sh?"
+            f"Did not find build arm_executor_runner or executor_runner in path {elf_path}, \
+            run setup_testing.sh or setup_testing_vkml.sh?"
         )
 
 
@@ -643,7 +654,7 @@ def get_elf_path(target_board):
         assert_elf_path_exists(elf_path)
     elif target_board == "vkml_emulation_layer":
         elf_path = os.path.join(
-            "cmake-out",
+            "arm_test/arm_executor_runner_vkml",
             "executor_runner",
         )
         assert_elf_path_exists(elf_path)

backends/arm/test/setup_testing.sh

@@ -1,6 +1,5 @@
 #!/usr/bin/env bash
 # Copyright 2024-2025 Arm Limited and/or its affiliates.
-# All rights reserved.
 #
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
@@ -13,4 +12,4 @@ build_executor_runner=${et_root_dir}/backends/arm/scripts/build_executor_runner.
 build_root_test_dir=${et_root_dir}/arm_test/arm_semihosting_executor_runner
 
 ${build_executor_runner} --pte=semihosting --target=ethos-u55-128 --output="${build_root_test_dir}_corstone-300"
-${build_executor_runner} --pte=semihosting --target=ethos-u85-128 --output="${build_root_test_dir}_corstone-320"
+${build_executor_runner} --pte=semihosting --target=ethos-u85-128 --output="${build_root_test_dir}_corstone-320"

backends/arm/test/setup_testing_vkml.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Copyright 2024-2025 Arm Limited and/or its affiliates.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -eu
+
+script_dir=$(realpath "$(dirname "${BASH_SOURCE[0]}")")
+et_root_dir=$(realpath "${script_dir}/../../..")
+build_executor_runner=${et_root_dir}/backends/arm/scripts/build_executor_runner_vkml.sh
+build_root_test_dir=${et_root_dir}/arm_test/arm_executor_runner_vkml
+
+${build_executor_runner} --output="${build_root_test_dir}"