From 70ed1b1ce0097157ec03e604e9e05016f55f3aa1 Mon Sep 17 00:00:00 2001 From: Eli Amesefe Date: Mon, 15 Sep 2025 15:29:00 -0700 Subject: [PATCH] custom fbcode serialize stage to run FVP internally on arm ops tests (#14070) Summary: update the codebase to allow a system whereby: - the test inventory in https://www.internalfb.com/code/fbsource/fbcode/executorch/backends/arm/test/ops/, which until now runs on open source CI can all be exercised internally on our internal diff-time sandcastle and CI runs - those tests can **transparently** switch between running an internal version of FVP using Meta's FVP runtimes, build configs and artifacts when run in fbsrouce, but then use the open-source FVp runtime when executed in github opens-ource CI bypass-github-export-checks bypass-github-pytorch-ci-checks bypass-github-executorch-ci-checks Reviewed By: digantdesai Differential Revision: D81159036 --- backends/arm/test/TARGETS | 22 +++++++- backends/arm/test/ops/test_tanh.py | 6 ++- backends/arm/test/targets.bzl | 3 +- backends/arm/test/tester/arm_tester.py | 49 +++-------------- backends/arm/test/tester/serialize.py | 75 ++++++++++++++++++++++++++ 5 files changed, 107 insertions(+), 48 deletions(-) create mode 100644 backends/arm/test/tester/serialize.py diff --git a/backends/arm/test/TARGETS b/backends/arm/test/TARGETS index 8ffad640d5a..ec35b63f8f6 100644 --- a/backends/arm/test/TARGETS +++ b/backends/arm/test/TARGETS @@ -40,8 +40,17 @@ runtime.python_library( ) runtime.python_library( - name = "arm_tester", - srcs = glob(["tester/*.py"]), + name = "arm_tester_serialize", + srcs = ["tester/serialize.py"], + deps = [ + "//executorch/backends/xnnpack/test/tester:tester", + "//executorch/devtools/backend_debug:delegation_info", + ] +) + +runtime.python_library( + name = "arm_tester_lib", + srcs = glob(["tester/*.py"], exclude = ["tester/serialize.py"]), deps = [ ":common", "//executorch/backends/xnnpack/test/tester:tester", @@ -55,4 +64,13 @@ runtime.python_library( ] ) + +runtime.python_library( + name = "arm_tester", + deps = [ + "//executorch/backends/arm/test:arm_tester_lib", + "//executorch/backends/arm/test:arm_tester_serialize", + ] +) + define_arm_tests() diff --git a/backends/arm/test/ops/test_tanh.py b/backends/arm/test/ops/test_tanh.py index 0e74618fd2f..f3f4df31d0e 100644 --- a/backends/arm/test/ops/test_tanh.py +++ b/backends/arm/test/ops/test_tanh.py @@ -70,25 +70,27 @@ def test_tanh_tosa_INT(test_data: Tuple): @common.parametrize("test_data", test_data_suite) +@common.XfailIfNoCorstone300 def test_tanh_u55_INT(test_data: Tuple): pipeline = EthosU55PipelineINT[input_t1]( Tanh(), (test_data(),), aten_op, exir_ops=[], - run_on_fvp=False, + run_on_fvp=True, ) pipeline.run() @common.parametrize("test_data", test_data_suite) +@common.XfailIfNoCorstone320 def test_tanh_u85_INT(test_data: Tuple): pipeline = EthosU85PipelineINT[input_t1]( Tanh(), (test_data(),), aten_op, exir_ops=[], - run_on_fvp=False, + run_on_fvp=True, ) pipeline.run() diff --git a/backends/arm/test/targets.bzl b/backends/arm/test/targets.bzl index 62bc5aef57a..f240855cdf4 100644 --- a/backends/arm/test/targets.bzl +++ b/backends/arm/test/targets.bzl @@ -1,4 +1,5 @@ # load("//caffe2/test/fb:defs.bzl", "define_tests") +load("@fbsource//tools/build_defs:fbsource_utils.bzl", "is_fbcode") load("@fbcode_macros//build_defs:python_pytest.bzl", "python_pytest") load("@bazel_skylib//lib:paths.bzl", "paths") @@ -59,7 +60,7 @@ def define_arm_tests(): "//executorch/kernels/quantized:custom_ops_generated_lib", ], deps = [ - "//executorch/backends/arm/test:arm_tester", + "//executorch/backends/arm/test/tester/fb:arm_tester_fb" if is_fbcode else "//executorch/backends/arm/test:arm_tester", "//executorch/backends/arm/test:conftest", "//executorch/backends/arm:ethosu", "//executorch/backends/arm/tosa:compile_spec", diff --git a/backends/arm/test/tester/arm_tester.py b/backends/arm/test/tester/arm_tester.py index a7c48e66a31..bb249644c47 100644 --- a/backends/arm/test/tester/arm_tester.py +++ b/backends/arm/test/tester/arm_tester.py @@ -7,7 +7,6 @@ import logging -import os from collections import Counter from pprint import pformat from typing import ( @@ -42,10 +41,7 @@ ) from executorch.backends.arm.test.runner_utils import ( dbg_tosa_fb_to_json, - get_elf_path, get_output_quantization_params, - get_target_board, - run_target, TosaReferenceModelDispatch, ) @@ -53,6 +49,7 @@ dump_error_output, print_error_diffs, ) +from executorch.backends.arm.test.tester.serialize import Serialize from executorch.backends.arm.tosa import TosaSpecification from executorch.backends.arm.tosa.compile_spec import TosaCompileSpec from executorch.backends.arm.tosa.mapping import extract_tensor_meta @@ -90,7 +87,6 @@ from torch.export.graph_signature import ExportGraphSignature, InputSpec, OutputSpec from torch.fx import Graph -from torch.utils._pytree import tree_flatten logger = logging.getLogger(__name__) @@ -179,43 +175,6 @@ def run( ) -class Serialize(tester.Serialize): - def __init__(self, compile_spec: ArmCompileSpec, timeout): - super().__init__() - self.timeout = timeout - self.executorch_program_manager: ExecutorchProgramManager | None - self.compile_spec = compile_spec - - def run(self, artifact: ExecutorchProgramManager, inputs=None) -> None: - super().run(artifact, inputs) - # Keep the entire ExecutorchProgramManager for execution. - self.executorch_program_manager = artifact - - def run_artifact(self, inputs): - if self.executorch_program_manager is None: - raise RuntimeError( - "Tried running artifact from Serialize stage without running the stage." - ) - inputs_flattened, _ = tree_flatten(inputs) - intermediate_path = self.compile_spec.get_intermediate_path() - target_board = get_target_board(self.compile_spec) - elf_path = get_elf_path(target_board) - - if not os.path.exists(elf_path): - raise FileNotFoundError( - f"Did not find build arm_executor_runner in path {elf_path}, run setup_testing.sh?" - ) - - return run_target( - self.executorch_program_manager, - inputs_flattened, - intermediate_path, - target_board, - elf_path, - self.timeout, - ) - - class ToExecutorch(tester.ToExecutorch): def run_artifact(self, inputs): with TosaReferenceModelDispatch(): @@ -419,7 +378,11 @@ def serialize( self, serialize_stage: Optional[Serialize] = None, timeout: int = 480 ): if serialize_stage is None: - serialize_stage = Serialize(self.compile_spec, timeout) + serialize_stage = Serialize( + compile_spec=self.compile_spec, + module=self.original_module, + timeout=timeout, + ) assert ( self.compile_spec.get_intermediate_path() is not None ), "Can't dump serialized file when compile specs do not contain an artifact path." diff --git a/backends/arm/test/tester/serialize.py b/backends/arm/test/tester/serialize.py new file mode 100644 index 00000000000..f0fd246b3a6 --- /dev/null +++ b/backends/arm/test/tester/serialize.py @@ -0,0 +1,75 @@ +# Copyright 2024-2025 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import logging +import os +from typing import Optional + +import executorch.backends.xnnpack.test.tester.tester as tester + +import torch.fx + +from executorch.backends.arm.common.arm_compile_spec import ArmCompileSpec + +from executorch.backends.arm.test.runner_utils import ( + get_elf_path, + get_target_board, + run_target, +) + +from executorch.exir import ExecutorchProgramManager +from torch.utils._pytree import tree_flatten + + +logger = logging.getLogger(__name__) + + +class Serialize(tester.Serialize): + def __init__( + self, + compile_spec: ArmCompileSpec, + module: Optional[torch.nn.Module], + timeout: int = 120, + ): + """ + Args: + compile_spec: CompileSpecs to be used for serialization. + module: Original Module to be used for serialization. Optional - can be used for reference output generation. + timeout: Timeout for fvp. Default is 120 seconds. + """ + super().__init__() + self.module = module + self.timeout = timeout + self.executorch_program_manager: ExecutorchProgramManager | None + self.compile_spec = compile_spec + + def run(self, artifact: ExecutorchProgramManager, inputs=None) -> None: + super().run(artifact, inputs) + # Keep the entire ExecutorchProgramManager for execution. + self.executorch_program_manager = artifact + + def run_artifact(self, inputs): + if self.executorch_program_manager is None: + raise RuntimeError( + "Tried running artifact from Serialize stage without running the stage." + ) + inputs_flattened, _ = tree_flatten(inputs) + intermediate_path = self.compile_spec.get_intermediate_path() + target_board = get_target_board(self.compile_spec) + elf_path = get_elf_path(target_board) + + if not os.path.exists(elf_path): + raise FileNotFoundError( + f"Did not find build arm_executor_runner in path {elf_path}, run setup_testing.sh?" + ) + + return run_target( + self.executorch_program_manager, + inputs_flattened, + intermediate_path, + target_board, + elf_path, + self.timeout, + )