custom fbcode serialize stage to run FVP internally on arm ops tests (#14070)

Summary:
Pull Request resolved: #14070

update the codebase to allow a system whereby:

- the test inventory in https://www.internalfb.com/code/fbsource/fbcode/executorch/backends/arm/test/ops/, which until now runs on open source CI can all be exercised internally on our internal diff-time sandcastle and CI runs
- those tests can **transparently** switch between running an internal version of FVP using Meta's FVP runtimes, build configs and artifacts when run in fbsrouce, but then use the open-source FVp runtime when executed in github opens-ource CI

Differential Revision: D81159036

Author: 3l1

backends/arm/test/TARGETS

@@ -36,8 +36,18 @@ runtime.python_library(
 )
 
 runtime.python_library(
-    name = "arm_tester",
-    srcs = glob(["tester/*.py"]),
+    name = "arm_tester_serialize",
+    srcs = ["tester/serialize.py"],
+    deps = [
+        "//executorch/backends/xnnpack/test/tester:tester",
+        "//executorch/backends/arm:tosa_mapping",
+        "//executorch/devtools/backend_debug:delegation_info",
+    ]
+)
+
+python_library(
+    name = "arm_tester_lib",
+    srcs = glob(["tester/*.py"], exclude = ["tester/serialize.py"]),
     deps = [
         ":common",
         "//executorch/backends/xnnpack/test/tester:tester",
@@ -51,4 +61,13 @@ runtime.python_library(
     ]
 )
 
+
+python_library(
+    name = "arm_tester",
+    deps = [
+        "//executorch/backends/arm/test:arm_tester_lib",
+        "//executorch/backends/arm/test:arm_tester_serialize",
+    ]
+)
+
 define_arm_tests()

backends/arm/test/ops/test_tanh.py

@@ -76,7 +76,7 @@ def test_tanh_u55_INT(test_data: Tuple):
         (test_data(),),
         aten_op,
         exir_ops=[],
-        run_on_fvp=False,
+        run_on_fvp=True,
     )
     pipeline.run()
 

backends/arm/test/targets.bzl

@@ -1,4 +1,5 @@
 # load("//caffe2/test/fb:defs.bzl", "define_tests")
+load("@fbsource//tools/build_defs:fbsource_utils.bzl", "is_fbcode")
 load("@fbcode_macros//build_defs:python_pytest.bzl", "python_pytest")
 load("@bazel_skylib//lib:paths.bzl", "paths")
 
@@ -36,7 +37,7 @@ def define_arm_tests():
     for test_file in test_files:
         test_file_name = paths.basename(test_file)
         test_name = test_file_name.replace("test_", "").replace(".py", "")
-
+        
         python_pytest(
             name = test_name,
             srcs = [test_file],
@@ -48,7 +49,7 @@ def define_arm_tests():
                 "//executorch/kernels/quantized:custom_ops_generated_lib",
             ],
             deps = [
-                "//executorch/backends/arm/test:arm_tester",
+                "//executorch/backends/arm/test/tester/fb:arm_tester_fb" if is_fbcode else "//executorch/backends/arm/test:arm_tester",
                 "//executorch/backends/arm/test:conftest",
                 "//executorch/exir:lib",
                 "fbsource//third-party/pypi/pytest:pytest",

backends/arm/test/tester/arm_tester.py

@@ -7,7 +7,6 @@
 
 import logging
 
-import os
 from collections import Counter
 from pprint import pformat
 from typing import (
@@ -47,17 +46,15 @@
 )
 from executorch.backends.arm.test.runner_utils import (
     dbg_tosa_fb_to_json,
-    get_elf_path,
     get_output_quantization_params,
-    get_target_board,
-    run_target,
     TosaReferenceModelDispatch,
 )
 
 from executorch.backends.arm.test.tester.analyze_output_utils import (
     dump_error_output,
     print_error_diffs,
 )
+from executorch.backends.arm.test.tester.serialize import Serialize
 from executorch.backends.arm.tosa import TosaSpecification
 from executorch.backends.arm.tosa.mapping import extract_tensor_meta
 from executorch.backends.arm.tosa.partitioner import TOSAPartitioner
@@ -96,7 +93,6 @@
 
 from torch.export.graph_signature import ExportGraphSignature, InputSpec, OutputSpec
 from torch.fx import Graph
-from torch.utils._pytree import tree_flatten
 
 
 logger = logging.getLogger(__name__)
@@ -184,44 +180,6 @@ def run(
             generate_etrecord=generate_etrecord,
         )
 
-
-class Serialize(tester.Serialize):
-    def __init__(self, compile_spec: list[CompileSpec], timeout):
-        super().__init__()
-        self.timeout = timeout
-        self.executorch_program_manager: ExecutorchProgramManager | None
-        self.compile_spec = compile_spec
-
-    def run(self, artifact: ExecutorchProgramManager, inputs=None) -> None:
-        super().run(artifact, inputs)
-        # Keep the entire ExecutorchProgramManager for execution.
-        self.executorch_program_manager = artifact
-
-    def run_artifact(self, inputs):
-        if self.executorch_program_manager is None:
-            raise RuntimeError(
-                "Tried running artifact from Serialize stage without running the stage."
-            )
-        inputs_flattened, _ = tree_flatten(inputs)
-        intermediate_path = get_intermediate_path(self.compile_spec)
-        target_board = get_target_board(self.compile_spec)
-        elf_path = get_elf_path(target_board)
-
-        if not os.path.exists(elf_path):
-            raise FileNotFoundError(
-                f"Did not find build arm_executor_runner in path {elf_path}, run setup_testing.sh?"
-            )
-
-        return run_target(
-            self.executorch_program_manager,
-            inputs_flattened,
-            intermediate_path,
-            target_board,
-            elf_path,
-            self.timeout,
-        )
-
-
 class ToExecutorch(tester.ToExecutorch):
     def run_artifact(self, inputs):
         with TosaReferenceModelDispatch():
@@ -423,7 +381,11 @@ def serialize(
         self, serialize_stage: Optional[Serialize] = None, timeout: int = 480
     ):
         if serialize_stage is None:
-            serialize_stage = Serialize(self.compile_spec, timeout)
+            serialize_stage = Serialize(
+                compile_spec=self.compile_spec,
+                module=self.original_module,
+                timeout=timeout
+            )
         assert (
             get_intermediate_path(self.compile_spec) is not None
         ), "Can't dump serialized file when compile specs do not contain an artifact path."

backends/arm/test/tester/serialize.py

@@ -0,0 +1,74 @@
+# Copyright 2024-2025 Arm Limited and/or its affiliates.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+import logging
+import os
+from typing import Optional
+
+import executorch.backends.xnnpack.test.tester.tester as tester
+
+import torch.fx
+
+from executorch.backends.arm.arm_backend import get_intermediate_path
+from executorch.backends.arm.test.runner_utils import (
+    get_elf_path,
+    get_target_board,
+    run_target,
+)
+
+from executorch.exir import ExecutorchProgramManager
+from executorch.exir.backend.compile_spec_schema import CompileSpec
+from torch.utils._pytree import tree_flatten
+
+
+logger = logging.getLogger(__name__)
+
+class Serialize(tester.Serialize):
+    def __init__(
+        self,
+        compile_spec: list[CompileSpec],
+        module: Optional[torch.nn.Module],
+        timeout: int = 120,
+    ):
+        """
+        Args:
+            compile_spec: CompileSpecs to be used for serialization.
+            module: Original Module to be used for serialization. Optional - can be used for reference output generation.
+            timeout: Timeout for fvp. Default is 120 seconds.
+        """
+        super().__init__()
+        self.module = module
+        self.timeout = timeout
+        self.executorch_program_manager: ExecutorchProgramManager | None
+        self.compile_spec = compile_spec
+
+    def run(self, artifact: ExecutorchProgramManager, inputs=None) -> None:
+        super().run(artifact, inputs)
+        # Keep the entire ExecutorchProgramManager for execution.
+        self.executorch_program_manager = artifact
+
+    def run_artifact(self, inputs):
+        if self.executorch_program_manager is None:
+            raise RuntimeError(
+                "Tried running artifact from Serialize stage without running the stage."
+            )
+        inputs_flattened, _ = tree_flatten(inputs)
+        intermediate_path = get_intermediate_path(self.compile_spec)
+        target_board = get_target_board(self.compile_spec)
+        elf_path = get_elf_path(target_board)
+
+        if not os.path.exists(elf_path):
+            raise FileNotFoundError(
+                f"Did not find build arm_executor_runner in path {elf_path}, run setup_testing.sh?"
+            )
+
+        return run_target(
+            self.executorch_program_manager,
+            inputs_flattened,
+            intermediate_path,
+            target_board,
+            elf_path,
+            self.timeout,
+        )