Arm backend: run on tosa_ref_model only when requested

backends/arm/test/misc/test_bn_relu_folding_qat.py

@@ -47,7 +47,14 @@ def forward(self, x: torch.Tensor):
 
 @common.parametrize("model", models)
 def test_qat_tosa_INT(model: torch.nn.Module):
-    pipeline = TosaPipelineINT[input_t1](model, model.test_data, [], [], qtol=1)
+    pipeline = TosaPipelineINT[input_t1](
+        model, 
+        model.test_data, 
+        [], 
+        [], 
+        qtol=1,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     tosa_version = conftest.get_option("tosa_version")
     tosa_profiles = {
         "1.0": common.TosaSpecification.create_from_string("TOSA-1.0+INT"),

backends/arm/test/misc/test_custom_partition.py

@@ -3,6 +3,7 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+import conftest
 import logging
 from typing import Tuple
 
@@ -50,7 +51,13 @@ def test_single_reject(caplog, test_data: input_t1):
     caplog.set_level(logging.INFO)
 
     module = CustomPartitioning()
-    pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[])
+    pipeline = TosaPipelineFP[input_t1](
+        module, 
+        test_data, 
+        [], 
+        exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     check = DontPartition(exir_ops.edge.aten.sigmoid.default)
     pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check])
     pipeline.change_args(
@@ -68,7 +75,13 @@ def test_single_reject(caplog, test_data: input_t1):
 @common.parametrize("test_data", CustomPartitioning.inputs)
 def test_multiple_reject(test_data: input_t1):
     module = CustomPartitioning()
-    pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[])
+    pipeline = TosaPipelineFP[input_t1](
+        module, 
+        test_data, 
+        [], 
+        exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     check = DontPartition(
         exir_ops.edge.aten.sigmoid.default, exir_ops.edge.aten.mul.Tensor
     )
@@ -90,7 +103,13 @@ def test_torch_op_reject(caplog, test_data: input_t1):
 
     module = CustomPartitioning()
     check = DontPartition(torch.ops.aten.sigmoid.default)
-    pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[])
+    pipeline = TosaPipelineFP[input_t1](
+        module, 
+        test_data, 
+        [], 
+        exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check])
     pipeline.change_args(
         "check_count.exir", {"torch.ops.higher_order.executorch_call_delegate": 2}
@@ -108,7 +127,13 @@ def test_torch_op_reject(caplog, test_data: input_t1):
 def test_string_op_reject(test_data: input_t1):
     module = CustomPartitioning()
     check = DontPartition("aten.sigmoid.default")
-    pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[])
+    pipeline = TosaPipelineFP[input_t1](
+        module, 
+        test_data, 
+        [], 
+        exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check])
     pipeline.change_args(
         "check_count.exir", {"torch.ops.higher_order.executorch_call_delegate": 2}
@@ -127,7 +152,13 @@ def test_name_reject(caplog, test_data: input_t1):
 
     module = CustomPartitioning()
     check = DontPartitionName("mul", "sigmoid", exact=False)
-    pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[])
+    pipeline = TosaPipelineFP[input_t1](
+        module, 
+        test_data, 
+        [], 
+        exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check])
     pipeline.change_args(
         "check_count.exir",
@@ -142,7 +173,13 @@ def test_name_reject(caplog, test_data: input_t1):
 def test_module_reject(test_data: input_t1):
     module = NestedModule()
     check = DontPartitionModule(module_name="CustomPartitioning")
-    pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[])
+    pipeline = TosaPipelineFP[input_t1](
+        module, 
+        test_data, 
+        [], 
+        exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check])
     pipeline.change_args(
         "check_count.exir",
@@ -158,7 +195,13 @@ def test_inexact_module_reject(caplog, test_data: input_t1):
 
     module = NestedModule()
     check = DontPartitionModule(module_name="Custom", exact=False)
-    pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[])
+    pipeline = TosaPipelineFP[input_t1](
+        module, 
+        test_data, 
+        [], 
+        exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check])
     pipeline.change_args(
         "check_count.exir",
@@ -173,7 +216,13 @@ def test_inexact_module_reject(caplog, test_data: input_t1):
 def test_module_instance_reject(test_data: input_t1):
     module = NestedModule()
     check = DontPartitionModule(instance_name="nested")
-    pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[])
+    pipeline = TosaPipelineFP[input_t1](
+        module, 
+        test_data, 
+        [], 
+        exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check])
     pipeline.change_args(
         "check_count.exir",

backends/arm/test/models/test_conformer.py

@@ -5,6 +5,7 @@
 
 from typing import Tuple
 
+import conftest
 import pytest
 
 import torch
@@ -56,6 +57,7 @@ def test_conformer_tosa_FP():
         TestConformer.model_example_inputs,
         aten_op=TestConformer.aten_ops,
         exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
         use_to_edge_transform_and_lower=True,
     )
     pipeline.run()
@@ -67,6 +69,7 @@ def test_conformer_tosa_INT():
         TestConformer.model_example_inputs,
         aten_op=TestConformer.aten_ops,
         exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
         use_to_edge_transform_and_lower=True,
     )
     pipeline.pop_stage("check_count.exir")

backends/arm/test/models/test_deit_tiny_arm.py

@@ -7,6 +7,7 @@
 
 from typing import Tuple
 
+import conftest
 import timm
 
 import torch
@@ -43,6 +44,7 @@ def test_deit_tiny_tosa_FP():
         model_inputs,
         aten_op=[],
         exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
         use_to_edge_transform_and_lower=True,
     )
     pipeline.run()
@@ -54,6 +56,7 @@ def test_deit_tiny_tosa_INT():
         model_inputs,
         aten_op=[],
         exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
         use_to_edge_transform_and_lower=True,
         atol=1.5,
         qtol=1,

backends/arm/test/models/test_llama.py

@@ -111,6 +111,7 @@ def test_llama_tosa_FP():
             llama_inputs,
             aten_op=[],
             exir_op=[],
+            run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
             use_to_edge_transform_and_lower=True,
             transform_passes=[InsertCastForOpsWithInt64InputPass()],
         )
@@ -129,6 +130,7 @@ def test_llama_tosa_INT():
             llama_inputs,
             aten_op=[],
             exir_op=[],
+            run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
             use_to_edge_transform_and_lower=True,
         )
         pipeline.run()

backends/arm/test/models/test_lstm_arm.py

@@ -5,6 +5,7 @@
 
 from typing import Tuple
 
+import conftest
 import torch
 
 from executorch.backends.arm.test import common
@@ -49,6 +50,7 @@ def test_lstm_tosa_FP():
         TestLSTM.model_example_inputs,
         aten_op=[],
         exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
         use_to_edge_transform_and_lower=True,
     )
     pipeline.change_args("run_method_and_compare_outputs", get_test_inputs(), atol=3e-1)
@@ -61,6 +63,7 @@ def test_lstm_tosa_INT():
         TestLSTM.model_example_inputs,
         aten_op=[],
         exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
         use_to_edge_transform_and_lower=True,
     )
     pipeline.change_args(

backends/arm/test/models/test_w2l_arm.py

@@ -8,6 +8,7 @@
 import unittest
 from typing import Tuple
 
+import conftest
 import pytest
 
 import torch
@@ -53,6 +54,7 @@ def test_w2l_tosa_FP():
         TestW2L.model_example_inputs,
         aten_op=[],
         exir_op=TestW2L.all_operators,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
         use_to_edge_transform_and_lower=True,
     )
     pipeline.run()
@@ -66,6 +68,7 @@ def test_w2l_tosa_INT():
         TestW2L.model_example_inputs,
         aten_op=[],
         exir_op=TestW2L.all_operators,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
         use_to_edge_transform_and_lower=True,
     )
     pipeline.run()

backends/arm/test/ops/test_abs.py

@@ -8,6 +8,7 @@
 
 from typing import Tuple
 
+import conftest
 import torch
 from executorch.backends.arm.test import common
 from executorch.backends.arm.test.tester.test_pipeline import (
@@ -41,13 +42,25 @@ def forward(self, x):
 
 @common.parametrize("test_data", Abs.test_parameters)
 def test_abs_tosa_FP(test_data: torch.Tensor):
-    pipeline = TosaPipelineFP[input_t1](Abs(), test_data(), aten_op, exir_op)
+    pipeline = TosaPipelineFP[input_t1](
+        Abs(), 
+        test_data(), 
+        aten_op, 
+        exir_op,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.run()
 
 
 @common.parametrize("test_data", Abs.test_parameters)
 def test_abs_tosa_INT(test_data: torch.Tensor):
-    pipeline = TosaPipelineINT[input_t1](Abs(), test_data(), aten_op, exir_op)
+    pipeline = TosaPipelineINT[input_t1](
+        Abs(), 
+        test_data(), 
+        aten_op, 
+        exir_op,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.run()
 
 

backends/arm/test/ops/test_acosh.py

@@ -4,6 +4,7 @@
 # LICENSE file in the root directory of this source tree.
 from typing import Tuple
 
+import conftest
 import pytest
 
 import torch
@@ -55,6 +56,7 @@ def test_acosh_tosa_FP(test_data: Tuple):
         (test_data(),),
         aten_op,
         exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
     )
     pipeline.run()
 
@@ -65,6 +67,7 @@ def test_acosh_tosa_INT(test_data: Tuple):
         Acosh(),
         (test_data(),),
         aten_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
     )
     pipeline.run()
 

backends/arm/test/ops/test_add.py

@@ -81,13 +81,25 @@ def forward(self, x: torch.Tensor, y: torch.Tensor):
 
 @common.parametrize("test_data", Add.test_data)
 def test_add_tensor_tosa_FP(test_data: input_t1):
-    pipeline = TosaPipelineFP[input_t1](Add(), test_data(), aten_op, exir_op)
+    pipeline = TosaPipelineFP[input_t1](
+        Add(), 
+        test_data(), 
+        aten_op, 
+        exir_op,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.run()
 
 
 @common.parametrize("test_data", Add.test_data)
 def test_add_tensor_tosa_INT(test_data: input_t1):
-    pipeline = TosaPipelineINT[input_t1](Add(), test_data(), aten_op, exir_op)
+    pipeline = TosaPipelineINT[input_t1](
+        Add(), 
+        test_data(), 
+        aten_op, 
+        exir_op,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.run()
 
 
@@ -146,25 +158,49 @@ def test_add_tensor_u85_INT(test_data: input_t1):
 
 @common.parametrize("test_data", Add2.test_data)
 def test_add_tensor_tosa_FP_2(test_data: input_t2):
-    pipeline = TosaPipelineFP[input_t2](Add2(), test_data(), aten_op, exir_op)
+    pipeline = TosaPipelineFP[input_t2](
+        Add2(), 
+        test_data(), 
+        aten_op, 
+        exir_op,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.run()
 
 
 @common.parametrize("test_data", Add3.test_data)
 def test_add_tensor_tosa_FP_3(test_data: input_t2):
-    pipeline = TosaPipelineFP[input_t2](Add3(), test_data(), aten_op, exir_op)
+    pipeline = TosaPipelineFP[input_t2](
+        Add3(), 
+        test_data(), 
+        aten_op, 
+        exir_op,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.run()
 
 
 @common.parametrize("test_data", Add3.test_data)
 def test_add_tensor_tosa_INT_3(test_data: input_t2):
-    pipeline = TosaPipelineINT[input_t2](Add3(), test_data(), aten_op, exir_op)
+    pipeline = TosaPipelineINT[input_t2](
+        Add3(), 
+        test_data(), 
+        aten_op, 
+        exir_op,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.run()
 
 
 @common.parametrize("test_data", Add2.test_data)
 def test_add_tensor_tosa_INT_2(test_data: input_t2):
-    pipeline = TosaPipelineINT[input_t2](Add2(), test_data(), aten_op, exir_op)
+    pipeline = TosaPipelineINT[input_t2](
+        Add2(), 
+        test_data(), 
+        aten_op, 
+        exir_op,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.run()