From 692ad68341e84d477ddd7ef786eb16b769a92255 Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Wed, 6 Aug 2025 13:34:20 -0700 Subject: [PATCH] Arm backend: run on tosa_ref_model only when requested (#13165) Summary: Pull Request resolved: https://github.com/pytorch/executorch/pull/13165 This is needed to support running the tests on setups where we may not have tosa reference model installed. The downside is, it may pass without doing e2e validaiton, which shouldn't happen on the CI where this is installed. Reviewed By: kirklandsign Differential Revision: D79741689 --- .../arm/test/misc/test_bn_relu_folding_qat.py | 9 ++- .../arm/test/misc/test_custom_partition.py | 65 ++++++++++++++++--- backends/arm/test/models/test_conformer.py | 3 + .../arm/test/models/test_deit_tiny_arm.py | 3 + backends/arm/test/models/test_llama.py | 2 + backends/arm/test/models/test_lstm_arm.py | 3 + backends/arm/test/models/test_w2l_arm.py | 3 + backends/arm/test/ops/test_abs.py | 17 ++++- backends/arm/test/ops/test_acosh.py | 3 + backends/arm/test/ops/test_add.py | 48 ++++++++++++-- backends/arm/test/ops/test_asin.py | 3 + backends/arm/test/ops/test_asinh.py | 3 + backends/arm/test/ops/test_atan.py | 3 + backends/arm/test/ops/test_atanh.py | 3 + backends/arm/test/ops/test_ceil.py | 3 + backends/arm/test/ops/test_conv2d.py | 3 + backends/arm/test/ops/test_div.py | 17 ++++- backends/arm/test/ops/test_exp.py | 3 + backends/arm/test/ops/test_floor.py | 3 + backends/arm/test/ops/test_leaky_relu.py | 3 + backends/arm/test/ops/test_log.py | 17 ++++- backends/arm/test/ops/test_matmul.py | 30 +++++++-- backends/arm/test/ops/test_mul.py | 7 ++ backends/arm/test/ops/test_neg.py | 17 ++++- backends/arm/test/ops/test_permute.py | 3 + backends/arm/test/ops/test_pow.py | 4 ++ backends/arm/test/ops/test_relu.py | 3 + backends/arm/test/ops/test_rshift.py | 5 ++ backends/arm/test/ops/test_scalars.py | 55 ++++++++-------- backends/arm/test/ops/test_sdpa.py | 20 ++++-- backends/arm/test/ops/test_sigmoid.py | 24 ++++++- backends/arm/test/ops/test_sin.py | 2 + backends/arm/test/ops/test_sinh.py | 8 ++- backends/arm/test/ops/test_slice.py | 13 +++- backends/arm/test/ops/test_sqrt.py | 3 + backends/arm/test/ops/test_sub.py | 5 ++ backends/arm/test/ops/test_tanh.py | 3 + .../test/passes/test_broadcast_args_pass.py | 2 + .../arm/test/passes/test_cast_int64_pass.py | 2 + .../test_convert_expand_copy_to_repeat.py | 2 + .../passes/test_convert_int_pow_to_muls.py | 2 + .../passes/test_convert_split_to_slice.py | 2 + .../arm/test/passes/test_convert_to_clamp.py | 3 + .../test_decompose_cosine_similarity_pass.py | 2 + .../test/passes/test_decompose_div_pass.py | 2 + .../passes/test_decompose_layernorm_pass.py | 2 + .../test_decompose_linalg_vector_norm_pass.py | 2 + .../passes/test_decompose_softmax_pass.py | 3 + .../test/passes/test_decompose_var_pass.py | 2 + .../test/passes/test_fuse_batchnorm_pass.py | 2 + .../passes/test_fuse_constant_ops_pass.py | 4 ++ .../test_fuse_equal_placeholders_ops_pass.py | 5 ++ .../test_insert_int64_to_int32_cast_pass.py | 2 + .../test/passes/test_insert_table_ops_pass.py | 2 + .../passes/test_int32_cast_embedding_pass.py | 2 + .../arm/test/passes/test_remove_clone_pass.py | 2 + backends/arm/test/passes/test_rescale_pass.py | 3 +- .../test_unsqueeze_before_repeat_pass.py | 2 + backends/arm/test/tester/test_pipeline.py | 18 +++-- 59 files changed, 416 insertions(+), 71 deletions(-) diff --git a/backends/arm/test/misc/test_bn_relu_folding_qat.py b/backends/arm/test/misc/test_bn_relu_folding_qat.py index c39c1694d0a..0b75561d15a 100644 --- a/backends/arm/test/misc/test_bn_relu_folding_qat.py +++ b/backends/arm/test/misc/test_bn_relu_folding_qat.py @@ -47,7 +47,14 @@ def forward(self, x: torch.Tensor): @common.parametrize("model", models) def test_qat_tosa_INT(model: torch.nn.Module): - pipeline = TosaPipelineINT[input_t1](model, model.test_data, [], [], qtol=1) + pipeline = TosaPipelineINT[input_t1]( + model, + model.test_data, + [], + [], + qtol=1, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) tosa_version = conftest.get_option("tosa_version") tosa_profiles = { "1.0": common.TosaSpecification.create_from_string("TOSA-1.0+INT"), diff --git a/backends/arm/test/misc/test_custom_partition.py b/backends/arm/test/misc/test_custom_partition.py index 6cdd63af7c9..f7285b678a4 100644 --- a/backends/arm/test/misc/test_custom_partition.py +++ b/backends/arm/test/misc/test_custom_partition.py @@ -3,6 +3,7 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. +import conftest import logging from typing import Tuple @@ -50,7 +51,13 @@ def test_single_reject(caplog, test_data: input_t1): caplog.set_level(logging.INFO) module = CustomPartitioning() - pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[]) + pipeline = TosaPipelineFP[input_t1]( + module, + test_data, + [], + exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) check = DontPartition(exir_ops.edge.aten.sigmoid.default) pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check]) pipeline.change_args( @@ -68,7 +75,13 @@ def test_single_reject(caplog, test_data: input_t1): @common.parametrize("test_data", CustomPartitioning.inputs) def test_multiple_reject(test_data: input_t1): module = CustomPartitioning() - pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[]) + pipeline = TosaPipelineFP[input_t1]( + module, + test_data, + [], + exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) check = DontPartition( exir_ops.edge.aten.sigmoid.default, exir_ops.edge.aten.mul.Tensor ) @@ -90,7 +103,13 @@ def test_torch_op_reject(caplog, test_data: input_t1): module = CustomPartitioning() check = DontPartition(torch.ops.aten.sigmoid.default) - pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[]) + pipeline = TosaPipelineFP[input_t1]( + module, + test_data, + [], + exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check]) pipeline.change_args( "check_count.exir", {"torch.ops.higher_order.executorch_call_delegate": 2} @@ -108,7 +127,13 @@ def test_torch_op_reject(caplog, test_data: input_t1): def test_string_op_reject(test_data: input_t1): module = CustomPartitioning() check = DontPartition("aten.sigmoid.default") - pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[]) + pipeline = TosaPipelineFP[input_t1]( + module, + test_data, + [], + exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check]) pipeline.change_args( "check_count.exir", {"torch.ops.higher_order.executorch_call_delegate": 2} @@ -127,7 +152,13 @@ def test_name_reject(caplog, test_data: input_t1): module = CustomPartitioning() check = DontPartitionName("mul", "sigmoid", exact=False) - pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[]) + pipeline = TosaPipelineFP[input_t1]( + module, + test_data, + [], + exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check]) pipeline.change_args( "check_count.exir", @@ -142,7 +173,13 @@ def test_name_reject(caplog, test_data: input_t1): def test_module_reject(test_data: input_t1): module = NestedModule() check = DontPartitionModule(module_name="CustomPartitioning") - pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[]) + pipeline = TosaPipelineFP[input_t1]( + module, + test_data, + [], + exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check]) pipeline.change_args( "check_count.exir", @@ -158,7 +195,13 @@ def test_inexact_module_reject(caplog, test_data: input_t1): module = NestedModule() check = DontPartitionModule(module_name="Custom", exact=False) - pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[]) + pipeline = TosaPipelineFP[input_t1]( + module, + test_data, + [], + exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check]) pipeline.change_args( "check_count.exir", @@ -173,7 +216,13 @@ def test_inexact_module_reject(caplog, test_data: input_t1): def test_module_instance_reject(test_data: input_t1): module = NestedModule() check = DontPartitionModule(instance_name="nested") - pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[]) + pipeline = TosaPipelineFP[input_t1]( + module, + test_data, + [], + exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check]) pipeline.change_args( "check_count.exir", diff --git a/backends/arm/test/models/test_conformer.py b/backends/arm/test/models/test_conformer.py index 6a66b25d27d..85404840d52 100644 --- a/backends/arm/test/models/test_conformer.py +++ b/backends/arm/test/models/test_conformer.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import pytest import torch @@ -56,6 +57,7 @@ def test_conformer_tosa_FP(): TestConformer.model_example_inputs, aten_op=TestConformer.aten_ops, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), use_to_edge_transform_and_lower=True, ) pipeline.run() @@ -67,6 +69,7 @@ def test_conformer_tosa_INT(): TestConformer.model_example_inputs, aten_op=TestConformer.aten_ops, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), use_to_edge_transform_and_lower=True, ) pipeline.pop_stage("check_count.exir") diff --git a/backends/arm/test/models/test_deit_tiny_arm.py b/backends/arm/test/models/test_deit_tiny_arm.py index 22685a079bd..e9a4efe8a48 100644 --- a/backends/arm/test/models/test_deit_tiny_arm.py +++ b/backends/arm/test/models/test_deit_tiny_arm.py @@ -7,6 +7,7 @@ from typing import Tuple +import conftest import timm import torch @@ -43,6 +44,7 @@ def test_deit_tiny_tosa_FP(): model_inputs, aten_op=[], exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), use_to_edge_transform_and_lower=True, ) pipeline.run() @@ -54,6 +56,7 @@ def test_deit_tiny_tosa_INT(): model_inputs, aten_op=[], exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), use_to_edge_transform_and_lower=True, atol=1.5, qtol=1, diff --git a/backends/arm/test/models/test_llama.py b/backends/arm/test/models/test_llama.py index 7732943d5fb..cfbca5503a7 100644 --- a/backends/arm/test/models/test_llama.py +++ b/backends/arm/test/models/test_llama.py @@ -111,6 +111,7 @@ def test_llama_tosa_FP(): llama_inputs, aten_op=[], exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), use_to_edge_transform_and_lower=True, transform_passes=[InsertCastForOpsWithInt64InputPass()], ) @@ -129,6 +130,7 @@ def test_llama_tosa_INT(): llama_inputs, aten_op=[], exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), use_to_edge_transform_and_lower=True, ) pipeline.run() diff --git a/backends/arm/test/models/test_lstm_arm.py b/backends/arm/test/models/test_lstm_arm.py index 1e63472f5f4..70ccd804fe7 100644 --- a/backends/arm/test/models/test_lstm_arm.py +++ b/backends/arm/test/models/test_lstm_arm.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common @@ -49,6 +50,7 @@ def test_lstm_tosa_FP(): TestLSTM.model_example_inputs, aten_op=[], exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), use_to_edge_transform_and_lower=True, ) pipeline.change_args("run_method_and_compare_outputs", get_test_inputs(), atol=3e-1) @@ -61,6 +63,7 @@ def test_lstm_tosa_INT(): TestLSTM.model_example_inputs, aten_op=[], exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), use_to_edge_transform_and_lower=True, ) pipeline.change_args( diff --git a/backends/arm/test/models/test_w2l_arm.py b/backends/arm/test/models/test_w2l_arm.py index 32b25a18fd8..c822540a188 100644 --- a/backends/arm/test/models/test_w2l_arm.py +++ b/backends/arm/test/models/test_w2l_arm.py @@ -8,6 +8,7 @@ import unittest from typing import Tuple +import conftest import pytest import torch @@ -53,6 +54,7 @@ def test_w2l_tosa_FP(): TestW2L.model_example_inputs, aten_op=[], exir_op=TestW2L.all_operators, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), use_to_edge_transform_and_lower=True, ) pipeline.run() @@ -66,6 +68,7 @@ def test_w2l_tosa_INT(): TestW2L.model_example_inputs, aten_op=[], exir_op=TestW2L.all_operators, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), use_to_edge_transform_and_lower=True, ) pipeline.run() diff --git a/backends/arm/test/ops/test_abs.py b/backends/arm/test/ops/test_abs.py index 4ebcf7393c1..4841bca799e 100644 --- a/backends/arm/test/ops/test_abs.py +++ b/backends/arm/test/ops/test_abs.py @@ -8,6 +8,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -41,13 +42,25 @@ def forward(self, x): @common.parametrize("test_data", Abs.test_parameters) def test_abs_tosa_FP(test_data: torch.Tensor): - pipeline = TosaPipelineFP[input_t1](Abs(), test_data(), aten_op, exir_op) + pipeline = TosaPipelineFP[input_t1]( + Abs(), + test_data(), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @common.parametrize("test_data", Abs.test_parameters) def test_abs_tosa_INT(test_data: torch.Tensor): - pipeline = TosaPipelineINT[input_t1](Abs(), test_data(), aten_op, exir_op) + pipeline = TosaPipelineINT[input_t1]( + Abs(), + test_data(), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() diff --git a/backends/arm/test/ops/test_acosh.py b/backends/arm/test/ops/test_acosh.py index 25ba2b1a83b..16e8b80f0ba 100644 --- a/backends/arm/test/ops/test_acosh.py +++ b/backends/arm/test/ops/test_acosh.py @@ -4,6 +4,7 @@ # LICENSE file in the root directory of this source tree. from typing import Tuple +import conftest import pytest import torch @@ -55,6 +56,7 @@ def test_acosh_tosa_FP(test_data: Tuple): (test_data(),), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -65,6 +67,7 @@ def test_acosh_tosa_INT(test_data: Tuple): Acosh(), (test_data(),), aten_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_add.py b/backends/arm/test/ops/test_add.py index 421ec0adc61..1fcf5623769 100644 --- a/backends/arm/test/ops/test_add.py +++ b/backends/arm/test/ops/test_add.py @@ -81,13 +81,25 @@ def forward(self, x: torch.Tensor, y: torch.Tensor): @common.parametrize("test_data", Add.test_data) def test_add_tensor_tosa_FP(test_data: input_t1): - pipeline = TosaPipelineFP[input_t1](Add(), test_data(), aten_op, exir_op) + pipeline = TosaPipelineFP[input_t1]( + Add(), + test_data(), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @common.parametrize("test_data", Add.test_data) def test_add_tensor_tosa_INT(test_data: input_t1): - pipeline = TosaPipelineINT[input_t1](Add(), test_data(), aten_op, exir_op) + pipeline = TosaPipelineINT[input_t1]( + Add(), + test_data(), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @@ -146,25 +158,49 @@ def test_add_tensor_u85_INT(test_data: input_t1): @common.parametrize("test_data", Add2.test_data) def test_add_tensor_tosa_FP_2(test_data: input_t2): - pipeline = TosaPipelineFP[input_t2](Add2(), test_data(), aten_op, exir_op) + pipeline = TosaPipelineFP[input_t2]( + Add2(), + test_data(), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @common.parametrize("test_data", Add3.test_data) def test_add_tensor_tosa_FP_3(test_data: input_t2): - pipeline = TosaPipelineFP[input_t2](Add3(), test_data(), aten_op, exir_op) + pipeline = TosaPipelineFP[input_t2]( + Add3(), + test_data(), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @common.parametrize("test_data", Add3.test_data) def test_add_tensor_tosa_INT_3(test_data: input_t2): - pipeline = TosaPipelineINT[input_t2](Add3(), test_data(), aten_op, exir_op) + pipeline = TosaPipelineINT[input_t2]( + Add3(), + test_data(), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @common.parametrize("test_data", Add2.test_data) def test_add_tensor_tosa_INT_2(test_data: input_t2): - pipeline = TosaPipelineINT[input_t2](Add2(), test_data(), aten_op, exir_op) + pipeline = TosaPipelineINT[input_t2]( + Add2(), + test_data(), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() diff --git a/backends/arm/test/ops/test_asin.py b/backends/arm/test/ops/test_asin.py index 9c37bddbd92..e1a34991b05 100644 --- a/backends/arm/test/ops/test_asin.py +++ b/backends/arm/test/ops/test_asin.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common @@ -44,6 +45,7 @@ def test_asin_tosa_FP(test_data: Tuple): (test_data(),), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -55,6 +57,7 @@ def test_asin_tosa_INT(test_data: Tuple): (test_data(),), aten_op=[], exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_asinh.py b/backends/arm/test/ops/test_asinh.py index 305c822601c..5342a7c58aa 100644 --- a/backends/arm/test/ops/test_asinh.py +++ b/backends/arm/test/ops/test_asinh.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common @@ -43,6 +44,7 @@ def test_asinh_tosa_FP(test_data: Tuple): (test_data(),), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -54,6 +56,7 @@ def test_asinh_tosa_INT(test_data: Tuple): (test_data(),), aten_op=[], exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_atan.py b/backends/arm/test/ops/test_atan.py index 51114d2800f..b873ef47a7e 100644 --- a/backends/arm/test/ops/test_atan.py +++ b/backends/arm/test/ops/test_atan.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common @@ -46,6 +47,7 @@ def test_atan_tosa_FP(test_data: Tuple): (test_data,), aten_op=aten_op, exir_op=exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -57,6 +59,7 @@ def test_atan_tosa_INT(test_data: Tuple): (test_data,), aten_op=aten_op, exir_op=exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_atanh.py b/backends/arm/test/ops/test_atanh.py index 12754a34646..1c82c792c54 100644 --- a/backends/arm/test/ops/test_atanh.py +++ b/backends/arm/test/ops/test_atanh.py @@ -6,6 +6,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -47,6 +48,7 @@ def test_atanh_tosa_FP(test_data: Tuple): (test_data,), aten_op=aten_op, exir_op=exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -58,6 +60,7 @@ def test_atanh_tosa_INT(test_data: Tuple): (test_data,), aten_op=aten_op, exir_op=exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_ceil.py b/backends/arm/test/ops/test_ceil.py index 64e9040a974..88c8b302124 100644 --- a/backends/arm/test/ops/test_ceil.py +++ b/backends/arm/test/ops/test_ceil.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -51,6 +52,7 @@ def test_ceil_tosa_FP(test_data: input_t1): (data,), module.aten_op, module.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -65,6 +67,7 @@ def test_ceil_tosa_INT(test_data: input_t1): module.exir_op, atol=0.06, rtol=0.01, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_conv2d.py b/backends/arm/test/ops/test_conv2d.py index 0d23d2a6c7e..ebf58be4876 100644 --- a/backends/arm/test/ops/test_conv2d.py +++ b/backends/arm/test/ops/test_conv2d.py @@ -6,6 +6,7 @@ from typing import List, Tuple, Union +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -408,6 +409,7 @@ def test_convolution_2d_tosa_FP(test_data): model.get_inputs(), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -422,6 +424,7 @@ def test_convolution_2d_tosa_INT(test_data): exir_op, per_channel_quantization=per_channel_quantization, qtol=1, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_div.py b/backends/arm/test/ops/test_div.py index 026939758a0..b1305fa5824 100644 --- a/backends/arm/test/ops/test_div.py +++ b/backends/arm/test/ops/test_div.py @@ -8,6 +8,7 @@ from typing import Optional, Tuple, Union +import conftest import torch from executorch.backends.arm.test import common @@ -91,13 +92,25 @@ def forward( @common.parametrize("test_data", test_data_suite) def test_div_tensor_tosa_FP(test_data: Tuple): - pipeline = TosaPipelineFP[input_t1](Div(), test_data(), aten_op, exir_op) + pipeline = TosaPipelineFP[input_t1]( + Div(), + test_data(), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @common.parametrize("test_data", test_data_suite) def test_div_tensor_tosa_INT(test_data: Tuple): - pipeline = TosaPipelineINT[input_t1](Div(), test_data(), aten_op=[], exir_op=[]) + pipeline = TosaPipelineINT[input_t1]( + Div(), + test_data(), + aten_op=[], + exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() diff --git a/backends/arm/test/ops/test_exp.py b/backends/arm/test/ops/test_exp.py index 6eaacc71d86..ef2910225fd 100644 --- a/backends/arm/test/ops/test_exp.py +++ b/backends/arm/test/ops/test_exp.py @@ -8,6 +8,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common @@ -45,6 +46,7 @@ def test_exp_tosa_FP(test_data: Tuple): (test_data(),), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -56,6 +58,7 @@ def test_exp_tosa_INT(test_data: Tuple): (test_data(),), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_floor.py b/backends/arm/test/ops/test_floor.py index c66ef1c5d27..b0f13c9b6af 100644 --- a/backends/arm/test/ops/test_floor.py +++ b/backends/arm/test/ops/test_floor.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -51,6 +52,7 @@ def test_floor_tosa_FP(test_data: input_t1): (data,), module.aten_op, module.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -65,6 +67,7 @@ def test_floor_tosa_INT(test_data: input_t1): module.exir_op, atol=0.06, rtol=0.01, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_leaky_relu.py b/backends/arm/test/ops/test_leaky_relu.py index c18255a73c0..41284173f10 100644 --- a/backends/arm/test/ops/test_leaky_relu.py +++ b/backends/arm/test/ops/test_leaky_relu.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -45,6 +46,7 @@ def test_leaky_relu_tosa_FP(test_data): data, [], use_to_edge_transform_and_lower=True, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.add_stage_after( "to_edge_transform_and_lower", pipeline.tester.check_not, [exir_op] @@ -60,6 +62,7 @@ def test_leaky_relu_tosa_INT(test_data): data, [], use_to_edge_transform_and_lower=True, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.add_stage_after("quantize", pipeline.tester.check_not, [aten_op]) pipeline.run() diff --git a/backends/arm/test/ops/test_log.py b/backends/arm/test/ops/test_log.py index 1ed5c57f1ab..04ffbae4fb3 100644 --- a/backends/arm/test/ops/test_log.py +++ b/backends/arm/test/ops/test_log.py @@ -8,6 +8,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common @@ -42,13 +43,25 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: @common.parametrize("test_data", test_data_suite) def test_log_tosa_FP(test_data: input_t1): - pipeline = TosaPipelineFP[input_t1](Log(), (test_data(),), aten_op, exir_op) + pipeline = TosaPipelineFP[input_t1]( + Log(), + (test_data(),), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @common.parametrize("test_data", test_data_suite) def test_log_tosa_INT(test_data: input_t1): - pipeline = TosaPipelineINT[input_t1](Log(), (test_data(),), aten_op, exir_op) + pipeline = TosaPipelineINT[input_t1]( + Log(), + (test_data(),), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() diff --git a/backends/arm/test/ops/test_matmul.py b/backends/arm/test/ops/test_matmul.py index d1a21684325..c80de7e2080 100644 --- a/backends/arm/test/ops/test_matmul.py +++ b/backends/arm/test/ops/test_matmul.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -62,14 +63,24 @@ def forward(self, x1: torch.Tensor, x2: torch.Tensor, x3: torch.Tensor): @common.parametrize("test_data", MatMul.test_data_generators) def test_matmul_tosa_FP(test_data: input_t1): - pipeline = TosaPipelineFP[input_t1](MatMul(), test_data(), aten_op_mm, exir_op_mm) + pipeline = TosaPipelineFP[input_t1]( + MatMul(), + test_data(), + aten_op_mm, + exir_op_mm, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @common.parametrize("test_data", MatMulSingleInput.test_data_generators) def test_matmul_single_input_tosa_FP(test_data: input_t1): pipeline = TosaPipelineFP[input_t1]( - MatMulSingleInput(), test_data(), aten_op_mm, exir_op_mm + MatMulSingleInput(), + test_data(), + aten_op_mm, + exir_op_mm, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -77,7 +88,11 @@ def test_matmul_single_input_tosa_FP(test_data: input_t1): @common.parametrize("test_data", MatMulCombo.test_data_generators) def test_matmul_combo_tosa_FP(test_data: input_t1): pipeline = TosaPipelineFP[input_t1]( - MatMulCombo(), test_data(), aten_op_mm, exir_op_mm + MatMulCombo(), + test_data(), + aten_op_mm, + exir_op_mm, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -85,7 +100,12 @@ def test_matmul_combo_tosa_FP(test_data: input_t1): @common.parametrize("test_data", MatMul.test_data_generators) def test_matmul_tosa_INT(test_data: input_t1): pipeline = TosaPipelineINT[input_t1]( - MatMul(), test_data(), aten_op_mm, exir_op_mm, qtol=1 + MatMul(), + test_data(), + aten_op_mm, + exir_op_mm, + qtol=1, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -98,6 +118,7 @@ def test_matmul_single_input_tosa_INT(test_data: input_t1): aten_op_mm, exir_op_mm, qtol=1, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -110,6 +131,7 @@ def test_matmul_combo_tosa_INT(test_data: input_t1): aten_op_mm, exir_op_mm, qtol=1, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_mul.py b/backends/arm/test/ops/test_mul.py index b0b7f5f4b7d..aa595ffde2a 100644 --- a/backends/arm/test/ops/test_mul.py +++ b/backends/arm/test/ops/test_mul.py @@ -8,6 +8,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common @@ -114,6 +115,7 @@ def test_mul_tensor_tosa_FP(test_data: torch.Tensor): test_data(), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -125,6 +127,7 @@ def test_mul_tensor_tosa_FP_diff_input_ranks(test_data: torch.Tensor): test_data(), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -136,6 +139,7 @@ def test_mul_tensor_tosa_FP_int32(test_data: torch.Tensor): test_data(), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -147,6 +151,7 @@ def test_mul_tensor_tosa_INT_diff_input_ranks(test_data: torch.Tensor): test_data(), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -158,6 +163,7 @@ def test_mul_tensor_tosa_INT(test_data: torch.Tensor): test_data(), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -169,6 +175,7 @@ def test_mul_tensor_tosa_INT_int32(test_data: torch.Tensor): test_data(), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.pop_stage("check.quant_nodes") pipeline.run() diff --git a/backends/arm/test/ops/test_neg.py b/backends/arm/test/ops/test_neg.py index 395a4815b62..4261a9f9987 100644 --- a/backends/arm/test/ops/test_neg.py +++ b/backends/arm/test/ops/test_neg.py @@ -6,6 +6,7 @@ from typing import Dict, Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -39,13 +40,25 @@ def forward(self, x: torch.Tensor): @common.parametrize("test_data", Neg.test_data) def test_neg_tosa_FP(test_data: input_t1): - pipeline = TosaPipelineFP[input_t1](Neg(), test_data, Neg.aten_op, Neg.exir_op) + pipeline = TosaPipelineFP[input_t1]( + Neg(), + test_data, + Neg.aten_op, + Neg.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @common.parametrize("test_data", Neg.test_data) def test_neg_tosa_INT(test_data: input_t1): - pipeline = TosaPipelineINT[input_t1](Neg(), test_data, Neg.aten_op, Neg.exir_op) + pipeline = TosaPipelineINT[input_t1]( + Neg(), + test_data, + Neg.aten_op, + Neg.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() diff --git a/backends/arm/test/ops/test_permute.py b/backends/arm/test/ops/test_permute.py index 57f7f9603a1..a38b427deb4 100644 --- a/backends/arm/test/ops/test_permute.py +++ b/backends/arm/test/ops/test_permute.py @@ -8,6 +8,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common @@ -56,6 +57,7 @@ def test_permute_tosa_FP(test_data: torch.Tensor): (test_data,), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -68,6 +70,7 @@ def test_permute_tosa_INT(test_data: torch.Tensor): (test_data,), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_pow.py b/backends/arm/test/ops/test_pow.py index 016c3e97265..eba7d53d1c0 100644 --- a/backends/arm/test/ops/test_pow.py +++ b/backends/arm/test/ops/test_pow.py @@ -6,6 +6,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -99,6 +100,7 @@ def test_pow_tensor_tensor_tosa_FP(test_data: Pow_TensorTensor.input_t): test_data(), Pow_TensorTensor.aten_op, Pow_TensorTensor.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -134,6 +136,7 @@ def test_pow_tensor_scalar_tosa_FP(test_data: Pow_TensorScalar.input_t): (base,), Pow_TensorScalar.aten_op, Pow_TensorScalar.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -146,6 +149,7 @@ def test_pow_tensor_scalar_tosa_INT(test_data: Pow_TensorScalar.input_t): (base,), Pow_TensorScalar.aten_op, Pow_TensorScalar.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_relu.py b/backends/arm/test/ops/test_relu.py index 0b29bc24e75..cd08a2bca5b 100644 --- a/backends/arm/test/ops/test_relu.py +++ b/backends/arm/test/ops/test_relu.py @@ -8,6 +8,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -50,6 +51,7 @@ def test_relu_tosa_FP(test_data: torch.Tensor): (test_data(),), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -61,6 +63,7 @@ def test_relu_tosa_INT(test_data: torch.Tensor): (test_data(),), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_rshift.py b/backends/arm/test/ops/test_rshift.py index e97bfb840ae..abeeb245860 100644 --- a/backends/arm/test/ops/test_rshift.py +++ b/backends/arm/test/ops/test_rshift.py @@ -3,6 +3,7 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.common import ( @@ -80,6 +81,7 @@ def test_bitwise_right_shift_scalar_tosa_FP_scalar(test_data): test_data(), RshiftScalar.torch_op_FP, RshiftScalar.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ).run() @@ -90,6 +92,7 @@ def test_bitwise_right_shift_tensor_tosa_INT_scalar(test_data): test_data(), RshiftScalar.torch_op_INT, RshiftScalar.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.pop_stage("check.quant_nodes") pipeline.run() @@ -165,6 +168,7 @@ def test_bitwise_right_shift_tensor_tosa_FP(test_data): test_data(), RshiftTensor.torch_op, RshiftTensor.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ).run() @@ -175,6 +179,7 @@ def test_bitwise_right_shift_tensor_tosa_INT(test_data): test_data(), RshiftTensor.torch_op, RshiftTensor.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.pop_stage("check.quant_nodes") pipeline.run() diff --git a/backends/arm/test/ops/test_scalars.py b/backends/arm/test/ops/test_scalars.py index 1243a522526..29f6a70a889 100644 --- a/backends/arm/test/ops/test_scalars.py +++ b/backends/arm/test/ops/test_scalars.py @@ -6,6 +6,7 @@ from typing import Tuple +import conftest import pytest import torch @@ -174,21 +175,21 @@ def forward(self, x): @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_add_tensor_tosa_FP_scalar(test_data): """Tests regular add with one scalar input.""" - pipeline = TosaPipelineFP[input_t1](Add(), test_data, aten_op=Add.aten_op) + pipeline = TosaPipelineFP[input_t1](Add(), test_data, aten_op=Add.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_add_tensor_tosa_FP_inplace(test_data): """Tests inplace add with one scalar input.""" - pipeline = TosaPipelineFP[input_t1](AddInplace(), test_data, aten_op=[]) + pipeline = TosaPipelineFP[input_t1](AddInplace(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_const_tests, xfails=xfails) def test_add_tensor_tosa_FP_const(test_data): """Tests regular add with one scalar input, with one of inputs constant.""" - pipeline = TosaPipelineFP[input_t1](AddConst(), test_data, aten_op=AddConst.aten_op) + pipeline = TosaPipelineFP[input_t1](AddConst(), test_data, aten_op=AddConst.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @@ -196,7 +197,7 @@ def test_add_tensor_tosa_FP_const(test_data): def test_add_scalar_tosa_FP(test_data): """Tests a scalar add with one scalar input.""" pipeline = TosaPipelineFP[input_t1]( - AddScalar(), test_data, aten_op=AddScalar.aten_op + AddScalar(), test_data, aten_op=AddScalar.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model") ) pipeline.run() @@ -205,14 +206,14 @@ def test_add_scalar_tosa_FP(test_data): @common.parametrize("test_data", tensor_scalar_tests) def test_add_tensor_tosa_INT_scalar(test_data): """Tests regular add with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](Add(), test_data, aten_op=[]) + pipeline = TosaPipelineINT[input_t1](Add(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests) def test_add_tensor_tosa_INT_inplace(test_data): """Tests inplace add with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](AddInplace(), test_data, aten_op=[]) + pipeline = TosaPipelineINT[input_t1](AddInplace(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @@ -220,7 +221,7 @@ def test_add_tensor_tosa_INT_inplace(test_data): def test_add_tensor_tosa_INT_const(test_data): """Tests regular add with one scalar input, with one of inputs constant.""" pipeline = TosaPipelineINT[input_t1]( - AddConst(), test_data, aten_op=AddConst.aten_op + AddConst(), test_data, aten_op=AddConst.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model") ) pipeline.run() @@ -228,7 +229,7 @@ def test_add_tensor_tosa_INT_const(test_data): @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_add_scalar_tosa_INT(test_data): """Tests a scalar add with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](AddScalar(), test_data, aten_op=Add.aten_op) + pipeline = TosaPipelineINT[input_t1](AddScalar(), test_data, aten_op=Add.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @@ -249,14 +250,14 @@ def test_add_scalar_u85_INT(): @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_sub_tensor_tosa_FP_scalar(test_data): """Tests regular sub with one scalar input.""" - pipeline = TosaPipelineFP[input_t1](Sub(), test_data, aten_op=Sub.aten_op) + pipeline = TosaPipelineFP[input_t1](Sub(), test_data, aten_op=Sub.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_sub_tensor_tosa_FP_inplace(test_data): """Tests inplace sub with one scalar input.""" - pipeline = TosaPipelineFP[input_t1](SubInplace(), test_data, aten_op=[]) + pipeline = TosaPipelineFP[input_t1](SubInplace(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @@ -264,7 +265,7 @@ def test_sub_tensor_tosa_FP_inplace(test_data): def test_sub_scalar_tosa_FP(test_data): """Tests a scalar sub with one scalar input.""" pipeline = TosaPipelineFP[input_t1]( - SubScalar(), test_data, aten_op=SubScalar.aten_op + SubScalar(), test_data, aten_op=SubScalar.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model") ) pipeline.run() @@ -273,21 +274,21 @@ def test_sub_scalar_tosa_FP(test_data): @common.parametrize("test_data", tensor_scalar_tests) def test_sub_tensor_tosa_INT_scalar(test_data): """Tests regular sub with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](Sub(), test_data, aten_op=[]) + pipeline = TosaPipelineINT[input_t1](Sub(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests) def test_sub_tensor_tosa_INT_inplace(test_data): """Tests inplace sub with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](SubInplace(), test_data, aten_op=[]) + pipeline = TosaPipelineINT[input_t1](SubInplace(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_sub_scalar_tosa_INT(test_data): """Tests a scalar sub with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](SubScalar(), test_data, aten_op=Sub.aten_op) + pipeline = TosaPipelineINT[input_t1](SubScalar(), test_data, aten_op=Sub.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @@ -306,14 +307,14 @@ def test_sub_scalar_u85_INT(): @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_mul_tensor_tosa_FP_scalar(test_data): """Tests regular mul with one scalar input.""" - pipeline = TosaPipelineFP[input_t1](Mul(), test_data, aten_op=Mul.aten_op) + pipeline = TosaPipelineFP[input_t1](Mul(), test_data, aten_op=Mul.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_mul_tensor_tosa_FP_inplace(test_data): """Tests inplace mul with one scalar input.""" - pipeline = TosaPipelineFP[input_t1](MulInplace(), test_data, aten_op=[]) + pipeline = TosaPipelineFP[input_t1](MulInplace(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @@ -321,7 +322,7 @@ def test_mul_tensor_tosa_FP_inplace(test_data): def test_mul_scalar_tosa_FP(test_data): """Tests a scalar mul with one scalar input.""" pipeline = TosaPipelineFP[input_t1]( - MulScalar(), test_data, aten_op=MulScalar.aten_op + MulScalar(), test_data, aten_op=MulScalar.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model") ) pipeline.run() @@ -330,21 +331,21 @@ def test_mul_scalar_tosa_FP(test_data): @common.parametrize("test_data", tensor_scalar_tests) def test_mul_tensor_tosa_INT_scalar(test_data): """Tests regular mul with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](Mul(), test_data, aten_op=[]) + pipeline = TosaPipelineINT[input_t1](Mul(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests) def test_mul_tensor_tosa_INT_inplace(test_data): """Tests inplace mul with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](MulInplace(), test_data, aten_op=[]) + pipeline = TosaPipelineINT[input_t1](MulInplace(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_mul_scalar_tosa_INT(test_data): """Tests a scalar mul with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](MulScalar(), test_data, aten_op=Mul.aten_op) + pipeline = TosaPipelineINT[input_t1](MulScalar(), test_data, aten_op=Mul.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @@ -363,14 +364,14 @@ def test_mul_scalar_u85_INT(): @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_div_tensor_tosa_FP_scalar(test_data): """Tests regular div with one scalar input.""" - pipeline = TosaPipelineFP[input_t1](Div(), test_data, aten_op=Div.aten_op) + pipeline = TosaPipelineFP[input_t1](Div(), test_data, aten_op=Div.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_div_tensor_tosa_FP_inplace(test_data): """Tests inplace div with one scalar input.""" - pipeline = TosaPipelineFP[input_t1](DivInplace(), test_data, aten_op=[]) + pipeline = TosaPipelineFP[input_t1](DivInplace(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @@ -378,7 +379,7 @@ def test_div_tensor_tosa_FP_inplace(test_data): def test_div_scalar_tosa_FP(test_data): """Tests a scalar div with one scalar input.""" pipeline = TosaPipelineFP[input_t1]( - DivScalar(), test_data, aten_op=DivScalar.aten_op + DivScalar(), test_data, aten_op=DivScalar.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model") ) pipeline.run() @@ -387,21 +388,21 @@ def test_div_scalar_tosa_FP(test_data): @common.parametrize("test_data", tensor_scalar_tests) def test_div_tensor_tosa_INT_scalar(test_data): """Tests regular div with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](Div(), test_data, aten_op=[]) + pipeline = TosaPipelineINT[input_t1](Div(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests) def test_div_tensor_tosa_INT_inplace(test_data): """Tests inplace div with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](DivInplace(), test_data, aten_op=[]) + pipeline = TosaPipelineINT[input_t1](DivInplace(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_div_scalar_tosa_INT(test_data): """Tests a scalar div with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](DivScalar(), test_data, aten_op=[]) + pipeline = TosaPipelineINT[input_t1](DivScalar(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @@ -422,6 +423,7 @@ def test_bitwise_right_shift_tensor_tosa_FP_inplace(): ShiftInplaceSub(), (torch.IntTensor(5),), aten_op="torch.ops.aten.__rshift__.Scalar", + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -431,6 +433,7 @@ def test_bitwise_right_shift_tensor_tosa_INT_inplace(): ShiftInplaceSub(), (torch.IntTensor(5),), aten_op="torch.ops.aten.bitwise_right_shift.Tensor", + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.pop_stage("check.quant_nodes") pipeline.run() diff --git a/backends/arm/test/ops/test_sdpa.py b/backends/arm/test/ops/test_sdpa.py index c4b05972f76..57d80931b77 100644 --- a/backends/arm/test/ops/test_sdpa.py +++ b/backends/arm/test/ops/test_sdpa.py @@ -6,6 +6,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test.tester.test_pipeline import ( @@ -29,17 +30,26 @@ def forward(self, query, key, value): def test_sdpa_FP(): test_input = tuple(torch.randn(1, 3, 197, 64) for x in range(3)) - pipeline = TosaPipelineFP[input_t](SDPA(), test_input, [], []) + pipeline = TosaPipelineFP[input_t]( + SDPA(), + test_input, + [], + [], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.pop_stage("check_count.exir") pipeline.run() def test_sdpa_INT(): test_input = tuple(torch.randn(1, 3, 197, 64) for x in range(3)) - pipeline = TosaPipelineINT[input_t](SDPA(), test_input, [], []) + pipeline = TosaPipelineINT[input_t]( + SDPA(), + test_input, + [], + [], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.pop_stage("check.quant_nodes") pipeline.pop_stage("check_count.exir") - pipeline.pop_stage( - "run_method_and_compare_outputs" - ) # TODO: reference is not quantized pipeline.run() diff --git a/backends/arm/test/ops/test_sigmoid.py b/backends/arm/test/ops/test_sigmoid.py index b4f8458574e..8113607de83 100644 --- a/backends/arm/test/ops/test_sigmoid.py +++ b/backends/arm/test/ops/test_sigmoid.py @@ -8,6 +8,8 @@ from typing import Tuple +import conftest + import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -70,12 +72,24 @@ def forward(self, x, y): @common.parametrize("test_data", test_data_suite) def test_sigmoid_tosa_FP(test_data: torch.Tensor): - TosaPipelineFP[input_t1](Sigmoid(), (test_data(),), aten_op, exir_op).run() + TosaPipelineFP[input_t1]( + Sigmoid(), + (test_data(),), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ).run() @common.parametrize("test_data", test_data_suite) def test_sigmoid_tosa_INT(test_data: torch.Tensor): - TosaPipelineINT[input_t1](Sigmoid(), (test_data(),), aten_op, exir_op).run() + TosaPipelineINT[input_t1]( + Sigmoid(), + (test_data(),), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ).run() def test_sigmoid_tosa_FP_add(): @@ -84,6 +98,7 @@ def test_sigmoid_tosa_FP_add(): (test_data_suite["zeros"](),), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ).run() @@ -93,6 +108,7 @@ def test_sigmoid_tosa_INT_add(): (test_data_suite["ramp"](),), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ).run() @@ -102,6 +118,7 @@ def test_sigmoid_tosa_FP_add_2(): (test_data_suite["zeros"](),), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ).run() @@ -111,6 +128,7 @@ def test_sigmoid_tosa_INT_add_2(): (test_data_suite["zeros"](),), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ).run() @@ -120,6 +138,7 @@ def test_sigmoid_tosa_FP_add_3(): (test_data_suite["randn_neg"](), test_data_suite["randn_pos"]()), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ).run() @@ -129,6 +148,7 @@ def test_sigmoid_tosa_INT_3(): (test_data_suite["randn_neg"](), test_data_suite["randn_pos"]()), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ).run() diff --git a/backends/arm/test/ops/test_sin.py b/backends/arm/test/ops/test_sin.py index 6f9037e1021..948ea23be25 100644 --- a/backends/arm/test/ops/test_sin.py +++ b/backends/arm/test/ops/test_sin.py @@ -43,6 +43,7 @@ def test_sin_tosa_FP(test_data: Tuple): (test_data,), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) if conftest.get_option("tosa_version") == "1.0": pipeline.run() @@ -55,6 +56,7 @@ def test_sin_tosa_INT(test_data: Tuple): (test_data,), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_sinh.py b/backends/arm/test/ops/test_sinh.py index ff486e6a4b8..751559e084f 100644 --- a/backends/arm/test/ops/test_sinh.py +++ b/backends/arm/test/ops/test_sinh.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -48,6 +49,7 @@ def test_sinh_tosa_FP(test_data: Tuple): (test_data,), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -55,7 +57,11 @@ def test_sinh_tosa_FP(test_data: Tuple): @common.parametrize("test_data", test_data_suite) def test_sinh_tosa_INT(test_data: Tuple): pipeline = TosaPipelineINT[input_t1]( - Sinh(), (test_data,), aten_op=aten_op, exir_op=exir_op + Sinh(), + (test_data,), + aten_op=aten_op, + exir_op=exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_slice.py b/backends/arm/test/ops/test_slice.py index 8fcf343dd57..d8719d1b274 100644 --- a/backends/arm/test/ops/test_slice.py +++ b/backends/arm/test/ops/test_slice.py @@ -7,8 +7,9 @@ from typing import Tuple -import torch +import conftest +import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -44,7 +45,13 @@ def forward(self, x: torch.Tensor, s: list[tuple[int, int]]): @common.parametrize("test_data", test_data_suite) def test_slice_tensor_tosa_FP(test_data: torch.Tensor): - pipeline = TosaPipelineFP[input_t1](Slice(), test_data(), aten_op, exir_op) + pipeline = TosaPipelineFP[input_t1]( + Slice(), + test_data(), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @@ -55,6 +62,7 @@ def test_slice_tensor_tosa_INT_nchw(test_data: torch.Tensor): test_data(), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -66,6 +74,7 @@ def test_slice_tensor_tosa_INT_nhwc(test_data: torch.Tensor): test_data(), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_sqrt.py b/backends/arm/test/ops/test_sqrt.py index ee554ce4fd2..b9b9429847f 100644 --- a/backends/arm/test/ops/test_sqrt.py +++ b/backends/arm/test/ops/test_sqrt.py @@ -6,6 +6,7 @@ from typing import Dict, Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -51,6 +52,7 @@ def test_sqrt_tosa_FP(test_data: Sqrt.input_t): test_data(), Sqrt.aten_op_FP, Sqrt.exir_op_FP, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -62,6 +64,7 @@ def test_sqrt_tosa_INT(test_data: Sqrt.input_t): test_data(), Sqrt.aten_op_INT, Sqrt.exir_op_INT, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_sub.py b/backends/arm/test/ops/test_sub.py index 09f5884b1c4..9e952abf051 100644 --- a/backends/arm/test/ops/test_sub.py +++ b/backends/arm/test/ops/test_sub.py @@ -7,6 +7,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -70,6 +71,7 @@ def test_sub_tensor_tosa_FP(test_data): test_data(), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -82,6 +84,7 @@ def test_sub_tensor_tosa_FP_2(test_data: Tuple[torch.Tensor, torch.Tensor]): test_data(), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -94,6 +97,7 @@ def test_sub_tensor_tosa_INT(test_data): test_data(), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -106,6 +110,7 @@ def test_sub_tensor_tosa_INT_2(test_data: Tuple[torch.Tensor, torch.Tensor]): test_data(), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_tanh.py b/backends/arm/test/ops/test_tanh.py index 1bd746d7b24..ceae4a85843 100644 --- a/backends/arm/test/ops/test_tanh.py +++ b/backends/arm/test/ops/test_tanh.py @@ -6,6 +6,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common @@ -46,6 +47,7 @@ def test_tanh_tosa_FP(test_data: Tuple): (test_data(),), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -57,6 +59,7 @@ def test_tanh_tosa_INT(test_data: Tuple): (test_data(),), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_broadcast_args_pass.py b/backends/arm/test/passes/test_broadcast_args_pass.py index 719a0ddd622..b675144cac7 100644 --- a/backends/arm/test/passes/test_broadcast_args_pass.py +++ b/backends/arm/test/passes/test_broadcast_args_pass.py @@ -6,6 +6,7 @@ import operator from typing import Tuple +import conftest import torch from executorch.backends.arm._passes import BroadcastArgsPass @@ -50,5 +51,6 @@ def test_multiple_broacasts_model(module: NeedsMultipleBroadcastsModel): ops_not_before_pass=ops_not_before_pass, ops_after_pass=ops_after_pass, pass_list=[BroadcastArgsPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_cast_int64_pass.py b/backends/arm/test/passes/test_cast_int64_pass.py index 7832fd87ed9..e4dc26ffae8 100644 --- a/backends/arm/test/passes/test_cast_int64_pass.py +++ b/backends/arm/test/passes/test_cast_int64_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes import CastInt64BuffersToInt32Pass @@ -39,6 +40,7 @@ def test_int64_model(test_data: input_t): ops_before_pass=op_checks, ops_after_pass=op_checks, passes_with_exported_program=[CastInt64BuffersToInt32Pass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_convert_expand_copy_to_repeat.py b/backends/arm/test/passes/test_convert_expand_copy_to_repeat.py index aa877c355bd..9746076cffc 100644 --- a/backends/arm/test/passes/test_convert_expand_copy_to_repeat.py +++ b/backends/arm/test/passes/test_convert_expand_copy_to_repeat.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.convert_expand_copy_to_repeat import ( ConvertExpandCopyToRepeatPass, @@ -47,5 +48,6 @@ def test_expand_to_repeat_tosa_INT(): "executorch_exir_dialects_edge__ops_aten_expand_copy_default" ], pass_list=[ConvertExpandCopyToRepeatPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_convert_int_pow_to_muls.py b/backends/arm/test/passes/test_convert_int_pow_to_muls.py index 4eeff845749..b66e9305fcc 100644 --- a/backends/arm/test/passes/test_convert_int_pow_to_muls.py +++ b/backends/arm/test/passes/test_convert_int_pow_to_muls.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes import ConvertIntPowToMuls @@ -69,5 +70,6 @@ def test_convert_pow_to_muls(data): }, ops_not_after_pass=["executorch_exir_dialects_edge__ops_pow_Tensor_Scalar"], pass_list=[ConvertIntPowToMuls], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_convert_split_to_slice.py b/backends/arm/test/passes/test_convert_split_to_slice.py index fba52308ff0..e04275f81d2 100644 --- a/backends/arm/test/passes/test_convert_split_to_slice.py +++ b/backends/arm/test/passes/test_convert_split_to_slice.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.convert_split_to_slice import ( ConvertSplitToSlicePass, @@ -63,5 +64,6 @@ def test_split_to_slice_tosa_INT(module): "executorch_exir_dialects_edge__ops_aten_split_with_sizes_copy_default" ], pass_list=[ConvertSplitToSlicePass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_convert_to_clamp.py b/backends/arm/test/passes/test_convert_to_clamp.py index cc854eeacd7..3c4b2943f16 100644 --- a/backends/arm/test/passes/test_convert_to_clamp.py +++ b/backends/arm/test/passes/test_convert_to_clamp.py @@ -6,6 +6,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.convert_to_clamp import ConvertToClampPass @@ -64,6 +65,7 @@ def test_tosa_FP_hardtahn(test_data: input_t): ops_after_pass=op_checks_after_pass, ops_not_after_pass=op_checks_not_after_pass, pass_list=[ConvertToClampPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -88,5 +90,6 @@ def test_tosa_FP_relu(test_data: input_t): ops_after_pass=op_checks_after_pass, ops_not_after_pass=op_checks_not_after_pass, pass_list=[ConvertToClampPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_decompose_cosine_similarity_pass.py b/backends/arm/test/passes/test_decompose_cosine_similarity_pass.py index 80a328f39c6..0aaa429e231 100644 --- a/backends/arm/test/passes/test_decompose_cosine_similarity_pass.py +++ b/backends/arm/test/passes/test_decompose_cosine_similarity_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.decompose_cosine_similarity_pass import ( @@ -49,5 +50,6 @@ def test_decompose_cosine_similarity_tosa_INT(module): ops_not_after_pass=None, pass_list=[DecomposeCosineSimilarityPass], quantize=True, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_decompose_div_pass.py b/backends/arm/test/passes/test_decompose_div_pass.py index b52e264bf11..585fbe19b16 100644 --- a/backends/arm/test/passes/test_decompose_div_pass.py +++ b/backends/arm/test/passes/test_decompose_div_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.decompose_div_pass import DecomposeDivPass @@ -61,5 +62,6 @@ def test_decompose_div_tosa_FP(module): }, ops_not_after_pass=["executorch_exir_dialects_edge__ops_aten_div_Tensor"], pass_list=[DecomposeDivPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_decompose_layernorm_pass.py b/backends/arm/test/passes/test_decompose_layernorm_pass.py index d3c2cd6efd7..de1c74be2ae 100644 --- a/backends/arm/test/passes/test_decompose_layernorm_pass.py +++ b/backends/arm/test/passes/test_decompose_layernorm_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.decompose_layernorm_pass import ( DecomposeLayerNormPass, @@ -65,5 +66,6 @@ def test_decompose_layernorm_tosa_FP(): "executorch_exir_dialects_edge__ops_aten_expand_copy_default" ], pass_list=[DecomposeLayerNormPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_decompose_linalg_vector_norm_pass.py b/backends/arm/test/passes/test_decompose_linalg_vector_norm_pass.py index 5b4c84edbfd..bd318e418b9 100644 --- a/backends/arm/test/passes/test_decompose_linalg_vector_norm_pass.py +++ b/backends/arm/test/passes/test_decompose_linalg_vector_norm_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.decompose_linalg_vector_norm_pass import ( @@ -87,5 +88,6 @@ def test_decompose_vector_norm_tosa_INT(module): "executorch_exir_dialects_edge__ops_aten_linarg_vector_norm_default", ], pass_list=[DecomposeLinearVectorNormPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_decompose_softmax_pass.py b/backends/arm/test/passes/test_decompose_softmax_pass.py index 3af1976e3f3..f3ea3128b01 100644 --- a/backends/arm/test/passes/test_decompose_softmax_pass.py +++ b/backends/arm/test/passes/test_decompose_softmax_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.decompose_softmax_pass import DecomposeSoftmaxPass @@ -70,6 +71,7 @@ def test_softmax_basic_tosa_FP(): }, ops_not_after_pass=["executorch_exir_dialects_edge__ops_aten__softmax_default"], pass_list=[DecomposeSoftmaxPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -99,5 +101,6 @@ def test_softmax_log_tosa_FP(): "executorch_exir_dialects_edge__ops_aten__log_softmax_default" ], pass_list=[DecomposeSoftmaxPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_decompose_var_pass.py b/backends/arm/test/passes/test_decompose_var_pass.py index c347a2f667c..ba5eb47b7b5 100644 --- a/backends/arm/test/passes/test_decompose_var_pass.py +++ b/backends/arm/test/passes/test_decompose_var_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.decompose_var_pass import DecomposeVarPass @@ -80,5 +81,6 @@ def test_decompose_var_tosa_FP(module): }, ops_not_after_pass=["executorch_exir_dialects_edge__ops_aten_var_correction"], pass_list=[DecomposeVarPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_fuse_batchnorm_pass.py b/backends/arm/test/passes/test_fuse_batchnorm_pass.py index 59fae7cafbd..2c0edf109a8 100644 --- a/backends/arm/test/passes/test_fuse_batchnorm_pass.py +++ b/backends/arm/test/passes/test_fuse_batchnorm_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.fuse_batchnorm2d_pass import FuseBatchnorm2DPass from executorch.backends.arm.test import common @@ -148,5 +149,6 @@ def test_fuse_batchnorm_tosa_FP(module: torch.nn.Module): ops_before_pass=module.ops_before_pass, ops_after_pass=module.ops_after_pass, passes_with_exported_program=[FuseBatchnorm2DPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_fuse_constant_ops_pass.py b/backends/arm/test/passes/test_fuse_constant_ops_pass.py index 1a318c5cd42..8947aade3f0 100644 --- a/backends/arm/test/passes/test_fuse_constant_ops_pass.py +++ b/backends/arm/test/passes/test_fuse_constant_ops_pass.py @@ -6,6 +6,7 @@ import operator from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.fuse_constant_ops_pass import ( ComputeConstantOpsAOT, @@ -124,6 +125,7 @@ def test_fuse_const_ops_tosa_FP(module: torch.nn.Module): ops_after_pass=module.ops_after_pass, ops_not_after_pass=module.ops_not_after_pass, passes_with_exported_program=[ComputeConstantOpsAOT, FuseConstantArgsPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -137,6 +139,7 @@ def test_fuse_const_ops_tosa_INT(module: torch.nn.Module): ops_before_pass=module.ops_before_pass, ops_after_pass=module.ops_after_pass, passes_with_exported_program=[ComputeConstantOpsAOT, FuseConstantArgsPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -150,5 +153,6 @@ def test_fuse_const_ops_tosa_BI_cat(module: torch.nn.Module): ops_before_pass=module.ops_before_pass, ops_after_pass=module.ops_after_pass, passes_with_exported_program=[ComputeConstantOpsAOT, FuseConstantArgsPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_fuse_equal_placeholders_ops_pass.py b/backends/arm/test/passes/test_fuse_equal_placeholders_ops_pass.py index f6e437ba034..ee71ea74a00 100644 --- a/backends/arm/test/passes/test_fuse_equal_placeholders_ops_pass.py +++ b/backends/arm/test/passes/test_fuse_equal_placeholders_ops_pass.py @@ -6,6 +6,8 @@ from copy import deepcopy from typing import Tuple +import conftest + import torch from executorch.backends.arm._passes.fuse_equal_placeholders_pass import ( FuseEqualPlaceholdersPass, @@ -86,6 +88,7 @@ def test_fuse_equal_placeholders_constants_tosa_FP(): ops_before_pass=module.ops_before_pass, ops_after_pass=module.ops_after_pass, passes_with_exported_program=[FuseEqualPlaceholdersPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -107,6 +110,7 @@ def test_fuse_equal_placeholders_state_dict_tosa_FP(): ops_before_pass=module.ops_before_pass, ops_after_pass=module.ops_after_pass, passes_with_exported_program=[FuseEqualPlaceholdersPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -135,5 +139,6 @@ def test_not_fuse_tensor_with_different_type_FP(): module, data, aten_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_insert_int64_to_int32_cast_pass.py b/backends/arm/test/passes/test_insert_int64_to_int32_cast_pass.py index da6eeb59459..5fd026cc89f 100644 --- a/backends/arm/test/passes/test_insert_int64_to_int32_cast_pass.py +++ b/backends/arm/test/passes/test_insert_int64_to_int32_cast_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes import InsertCastForOpsWithInt64InputPass @@ -41,6 +42,7 @@ def test_int64_model_tosa_FP(): ops_before_pass=op_checks_before, ops_after_pass=op_checks_after, pass_list=[InsertCastForOpsWithInt64InputPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.pop_stage(-1) # Do not compare output pipeline.run() diff --git a/backends/arm/test/passes/test_insert_table_ops_pass.py b/backends/arm/test/passes/test_insert_table_ops_pass.py index 029942dd659..d80a275cdd2 100644 --- a/backends/arm/test/passes/test_insert_table_ops_pass.py +++ b/backends/arm/test/passes/test_insert_table_ops_pass.py @@ -6,6 +6,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.fold_qdq_with_annotated_qparams_pass import ( FoldAndAnnotateQParamsPass, @@ -42,6 +43,7 @@ def test_insert_table_tosa_INT(test_data: input_t): ops_not_after_pass=["aten_sigmoid_default"], pass_list=[FoldAndAnnotateQParamsPass], passes_with_exported_program=[InsertTableOpsPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.pop_stage(-1) # Do not compare output diff --git a/backends/arm/test/passes/test_int32_cast_embedding_pass.py b/backends/arm/test/passes/test_int32_cast_embedding_pass.py index 7adca527d75..35b940cc0ea 100644 --- a/backends/arm/test/passes/test_int32_cast_embedding_pass.py +++ b/backends/arm/test/passes/test_int32_cast_embedding_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes import DecomposeEmbeddingPass @@ -41,6 +42,7 @@ def test_int64_model_tosa_FP(): ops_before_pass=op_checks_before, ops_after_pass=op_checks_after, pass_list=[DecomposeEmbeddingPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.pop_stage(-1) # Do not compare output pipeline.run() diff --git a/backends/arm/test/passes/test_remove_clone_pass.py b/backends/arm/test/passes/test_remove_clone_pass.py index dea0bb06f5e..bfbdbbd5782 100755 --- a/backends/arm/test/passes/test_remove_clone_pass.py +++ b/backends/arm/test/passes/test_remove_clone_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.remove_clone_pass import RemoveClonePass @@ -39,5 +40,6 @@ def test_remove_clone_tosa_INT(): }, ops_not_after_pass=["executorch_exir_dialects_edge__ops_aten_clone_default"], pass_list=[RemoveClonePass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_rescale_pass.py b/backends/arm/test/passes/test_rescale_pass.py index 0fe72f6d1fe..09315dd1b3c 100644 --- a/backends/arm/test/passes/test_rescale_pass.py +++ b/backends/arm/test/passes/test_rescale_pass.py @@ -125,9 +125,8 @@ def test_quantized_rescale_tosa_bi(test_data: tuple[torch.Tensor, torch.Tensor]) test_data=test_data, aten_op=[], exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) - if not conftest.is_option_enabled("tosa_ref_model"): - pipeline.pop_stage("run_method_and_compare_outputs") pipeline.run() diff --git a/backends/arm/test/passes/test_unsqueeze_before_repeat_pass.py b/backends/arm/test/passes/test_unsqueeze_before_repeat_pass.py index fc405e21f2a..127872b60dd 100644 --- a/backends/arm/test/passes/test_unsqueeze_before_repeat_pass.py +++ b/backends/arm/test/passes/test_unsqueeze_before_repeat_pass.py @@ -5,6 +5,7 @@ from typing import Dict, Tuple +import conftest import torch from executorch.backends.arm._passes import UnsqueezeBeforeRepeatPass from executorch.backends.arm.test import common @@ -54,6 +55,7 @@ def test_unsqueeze_before_repeat_tosa_FP(test_data: input_t): ops_after_pass=ops_after_pass, ops_not_after_pass=ops_not_after_pass, pass_list=[UnsqueezeBeforeRepeatPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.pop_stage(-1) # Do not compare output pipeline.run() diff --git a/backends/arm/test/tester/test_pipeline.py b/backends/arm/test/tester/test_pipeline.py index fb9f05444e5..e86cca02552 100644 --- a/backends/arm/test/tester/test_pipeline.py +++ b/backends/arm/test/tester/test_pipeline.py @@ -637,6 +637,7 @@ class PassPipeline(BasePipelineMaker, Generic[T]): pass_functions: List of functions applied directly to the exported program. passes_with_exported_program: List of passes initiated with an exported_program. custom_path : Path to dump intermediate artifacts such as tosa and pte to. + run_on_tosa_ref_model: Set to true to test the tosa file on the TOSA reference model. Passes are run in order pass_list -> pass_functions -> passes_with_exported_program. See arm_tester.RunPasses() for more information. @@ -655,6 +656,7 @@ def __init__( pass_functions: Optional[List[Callable]] = None, passes_with_exported_program: Optional[List[Type[ExportPass]]] = None, custom_path: str = None, + run_on_tosa_ref_model: bool = True, ): tosa_profiles = { "1.0": TosaSpecification.create_from_string( @@ -701,7 +703,8 @@ def __init__( self.add_stage(self.tester.check_count, ops_after_pass, suffix="after") if ops_not_after_pass: self.add_stage(self.tester.check_not, ops_not_after_pass, suffix="after") - self.add_stage(self.tester.run_method_and_compare_outputs) + if run_on_tosa_ref_model: + self.add_stage(self.tester.run_method_and_compare_outputs) class TransformAnnotationPassPipeline(BasePipelineMaker, Generic[T]): @@ -713,6 +716,7 @@ class TransformAnnotationPassPipeline(BasePipelineMaker, Generic[T]): test_data: Data used for testing the module. custom_path : Path to dump intermediate artifacts such as tosa and pte to. + run_on_tosa_ref_model: Set to true to test the tosa file on the TOSA reference model. """ @@ -721,6 +725,7 @@ def __init__( module: torch.nn.Module, test_data: T, custom_path: str = None, + run_on_tosa_ref_model: bool = True, ): tosa_profiles = { "1.0": TosaSpecification.create_from_string("TOSA-1.0+INT"), @@ -748,11 +753,12 @@ def __init__( self.pop_stage("to_executorch") self.pop_stage("to_edge_transform_and_lower") self.pop_stage("check.aten") - self.add_stage( - self.tester.run_method_and_compare_outputs, - inputs=test_data, - run_eager_mode=True, - ) + if run_on_tosa_ref_model: + self.add_stage( + self.tester.run_method_and_compare_outputs, + inputs=test_data, + run_eager_mode=True, + ) class OpNotSupportedPipeline(BasePipelineMaker, Generic[T]):