diff --git a/backends/arm/test/misc/test_bn_relu_folding_qat.py b/backends/arm/test/misc/test_bn_relu_folding_qat.py index c39c1694d0a..0b75561d15a 100644 --- a/backends/arm/test/misc/test_bn_relu_folding_qat.py +++ b/backends/arm/test/misc/test_bn_relu_folding_qat.py @@ -47,7 +47,14 @@ def forward(self, x: torch.Tensor): @common.parametrize("model", models) def test_qat_tosa_INT(model: torch.nn.Module): - pipeline = TosaPipelineINT[input_t1](model, model.test_data, [], [], qtol=1) + pipeline = TosaPipelineINT[input_t1]( + model, + model.test_data, + [], + [], + qtol=1, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) tosa_version = conftest.get_option("tosa_version") tosa_profiles = { "1.0": common.TosaSpecification.create_from_string("TOSA-1.0+INT"), diff --git a/backends/arm/test/misc/test_custom_partition.py b/backends/arm/test/misc/test_custom_partition.py index 6cdd63af7c9..f7285b678a4 100644 --- a/backends/arm/test/misc/test_custom_partition.py +++ b/backends/arm/test/misc/test_custom_partition.py @@ -3,6 +3,7 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. +import conftest import logging from typing import Tuple @@ -50,7 +51,13 @@ def test_single_reject(caplog, test_data: input_t1): caplog.set_level(logging.INFO) module = CustomPartitioning() - pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[]) + pipeline = TosaPipelineFP[input_t1]( + module, + test_data, + [], + exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) check = DontPartition(exir_ops.edge.aten.sigmoid.default) pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check]) pipeline.change_args( @@ -68,7 +75,13 @@ def test_single_reject(caplog, test_data: input_t1): @common.parametrize("test_data", CustomPartitioning.inputs) def test_multiple_reject(test_data: input_t1): module = CustomPartitioning() - pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[]) + pipeline = TosaPipelineFP[input_t1]( + module, + test_data, + [], + exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) check = DontPartition( exir_ops.edge.aten.sigmoid.default, exir_ops.edge.aten.mul.Tensor ) @@ -90,7 +103,13 @@ def test_torch_op_reject(caplog, test_data: input_t1): module = CustomPartitioning() check = DontPartition(torch.ops.aten.sigmoid.default) - pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[]) + pipeline = TosaPipelineFP[input_t1]( + module, + test_data, + [], + exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check]) pipeline.change_args( "check_count.exir", {"torch.ops.higher_order.executorch_call_delegate": 2} @@ -108,7 +127,13 @@ def test_torch_op_reject(caplog, test_data: input_t1): def test_string_op_reject(test_data: input_t1): module = CustomPartitioning() check = DontPartition("aten.sigmoid.default") - pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[]) + pipeline = TosaPipelineFP[input_t1]( + module, + test_data, + [], + exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check]) pipeline.change_args( "check_count.exir", {"torch.ops.higher_order.executorch_call_delegate": 2} @@ -127,7 +152,13 @@ def test_name_reject(caplog, test_data: input_t1): module = CustomPartitioning() check = DontPartitionName("mul", "sigmoid", exact=False) - pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[]) + pipeline = TosaPipelineFP[input_t1]( + module, + test_data, + [], + exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check]) pipeline.change_args( "check_count.exir", @@ -142,7 +173,13 @@ def test_name_reject(caplog, test_data: input_t1): def test_module_reject(test_data: input_t1): module = NestedModule() check = DontPartitionModule(module_name="CustomPartitioning") - pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[]) + pipeline = TosaPipelineFP[input_t1]( + module, + test_data, + [], + exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check]) pipeline.change_args( "check_count.exir", @@ -158,7 +195,13 @@ def test_inexact_module_reject(caplog, test_data: input_t1): module = NestedModule() check = DontPartitionModule(module_name="Custom", exact=False) - pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[]) + pipeline = TosaPipelineFP[input_t1]( + module, + test_data, + [], + exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check]) pipeline.change_args( "check_count.exir", @@ -173,7 +216,13 @@ def test_inexact_module_reject(caplog, test_data: input_t1): def test_module_instance_reject(test_data: input_t1): module = NestedModule() check = DontPartitionModule(instance_name="nested") - pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[]) + pipeline = TosaPipelineFP[input_t1]( + module, + test_data, + [], + exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check]) pipeline.change_args( "check_count.exir", diff --git a/backends/arm/test/models/test_conformer.py b/backends/arm/test/models/test_conformer.py index 6a66b25d27d..85404840d52 100644 --- a/backends/arm/test/models/test_conformer.py +++ b/backends/arm/test/models/test_conformer.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import pytest import torch @@ -56,6 +57,7 @@ def test_conformer_tosa_FP(): TestConformer.model_example_inputs, aten_op=TestConformer.aten_ops, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), use_to_edge_transform_and_lower=True, ) pipeline.run() @@ -67,6 +69,7 @@ def test_conformer_tosa_INT(): TestConformer.model_example_inputs, aten_op=TestConformer.aten_ops, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), use_to_edge_transform_and_lower=True, ) pipeline.pop_stage("check_count.exir") diff --git a/backends/arm/test/models/test_deit_tiny_arm.py b/backends/arm/test/models/test_deit_tiny_arm.py index 22685a079bd..e9a4efe8a48 100644 --- a/backends/arm/test/models/test_deit_tiny_arm.py +++ b/backends/arm/test/models/test_deit_tiny_arm.py @@ -7,6 +7,7 @@ from typing import Tuple +import conftest import timm import torch @@ -43,6 +44,7 @@ def test_deit_tiny_tosa_FP(): model_inputs, aten_op=[], exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), use_to_edge_transform_and_lower=True, ) pipeline.run() @@ -54,6 +56,7 @@ def test_deit_tiny_tosa_INT(): model_inputs, aten_op=[], exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), use_to_edge_transform_and_lower=True, atol=1.5, qtol=1, diff --git a/backends/arm/test/models/test_llama.py b/backends/arm/test/models/test_llama.py index 7732943d5fb..cfbca5503a7 100644 --- a/backends/arm/test/models/test_llama.py +++ b/backends/arm/test/models/test_llama.py @@ -111,6 +111,7 @@ def test_llama_tosa_FP(): llama_inputs, aten_op=[], exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), use_to_edge_transform_and_lower=True, transform_passes=[InsertCastForOpsWithInt64InputPass()], ) @@ -129,6 +130,7 @@ def test_llama_tosa_INT(): llama_inputs, aten_op=[], exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), use_to_edge_transform_and_lower=True, ) pipeline.run() diff --git a/backends/arm/test/models/test_lstm_arm.py b/backends/arm/test/models/test_lstm_arm.py index 1e63472f5f4..70ccd804fe7 100644 --- a/backends/arm/test/models/test_lstm_arm.py +++ b/backends/arm/test/models/test_lstm_arm.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common @@ -49,6 +50,7 @@ def test_lstm_tosa_FP(): TestLSTM.model_example_inputs, aten_op=[], exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), use_to_edge_transform_and_lower=True, ) pipeline.change_args("run_method_and_compare_outputs", get_test_inputs(), atol=3e-1) @@ -61,6 +63,7 @@ def test_lstm_tosa_INT(): TestLSTM.model_example_inputs, aten_op=[], exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), use_to_edge_transform_and_lower=True, ) pipeline.change_args( diff --git a/backends/arm/test/models/test_w2l_arm.py b/backends/arm/test/models/test_w2l_arm.py index 32b25a18fd8..c822540a188 100644 --- a/backends/arm/test/models/test_w2l_arm.py +++ b/backends/arm/test/models/test_w2l_arm.py @@ -8,6 +8,7 @@ import unittest from typing import Tuple +import conftest import pytest import torch @@ -53,6 +54,7 @@ def test_w2l_tosa_FP(): TestW2L.model_example_inputs, aten_op=[], exir_op=TestW2L.all_operators, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), use_to_edge_transform_and_lower=True, ) pipeline.run() @@ -66,6 +68,7 @@ def test_w2l_tosa_INT(): TestW2L.model_example_inputs, aten_op=[], exir_op=TestW2L.all_operators, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), use_to_edge_transform_and_lower=True, ) pipeline.run() diff --git a/backends/arm/test/ops/test_abs.py b/backends/arm/test/ops/test_abs.py index 4ebcf7393c1..4841bca799e 100644 --- a/backends/arm/test/ops/test_abs.py +++ b/backends/arm/test/ops/test_abs.py @@ -8,6 +8,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -41,13 +42,25 @@ def forward(self, x): @common.parametrize("test_data", Abs.test_parameters) def test_abs_tosa_FP(test_data: torch.Tensor): - pipeline = TosaPipelineFP[input_t1](Abs(), test_data(), aten_op, exir_op) + pipeline = TosaPipelineFP[input_t1]( + Abs(), + test_data(), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @common.parametrize("test_data", Abs.test_parameters) def test_abs_tosa_INT(test_data: torch.Tensor): - pipeline = TosaPipelineINT[input_t1](Abs(), test_data(), aten_op, exir_op) + pipeline = TosaPipelineINT[input_t1]( + Abs(), + test_data(), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() diff --git a/backends/arm/test/ops/test_acosh.py b/backends/arm/test/ops/test_acosh.py index 25ba2b1a83b..16e8b80f0ba 100644 --- a/backends/arm/test/ops/test_acosh.py +++ b/backends/arm/test/ops/test_acosh.py @@ -4,6 +4,7 @@ # LICENSE file in the root directory of this source tree. from typing import Tuple +import conftest import pytest import torch @@ -55,6 +56,7 @@ def test_acosh_tosa_FP(test_data: Tuple): (test_data(),), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -65,6 +67,7 @@ def test_acosh_tosa_INT(test_data: Tuple): Acosh(), (test_data(),), aten_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_add.py b/backends/arm/test/ops/test_add.py index 421ec0adc61..1fcf5623769 100644 --- a/backends/arm/test/ops/test_add.py +++ b/backends/arm/test/ops/test_add.py @@ -81,13 +81,25 @@ def forward(self, x: torch.Tensor, y: torch.Tensor): @common.parametrize("test_data", Add.test_data) def test_add_tensor_tosa_FP(test_data: input_t1): - pipeline = TosaPipelineFP[input_t1](Add(), test_data(), aten_op, exir_op) + pipeline = TosaPipelineFP[input_t1]( + Add(), + test_data(), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @common.parametrize("test_data", Add.test_data) def test_add_tensor_tosa_INT(test_data: input_t1): - pipeline = TosaPipelineINT[input_t1](Add(), test_data(), aten_op, exir_op) + pipeline = TosaPipelineINT[input_t1]( + Add(), + test_data(), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @@ -146,25 +158,49 @@ def test_add_tensor_u85_INT(test_data: input_t1): @common.parametrize("test_data", Add2.test_data) def test_add_tensor_tosa_FP_2(test_data: input_t2): - pipeline = TosaPipelineFP[input_t2](Add2(), test_data(), aten_op, exir_op) + pipeline = TosaPipelineFP[input_t2]( + Add2(), + test_data(), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @common.parametrize("test_data", Add3.test_data) def test_add_tensor_tosa_FP_3(test_data: input_t2): - pipeline = TosaPipelineFP[input_t2](Add3(), test_data(), aten_op, exir_op) + pipeline = TosaPipelineFP[input_t2]( + Add3(), + test_data(), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @common.parametrize("test_data", Add3.test_data) def test_add_tensor_tosa_INT_3(test_data: input_t2): - pipeline = TosaPipelineINT[input_t2](Add3(), test_data(), aten_op, exir_op) + pipeline = TosaPipelineINT[input_t2]( + Add3(), + test_data(), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @common.parametrize("test_data", Add2.test_data) def test_add_tensor_tosa_INT_2(test_data: input_t2): - pipeline = TosaPipelineINT[input_t2](Add2(), test_data(), aten_op, exir_op) + pipeline = TosaPipelineINT[input_t2]( + Add2(), + test_data(), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() diff --git a/backends/arm/test/ops/test_asin.py b/backends/arm/test/ops/test_asin.py index 9c37bddbd92..e1a34991b05 100644 --- a/backends/arm/test/ops/test_asin.py +++ b/backends/arm/test/ops/test_asin.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common @@ -44,6 +45,7 @@ def test_asin_tosa_FP(test_data: Tuple): (test_data(),), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -55,6 +57,7 @@ def test_asin_tosa_INT(test_data: Tuple): (test_data(),), aten_op=[], exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_asinh.py b/backends/arm/test/ops/test_asinh.py index 305c822601c..5342a7c58aa 100644 --- a/backends/arm/test/ops/test_asinh.py +++ b/backends/arm/test/ops/test_asinh.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common @@ -43,6 +44,7 @@ def test_asinh_tosa_FP(test_data: Tuple): (test_data(),), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -54,6 +56,7 @@ def test_asinh_tosa_INT(test_data: Tuple): (test_data(),), aten_op=[], exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_atan.py b/backends/arm/test/ops/test_atan.py index 51114d2800f..b873ef47a7e 100644 --- a/backends/arm/test/ops/test_atan.py +++ b/backends/arm/test/ops/test_atan.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common @@ -46,6 +47,7 @@ def test_atan_tosa_FP(test_data: Tuple): (test_data,), aten_op=aten_op, exir_op=exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -57,6 +59,7 @@ def test_atan_tosa_INT(test_data: Tuple): (test_data,), aten_op=aten_op, exir_op=exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_atanh.py b/backends/arm/test/ops/test_atanh.py index 12754a34646..1c82c792c54 100644 --- a/backends/arm/test/ops/test_atanh.py +++ b/backends/arm/test/ops/test_atanh.py @@ -6,6 +6,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -47,6 +48,7 @@ def test_atanh_tosa_FP(test_data: Tuple): (test_data,), aten_op=aten_op, exir_op=exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -58,6 +60,7 @@ def test_atanh_tosa_INT(test_data: Tuple): (test_data,), aten_op=aten_op, exir_op=exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_ceil.py b/backends/arm/test/ops/test_ceil.py index 64e9040a974..88c8b302124 100644 --- a/backends/arm/test/ops/test_ceil.py +++ b/backends/arm/test/ops/test_ceil.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -51,6 +52,7 @@ def test_ceil_tosa_FP(test_data: input_t1): (data,), module.aten_op, module.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -65,6 +67,7 @@ def test_ceil_tosa_INT(test_data: input_t1): module.exir_op, atol=0.06, rtol=0.01, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_conv2d.py b/backends/arm/test/ops/test_conv2d.py index 0d23d2a6c7e..ebf58be4876 100644 --- a/backends/arm/test/ops/test_conv2d.py +++ b/backends/arm/test/ops/test_conv2d.py @@ -6,6 +6,7 @@ from typing import List, Tuple, Union +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -408,6 +409,7 @@ def test_convolution_2d_tosa_FP(test_data): model.get_inputs(), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -422,6 +424,7 @@ def test_convolution_2d_tosa_INT(test_data): exir_op, per_channel_quantization=per_channel_quantization, qtol=1, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_div.py b/backends/arm/test/ops/test_div.py index 026939758a0..b1305fa5824 100644 --- a/backends/arm/test/ops/test_div.py +++ b/backends/arm/test/ops/test_div.py @@ -8,6 +8,7 @@ from typing import Optional, Tuple, Union +import conftest import torch from executorch.backends.arm.test import common @@ -91,13 +92,25 @@ def forward( @common.parametrize("test_data", test_data_suite) def test_div_tensor_tosa_FP(test_data: Tuple): - pipeline = TosaPipelineFP[input_t1](Div(), test_data(), aten_op, exir_op) + pipeline = TosaPipelineFP[input_t1]( + Div(), + test_data(), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @common.parametrize("test_data", test_data_suite) def test_div_tensor_tosa_INT(test_data: Tuple): - pipeline = TosaPipelineINT[input_t1](Div(), test_data(), aten_op=[], exir_op=[]) + pipeline = TosaPipelineINT[input_t1]( + Div(), + test_data(), + aten_op=[], + exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() diff --git a/backends/arm/test/ops/test_exp.py b/backends/arm/test/ops/test_exp.py index 6eaacc71d86..ef2910225fd 100644 --- a/backends/arm/test/ops/test_exp.py +++ b/backends/arm/test/ops/test_exp.py @@ -8,6 +8,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common @@ -45,6 +46,7 @@ def test_exp_tosa_FP(test_data: Tuple): (test_data(),), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -56,6 +58,7 @@ def test_exp_tosa_INT(test_data: Tuple): (test_data(),), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_floor.py b/backends/arm/test/ops/test_floor.py index c66ef1c5d27..b0f13c9b6af 100644 --- a/backends/arm/test/ops/test_floor.py +++ b/backends/arm/test/ops/test_floor.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -51,6 +52,7 @@ def test_floor_tosa_FP(test_data: input_t1): (data,), module.aten_op, module.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -65,6 +67,7 @@ def test_floor_tosa_INT(test_data: input_t1): module.exir_op, atol=0.06, rtol=0.01, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_leaky_relu.py b/backends/arm/test/ops/test_leaky_relu.py index c18255a73c0..41284173f10 100644 --- a/backends/arm/test/ops/test_leaky_relu.py +++ b/backends/arm/test/ops/test_leaky_relu.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -45,6 +46,7 @@ def test_leaky_relu_tosa_FP(test_data): data, [], use_to_edge_transform_and_lower=True, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.add_stage_after( "to_edge_transform_and_lower", pipeline.tester.check_not, [exir_op] @@ -60,6 +62,7 @@ def test_leaky_relu_tosa_INT(test_data): data, [], use_to_edge_transform_and_lower=True, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.add_stage_after("quantize", pipeline.tester.check_not, [aten_op]) pipeline.run() diff --git a/backends/arm/test/ops/test_log.py b/backends/arm/test/ops/test_log.py index 1ed5c57f1ab..04ffbae4fb3 100644 --- a/backends/arm/test/ops/test_log.py +++ b/backends/arm/test/ops/test_log.py @@ -8,6 +8,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common @@ -42,13 +43,25 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: @common.parametrize("test_data", test_data_suite) def test_log_tosa_FP(test_data: input_t1): - pipeline = TosaPipelineFP[input_t1](Log(), (test_data(),), aten_op, exir_op) + pipeline = TosaPipelineFP[input_t1]( + Log(), + (test_data(),), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @common.parametrize("test_data", test_data_suite) def test_log_tosa_INT(test_data: input_t1): - pipeline = TosaPipelineINT[input_t1](Log(), (test_data(),), aten_op, exir_op) + pipeline = TosaPipelineINT[input_t1]( + Log(), + (test_data(),), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() diff --git a/backends/arm/test/ops/test_matmul.py b/backends/arm/test/ops/test_matmul.py index d1a21684325..c80de7e2080 100644 --- a/backends/arm/test/ops/test_matmul.py +++ b/backends/arm/test/ops/test_matmul.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -62,14 +63,24 @@ def forward(self, x1: torch.Tensor, x2: torch.Tensor, x3: torch.Tensor): @common.parametrize("test_data", MatMul.test_data_generators) def test_matmul_tosa_FP(test_data: input_t1): - pipeline = TosaPipelineFP[input_t1](MatMul(), test_data(), aten_op_mm, exir_op_mm) + pipeline = TosaPipelineFP[input_t1]( + MatMul(), + test_data(), + aten_op_mm, + exir_op_mm, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @common.parametrize("test_data", MatMulSingleInput.test_data_generators) def test_matmul_single_input_tosa_FP(test_data: input_t1): pipeline = TosaPipelineFP[input_t1]( - MatMulSingleInput(), test_data(), aten_op_mm, exir_op_mm + MatMulSingleInput(), + test_data(), + aten_op_mm, + exir_op_mm, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -77,7 +88,11 @@ def test_matmul_single_input_tosa_FP(test_data: input_t1): @common.parametrize("test_data", MatMulCombo.test_data_generators) def test_matmul_combo_tosa_FP(test_data: input_t1): pipeline = TosaPipelineFP[input_t1]( - MatMulCombo(), test_data(), aten_op_mm, exir_op_mm + MatMulCombo(), + test_data(), + aten_op_mm, + exir_op_mm, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -85,7 +100,12 @@ def test_matmul_combo_tosa_FP(test_data: input_t1): @common.parametrize("test_data", MatMul.test_data_generators) def test_matmul_tosa_INT(test_data: input_t1): pipeline = TosaPipelineINT[input_t1]( - MatMul(), test_data(), aten_op_mm, exir_op_mm, qtol=1 + MatMul(), + test_data(), + aten_op_mm, + exir_op_mm, + qtol=1, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -98,6 +118,7 @@ def test_matmul_single_input_tosa_INT(test_data: input_t1): aten_op_mm, exir_op_mm, qtol=1, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -110,6 +131,7 @@ def test_matmul_combo_tosa_INT(test_data: input_t1): aten_op_mm, exir_op_mm, qtol=1, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_mul.py b/backends/arm/test/ops/test_mul.py index b0b7f5f4b7d..aa595ffde2a 100644 --- a/backends/arm/test/ops/test_mul.py +++ b/backends/arm/test/ops/test_mul.py @@ -8,6 +8,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common @@ -114,6 +115,7 @@ def test_mul_tensor_tosa_FP(test_data: torch.Tensor): test_data(), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -125,6 +127,7 @@ def test_mul_tensor_tosa_FP_diff_input_ranks(test_data: torch.Tensor): test_data(), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -136,6 +139,7 @@ def test_mul_tensor_tosa_FP_int32(test_data: torch.Tensor): test_data(), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -147,6 +151,7 @@ def test_mul_tensor_tosa_INT_diff_input_ranks(test_data: torch.Tensor): test_data(), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -158,6 +163,7 @@ def test_mul_tensor_tosa_INT(test_data: torch.Tensor): test_data(), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -169,6 +175,7 @@ def test_mul_tensor_tosa_INT_int32(test_data: torch.Tensor): test_data(), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.pop_stage("check.quant_nodes") pipeline.run() diff --git a/backends/arm/test/ops/test_neg.py b/backends/arm/test/ops/test_neg.py index 395a4815b62..4261a9f9987 100644 --- a/backends/arm/test/ops/test_neg.py +++ b/backends/arm/test/ops/test_neg.py @@ -6,6 +6,7 @@ from typing import Dict, Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -39,13 +40,25 @@ def forward(self, x: torch.Tensor): @common.parametrize("test_data", Neg.test_data) def test_neg_tosa_FP(test_data: input_t1): - pipeline = TosaPipelineFP[input_t1](Neg(), test_data, Neg.aten_op, Neg.exir_op) + pipeline = TosaPipelineFP[input_t1]( + Neg(), + test_data, + Neg.aten_op, + Neg.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @common.parametrize("test_data", Neg.test_data) def test_neg_tosa_INT(test_data: input_t1): - pipeline = TosaPipelineINT[input_t1](Neg(), test_data, Neg.aten_op, Neg.exir_op) + pipeline = TosaPipelineINT[input_t1]( + Neg(), + test_data, + Neg.aten_op, + Neg.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() diff --git a/backends/arm/test/ops/test_permute.py b/backends/arm/test/ops/test_permute.py index 57f7f9603a1..a38b427deb4 100644 --- a/backends/arm/test/ops/test_permute.py +++ b/backends/arm/test/ops/test_permute.py @@ -8,6 +8,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common @@ -56,6 +57,7 @@ def test_permute_tosa_FP(test_data: torch.Tensor): (test_data,), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -68,6 +70,7 @@ def test_permute_tosa_INT(test_data: torch.Tensor): (test_data,), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_pow.py b/backends/arm/test/ops/test_pow.py index 016c3e97265..eba7d53d1c0 100644 --- a/backends/arm/test/ops/test_pow.py +++ b/backends/arm/test/ops/test_pow.py @@ -6,6 +6,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -99,6 +100,7 @@ def test_pow_tensor_tensor_tosa_FP(test_data: Pow_TensorTensor.input_t): test_data(), Pow_TensorTensor.aten_op, Pow_TensorTensor.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -134,6 +136,7 @@ def test_pow_tensor_scalar_tosa_FP(test_data: Pow_TensorScalar.input_t): (base,), Pow_TensorScalar.aten_op, Pow_TensorScalar.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -146,6 +149,7 @@ def test_pow_tensor_scalar_tosa_INT(test_data: Pow_TensorScalar.input_t): (base,), Pow_TensorScalar.aten_op, Pow_TensorScalar.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_relu.py b/backends/arm/test/ops/test_relu.py index 0b29bc24e75..cd08a2bca5b 100644 --- a/backends/arm/test/ops/test_relu.py +++ b/backends/arm/test/ops/test_relu.py @@ -8,6 +8,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -50,6 +51,7 @@ def test_relu_tosa_FP(test_data: torch.Tensor): (test_data(),), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -61,6 +63,7 @@ def test_relu_tosa_INT(test_data: torch.Tensor): (test_data(),), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_rshift.py b/backends/arm/test/ops/test_rshift.py index e97bfb840ae..abeeb245860 100644 --- a/backends/arm/test/ops/test_rshift.py +++ b/backends/arm/test/ops/test_rshift.py @@ -3,6 +3,7 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.common import ( @@ -80,6 +81,7 @@ def test_bitwise_right_shift_scalar_tosa_FP_scalar(test_data): test_data(), RshiftScalar.torch_op_FP, RshiftScalar.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ).run() @@ -90,6 +92,7 @@ def test_bitwise_right_shift_tensor_tosa_INT_scalar(test_data): test_data(), RshiftScalar.torch_op_INT, RshiftScalar.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.pop_stage("check.quant_nodes") pipeline.run() @@ -165,6 +168,7 @@ def test_bitwise_right_shift_tensor_tosa_FP(test_data): test_data(), RshiftTensor.torch_op, RshiftTensor.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ).run() @@ -175,6 +179,7 @@ def test_bitwise_right_shift_tensor_tosa_INT(test_data): test_data(), RshiftTensor.torch_op, RshiftTensor.exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.pop_stage("check.quant_nodes") pipeline.run() diff --git a/backends/arm/test/ops/test_scalars.py b/backends/arm/test/ops/test_scalars.py index 1243a522526..29f6a70a889 100644 --- a/backends/arm/test/ops/test_scalars.py +++ b/backends/arm/test/ops/test_scalars.py @@ -6,6 +6,7 @@ from typing import Tuple +import conftest import pytest import torch @@ -174,21 +175,21 @@ def forward(self, x): @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_add_tensor_tosa_FP_scalar(test_data): """Tests regular add with one scalar input.""" - pipeline = TosaPipelineFP[input_t1](Add(), test_data, aten_op=Add.aten_op) + pipeline = TosaPipelineFP[input_t1](Add(), test_data, aten_op=Add.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_add_tensor_tosa_FP_inplace(test_data): """Tests inplace add with one scalar input.""" - pipeline = TosaPipelineFP[input_t1](AddInplace(), test_data, aten_op=[]) + pipeline = TosaPipelineFP[input_t1](AddInplace(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_const_tests, xfails=xfails) def test_add_tensor_tosa_FP_const(test_data): """Tests regular add with one scalar input, with one of inputs constant.""" - pipeline = TosaPipelineFP[input_t1](AddConst(), test_data, aten_op=AddConst.aten_op) + pipeline = TosaPipelineFP[input_t1](AddConst(), test_data, aten_op=AddConst.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @@ -196,7 +197,7 @@ def test_add_tensor_tosa_FP_const(test_data): def test_add_scalar_tosa_FP(test_data): """Tests a scalar add with one scalar input.""" pipeline = TosaPipelineFP[input_t1]( - AddScalar(), test_data, aten_op=AddScalar.aten_op + AddScalar(), test_data, aten_op=AddScalar.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model") ) pipeline.run() @@ -205,14 +206,14 @@ def test_add_scalar_tosa_FP(test_data): @common.parametrize("test_data", tensor_scalar_tests) def test_add_tensor_tosa_INT_scalar(test_data): """Tests regular add with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](Add(), test_data, aten_op=[]) + pipeline = TosaPipelineINT[input_t1](Add(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests) def test_add_tensor_tosa_INT_inplace(test_data): """Tests inplace add with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](AddInplace(), test_data, aten_op=[]) + pipeline = TosaPipelineINT[input_t1](AddInplace(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @@ -220,7 +221,7 @@ def test_add_tensor_tosa_INT_inplace(test_data): def test_add_tensor_tosa_INT_const(test_data): """Tests regular add with one scalar input, with one of inputs constant.""" pipeline = TosaPipelineINT[input_t1]( - AddConst(), test_data, aten_op=AddConst.aten_op + AddConst(), test_data, aten_op=AddConst.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model") ) pipeline.run() @@ -228,7 +229,7 @@ def test_add_tensor_tosa_INT_const(test_data): @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_add_scalar_tosa_INT(test_data): """Tests a scalar add with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](AddScalar(), test_data, aten_op=Add.aten_op) + pipeline = TosaPipelineINT[input_t1](AddScalar(), test_data, aten_op=Add.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @@ -249,14 +250,14 @@ def test_add_scalar_u85_INT(): @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_sub_tensor_tosa_FP_scalar(test_data): """Tests regular sub with one scalar input.""" - pipeline = TosaPipelineFP[input_t1](Sub(), test_data, aten_op=Sub.aten_op) + pipeline = TosaPipelineFP[input_t1](Sub(), test_data, aten_op=Sub.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_sub_tensor_tosa_FP_inplace(test_data): """Tests inplace sub with one scalar input.""" - pipeline = TosaPipelineFP[input_t1](SubInplace(), test_data, aten_op=[]) + pipeline = TosaPipelineFP[input_t1](SubInplace(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @@ -264,7 +265,7 @@ def test_sub_tensor_tosa_FP_inplace(test_data): def test_sub_scalar_tosa_FP(test_data): """Tests a scalar sub with one scalar input.""" pipeline = TosaPipelineFP[input_t1]( - SubScalar(), test_data, aten_op=SubScalar.aten_op + SubScalar(), test_data, aten_op=SubScalar.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model") ) pipeline.run() @@ -273,21 +274,21 @@ def test_sub_scalar_tosa_FP(test_data): @common.parametrize("test_data", tensor_scalar_tests) def test_sub_tensor_tosa_INT_scalar(test_data): """Tests regular sub with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](Sub(), test_data, aten_op=[]) + pipeline = TosaPipelineINT[input_t1](Sub(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests) def test_sub_tensor_tosa_INT_inplace(test_data): """Tests inplace sub with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](SubInplace(), test_data, aten_op=[]) + pipeline = TosaPipelineINT[input_t1](SubInplace(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_sub_scalar_tosa_INT(test_data): """Tests a scalar sub with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](SubScalar(), test_data, aten_op=Sub.aten_op) + pipeline = TosaPipelineINT[input_t1](SubScalar(), test_data, aten_op=Sub.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @@ -306,14 +307,14 @@ def test_sub_scalar_u85_INT(): @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_mul_tensor_tosa_FP_scalar(test_data): """Tests regular mul with one scalar input.""" - pipeline = TosaPipelineFP[input_t1](Mul(), test_data, aten_op=Mul.aten_op) + pipeline = TosaPipelineFP[input_t1](Mul(), test_data, aten_op=Mul.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_mul_tensor_tosa_FP_inplace(test_data): """Tests inplace mul with one scalar input.""" - pipeline = TosaPipelineFP[input_t1](MulInplace(), test_data, aten_op=[]) + pipeline = TosaPipelineFP[input_t1](MulInplace(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @@ -321,7 +322,7 @@ def test_mul_tensor_tosa_FP_inplace(test_data): def test_mul_scalar_tosa_FP(test_data): """Tests a scalar mul with one scalar input.""" pipeline = TosaPipelineFP[input_t1]( - MulScalar(), test_data, aten_op=MulScalar.aten_op + MulScalar(), test_data, aten_op=MulScalar.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model") ) pipeline.run() @@ -330,21 +331,21 @@ def test_mul_scalar_tosa_FP(test_data): @common.parametrize("test_data", tensor_scalar_tests) def test_mul_tensor_tosa_INT_scalar(test_data): """Tests regular mul with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](Mul(), test_data, aten_op=[]) + pipeline = TosaPipelineINT[input_t1](Mul(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests) def test_mul_tensor_tosa_INT_inplace(test_data): """Tests inplace mul with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](MulInplace(), test_data, aten_op=[]) + pipeline = TosaPipelineINT[input_t1](MulInplace(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_mul_scalar_tosa_INT(test_data): """Tests a scalar mul with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](MulScalar(), test_data, aten_op=Mul.aten_op) + pipeline = TosaPipelineINT[input_t1](MulScalar(), test_data, aten_op=Mul.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @@ -363,14 +364,14 @@ def test_mul_scalar_u85_INT(): @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_div_tensor_tosa_FP_scalar(test_data): """Tests regular div with one scalar input.""" - pipeline = TosaPipelineFP[input_t1](Div(), test_data, aten_op=Div.aten_op) + pipeline = TosaPipelineFP[input_t1](Div(), test_data, aten_op=Div.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_div_tensor_tosa_FP_inplace(test_data): """Tests inplace div with one scalar input.""" - pipeline = TosaPipelineFP[input_t1](DivInplace(), test_data, aten_op=[]) + pipeline = TosaPipelineFP[input_t1](DivInplace(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @@ -378,7 +379,7 @@ def test_div_tensor_tosa_FP_inplace(test_data): def test_div_scalar_tosa_FP(test_data): """Tests a scalar div with one scalar input.""" pipeline = TosaPipelineFP[input_t1]( - DivScalar(), test_data, aten_op=DivScalar.aten_op + DivScalar(), test_data, aten_op=DivScalar.aten_op, run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model") ) pipeline.run() @@ -387,21 +388,21 @@ def test_div_scalar_tosa_FP(test_data): @common.parametrize("test_data", tensor_scalar_tests) def test_div_tensor_tosa_INT_scalar(test_data): """Tests regular div with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](Div(), test_data, aten_op=[]) + pipeline = TosaPipelineINT[input_t1](Div(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests) def test_div_tensor_tosa_INT_inplace(test_data): """Tests inplace div with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](DivInplace(), test_data, aten_op=[]) + pipeline = TosaPipelineINT[input_t1](DivInplace(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) def test_div_scalar_tosa_INT(test_data): """Tests a scalar div with one scalar input.""" - pipeline = TosaPipelineINT[input_t1](DivScalar(), test_data, aten_op=[]) + pipeline = TosaPipelineINT[input_t1](DivScalar(), test_data, aten_op=[], run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model")) pipeline.run() @@ -422,6 +423,7 @@ def test_bitwise_right_shift_tensor_tosa_FP_inplace(): ShiftInplaceSub(), (torch.IntTensor(5),), aten_op="torch.ops.aten.__rshift__.Scalar", + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -431,6 +433,7 @@ def test_bitwise_right_shift_tensor_tosa_INT_inplace(): ShiftInplaceSub(), (torch.IntTensor(5),), aten_op="torch.ops.aten.bitwise_right_shift.Tensor", + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.pop_stage("check.quant_nodes") pipeline.run() diff --git a/backends/arm/test/ops/test_sdpa.py b/backends/arm/test/ops/test_sdpa.py index c4b05972f76..57d80931b77 100644 --- a/backends/arm/test/ops/test_sdpa.py +++ b/backends/arm/test/ops/test_sdpa.py @@ -6,6 +6,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test.tester.test_pipeline import ( @@ -29,17 +30,26 @@ def forward(self, query, key, value): def test_sdpa_FP(): test_input = tuple(torch.randn(1, 3, 197, 64) for x in range(3)) - pipeline = TosaPipelineFP[input_t](SDPA(), test_input, [], []) + pipeline = TosaPipelineFP[input_t]( + SDPA(), + test_input, + [], + [], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.pop_stage("check_count.exir") pipeline.run() def test_sdpa_INT(): test_input = tuple(torch.randn(1, 3, 197, 64) for x in range(3)) - pipeline = TosaPipelineINT[input_t](SDPA(), test_input, [], []) + pipeline = TosaPipelineINT[input_t]( + SDPA(), + test_input, + [], + [], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.pop_stage("check.quant_nodes") pipeline.pop_stage("check_count.exir") - pipeline.pop_stage( - "run_method_and_compare_outputs" - ) # TODO: reference is not quantized pipeline.run() diff --git a/backends/arm/test/ops/test_sigmoid.py b/backends/arm/test/ops/test_sigmoid.py index b4f8458574e..8113607de83 100644 --- a/backends/arm/test/ops/test_sigmoid.py +++ b/backends/arm/test/ops/test_sigmoid.py @@ -8,6 +8,8 @@ from typing import Tuple +import conftest + import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -70,12 +72,24 @@ def forward(self, x, y): @common.parametrize("test_data", test_data_suite) def test_sigmoid_tosa_FP(test_data: torch.Tensor): - TosaPipelineFP[input_t1](Sigmoid(), (test_data(),), aten_op, exir_op).run() + TosaPipelineFP[input_t1]( + Sigmoid(), + (test_data(),), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ).run() @common.parametrize("test_data", test_data_suite) def test_sigmoid_tosa_INT(test_data: torch.Tensor): - TosaPipelineINT[input_t1](Sigmoid(), (test_data(),), aten_op, exir_op).run() + TosaPipelineINT[input_t1]( + Sigmoid(), + (test_data(),), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ).run() def test_sigmoid_tosa_FP_add(): @@ -84,6 +98,7 @@ def test_sigmoid_tosa_FP_add(): (test_data_suite["zeros"](),), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ).run() @@ -93,6 +108,7 @@ def test_sigmoid_tosa_INT_add(): (test_data_suite["ramp"](),), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ).run() @@ -102,6 +118,7 @@ def test_sigmoid_tosa_FP_add_2(): (test_data_suite["zeros"](),), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ).run() @@ -111,6 +128,7 @@ def test_sigmoid_tosa_INT_add_2(): (test_data_suite["zeros"](),), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ).run() @@ -120,6 +138,7 @@ def test_sigmoid_tosa_FP_add_3(): (test_data_suite["randn_neg"](), test_data_suite["randn_pos"]()), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ).run() @@ -129,6 +148,7 @@ def test_sigmoid_tosa_INT_3(): (test_data_suite["randn_neg"](), test_data_suite["randn_pos"]()), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ).run() diff --git a/backends/arm/test/ops/test_sin.py b/backends/arm/test/ops/test_sin.py index 6f9037e1021..948ea23be25 100644 --- a/backends/arm/test/ops/test_sin.py +++ b/backends/arm/test/ops/test_sin.py @@ -43,6 +43,7 @@ def test_sin_tosa_FP(test_data: Tuple): (test_data,), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) if conftest.get_option("tosa_version") == "1.0": pipeline.run() @@ -55,6 +56,7 @@ def test_sin_tosa_INT(test_data: Tuple): (test_data,), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_sinh.py b/backends/arm/test/ops/test_sinh.py index ff486e6a4b8..751559e084f 100644 --- a/backends/arm/test/ops/test_sinh.py +++ b/backends/arm/test/ops/test_sinh.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -48,6 +49,7 @@ def test_sinh_tosa_FP(test_data: Tuple): (test_data,), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -55,7 +57,11 @@ def test_sinh_tosa_FP(test_data: Tuple): @common.parametrize("test_data", test_data_suite) def test_sinh_tosa_INT(test_data: Tuple): pipeline = TosaPipelineINT[input_t1]( - Sinh(), (test_data,), aten_op=aten_op, exir_op=exir_op + Sinh(), + (test_data,), + aten_op=aten_op, + exir_op=exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_slice.py b/backends/arm/test/ops/test_slice.py index 8fcf343dd57..d8719d1b274 100644 --- a/backends/arm/test/ops/test_slice.py +++ b/backends/arm/test/ops/test_slice.py @@ -7,8 +7,9 @@ from typing import Tuple -import torch +import conftest +import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -44,7 +45,13 @@ def forward(self, x: torch.Tensor, s: list[tuple[int, int]]): @common.parametrize("test_data", test_data_suite) def test_slice_tensor_tosa_FP(test_data: torch.Tensor): - pipeline = TosaPipelineFP[input_t1](Slice(), test_data(), aten_op, exir_op) + pipeline = TosaPipelineFP[input_t1]( + Slice(), + test_data(), + aten_op, + exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), + ) pipeline.run() @@ -55,6 +62,7 @@ def test_slice_tensor_tosa_INT_nchw(test_data: torch.Tensor): test_data(), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -66,6 +74,7 @@ def test_slice_tensor_tosa_INT_nhwc(test_data: torch.Tensor): test_data(), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_sqrt.py b/backends/arm/test/ops/test_sqrt.py index ee554ce4fd2..b9b9429847f 100644 --- a/backends/arm/test/ops/test_sqrt.py +++ b/backends/arm/test/ops/test_sqrt.py @@ -6,6 +6,7 @@ from typing import Dict, Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -51,6 +52,7 @@ def test_sqrt_tosa_FP(test_data: Sqrt.input_t): test_data(), Sqrt.aten_op_FP, Sqrt.exir_op_FP, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -62,6 +64,7 @@ def test_sqrt_tosa_INT(test_data: Sqrt.input_t): test_data(), Sqrt.aten_op_INT, Sqrt.exir_op_INT, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_sub.py b/backends/arm/test/ops/test_sub.py index 09f5884b1c4..9e952abf051 100644 --- a/backends/arm/test/ops/test_sub.py +++ b/backends/arm/test/ops/test_sub.py @@ -7,6 +7,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( @@ -70,6 +71,7 @@ def test_sub_tensor_tosa_FP(test_data): test_data(), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -82,6 +84,7 @@ def test_sub_tensor_tosa_FP_2(test_data: Tuple[torch.Tensor, torch.Tensor]): test_data(), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -94,6 +97,7 @@ def test_sub_tensor_tosa_INT(test_data): test_data(), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -106,6 +110,7 @@ def test_sub_tensor_tosa_INT_2(test_data: Tuple[torch.Tensor, torch.Tensor]): test_data(), aten_op, exir_op, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/ops/test_tanh.py b/backends/arm/test/ops/test_tanh.py index 1bd746d7b24..ceae4a85843 100644 --- a/backends/arm/test/ops/test_tanh.py +++ b/backends/arm/test/ops/test_tanh.py @@ -6,6 +6,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm.test import common @@ -46,6 +47,7 @@ def test_tanh_tosa_FP(test_data: Tuple): (test_data(),), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -57,6 +59,7 @@ def test_tanh_tosa_INT(test_data: Tuple): (test_data(),), aten_op, exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_broadcast_args_pass.py b/backends/arm/test/passes/test_broadcast_args_pass.py index 719a0ddd622..b675144cac7 100644 --- a/backends/arm/test/passes/test_broadcast_args_pass.py +++ b/backends/arm/test/passes/test_broadcast_args_pass.py @@ -6,6 +6,7 @@ import operator from typing import Tuple +import conftest import torch from executorch.backends.arm._passes import BroadcastArgsPass @@ -50,5 +51,6 @@ def test_multiple_broacasts_model(module: NeedsMultipleBroadcastsModel): ops_not_before_pass=ops_not_before_pass, ops_after_pass=ops_after_pass, pass_list=[BroadcastArgsPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_cast_int64_pass.py b/backends/arm/test/passes/test_cast_int64_pass.py index 7832fd87ed9..e4dc26ffae8 100644 --- a/backends/arm/test/passes/test_cast_int64_pass.py +++ b/backends/arm/test/passes/test_cast_int64_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes import CastInt64BuffersToInt32Pass @@ -39,6 +40,7 @@ def test_int64_model(test_data: input_t): ops_before_pass=op_checks, ops_after_pass=op_checks, passes_with_exported_program=[CastInt64BuffersToInt32Pass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_convert_expand_copy_to_repeat.py b/backends/arm/test/passes/test_convert_expand_copy_to_repeat.py index aa877c355bd..9746076cffc 100644 --- a/backends/arm/test/passes/test_convert_expand_copy_to_repeat.py +++ b/backends/arm/test/passes/test_convert_expand_copy_to_repeat.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.convert_expand_copy_to_repeat import ( ConvertExpandCopyToRepeatPass, @@ -47,5 +48,6 @@ def test_expand_to_repeat_tosa_INT(): "executorch_exir_dialects_edge__ops_aten_expand_copy_default" ], pass_list=[ConvertExpandCopyToRepeatPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_convert_int_pow_to_muls.py b/backends/arm/test/passes/test_convert_int_pow_to_muls.py index 4eeff845749..b66e9305fcc 100644 --- a/backends/arm/test/passes/test_convert_int_pow_to_muls.py +++ b/backends/arm/test/passes/test_convert_int_pow_to_muls.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes import ConvertIntPowToMuls @@ -69,5 +70,6 @@ def test_convert_pow_to_muls(data): }, ops_not_after_pass=["executorch_exir_dialects_edge__ops_pow_Tensor_Scalar"], pass_list=[ConvertIntPowToMuls], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_convert_split_to_slice.py b/backends/arm/test/passes/test_convert_split_to_slice.py index fba52308ff0..e04275f81d2 100644 --- a/backends/arm/test/passes/test_convert_split_to_slice.py +++ b/backends/arm/test/passes/test_convert_split_to_slice.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.convert_split_to_slice import ( ConvertSplitToSlicePass, @@ -63,5 +64,6 @@ def test_split_to_slice_tosa_INT(module): "executorch_exir_dialects_edge__ops_aten_split_with_sizes_copy_default" ], pass_list=[ConvertSplitToSlicePass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_convert_to_clamp.py b/backends/arm/test/passes/test_convert_to_clamp.py index cc854eeacd7..3c4b2943f16 100644 --- a/backends/arm/test/passes/test_convert_to_clamp.py +++ b/backends/arm/test/passes/test_convert_to_clamp.py @@ -6,6 +6,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.convert_to_clamp import ConvertToClampPass @@ -64,6 +65,7 @@ def test_tosa_FP_hardtahn(test_data: input_t): ops_after_pass=op_checks_after_pass, ops_not_after_pass=op_checks_not_after_pass, pass_list=[ConvertToClampPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -88,5 +90,6 @@ def test_tosa_FP_relu(test_data: input_t): ops_after_pass=op_checks_after_pass, ops_not_after_pass=op_checks_not_after_pass, pass_list=[ConvertToClampPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_decompose_cosine_similarity_pass.py b/backends/arm/test/passes/test_decompose_cosine_similarity_pass.py index 80a328f39c6..0aaa429e231 100644 --- a/backends/arm/test/passes/test_decompose_cosine_similarity_pass.py +++ b/backends/arm/test/passes/test_decompose_cosine_similarity_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.decompose_cosine_similarity_pass import ( @@ -49,5 +50,6 @@ def test_decompose_cosine_similarity_tosa_INT(module): ops_not_after_pass=None, pass_list=[DecomposeCosineSimilarityPass], quantize=True, + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_decompose_div_pass.py b/backends/arm/test/passes/test_decompose_div_pass.py index b52e264bf11..585fbe19b16 100644 --- a/backends/arm/test/passes/test_decompose_div_pass.py +++ b/backends/arm/test/passes/test_decompose_div_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.decompose_div_pass import DecomposeDivPass @@ -61,5 +62,6 @@ def test_decompose_div_tosa_FP(module): }, ops_not_after_pass=["executorch_exir_dialects_edge__ops_aten_div_Tensor"], pass_list=[DecomposeDivPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_decompose_layernorm_pass.py b/backends/arm/test/passes/test_decompose_layernorm_pass.py index d3c2cd6efd7..de1c74be2ae 100644 --- a/backends/arm/test/passes/test_decompose_layernorm_pass.py +++ b/backends/arm/test/passes/test_decompose_layernorm_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.decompose_layernorm_pass import ( DecomposeLayerNormPass, @@ -65,5 +66,6 @@ def test_decompose_layernorm_tosa_FP(): "executorch_exir_dialects_edge__ops_aten_expand_copy_default" ], pass_list=[DecomposeLayerNormPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_decompose_linalg_vector_norm_pass.py b/backends/arm/test/passes/test_decompose_linalg_vector_norm_pass.py index 5b4c84edbfd..bd318e418b9 100644 --- a/backends/arm/test/passes/test_decompose_linalg_vector_norm_pass.py +++ b/backends/arm/test/passes/test_decompose_linalg_vector_norm_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.decompose_linalg_vector_norm_pass import ( @@ -87,5 +88,6 @@ def test_decompose_vector_norm_tosa_INT(module): "executorch_exir_dialects_edge__ops_aten_linarg_vector_norm_default", ], pass_list=[DecomposeLinearVectorNormPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_decompose_softmax_pass.py b/backends/arm/test/passes/test_decompose_softmax_pass.py index 3af1976e3f3..f3ea3128b01 100644 --- a/backends/arm/test/passes/test_decompose_softmax_pass.py +++ b/backends/arm/test/passes/test_decompose_softmax_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.decompose_softmax_pass import DecomposeSoftmaxPass @@ -70,6 +71,7 @@ def test_softmax_basic_tosa_FP(): }, ops_not_after_pass=["executorch_exir_dialects_edge__ops_aten__softmax_default"], pass_list=[DecomposeSoftmaxPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -99,5 +101,6 @@ def test_softmax_log_tosa_FP(): "executorch_exir_dialects_edge__ops_aten__log_softmax_default" ], pass_list=[DecomposeSoftmaxPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_decompose_var_pass.py b/backends/arm/test/passes/test_decompose_var_pass.py index c347a2f667c..ba5eb47b7b5 100644 --- a/backends/arm/test/passes/test_decompose_var_pass.py +++ b/backends/arm/test/passes/test_decompose_var_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.decompose_var_pass import DecomposeVarPass @@ -80,5 +81,6 @@ def test_decompose_var_tosa_FP(module): }, ops_not_after_pass=["executorch_exir_dialects_edge__ops_aten_var_correction"], pass_list=[DecomposeVarPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_fuse_batchnorm_pass.py b/backends/arm/test/passes/test_fuse_batchnorm_pass.py index 59fae7cafbd..2c0edf109a8 100644 --- a/backends/arm/test/passes/test_fuse_batchnorm_pass.py +++ b/backends/arm/test/passes/test_fuse_batchnorm_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.fuse_batchnorm2d_pass import FuseBatchnorm2DPass from executorch.backends.arm.test import common @@ -148,5 +149,6 @@ def test_fuse_batchnorm_tosa_FP(module: torch.nn.Module): ops_before_pass=module.ops_before_pass, ops_after_pass=module.ops_after_pass, passes_with_exported_program=[FuseBatchnorm2DPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_fuse_constant_ops_pass.py b/backends/arm/test/passes/test_fuse_constant_ops_pass.py index 1a318c5cd42..8947aade3f0 100644 --- a/backends/arm/test/passes/test_fuse_constant_ops_pass.py +++ b/backends/arm/test/passes/test_fuse_constant_ops_pass.py @@ -6,6 +6,7 @@ import operator from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.fuse_constant_ops_pass import ( ComputeConstantOpsAOT, @@ -124,6 +125,7 @@ def test_fuse_const_ops_tosa_FP(module: torch.nn.Module): ops_after_pass=module.ops_after_pass, ops_not_after_pass=module.ops_not_after_pass, passes_with_exported_program=[ComputeConstantOpsAOT, FuseConstantArgsPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -137,6 +139,7 @@ def test_fuse_const_ops_tosa_INT(module: torch.nn.Module): ops_before_pass=module.ops_before_pass, ops_after_pass=module.ops_after_pass, passes_with_exported_program=[ComputeConstantOpsAOT, FuseConstantArgsPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -150,5 +153,6 @@ def test_fuse_const_ops_tosa_BI_cat(module: torch.nn.Module): ops_before_pass=module.ops_before_pass, ops_after_pass=module.ops_after_pass, passes_with_exported_program=[ComputeConstantOpsAOT, FuseConstantArgsPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_fuse_equal_placeholders_ops_pass.py b/backends/arm/test/passes/test_fuse_equal_placeholders_ops_pass.py index f6e437ba034..ee71ea74a00 100644 --- a/backends/arm/test/passes/test_fuse_equal_placeholders_ops_pass.py +++ b/backends/arm/test/passes/test_fuse_equal_placeholders_ops_pass.py @@ -6,6 +6,8 @@ from copy import deepcopy from typing import Tuple +import conftest + import torch from executorch.backends.arm._passes.fuse_equal_placeholders_pass import ( FuseEqualPlaceholdersPass, @@ -86,6 +88,7 @@ def test_fuse_equal_placeholders_constants_tosa_FP(): ops_before_pass=module.ops_before_pass, ops_after_pass=module.ops_after_pass, passes_with_exported_program=[FuseEqualPlaceholdersPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -107,6 +110,7 @@ def test_fuse_equal_placeholders_state_dict_tosa_FP(): ops_before_pass=module.ops_before_pass, ops_after_pass=module.ops_after_pass, passes_with_exported_program=[FuseEqualPlaceholdersPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() @@ -135,5 +139,6 @@ def test_not_fuse_tensor_with_different_type_FP(): module, data, aten_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_insert_int64_to_int32_cast_pass.py b/backends/arm/test/passes/test_insert_int64_to_int32_cast_pass.py index da6eeb59459..5fd026cc89f 100644 --- a/backends/arm/test/passes/test_insert_int64_to_int32_cast_pass.py +++ b/backends/arm/test/passes/test_insert_int64_to_int32_cast_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes import InsertCastForOpsWithInt64InputPass @@ -41,6 +42,7 @@ def test_int64_model_tosa_FP(): ops_before_pass=op_checks_before, ops_after_pass=op_checks_after, pass_list=[InsertCastForOpsWithInt64InputPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.pop_stage(-1) # Do not compare output pipeline.run() diff --git a/backends/arm/test/passes/test_insert_table_ops_pass.py b/backends/arm/test/passes/test_insert_table_ops_pass.py index 029942dd659..d80a275cdd2 100644 --- a/backends/arm/test/passes/test_insert_table_ops_pass.py +++ b/backends/arm/test/passes/test_insert_table_ops_pass.py @@ -6,6 +6,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.fold_qdq_with_annotated_qparams_pass import ( FoldAndAnnotateQParamsPass, @@ -42,6 +43,7 @@ def test_insert_table_tosa_INT(test_data: input_t): ops_not_after_pass=["aten_sigmoid_default"], pass_list=[FoldAndAnnotateQParamsPass], passes_with_exported_program=[InsertTableOpsPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.pop_stage(-1) # Do not compare output diff --git a/backends/arm/test/passes/test_int32_cast_embedding_pass.py b/backends/arm/test/passes/test_int32_cast_embedding_pass.py index 7adca527d75..35b940cc0ea 100644 --- a/backends/arm/test/passes/test_int32_cast_embedding_pass.py +++ b/backends/arm/test/passes/test_int32_cast_embedding_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes import DecomposeEmbeddingPass @@ -41,6 +42,7 @@ def test_int64_model_tosa_FP(): ops_before_pass=op_checks_before, ops_after_pass=op_checks_after, pass_list=[DecomposeEmbeddingPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.pop_stage(-1) # Do not compare output pipeline.run() diff --git a/backends/arm/test/passes/test_remove_clone_pass.py b/backends/arm/test/passes/test_remove_clone_pass.py index dea0bb06f5e..bfbdbbd5782 100755 --- a/backends/arm/test/passes/test_remove_clone_pass.py +++ b/backends/arm/test/passes/test_remove_clone_pass.py @@ -5,6 +5,7 @@ from typing import Tuple +import conftest import torch from executorch.backends.arm._passes.remove_clone_pass import RemoveClonePass @@ -39,5 +40,6 @@ def test_remove_clone_tosa_INT(): }, ops_not_after_pass=["executorch_exir_dialects_edge__ops_aten_clone_default"], pass_list=[RemoveClonePass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.run() diff --git a/backends/arm/test/passes/test_rescale_pass.py b/backends/arm/test/passes/test_rescale_pass.py index 0fe72f6d1fe..09315dd1b3c 100644 --- a/backends/arm/test/passes/test_rescale_pass.py +++ b/backends/arm/test/passes/test_rescale_pass.py @@ -125,9 +125,8 @@ def test_quantized_rescale_tosa_bi(test_data: tuple[torch.Tensor, torch.Tensor]) test_data=test_data, aten_op=[], exir_op=[], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) - if not conftest.is_option_enabled("tosa_ref_model"): - pipeline.pop_stage("run_method_and_compare_outputs") pipeline.run() diff --git a/backends/arm/test/passes/test_unsqueeze_before_repeat_pass.py b/backends/arm/test/passes/test_unsqueeze_before_repeat_pass.py index fc405e21f2a..127872b60dd 100644 --- a/backends/arm/test/passes/test_unsqueeze_before_repeat_pass.py +++ b/backends/arm/test/passes/test_unsqueeze_before_repeat_pass.py @@ -5,6 +5,7 @@ from typing import Dict, Tuple +import conftest import torch from executorch.backends.arm._passes import UnsqueezeBeforeRepeatPass from executorch.backends.arm.test import common @@ -54,6 +55,7 @@ def test_unsqueeze_before_repeat_tosa_FP(test_data: input_t): ops_after_pass=ops_after_pass, ops_not_after_pass=ops_not_after_pass, pass_list=[UnsqueezeBeforeRepeatPass], + run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"), ) pipeline.pop_stage(-1) # Do not compare output pipeline.run() diff --git a/backends/arm/test/tester/test_pipeline.py b/backends/arm/test/tester/test_pipeline.py index fb9f05444e5..e86cca02552 100644 --- a/backends/arm/test/tester/test_pipeline.py +++ b/backends/arm/test/tester/test_pipeline.py @@ -637,6 +637,7 @@ class PassPipeline(BasePipelineMaker, Generic[T]): pass_functions: List of functions applied directly to the exported program. passes_with_exported_program: List of passes initiated with an exported_program. custom_path : Path to dump intermediate artifacts such as tosa and pte to. + run_on_tosa_ref_model: Set to true to test the tosa file on the TOSA reference model. Passes are run in order pass_list -> pass_functions -> passes_with_exported_program. See arm_tester.RunPasses() for more information. @@ -655,6 +656,7 @@ def __init__( pass_functions: Optional[List[Callable]] = None, passes_with_exported_program: Optional[List[Type[ExportPass]]] = None, custom_path: str = None, + run_on_tosa_ref_model: bool = True, ): tosa_profiles = { "1.0": TosaSpecification.create_from_string( @@ -701,7 +703,8 @@ def __init__( self.add_stage(self.tester.check_count, ops_after_pass, suffix="after") if ops_not_after_pass: self.add_stage(self.tester.check_not, ops_not_after_pass, suffix="after") - self.add_stage(self.tester.run_method_and_compare_outputs) + if run_on_tosa_ref_model: + self.add_stage(self.tester.run_method_and_compare_outputs) class TransformAnnotationPassPipeline(BasePipelineMaker, Generic[T]): @@ -713,6 +716,7 @@ class TransformAnnotationPassPipeline(BasePipelineMaker, Generic[T]): test_data: Data used for testing the module. custom_path : Path to dump intermediate artifacts such as tosa and pte to. + run_on_tosa_ref_model: Set to true to test the tosa file on the TOSA reference model. """ @@ -721,6 +725,7 @@ def __init__( module: torch.nn.Module, test_data: T, custom_path: str = None, + run_on_tosa_ref_model: bool = True, ): tosa_profiles = { "1.0": TosaSpecification.create_from_string("TOSA-1.0+INT"), @@ -748,11 +753,12 @@ def __init__( self.pop_stage("to_executorch") self.pop_stage("to_edge_transform_and_lower") self.pop_stage("check.aten") - self.add_stage( - self.tester.run_method_and_compare_outputs, - inputs=test_data, - run_eager_mode=True, - ) + if run_on_tosa_ref_model: + self.add_stage( + self.tester.run_method_and_compare_outputs, + inputs=test_data, + run_eager_mode=True, + ) class OpNotSupportedPipeline(BasePipelineMaker, Generic[T]):