diff --git a/backends/arm/test/models/test_conformer.py b/backends/arm/test/models/test_conformer.py index e3b9bc21ebf..6a66b25d27d 100644 --- a/backends/arm/test/models/test_conformer.py +++ b/backends/arm/test/models/test_conformer.py @@ -15,6 +15,7 @@ EthosU85PipelineINT, TosaPipelineFP, TosaPipelineINT, + VgfPipeline, ) from torchaudio.models import Conformer @@ -124,3 +125,40 @@ def test_conformer_u85_INT(): atol=5.0, ) pipeline.run() + + +@common.SkipIfNoModelConverter +def test_conformer_vgf_INT(): + pipeline = VgfPipeline[input_t]( + TestConformer.conformer, + TestConformer.model_example_inputs, + aten_op=TestConformer.aten_ops, + exir_op=[], + tosa_version="TOSA-1.0+INT", + use_to_edge_transform_and_lower=True, + ) + pipeline.pop_stage("check_count.exir") + + # TODO: MLETORCH-1167 Create Vulkan backend e2e tests + # pipeline.change_args( + # "run_method_and_compare_outputs", + # get_test_inputs( + # TestConformer.dim, TestConformer.lengths, TestConformer.num_examples + # ), + # rtol=1.0, + # atol=3.0, + # ) + pipeline.run() + + +@common.SkipIfNoModelConverter +def test_conformer_vgf_FP(): + pipeline = VgfPipeline[input_t]( + TestConformer.conformer, + TestConformer.model_example_inputs, + aten_op=TestConformer.aten_ops, + exir_op=[], + tosa_version="TOSA-1.0+FP", + use_to_edge_transform_and_lower=True, + ) + pipeline.run() diff --git a/backends/arm/test/models/test_deit_tiny_arm.py b/backends/arm/test/models/test_deit_tiny_arm.py index 4d7f8c925f2..22685a079bd 100644 --- a/backends/arm/test/models/test_deit_tiny_arm.py +++ b/backends/arm/test/models/test_deit_tiny_arm.py @@ -11,9 +11,12 @@ import torch +from executorch.backends.arm.test import common + from executorch.backends.arm.test.tester.test_pipeline import ( TosaPipelineFP, TosaPipelineINT, + VgfPipeline, ) from timm.data import IMAGENET_INCEPTION_MEAN, IMAGENET_INCEPTION_STD @@ -56,3 +59,31 @@ def test_deit_tiny_tosa_INT(): qtol=1, ) pipeline.run() + + +@common.SkipIfNoModelConverter +def test_deit_tiny_vgf_INT(): + pipeline = VgfPipeline[input_t]( + deit_tiny, + model_inputs, + aten_op=[], + exir_op=[], + tosa_version="TOSA-1.0+INT", + use_to_edge_transform_and_lower=True, + atol=1.5, + qtol=1, + ) + pipeline.run() + + +@common.SkipIfNoModelConverter +def test_deit_tiny_vgf_FP(): + pipeline = VgfPipeline[input_t]( + deit_tiny, + model_inputs, + aten_op=[], + exir_op=[], + tosa_version="TOSA-1.0+FP", + use_to_edge_transform_and_lower=True, + ) + pipeline.run() diff --git a/backends/arm/test/models/test_dl3_arm.py b/backends/arm/test/models/test_dl3_arm.py index 433948d15b0..2000ac34794 100644 --- a/backends/arm/test/models/test_dl3_arm.py +++ b/backends/arm/test/models/test_dl3_arm.py @@ -16,6 +16,7 @@ EthosU85PipelineINT, TosaPipelineFP, TosaPipelineINT, + VgfPipeline, ) from executorch.examples.models import deeplab_v3 @@ -87,3 +88,37 @@ def test_dl3_u85_INT(): "run_method_and_compare_outputs", rtol=1.0, atol=1.0 ) # TODO: MLETORCH-1036 decrease tolerance pipeline.run() + + +@common.SkipIfNoModelConverter +def test_dl3_vgf_INT(): + pipeline = VgfPipeline[input_t]( + TestDl3.dl3, + TestDl3.model_example_inputs, + aten_op=[], + exir_op=[], + tosa_version="TOSA-1.0+INT", + use_to_edge_transform_and_lower=True, + ) + # TODO: MLETORCH-1167 Create Vulkan backend e2e tests + # pipeline.change_args( + # "run_method_and_compare_outputs", rtol=1.0, atol=1.0 + # ) + pipeline.run() + + +@common.SkipIfNoModelConverter +def test_dl3_vgf_FP(): + pipeline = VgfPipeline[input_t]( + TestDl3.dl3, + TestDl3.model_example_inputs, + aten_op=[], + exir_op=[], + tosa_version="TOSA-1.0+FP", + use_to_edge_transform_and_lower=True, + ) + # TODO: MLETORCH-1167 Create Vulkan backend e2e tests + # pipeline.change_args( + # "run_method_and_compare_outputs", rtol=1.0, atol=1.0 + # ) + pipeline.run() diff --git a/backends/arm/test/models/test_llama.py b/backends/arm/test/models/test_llama.py index ee9750f853c..7732943d5fb 100644 --- a/backends/arm/test/models/test_llama.py +++ b/backends/arm/test/models/test_llama.py @@ -17,10 +17,11 @@ import torch from executorch.backends.arm._passes import InsertCastForOpsWithInt64InputPass -from executorch.backends.arm.test import conftest +from executorch.backends.arm.test import common, conftest from executorch.backends.arm.test.tester.test_pipeline import ( TosaPipelineFP, TosaPipelineINT, + VgfPipeline, ) from executorch.examples.models.llama.export_llama_lib import ( build_args_parser, @@ -131,3 +132,42 @@ def test_llama_tosa_INT(): use_to_edge_transform_and_lower=True, ) pipeline.run() + + +@common.SkipIfNoModelConverter +def test_llama_vgf_FP(): + llama_model, llama_inputs, llama_meta = TestLlama().prepare_model() + + if llama_model is None or llama_inputs is None: + pytest.skip("Missing model and/or input files") + + with torch.no_grad(): + pipeline = VgfPipeline[input_t]( + llama_model, + llama_inputs, + aten_op=[], + exir_op=[], + tosa_version="TOSA-1.0+FP", + use_to_edge_transform_and_lower=True, + ) + pipeline.run() + + +@common.SkipIfNoModelConverter +def test_llama_vgf_INT(): + llama_model, llama_inputs, llama_meta = TestLlama().prepare_model() + + if llama_model is None or llama_inputs is None: + pytest.skip("Missing model and/or input files") + + with torch.no_grad(): + pipeline = VgfPipeline[input_t]( + llama_model, + llama_inputs, + aten_op=[], + exir_op=[], + tosa_version="TOSA-1.0+INT", + use_to_edge_transform_and_lower=True, + transform_passes=[InsertCastForOpsWithInt64InputPass()], + ) + pipeline.run() diff --git a/backends/arm/test/models/test_lstm_arm.py b/backends/arm/test/models/test_lstm_arm.py index bb9b92a0f7d..1e63472f5f4 100644 --- a/backends/arm/test/models/test_lstm_arm.py +++ b/backends/arm/test/models/test_lstm_arm.py @@ -13,6 +13,7 @@ EthosU85PipelineINT, TosaPipelineFP, TosaPipelineINT, + VgfPipeline, ) from torch.nn.quantizable.modules import rnn @@ -98,3 +99,37 @@ def test_lstm_u85_INT(): "run_method_and_compare_outputs", get_test_inputs(), atol=3e-1, qtol=1.0 ) pipeline.run() + + +@common.SkipIfNoModelConverter +def test_lstm_vgf_INT(): + pipeline = VgfPipeline[input_t]( + TestLSTM.lstm, + TestLSTM.model_example_inputs, + aten_op=[], + exir_op=[], + tosa_version="TOSA-1.0+INT", + use_to_edge_transform_and_lower=True, + ) + # TODO: MLETORCH-1167 Create Vulkan backend e2e tests + # pipeline.change_args( + # "run_method_and_compare_outputs", get_test_inputs(), atol=3e-1, qtol=1.0 + # ) + pipeline.run() + + +@common.SkipIfNoModelConverter +def test_lstm_vgf_FP(): + pipeline = VgfPipeline[input_t]( + TestLSTM.lstm, + TestLSTM.model_example_inputs, + aten_op=[], + exir_op=[], + tosa_version="TOSA-1.0+FP", + use_to_edge_transform_and_lower=True, + ) + # TODO: MLETORCH-1167 Create Vulkan backend e2e tests + # pipeline.change_args( + # "run_method_and_compare_outputs", get_test_inputs(), atol=3e-1, qtol=1.0 + # ) + pipeline.run() diff --git a/backends/arm/test/models/test_mobilenet_v2_arm.py b/backends/arm/test/models/test_mobilenet_v2_arm.py index 090d7f849d3..d4e3bbc8e28 100644 --- a/backends/arm/test/models/test_mobilenet_v2_arm.py +++ b/backends/arm/test/models/test_mobilenet_v2_arm.py @@ -16,6 +16,7 @@ EthosU85PipelineINT, TosaPipelineFP, TosaPipelineINT, + VgfPipeline, ) from torchvision import models, transforms # type: ignore[import-untyped] @@ -94,3 +95,41 @@ def test_mv2_u85_INT(per_channel_quantization): qtol=1, ) pipeline.run() + + +@common.SkipIfNoModelConverter +@common.parametrize("per_channel_quantization", quant_test_data) +def test_mv2_vgf_INT(per_channel_quantization): + pipeline = VgfPipeline[input_t]( + mv2, + model_inputs, + aten_op=[], + exir_op=[], + tosa_version="TOSA-1.0+INT", + use_to_edge_transform_and_lower=True, + per_channel_quantization=per_channel_quantization, + atol=0.25, + qtol=1, + ) + # TODO: MLETORCH-1167 Create Vulkan backend e2e tests + # pipeline.change_args( + # "run_method_and_compare_outputs", get_test_inputs(), atol=3e-1, qtol=1.0 + # ) + pipeline.run() + + +@common.SkipIfNoModelConverter +def test_mv2_vgf_FP(): + pipeline = VgfPipeline[input_t]( + mv2, + model_inputs, + aten_op=[], + exir_op=[], + tosa_version="TOSA-1.0+FP", + use_to_edge_transform_and_lower=True, + ) + # TODO: MLETORCH-1167 Create Vulkan backend e2e tests + # pipeline.change_args( + # "run_method_and_compare_outputs", get_test_inputs(), atol=3e-1, qtol=1.0 + # ) # TODO: MLETORCH-1036 decrease tolerance + pipeline.run() diff --git a/backends/arm/test/models/test_mobilenet_v3_arm.py b/backends/arm/test/models/test_mobilenet_v3_arm.py index c43f20b2884..0dcbd9757ac 100644 --- a/backends/arm/test/models/test_mobilenet_v3_arm.py +++ b/backends/arm/test/models/test_mobilenet_v3_arm.py @@ -15,6 +15,7 @@ EthosU85PipelineINT, TosaPipelineFP, TosaPipelineINT, + VgfPipeline, ) from torchvision import models, transforms @@ -82,3 +83,32 @@ def test_mv3_u85_INT(): qtol=1, ) pipeline.run() + + +@common.SkipIfNoModelConverter +@pytest.mark.slow +def test_mv3_vgf_INT(): + pipeline = VgfPipeline[input_t]( + mv3, + model_inputs, + aten_op=[], + exir_op=[], + tosa_version="TOSA-1.0+INT", + use_to_edge_transform_and_lower=True, + atol=0.5, + qtol=1, + ) + pipeline.run() + + +@common.SkipIfNoModelConverter +def test_mv3_vgf_FP(): + pipeline = VgfPipeline[input_t]( + mv3, + model_inputs, + aten_op=[], + exir_op=[], + tosa_version="TOSA-1.0+FP", + use_to_edge_transform_and_lower=True, + ) + pipeline.run() diff --git a/backends/arm/test/models/test_w2l_arm.py b/backends/arm/test/models/test_w2l_arm.py index fa19a3b97e4..32b25a18fd8 100644 --- a/backends/arm/test/models/test_w2l_arm.py +++ b/backends/arm/test/models/test_w2l_arm.py @@ -17,6 +17,7 @@ EthosU85PipelineINT, TosaPipelineFP, TosaPipelineINT, + VgfPipeline, ) from torchaudio import models @@ -101,3 +102,30 @@ def test_w2l_u85_INT(): run_on_fvp=True, ) pipeline.run() + + +@common.SkipIfNoModelConverter +@pytest.mark.slow +def test_w2l_vgf_INT(): + pipeline = VgfPipeline[input_t]( + TestW2L.w2l, + TestW2L.model_example_inputs, + aten_op=[], + exir_op=TestW2L.all_operators, + tosa_version="TOSA-1.0+INT", + use_to_edge_transform_and_lower=True, + ) + pipeline.run() + + +@common.SkipIfNoModelConverter +def test_w2l_vgf_FP(): + pipeline = VgfPipeline[input_t]( + TestW2L.w2l, + TestW2L.model_example_inputs, + aten_op=[], + exir_op=TestW2L.all_operators, + tosa_version="TOSA-1.0+FP", + use_to_edge_transform_and_lower=True, + ) + pipeline.run()