Skip to content

Arm backend: Refactor models to allow for TOSA 1.0 #10904

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
May 20, 2025
Merged

Arm backend: Refactor models to allow for TOSA 1.0 #10904

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
735d97b
Arm backend: Refactor models to allow for TOSA 1.0
SaoirseARM Apr 17, 2025
f8b199d
Merge branch 'main' into toupstream/model_update
zingo May 16, 2025
90d9b10
Arm Backend: Skip corestone tests for dl3
SaoirseARM May 16, 2025
5b5ecce
Merge remote-tracking branch 'origin/main' into toupstream/model_update
SaoirseARM May 19, 2025
c06093a
Arm backend: Increase tolerarce for dl3 test case
SaoirseARM May 19, 2025
b3a91fe
Merge branch 'main' into toupstream/model_update
zingo May 19, 2025
d45fe19
Merge branch 'main' into toupstream/model_update
zingo May 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
200 changes: 98 additions & 102 deletions backends/arm/test/models/test_conformer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,32 +3,37 @@
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.

import unittest
from typing import Tuple

import pytest

import torch
from executorch.backends.arm.test import common, conftest

from executorch.backends.arm.test.tester.arm_tester import ArmTester
from executorch.backends.arm.test import common
from executorch.backends.arm.test.tester.test_pipeline import (
EthosU55PipelineBI,
EthosU85PipelineBI,
TosaPipelineBI,
TosaPipelineMI,
)

from torchaudio.models import Conformer

input_t = Tuple[torch.Tensor, torch.IntTensor] # Input x, y


def get_test_inputs(dim, lengths, num_examples):
return (torch.rand(num_examples, int(lengths.max()), dim), lengths)


class TestConformer(unittest.TestCase):
class TestConformer:
"""Tests Torchaudio Conformer"""

# Adjust nbr below as we increase op support. Note: most of the delegates
# calls are directly consecutive to each other in the .pte. The reason
# for that is some assert ops are removed by passes in the
# .to_executorch step, i.e. after Arm partitioner.
ops_after_partitioner = {
"executorch_exir_dialects_edge__ops_aten_max_default": 1,
"torch.ops.aten._assert_scalar.default": 7,
"torch.ops.aten._local_scalar_dense.default": 1,
}
aten_ops = ["torch.ops.aten._assert_scalar.default"]

dim = 16
num_examples = 10
Expand All @@ -43,96 +48,87 @@ class TestConformer(unittest.TestCase):
)
conformer = conformer.eval()

def test_conformer_tosa_MI(self):
(
ArmTester(
self.conformer,
example_inputs=self.model_example_inputs,
compile_spec=common.get_tosa_compile_spec(tosa_spec="TOSA-0.80+MI"),
)
.export()
.to_edge_transform_and_lower()
.dump_operator_distribution()
.check_count(self.ops_after_partitioner)
.to_executorch()
# TODO(MLETORCH-632): Fix numerical errors
.run_method_and_compare_outputs(
rtol=1.0,
atol=5.0,
inputs=get_test_inputs(self.dim, self.lengths, self.num_examples),
)
)

def test_conformer_tosa_BI(self):
(
ArmTester(
self.conformer,
example_inputs=self.model_example_inputs,
compile_spec=common.get_tosa_compile_spec(tosa_spec="TOSA-0.80+BI"),
)
.quantize()
.export()
.to_edge_transform_and_lower()
.to_executorch()
.run_method_and_compare_outputs(
qtol=1.0,
rtol=1.0,
atol=5.0,
inputs=get_test_inputs(self.dim, self.lengths, self.num_examples),
)
)

def test_conformer_u55_BI(self):
tester = (
ArmTester(
self.conformer,
example_inputs=self.model_example_inputs,
compile_spec=common.get_u55_compile_spec(),
)
.quantize()
.export()
.to_edge_transform_and_lower()
.to_executorch()
.serialize()
)

if conftest.is_option_enabled("corstone_fvp"):
try:
tester.run_method_and_compare_outputs(
qtol=1.0,
rtol=1.0,
atol=5.0,
inputs=get_test_inputs(self.dim, self.lengths, self.num_examples),
)
self.fail(
"TODO(MLETORCH-635): Expected failure under FVP option, but test passed."
)
except Exception:
pass

def test_conformer_u85_BI(self):
tester = (
ArmTester(
self.conformer,
example_inputs=self.model_example_inputs,
compile_spec=common.get_u85_compile_spec(),
)
.quantize()
.export()
.to_edge_transform_and_lower()
.to_executorch()
.serialize()
)
if conftest.is_option_enabled("corstone_fvp"):
try:
tester.run_method_and_compare_outputs(
qtol=1.0,
rtol=1.0,
atol=5.0,
inputs=get_test_inputs(self.dim, self.lengths, self.num_examples),
)
self.fail(
"TODO(MLETORCH-635): Expected failure under FVP option, but test passed."
)
except Exception:
pass

def test_conformer_tosa_MI():
pipeline = TosaPipelineMI[input_t](
TestConformer.conformer,
TestConformer.model_example_inputs,
aten_op=TestConformer.aten_ops,
exir_op=[],
use_to_edge_transform_and_lower=True,
)
pipeline.change_args(
"run_method_and_compare_outputs",
get_test_inputs(
TestConformer.dim, TestConformer.lengths, TestConformer.num_examples
),
rtol=1.0,
atol=5.0,
)
pipeline.run()


def test_conformer_tosa_BI():
pipeline = TosaPipelineBI[input_t](
TestConformer.conformer,
TestConformer.model_example_inputs,
aten_op=TestConformer.aten_ops,
exir_op=[],
use_to_edge_transform_and_lower=True,
)
pipeline.pop_stage("check_count.exir")
pipeline.change_args(
"run_method_and_compare_outputs",
get_test_inputs(
TestConformer.dim, TestConformer.lengths, TestConformer.num_examples
),
rtol=1.0,
atol=5.0,
)
pipeline.run()


@common.XfailIfNoCorstone300
@pytest.mark.xfail(
reason="TODO(MLETORCH-635): Expected failure under FVP option, but test passed."
)
def test_conformer_u55_BI():
pipeline = EthosU55PipelineBI[input_t](
TestConformer.conformer,
TestConformer.model_example_inputs,
aten_ops=TestConformer.aten_ops,
exir_ops=[],
use_to_edge_transform_and_lower=True,
run_on_fvp=True,
)
pipeline.change_args(
"run_method_and_compare_outputs",
get_test_inputs(
TestConformer.dim, TestConformer.lengths, TestConformer.num_examples
),
rtol=1.0,
atol=5.0,
)
pipeline.run()


@common.XfailIfNoCorstone320
@pytest.mark.xfail(reason="All IO needs to have the same data type (MLETORCH-635)")
def test_conformer_u85_BI():
pipeline = EthosU85PipelineBI[input_t](
TestConformer.conformer,
TestConformer.model_example_inputs,
aten_ops=TestConformer.aten_ops,
exir_ops=[],
use_to_edge_transform_and_lower=True,
run_on_fvp=True,
)
pipeline.change_args(
"run_method_and_compare_outputs",
get_test_inputs(
TestConformer.dim, TestConformer.lengths, TestConformer.num_examples
),
rtol=1.0,
atol=5.0,
)
pipeline.run()
147 changes: 71 additions & 76 deletions backends/arm/test/models/test_dl3_arm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,92 +3,87 @@
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.

import unittest
from typing import Tuple

import pytest

from executorch.backends.arm.test import common, conftest
import torch

from executorch.backends.arm.test import common

from executorch.backends.arm.test.tester.test_pipeline import (
EthosU55PipelineBI,
EthosU85PipelineBI,
TosaPipelineBI,
TosaPipelineMI,
)

from executorch.backends.arm.test.tester.arm_tester import ArmTester
from executorch.examples.models import deeplab_v3

input_t = Tuple[torch.Tensor] # Input x


class TestDl3(unittest.TestCase):
class TestDl3:
"""Tests DeepLabv3."""

dl3 = deeplab_v3.DeepLabV3ResNet50Model()
model_example_inputs = dl3.get_example_inputs()
dl3 = dl3.get_eager_model()

@unittest.expectedFailure
def test_dl3_tosa_MI(self):
(
ArmTester(
self.dl3,
example_inputs=self.model_example_inputs,
compile_spec=common.get_tosa_compile_spec("TOSA-0.80+MI"),
)
.export()
.to_edge_transform_and_lower()
.to_executorch()
.run_method_and_compare_outputs(inputs=self.dl3.get_example_inputs())
)

@unittest.expectedFailure
def test_dl3_tosa_BI(self):
(
ArmTester(
self.dl3,
example_inputs=self.model_example_inputs,
compile_spec=common.get_tosa_compile_spec("TOSA-0.80+BI"),
)
.quantize()
.export()
.to_edge_transform_and_lower()
.to_executorch()
.run_method_and_compare_outputs(
atol=1.0, qtol=1, inputs=self.dl3.get_example_inputs()
)
)

@pytest.mark.slow
@pytest.mark.corstone_fvp
@unittest.skip
def test_dl3_u55_BI(self):
tester = (
ArmTester(
self.dl3,
example_inputs=self.model_example_inputs,
compile_spec=common.get_u55_compile_spec(),
)
.quantize()
.export()
.to_edge_transform_and_lower()
.to_executorch()
.serialize()
)
if conftest.is_option_enabled("corstone_fvp"):
tester.run_method_and_compare_outputs(
atol=1.0, qtol=1, inputs=self.dl3.get_example_inputs()
)

@pytest.mark.slow
@pytest.mark.corstone_fvp
@unittest.skip
def test_dl3_u85_BI(self):
tester = (
ArmTester(
self.dl3,
example_inputs=self.model_example_inputs,
compile_spec=common.get_u85_compile_spec(),
)
.quantize()
.export()
.to_edge_transform_and_lower()
.to_executorch()
.serialize()
)
if conftest.is_option_enabled("corstone_fvp"):
tester.run_method_and_compare_outputs(
atol=1.0, qtol=1, inputs=self.dl3.get_example_inputs()
)

def test_dl3_tosa_MI():
pipeline = TosaPipelineMI[input_t](
TestDl3.dl3,
TestDl3.model_example_inputs,
aten_op=[],
exir_op=[],
)
pipeline.change_args(
"run_method_and_compare_outputs", rtol=1.0, atol=1.0
) # TODO: MLETORCH-1036 decrease tolerance
pipeline.run()


def test_dl3_tosa_BI():
pipeline = TosaPipelineBI[input_t](
TestDl3.dl3,
TestDl3.model_example_inputs,
aten_op=[],
exir_op=[],
)
pipeline.change_args(
"run_method_and_compare_outputs", rtol=1.0, atol=1.0
) # TODO: MLETORCH-1036 decrease tolerance
pipeline.run()


@common.XfailIfNoCorstone300
@pytest.mark.skip(reason="upsample_bilinear2d operator is not supported on U55")
def test_dl3_u55_BI():
pipeline = EthosU55PipelineBI[input_t](
TestDl3.dl3,
TestDl3.model_example_inputs,
aten_ops=[],
exir_ops=[],
run_on_fvp=True,
)
pipeline.change_args(
"run_method_and_compare_outputs", rtol=1.0, atol=1.0
) # TODO: MLETORCH-1036 decrease tolerance
pipeline.run()


@common.XfailIfNoCorstone320
@pytest.mark.skip(reason="Runs out of memory on U85")
def test_dl3_u85_BI():
pipeline = EthosU85PipelineBI[input_t](
TestDl3.dl3,
TestDl3.model_example_inputs,
aten_ops=[],
exir_ops=[],
run_on_fvp=True,
)
pipeline.change_args(
"run_method_and_compare_outputs", rtol=1.0, atol=1.0
) # TODO: MLETORCH-1036 decrease tolerance
pipeline.run()
Loading
Loading