From cc6e4c5279b06a107ffe22bc0eee5ee6fd072ddb Mon Sep 17 00:00:00 2001 From: Saoirse Stewart Date: Thu, 24 Oct 2024 15:59:16 +0100 Subject: [PATCH 1/2] Add LSTM unit test to arm backend --- backends/arm/test/models/test_lstm_arm.py | 99 +++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 backends/arm/test/models/test_lstm_arm.py diff --git a/backends/arm/test/models/test_lstm_arm.py b/backends/arm/test/models/test_lstm_arm.py new file mode 100644 index 00000000000..93a36b9edef --- /dev/null +++ b/backends/arm/test/models/test_lstm_arm.py @@ -0,0 +1,99 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# Copyright 2025 Arm Limited and/or its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + + +import unittest + +import torch + +from executorch.backends.arm.test import common, conftest + +from executorch.backends.arm.test.tester.arm_tester import ArmTester + +from executorch.exir import EdgeCompileConfig + +from torch.nn.quantizable.modules import rnn + + +class TestLSTM(unittest.TestCase): + """Tests LSTM.""" + + lstm = rnn.LSTM(10, 20, 2) + lstm = lstm.eval() + + input_tensor = torch.randn(5, 3, 10) + h0 = torch.randn(2, 3, 20) + c0 = torch.randn(2, 3, 20) + + model_inputs = (input_tensor, (h0, c0)) + + _edge_compile_config = EdgeCompileConfig( + _skip_dim_order=True, # TODO(T182928844): Delegate dim order op to backend. + ) + + def test_lstm_tosa_MI(self): + ( + ArmTester( + self.lstm, + example_inputs=self.model_inputs, + compile_spec=common.get_tosa_compile_spec("TOSA-0.80+MI"), + ) + .export() + .to_edge_transform_and_lower(edge_compile_config=self._edge_compile_config) + .to_executorch() + .run_method_and_compare_outputs(inputs=self.model_inputs) + ) + + def test_lstm_tosa_BI(self): + ( + ArmTester( + self.lstm, + example_inputs=self.model_inputs, + compile_spec=common.get_tosa_compile_spec("TOSA-0.80+BI"), + ) + .quantize() + .export() + .to_edge_transform_and_lower(edge_compile_config=self._edge_compile_config) + .to_executorch() + .run_method_and_compare_outputs(atol=3e-1, qtol=1, inputs=self.model_inputs) + ) + + def test_lstm_u55_BI(self): + tester = ( + ArmTester( + self.lstm, + example_inputs=self.model_inputs, + compile_spec=common.get_u55_compile_spec(), + ) + .quantize() + .export() + .to_edge_transform_and_lower(edge_compile_config=self._edge_compile_config) + .to_executorch() + .serialize() + ) + if conftest.is_option_enabled("corstone_fvp"): + tester.run_method_and_compare_outputs( + atol=3e-1, qtol=1, inputs=self.model_inputs + ) + + def test_lstm_u85_BI(self): + tester = ( + ArmTester( + self.lstm, + example_inputs=self.model_inputs, + compile_spec=common.get_u85_compile_spec(), + ) + .quantize() + .export() + .to_edge_transform_and_lower(edge_compile_config=self._edge_compile_config) + .to_executorch() + .serialize() + ) + if conftest.is_option_enabled("corstone_fvp"): + tester.run_method_and_compare_outputs( + atol=3e-1, qtol=1, inputs=self.model_inputs + ) From 6cb41edf37b03f39ab5fbf8a5a5c0a1b1deb6370 Mon Sep 17 00:00:00 2001 From: Saoirse Stewart Date: Thu, 23 Jan 2025 11:02:33 +0000 Subject: [PATCH 2/2] Addressing PR comments for LSTM - Adding check that number of delegates=1 --- backends/arm/test/models/test_lstm_arm.py | 26 ++++++++++++----------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/backends/arm/test/models/test_lstm_arm.py b/backends/arm/test/models/test_lstm_arm.py index 93a36b9edef..3e859ecbf70 100644 --- a/backends/arm/test/models/test_lstm_arm.py +++ b/backends/arm/test/models/test_lstm_arm.py @@ -11,17 +11,19 @@ import torch from executorch.backends.arm.test import common, conftest - from executorch.backends.arm.test.tester.arm_tester import ArmTester -from executorch.exir import EdgeCompileConfig - from torch.nn.quantizable.modules import rnn class TestLSTM(unittest.TestCase): - """Tests LSTM.""" + """Tests quantizable LSTM module.""" + """ + Currently only the quantizable LSTM module has been verified with the arm backend. + There may be plans to update this to use torch.nn.LSTM. + TODO: MLETORCH-622 + """ lstm = rnn.LSTM(10, 20, 2) lstm = lstm.eval() @@ -31,10 +33,6 @@ class TestLSTM(unittest.TestCase): model_inputs = (input_tensor, (h0, c0)) - _edge_compile_config = EdgeCompileConfig( - _skip_dim_order=True, # TODO(T182928844): Delegate dim order op to backend. - ) - def test_lstm_tosa_MI(self): ( ArmTester( @@ -43,7 +41,8 @@ def test_lstm_tosa_MI(self): compile_spec=common.get_tosa_compile_spec("TOSA-0.80+MI"), ) .export() - .to_edge_transform_and_lower(edge_compile_config=self._edge_compile_config) + .to_edge_transform_and_lower() + .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .to_executorch() .run_method_and_compare_outputs(inputs=self.model_inputs) ) @@ -57,7 +56,8 @@ def test_lstm_tosa_BI(self): ) .quantize() .export() - .to_edge_transform_and_lower(edge_compile_config=self._edge_compile_config) + .to_edge_transform_and_lower() + .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .to_executorch() .run_method_and_compare_outputs(atol=3e-1, qtol=1, inputs=self.model_inputs) ) @@ -71,7 +71,8 @@ def test_lstm_u55_BI(self): ) .quantize() .export() - .to_edge_transform_and_lower(edge_compile_config=self._edge_compile_config) + .to_edge_transform_and_lower() + .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .to_executorch() .serialize() ) @@ -89,7 +90,8 @@ def test_lstm_u85_BI(self): ) .quantize() .export() - .to_edge_transform_and_lower(edge_compile_config=self._edge_compile_config) + .to_edge_transform_and_lower() + .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .to_executorch() .serialize() )