From 5a222825967b846c36ea2c6232fc5eaa47f29002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5ns=20Nilsson?= Date: Tue, 25 Mar 2025 14:39:25 +0100 Subject: [PATCH 1/2] Arm backend: fix test_llama.py Number of delegates and tolerance has changed so update it. --- backends/arm/test/models/test_llama.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/backends/arm/test/models/test_llama.py b/backends/arm/test/models/test_llama.py index 973f62d2724..aa9937e27f1 100644 --- a/backends/arm/test/models/test_llama.py +++ b/backends/arm/test/models/test_llama.py @@ -52,7 +52,7 @@ def prepare_model(self): params_file, str ), "invalid input for --llama_inputs" else: - logging.warning( + logger.warning( "Skipping Llama test because of lack of input. To run use --llama_inputs <.pt> <.json>" ) return None, None, None @@ -61,6 +61,8 @@ def prepare_model(self): params_file ), "Invalid file paths" + logger.info("Running test_llama.py") + # TODO: Enable key value cache args = [ "--disable_dynamic_shape", @@ -112,9 +114,9 @@ def test_llama_tosa_MI(self): ) .export() .to_edge_transform_and_lower() - .check_count({"torch.ops.higher_order.executorch_call_delegate": 14}) + .check_count({"torch.ops.higher_order.executorch_call_delegate": 26}) .to_executorch() .run_method_and_compare_outputs( - inputs=llama_inputs, atol=1.8, rtol=0.01 # TODO: decrease tolerance + inputs=llama_inputs, atol=4.3, rtol=1.1 # TODO: MLETORCH-825 decrease tolerance ) ) From e5ba52581f0c192d4635519f70960a049e0485d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5ns=20Nilsson?= Date: Wed, 26 Mar 2025 09:58:15 +0100 Subject: [PATCH 2/2] Fix lintrunner problem --- backends/arm/test/models/test_llama.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backends/arm/test/models/test_llama.py b/backends/arm/test/models/test_llama.py index aa9937e27f1..2656c12417d 100644 --- a/backends/arm/test/models/test_llama.py +++ b/backends/arm/test/models/test_llama.py @@ -117,6 +117,8 @@ def test_llama_tosa_MI(self): .check_count({"torch.ops.higher_order.executorch_call_delegate": 26}) .to_executorch() .run_method_and_compare_outputs( - inputs=llama_inputs, atol=4.3, rtol=1.1 # TODO: MLETORCH-825 decrease tolerance + inputs=llama_inputs, + atol=4.3, + rtol=1.1, # TODO: MLETORCH-825 decrease tolerance ) )