From b4396116c334b69dfa591f54d82d4018c80178f3 Mon Sep 17 00:00:00 2001 From: James Edgar Date: Fri, 14 Nov 2025 16:06:30 +0000 Subject: [PATCH] fix: Performance test failure when vela backend installed Running test_backend_vela_performance.py directly via "pytest" with the vela backend installed results in two failures. If the backend is not installed there are no failures. If the tests are run via "tox" then there are no failures. The intent of the failing test is: given layer performance data in CSV format, ensure that we can parse this to create a valid LayerwisePerfInfo object. However, the parsing code appears to have updated to cope with different versions of the underlying vela backend tool; versions that produce data with different field names (column headings). The test is failing because it checks the __repr__ for the layer performance data classes, which now contain a list of field name aliases ["TFLite_operator", "Original Operator"] instead of a single name. After discussion with Wojciech our conclusion was that the test should use direct attribute checks. It should specify both the list of field names and the corresponding list of actual fields (attributes) to validate. The code under test, performance.py, should also be updated as the __repr__ functions for the layerwise performance data classes are non-standard and only used in the tests. They should be removed entirely. Note that a temporary fix for these failures was submitted as part of MLIA-1423. Resolves: MLIA-1552 Change-Id: I3d4cb9ea7bab71857690d6abf9105d5a23d3fade Signed-off-by: James Edgar Reviewed-on: https://eu-gerrit-2.euhpc.arm.com/c/ml/ecosystem/mlia/+/1149746 Tested-by: expkit Reviewed-by: Wojciech Boncela IP-review: Isabella Gottardi Reviewed-on: https://eu-gerrit-2.euhpc.arm.com/c/ml/ecosystem/mlia/+/1151451 Reviewed-by: Isabella Gottardi --- src/mlia/backend/vela/performance.py | 21 +--- tests/test_backend_vela_performance.py | 137 +++++++++++++++++-------- 2 files changed, 94 insertions(+), 64 deletions(-) diff --git a/src/mlia/backend/vela/performance.py b/src/mlia/backend/vela/performance.py index 882d917..17cf00e 100644 --- a/src/mlia/backend/vela/performance.py +++ b/src/mlia/backend/vela/performance.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022-2024, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2025, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Vela performance module.""" from __future__ import annotations @@ -57,16 +57,6 @@ class LayerPerfInfo: # pylint: disable=too-many-instance-attributes mac_count: int util_mac_percentage: float - def __repr__(self) -> str: - """Return String Representation of LayerPerfInfo object.""" - header_values = {key: value for key, value, _ in layer_metrics} - string_to_check = "" - for field in fields(self): - string_to_check += ( - f"{header_values[field.name]}: {getattr(self, field.name)}, " - ) - return string_to_check - @dataclass class LayerwisePerfInfo: @@ -74,14 +64,6 @@ class LayerwisePerfInfo: layerwise_info: list[LayerPerfInfo] - def __repr__(self) -> str: - """Return String Representation of LayerwisePerfInfo object.""" - strings_to_check_layerwise_object = "" - for layer in self.layerwise_info: - string_to_check = repr(layer) - strings_to_check_layerwise_object += string_to_check - return strings_to_check_layerwise_object - complete_layer_metrics = [ ("tflite_operator", ["TFLite_operator", "Original Operator"], "TFLite Operator"), @@ -112,6 +94,7 @@ def __repr__(self) -> str: for layer_metric in complete_layer_metrics if layer_metric[0] in OUTPUT_METRICS ] + layer_metrics.sort(key=lambda e: OUTPUT_METRICS.index(e[0])) diff --git a/tests/test_backend_vela_performance.py b/tests/test_backend_vela_performance.py index 6665e8a..88b9c4d 100644 --- a/tests/test_backend_vela_performance.py +++ b/tests/test_backend_vela_performance.py @@ -20,11 +20,14 @@ from mlia.backend.vela.performance import estimate_performance # noqa: E402 from mlia.backend.vela.performance import layer_metrics # noqa: E402 from mlia.backend.vela.performance import LayerwisePerfInfo # noqa: E402 +from mlia.backend.vela.performance import LayerPerfInfo # noqa: E402 from mlia.backend.vela.performance import parse_layerwise_perf_csv # noqa: E402 from mlia.backend.vela.performance import PerformanceMetrics # noqa: E402 from mlia.target.ethos_u.config import EthosUConfiguration # noqa: E402 from mlia.utils.filesystem import recreate_directory # noqa: E402 +from typing import get_type_hints + def test_estimate_performance(test_tflite_model: Path) -> None: """Test getting performance estimations.""" @@ -62,12 +65,6 @@ def test_estimate_performance_csv_parser_called( MAX_POOL_2D,MaxPool,10944,50.10989010989011,2992.0,7.22147132651091,1330.0,2992.0,0.0,0.0,0.0,6912,0.819252432155658,0.9024064171122994,sequential/max_pooling2d/MaxPool """.strip() -LAYERWISE_TMP_DATA_MISSING_HEADER_STR = """ -TFLite_operator,NNG Operator,Peak%,Op Cycles,Network%,NPU,SRAM AC,DRAM AC,OnFlash AC,OffFlash AC,MAC Count,Network%,Util%,Name -CONV_2D,Conv2DBias,54.65201465201465,7312.0,17.648194632168373,7312.0,2000.0,0.0,0.0,0.0,73008,8.653353814644136,3.9002666849015313,sequential/conv1/Relu;sequential/conv1/Conv2D -MAX_POOL_2D,MaxPool,50.10989010989011,2992.0,7.22147132651091,1330.0,2992.0,0.0,0.0,0.0,6912,0.819252432155658,0.9024064171122994,sequential/max_pooling2d/MaxPool -""".strip() - LAYERWISE_MULTI_HEADER_TMP_DATA_STR = """ TFLite_operator,NNG Operator,SRAM Usage,Peak%,Op Cycles,Network%,NPU,SRAM AC,DRAM AC,OnFlash AC,OffFlash AC,MAC Count,Network%,Util%,Name CONV_2D,Conv2DBias,11936,54.65201465201465,7312.0,17.648194632168373,7312.0,2000.0,0.0,0.0,0.0,73008,8.653353814644136,3.9002666849015313,sequential/conv1/Relu;sequential/conv1/Conv2D @@ -75,54 +72,104 @@ def test_estimate_performance_csv_parser_called( MAX_POOL_2D,MaxPool,10944,50.10989010989011,2992.0,7.22147132651091,1330.0,2992.0,0.0,0.0,0.0,6912,0.819252432155658,0.9024064171122994,sequential/max_pooling2d/MaxPool """.strip() +LAYERWISE_ALT_ALIAS_TMP_DATA_STR = """ +Original Operator,NNG Operator,Staging Usage,Peak%,Op Cycles,Network%,NPU,SRAM AC,DRAM AC,OnFlash AC,OffFlash AC,MAC Count,Network% (MAC),Util% (MAC),Name +CONV_2D,Conv2DBias,11936,54.65201465201465,7312.0,17.648194632168373,7312.0,2000.0,0.0,0.0,0.0,73008,8.653353814644136,3.9002666849015313,sequential/conv1/Relu;sequential/conv1/Conv2D +MAX_POOL_2D,MaxPool,10944,50.10989010989011,2992.0,7.22147132651091,1330.0,2992.0,0.0,0.0,0.0,6912,0.819252432155658,0.9024064171122994,sequential/max_pooling2d/MaxPool +""".strip() -TMP_DATA_EXPECTED_STRING = "\ -Name: sequential/conv1/Relu;sequential/conv1/Conv2D, \ -TFLite_operator: CONV_2D, \ -SRAM Usage: 11936, \ -Op Cycles: 7312, \ -NPU: 7312, \ -SRAM AC: 2000, \ -DRAM AC: 0, \ -OnFlash AC: 0, \ -OffFlash AC: 0, \ -MAC Count: 73008, \ -Util%: 3.9002666849015313, \ -\ -Name: sequential/max_pooling2d/MaxPool, \ -TFLite_operator: MAX_POOL_2D, \ -SRAM Usage: 10944, \ -Op Cycles: 2992, \ -NPU: 1330, \ -SRAM AC: 2992, \ -DRAM AC: 0, \ -OnFlash AC: 0, \ -OffFlash AC: 0, \ -MAC Count: 6912, \ -Util%: 0.9024064171122994, \ -" +LAYERWISE_MIXED_ALIAS_TMP_DATA_STR = """ +TFLite_operator,NNG Operator,Staging Usage,Peak%,Op Cycles,Network%,NPU,SRAM AC,DRAM AC,OnFlash AC,OffFlash AC,MAC Count,Network% (1),Util% (MAC),Name +CONV_2D,Conv2DBias,11936,54.65201465201465,7312.0,17.648194632168373,7312.0,2000.0,0.0,0.0,0.0,73008,8.653353814644136,3.9002666849015313,sequential/conv1/Relu;sequential/conv1/Conv2D +MAX_POOL_2D,MaxPool,10944,50.10989010989011,2992.0,7.22147132651091,1330.0,2992.0,0.0,0.0,0.0,6912,0.819252432155658,0.9024064171122994,sequential/max_pooling2d/MaxPool +""".strip() + +EXPECTED_ROWS = [ + { + "name": "sequential/conv1/Relu;sequential/conv1/Conv2D", + "tflite_operator": "CONV_2D", + "sram_usage": 11936, + "op_cycles": 7312, + "npu_cycles": 7312, + "sram_access_cycles": 2000, + "dram_access_cycles": 0, + "on_chip_flash_access_cycles": 0, + "off_chip_flash_access_cycles": 0, + "mac_count": 73008, + "util_mac_percentage": 3.9002666849015313, + }, + { + "name": "sequential/max_pooling2d/MaxPool", + "tflite_operator": "MAX_POOL_2D", + "sram_usage": 10944, + "op_cycles": 2992, + "npu_cycles": 1330, + "sram_access_cycles": 2992, + "dram_access_cycles": 0, + "on_chip_flash_access_cycles": 0, + "off_chip_flash_access_cycles": 0, + "mac_count": 6912, + "util_mac_percentage": 0.9024064171122994, + }, +] @pytest.mark.parametrize( - "input_csv_content, expected_output", + "input_csv_content", [ - (LAYERWISE_TMP_DATA_STR, TMP_DATA_EXPECTED_STRING), - ( - LAYERWISE_MULTI_HEADER_TMP_DATA_STR, - TMP_DATA_EXPECTED_STRING, - ), + LAYERWISE_TMP_DATA_STR, + LAYERWISE_MULTI_HEADER_TMP_DATA_STR, + LAYERWISE_ALT_ALIAS_TMP_DATA_STR, + LAYERWISE_MIXED_ALIAS_TMP_DATA_STR, ], + ids=["single-header", "multi-header", "alt-aliases", "mixed-aliases"], ) -def test_estimate_performance_parse_layerwise_csv_file( - test_csv_file: Path, input_csv_content: str, expected_output: str +def test_parse_layerwise_csv_populates_fields_correctly( + test_csv_file: Path, input_csv_content: str ) -> None: - """Test that parsing a csv file produces a LayerwisePerfInfo object.""" - with open(test_csv_file, "w", encoding="utf8") as csv_file: + """Ensure that parse_layerwise_perf_csv + populates LayerPerfInfo objects correctly.""" + + # Create the test file and parse it + with open(test_csv_file, "w", encoding="utf8", newline="") as csv_file: csv_file.write(input_csv_content) - layerwise_object = parse_layerwise_perf_csv(test_csv_file, layer_metrics) - strings_to_check_layerwise_object = repr(layerwise_object) - assert isinstance(layerwise_object, LayerwisePerfInfo) - assert expected_output == strings_to_check_layerwise_object + layerwise = parse_layerwise_perf_csv(test_csv_file, layer_metrics) + assert isinstance(layerwise, LayerwisePerfInfo) + + items = layerwise.layerwise_info + assert items, "No parsed layers found" + assert len(items) == len( + EXPECTED_ROWS + ), f"Row count mismatch: got {len(items)} vs expected {len(EXPECTED_ROWS)}" + + # Guard against out-of-date EXPECTED_ROWS + hints = get_type_hints(LayerPerfInfo) + dc_keys = set(hints.keys()) + for row in EXPECTED_ROWS: + exp_keys = set(row.keys()) + assert ( + exp_keys == dc_keys + ), f"EXPECTED_ROWS keys != dataclass fields:\n{exp_keys ^ dc_keys}" + + # Check we got the expected values, with the appropriate types + for got, exp in zip(items, EXPECTED_ROWS): + for field_name, expected_type in hints.items(): + got_val = getattr(got, field_name) + exp_val = exp[field_name] + assert isinstance( + got_val, expected_type + ), f"{field_name} has wrong type: {type(got_val)} != {expected_type}" + if expected_type is float: + assert got_val == pytest.approx(exp_val) + else: + assert got_val == exp_val + + +LAYERWISE_TMP_DATA_MISSING_HEADER_STR = """ +TFLite_operator,NNG Operator,Peak%,Op Cycles,Network%,NPU,SRAM AC,DRAM AC,OnFlash AC,OffFlash AC,MAC Count,Network%,Util%,Name +CONV_2D,Conv2DBias,54.65201465201465,7312.0,17.648194632168373,7312.0,2000.0,0.0,0.0,0.0,73008,8.653353814644136,3.9002666849015313,sequential/conv1/Relu;sequential/conv1/Conv2D +MAX_POOL_2D,MaxPool,50.10989010989011,2992.0,7.22147132651091,1330.0,2992.0,0.0,0.0,0.0,6912,0.819252432155658,0.9024064171122994,sequential/max_pooling2d/MaxPool +""".strip() def test_estimate_performance_parse_layerwise_csv_file_with_missing_headers(