diff --git a/src/mlia/backend/vela/performance.py b/src/mlia/backend/vela/performance.py index 882d917..17cf00e 100644 --- a/src/mlia/backend/vela/performance.py +++ b/src/mlia/backend/vela/performance.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022-2024, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2025, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Vela performance module.""" from __future__ import annotations @@ -57,16 +57,6 @@ class LayerPerfInfo: # pylint: disable=too-many-instance-attributes mac_count: int util_mac_percentage: float - def __repr__(self) -> str: - """Return String Representation of LayerPerfInfo object.""" - header_values = {key: value for key, value, _ in layer_metrics} - string_to_check = "" - for field in fields(self): - string_to_check += ( - f"{header_values[field.name]}: {getattr(self, field.name)}, " - ) - return string_to_check - @dataclass class LayerwisePerfInfo: @@ -74,14 +64,6 @@ class LayerwisePerfInfo: layerwise_info: list[LayerPerfInfo] - def __repr__(self) -> str: - """Return String Representation of LayerwisePerfInfo object.""" - strings_to_check_layerwise_object = "" - for layer in self.layerwise_info: - string_to_check = repr(layer) - strings_to_check_layerwise_object += string_to_check - return strings_to_check_layerwise_object - complete_layer_metrics = [ ("tflite_operator", ["TFLite_operator", "Original Operator"], "TFLite Operator"), @@ -112,6 +94,7 @@ def __repr__(self) -> str: for layer_metric in complete_layer_metrics if layer_metric[0] in OUTPUT_METRICS ] + layer_metrics.sort(key=lambda e: OUTPUT_METRICS.index(e[0])) diff --git a/tests/test_backend_vela_performance.py b/tests/test_backend_vela_performance.py index 6665e8a..88b9c4d 100644 --- a/tests/test_backend_vela_performance.py +++ b/tests/test_backend_vela_performance.py @@ -20,11 +20,14 @@ from mlia.backend.vela.performance import estimate_performance # noqa: E402 from mlia.backend.vela.performance import layer_metrics # noqa: E402 from mlia.backend.vela.performance import LayerwisePerfInfo # noqa: E402 +from mlia.backend.vela.performance import LayerPerfInfo # noqa: E402 from mlia.backend.vela.performance import parse_layerwise_perf_csv # noqa: E402 from mlia.backend.vela.performance import PerformanceMetrics # noqa: E402 from mlia.target.ethos_u.config import EthosUConfiguration # noqa: E402 from mlia.utils.filesystem import recreate_directory # noqa: E402 +from typing import get_type_hints + def test_estimate_performance(test_tflite_model: Path) -> None: """Test getting performance estimations.""" @@ -62,12 +65,6 @@ def test_estimate_performance_csv_parser_called( MAX_POOL_2D,MaxPool,10944,50.10989010989011,2992.0,7.22147132651091,1330.0,2992.0,0.0,0.0,0.0,6912,0.819252432155658,0.9024064171122994,sequential/max_pooling2d/MaxPool """.strip() -LAYERWISE_TMP_DATA_MISSING_HEADER_STR = """ -TFLite_operator,NNG Operator,Peak%,Op Cycles,Network%,NPU,SRAM AC,DRAM AC,OnFlash AC,OffFlash AC,MAC Count,Network%,Util%,Name -CONV_2D,Conv2DBias,54.65201465201465,7312.0,17.648194632168373,7312.0,2000.0,0.0,0.0,0.0,73008,8.653353814644136,3.9002666849015313,sequential/conv1/Relu;sequential/conv1/Conv2D -MAX_POOL_2D,MaxPool,50.10989010989011,2992.0,7.22147132651091,1330.0,2992.0,0.0,0.0,0.0,6912,0.819252432155658,0.9024064171122994,sequential/max_pooling2d/MaxPool -""".strip() - LAYERWISE_MULTI_HEADER_TMP_DATA_STR = """ TFLite_operator,NNG Operator,SRAM Usage,Peak%,Op Cycles,Network%,NPU,SRAM AC,DRAM AC,OnFlash AC,OffFlash AC,MAC Count,Network%,Util%,Name CONV_2D,Conv2DBias,11936,54.65201465201465,7312.0,17.648194632168373,7312.0,2000.0,0.0,0.0,0.0,73008,8.653353814644136,3.9002666849015313,sequential/conv1/Relu;sequential/conv1/Conv2D @@ -75,54 +72,104 @@ def test_estimate_performance_csv_parser_called( MAX_POOL_2D,MaxPool,10944,50.10989010989011,2992.0,7.22147132651091,1330.0,2992.0,0.0,0.0,0.0,6912,0.819252432155658,0.9024064171122994,sequential/max_pooling2d/MaxPool """.strip() +LAYERWISE_ALT_ALIAS_TMP_DATA_STR = """ +Original Operator,NNG Operator,Staging Usage,Peak%,Op Cycles,Network%,NPU,SRAM AC,DRAM AC,OnFlash AC,OffFlash AC,MAC Count,Network% (MAC),Util% (MAC),Name +CONV_2D,Conv2DBias,11936,54.65201465201465,7312.0,17.648194632168373,7312.0,2000.0,0.0,0.0,0.0,73008,8.653353814644136,3.9002666849015313,sequential/conv1/Relu;sequential/conv1/Conv2D +MAX_POOL_2D,MaxPool,10944,50.10989010989011,2992.0,7.22147132651091,1330.0,2992.0,0.0,0.0,0.0,6912,0.819252432155658,0.9024064171122994,sequential/max_pooling2d/MaxPool +""".strip() -TMP_DATA_EXPECTED_STRING = "\ -Name: sequential/conv1/Relu;sequential/conv1/Conv2D, \ -TFLite_operator: CONV_2D, \ -SRAM Usage: 11936, \ -Op Cycles: 7312, \ -NPU: 7312, \ -SRAM AC: 2000, \ -DRAM AC: 0, \ -OnFlash AC: 0, \ -OffFlash AC: 0, \ -MAC Count: 73008, \ -Util%: 3.9002666849015313, \ -\ -Name: sequential/max_pooling2d/MaxPool, \ -TFLite_operator: MAX_POOL_2D, \ -SRAM Usage: 10944, \ -Op Cycles: 2992, \ -NPU: 1330, \ -SRAM AC: 2992, \ -DRAM AC: 0, \ -OnFlash AC: 0, \ -OffFlash AC: 0, \ -MAC Count: 6912, \ -Util%: 0.9024064171122994, \ -" +LAYERWISE_MIXED_ALIAS_TMP_DATA_STR = """ +TFLite_operator,NNG Operator,Staging Usage,Peak%,Op Cycles,Network%,NPU,SRAM AC,DRAM AC,OnFlash AC,OffFlash AC,MAC Count,Network% (1),Util% (MAC),Name +CONV_2D,Conv2DBias,11936,54.65201465201465,7312.0,17.648194632168373,7312.0,2000.0,0.0,0.0,0.0,73008,8.653353814644136,3.9002666849015313,sequential/conv1/Relu;sequential/conv1/Conv2D +MAX_POOL_2D,MaxPool,10944,50.10989010989011,2992.0,7.22147132651091,1330.0,2992.0,0.0,0.0,0.0,6912,0.819252432155658,0.9024064171122994,sequential/max_pooling2d/MaxPool +""".strip() + +EXPECTED_ROWS = [ + { + "name": "sequential/conv1/Relu;sequential/conv1/Conv2D", + "tflite_operator": "CONV_2D", + "sram_usage": 11936, + "op_cycles": 7312, + "npu_cycles": 7312, + "sram_access_cycles": 2000, + "dram_access_cycles": 0, + "on_chip_flash_access_cycles": 0, + "off_chip_flash_access_cycles": 0, + "mac_count": 73008, + "util_mac_percentage": 3.9002666849015313, + }, + { + "name": "sequential/max_pooling2d/MaxPool", + "tflite_operator": "MAX_POOL_2D", + "sram_usage": 10944, + "op_cycles": 2992, + "npu_cycles": 1330, + "sram_access_cycles": 2992, + "dram_access_cycles": 0, + "on_chip_flash_access_cycles": 0, + "off_chip_flash_access_cycles": 0, + "mac_count": 6912, + "util_mac_percentage": 0.9024064171122994, + }, +] @pytest.mark.parametrize( - "input_csv_content, expected_output", + "input_csv_content", [ - (LAYERWISE_TMP_DATA_STR, TMP_DATA_EXPECTED_STRING), - ( - LAYERWISE_MULTI_HEADER_TMP_DATA_STR, - TMP_DATA_EXPECTED_STRING, - ), + LAYERWISE_TMP_DATA_STR, + LAYERWISE_MULTI_HEADER_TMP_DATA_STR, + LAYERWISE_ALT_ALIAS_TMP_DATA_STR, + LAYERWISE_MIXED_ALIAS_TMP_DATA_STR, ], + ids=["single-header", "multi-header", "alt-aliases", "mixed-aliases"], ) -def test_estimate_performance_parse_layerwise_csv_file( - test_csv_file: Path, input_csv_content: str, expected_output: str +def test_parse_layerwise_csv_populates_fields_correctly( + test_csv_file: Path, input_csv_content: str ) -> None: - """Test that parsing a csv file produces a LayerwisePerfInfo object.""" - with open(test_csv_file, "w", encoding="utf8") as csv_file: + """Ensure that parse_layerwise_perf_csv + populates LayerPerfInfo objects correctly.""" + + # Create the test file and parse it + with open(test_csv_file, "w", encoding="utf8", newline="") as csv_file: csv_file.write(input_csv_content) - layerwise_object = parse_layerwise_perf_csv(test_csv_file, layer_metrics) - strings_to_check_layerwise_object = repr(layerwise_object) - assert isinstance(layerwise_object, LayerwisePerfInfo) - assert expected_output == strings_to_check_layerwise_object + layerwise = parse_layerwise_perf_csv(test_csv_file, layer_metrics) + assert isinstance(layerwise, LayerwisePerfInfo) + + items = layerwise.layerwise_info + assert items, "No parsed layers found" + assert len(items) == len( + EXPECTED_ROWS + ), f"Row count mismatch: got {len(items)} vs expected {len(EXPECTED_ROWS)}" + + # Guard against out-of-date EXPECTED_ROWS + hints = get_type_hints(LayerPerfInfo) + dc_keys = set(hints.keys()) + for row in EXPECTED_ROWS: + exp_keys = set(row.keys()) + assert ( + exp_keys == dc_keys + ), f"EXPECTED_ROWS keys != dataclass fields:\n{exp_keys ^ dc_keys}" + + # Check we got the expected values, with the appropriate types + for got, exp in zip(items, EXPECTED_ROWS): + for field_name, expected_type in hints.items(): + got_val = getattr(got, field_name) + exp_val = exp[field_name] + assert isinstance( + got_val, expected_type + ), f"{field_name} has wrong type: {type(got_val)} != {expected_type}" + if expected_type is float: + assert got_val == pytest.approx(exp_val) + else: + assert got_val == exp_val + + +LAYERWISE_TMP_DATA_MISSING_HEADER_STR = """ +TFLite_operator,NNG Operator,Peak%,Op Cycles,Network%,NPU,SRAM AC,DRAM AC,OnFlash AC,OffFlash AC,MAC Count,Network%,Util%,Name +CONV_2D,Conv2DBias,54.65201465201465,7312.0,17.648194632168373,7312.0,2000.0,0.0,0.0,0.0,73008,8.653353814644136,3.9002666849015313,sequential/conv1/Relu;sequential/conv1/Conv2D +MAX_POOL_2D,MaxPool,50.10989010989011,2992.0,7.22147132651091,1330.0,2992.0,0.0,0.0,0.0,6912,0.819252432155658,0.9024064171122994,sequential/max_pooling2d/MaxPool +""".strip() def test_estimate_performance_parse_layerwise_csv_file_with_missing_headers(