Skip to content

fix: Performance test failure when vela backend installed #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jedgar-arm merged 1 commit into from
Nov 19, 2025
Merged

fix: Performance test failure when vela backend installed #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 2 additions & 19 deletions src/mlia/backend/vela/performance.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: Copyright 2022-2024, Arm Limited and/or its affiliates.
# SPDX-FileCopyrightText: Copyright 2022-2025, Arm Limited and/or its affiliates.
# SPDX-License-Identifier: Apache-2.0
"""Vela performance module."""
from __future__ import annotations
Expand Down Expand Up @@ -57,31 +57,13 @@ class LayerPerfInfo: # pylint: disable=too-many-instance-attributes
mac_count: int
util_mac_percentage: float

def __repr__(self) -> str:
"""Return String Representation of LayerPerfInfo object."""
header_values = {key: value for key, value, _ in layer_metrics}
string_to_check = ""
for field in fields(self):
string_to_check += (
f"{header_values[field.name]}: {getattr(self, field.name)}, "
)
return string_to_check


@dataclass
class LayerwisePerfInfo:
"""Contains all the per-layer metrics from the per-layer csv file from Vela."""

layerwise_info: list[LayerPerfInfo]

def __repr__(self) -> str:
"""Return String Representation of LayerwisePerfInfo object."""
strings_to_check_layerwise_object = ""
for layer in self.layerwise_info:
string_to_check = repr(layer)
strings_to_check_layerwise_object += string_to_check
return strings_to_check_layerwise_object


complete_layer_metrics = [
("tflite_operator", ["TFLite_operator", "Original Operator"], "TFLite Operator"),
Expand Down Expand Up @@ -112,6 +94,7 @@ def __repr__(self) -> str:
for layer_metric in complete_layer_metrics
if layer_metric[0] in OUTPUT_METRICS
]

layer_metrics.sort(key=lambda e: OUTPUT_METRICS.index(e[0]))


Expand Down
137 changes: 92 additions & 45 deletions tests/test_backend_vela_performance.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,14 @@
from mlia.backend.vela.performance import estimate_performance # noqa: E402
from mlia.backend.vela.performance import layer_metrics # noqa: E402
from mlia.backend.vela.performance import LayerwisePerfInfo # noqa: E402
from mlia.backend.vela.performance import LayerPerfInfo # noqa: E402
from mlia.backend.vela.performance import parse_layerwise_perf_csv # noqa: E402
from mlia.backend.vela.performance import PerformanceMetrics # noqa: E402
from mlia.target.ethos_u.config import EthosUConfiguration # noqa: E402
from mlia.utils.filesystem import recreate_directory # noqa: E402

from typing import get_type_hints


def test_estimate_performance(test_tflite_model: Path) -> None:
"""Test getting performance estimations."""
Expand Down Expand Up @@ -62,67 +65,111 @@ def test_estimate_performance_csv_parser_called(
MAX_POOL_2D,MaxPool,10944,50.10989010989011,2992.0,7.22147132651091,1330.0,2992.0,0.0,0.0,0.0,6912,0.819252432155658,0.9024064171122994,sequential/max_pooling2d/MaxPool
""".strip()

LAYERWISE_TMP_DATA_MISSING_HEADER_STR = """
TFLite_operator,NNG Operator,Peak%,Op Cycles,Network%,NPU,SRAM AC,DRAM AC,OnFlash AC,OffFlash AC,MAC Count,Network%,Util%,Name
CONV_2D,Conv2DBias,54.65201465201465,7312.0,17.648194632168373,7312.0,2000.0,0.0,0.0,0.0,73008,8.653353814644136,3.9002666849015313,sequential/conv1/Relu;sequential/conv1/Conv2D
MAX_POOL_2D,MaxPool,50.10989010989011,2992.0,7.22147132651091,1330.0,2992.0,0.0,0.0,0.0,6912,0.819252432155658,0.9024064171122994,sequential/max_pooling2d/MaxPool
""".strip()

LAYERWISE_MULTI_HEADER_TMP_DATA_STR = """
TFLite_operator,NNG Operator,SRAM Usage,Peak%,Op Cycles,Network%,NPU,SRAM AC,DRAM AC,OnFlash AC,OffFlash AC,MAC Count,Network%,Util%,Name
CONV_2D,Conv2DBias,11936,54.65201465201465,7312.0,17.648194632168373,7312.0,2000.0,0.0,0.0,0.0,73008,8.653353814644136,3.9002666849015313,sequential/conv1/Relu;sequential/conv1/Conv2D
TFLite_operator,NNG Operator,SRAM Usage,Peak%,Op Cycles,Network%,NPU,SRAM AC,DRAM AC,OnFlash AC,OffFlash AC,MAC Count,Network%,Util%,Name
MAX_POOL_2D,MaxPool,10944,50.10989010989011,2992.0,7.22147132651091,1330.0,2992.0,0.0,0.0,0.0,6912,0.819252432155658,0.9024064171122994,sequential/max_pooling2d/MaxPool
""".strip()

LAYERWISE_ALT_ALIAS_TMP_DATA_STR = """
Original Operator,NNG Operator,Staging Usage,Peak%,Op Cycles,Network%,NPU,SRAM AC,DRAM AC,OnFlash AC,OffFlash AC,MAC Count,Network% (MAC),Util% (MAC),Name
CONV_2D,Conv2DBias,11936,54.65201465201465,7312.0,17.648194632168373,7312.0,2000.0,0.0,0.0,0.0,73008,8.653353814644136,3.9002666849015313,sequential/conv1/Relu;sequential/conv1/Conv2D
MAX_POOL_2D,MaxPool,10944,50.10989010989011,2992.0,7.22147132651091,1330.0,2992.0,0.0,0.0,0.0,6912,0.819252432155658,0.9024064171122994,sequential/max_pooling2d/MaxPool
""".strip()

TMP_DATA_EXPECTED_STRING = "\
Name: sequential/conv1/Relu;sequential/conv1/Conv2D, \
TFLite_operator: CONV_2D, \
SRAM Usage: 11936, \
Op Cycles: 7312, \
NPU: 7312, \
SRAM AC: 2000, \
DRAM AC: 0, \
OnFlash AC: 0, \
OffFlash AC: 0, \
MAC Count: 73008, \
Util%: 3.9002666849015313, \
\
Name: sequential/max_pooling2d/MaxPool, \
TFLite_operator: MAX_POOL_2D, \
SRAM Usage: 10944, \
Op Cycles: 2992, \
NPU: 1330, \
SRAM AC: 2992, \
DRAM AC: 0, \
OnFlash AC: 0, \
OffFlash AC: 0, \
MAC Count: 6912, \
Util%: 0.9024064171122994, \
"
LAYERWISE_MIXED_ALIAS_TMP_DATA_STR = """
TFLite_operator,NNG Operator,Staging Usage,Peak%,Op Cycles,Network%,NPU,SRAM AC,DRAM AC,OnFlash AC,OffFlash AC,MAC Count,Network% (1),Util% (MAC),Name
CONV_2D,Conv2DBias,11936,54.65201465201465,7312.0,17.648194632168373,7312.0,2000.0,0.0,0.0,0.0,73008,8.653353814644136,3.9002666849015313,sequential/conv1/Relu;sequential/conv1/Conv2D
MAX_POOL_2D,MaxPool,10944,50.10989010989011,2992.0,7.22147132651091,1330.0,2992.0,0.0,0.0,0.0,6912,0.819252432155658,0.9024064171122994,sequential/max_pooling2d/MaxPool
""".strip()

EXPECTED_ROWS = [
{
"name": "sequential/conv1/Relu;sequential/conv1/Conv2D",
"tflite_operator": "CONV_2D",
"sram_usage": 11936,
"op_cycles": 7312,
"npu_cycles": 7312,
"sram_access_cycles": 2000,
"dram_access_cycles": 0,
"on_chip_flash_access_cycles": 0,
"off_chip_flash_access_cycles": 0,
"mac_count": 73008,
"util_mac_percentage": 3.9002666849015313,
},
{
"name": "sequential/max_pooling2d/MaxPool",
"tflite_operator": "MAX_POOL_2D",
"sram_usage": 10944,
"op_cycles": 2992,
"npu_cycles": 1330,
"sram_access_cycles": 2992,
"dram_access_cycles": 0,
"on_chip_flash_access_cycles": 0,
"off_chip_flash_access_cycles": 0,
"mac_count": 6912,
"util_mac_percentage": 0.9024064171122994,
},
]


@pytest.mark.parametrize(
"input_csv_content, expected_output",
"input_csv_content",
[
(LAYERWISE_TMP_DATA_STR, TMP_DATA_EXPECTED_STRING),
(
LAYERWISE_MULTI_HEADER_TMP_DATA_STR,
TMP_DATA_EXPECTED_STRING,
),
LAYERWISE_TMP_DATA_STR,
LAYERWISE_MULTI_HEADER_TMP_DATA_STR,
LAYERWISE_ALT_ALIAS_TMP_DATA_STR,
LAYERWISE_MIXED_ALIAS_TMP_DATA_STR,
],
ids=["single-header", "multi-header", "alt-aliases", "mixed-aliases"],
)
def test_estimate_performance_parse_layerwise_csv_file(
test_csv_file: Path, input_csv_content: str, expected_output: str
def test_parse_layerwise_csv_populates_fields_correctly(
test_csv_file: Path, input_csv_content: str
) -> None:
"""Test that parsing a csv file produces a LayerwisePerfInfo object."""
with open(test_csv_file, "w", encoding="utf8") as csv_file:
"""Ensure that parse_layerwise_perf_csv
populates LayerPerfInfo objects correctly."""

# Create the test file and parse it
with open(test_csv_file, "w", encoding="utf8", newline="") as csv_file:
csv_file.write(input_csv_content)
layerwise_object = parse_layerwise_perf_csv(test_csv_file, layer_metrics)
strings_to_check_layerwise_object = repr(layerwise_object)
assert isinstance(layerwise_object, LayerwisePerfInfo)
assert expected_output == strings_to_check_layerwise_object
layerwise = parse_layerwise_perf_csv(test_csv_file, layer_metrics)
assert isinstance(layerwise, LayerwisePerfInfo)

items = layerwise.layerwise_info
assert items, "No parsed layers found"
assert len(items) == len(
EXPECTED_ROWS
), f"Row count mismatch: got {len(items)} vs expected {len(EXPECTED_ROWS)}"

# Guard against out-of-date EXPECTED_ROWS
hints = get_type_hints(LayerPerfInfo)
dc_keys = set(hints.keys())
for row in EXPECTED_ROWS:
exp_keys = set(row.keys())
assert (
exp_keys == dc_keys
), f"EXPECTED_ROWS keys != dataclass fields:\n{exp_keys ^ dc_keys}"

# Check we got the expected values, with the appropriate types
for got, exp in zip(items, EXPECTED_ROWS):
for field_name, expected_type in hints.items():
got_val = getattr(got, field_name)
exp_val = exp[field_name]
assert isinstance(
got_val, expected_type
), f"{field_name} has wrong type: {type(got_val)} != {expected_type}"
if expected_type is float:
assert got_val == pytest.approx(exp_val)
else:
assert got_val == exp_val


LAYERWISE_TMP_DATA_MISSING_HEADER_STR = """
TFLite_operator,NNG Operator,Peak%,Op Cycles,Network%,NPU,SRAM AC,DRAM AC,OnFlash AC,OffFlash AC,MAC Count,Network%,Util%,Name
CONV_2D,Conv2DBias,54.65201465201465,7312.0,17.648194632168373,7312.0,2000.0,0.0,0.0,0.0,73008,8.653353814644136,3.9002666849015313,sequential/conv1/Relu;sequential/conv1/Conv2D
MAX_POOL_2D,MaxPool,50.10989010989011,2992.0,7.22147132651091,1330.0,2992.0,0.0,0.0,0.0,6912,0.819252432155658,0.9024064171122994,sequential/max_pooling2d/MaxPool
""".strip()


def test_estimate_performance_parse_layerwise_csv_file_with_missing_headers(
Expand Down