From cfc995e4800580343e2a0ba8a77ba398d6cdab7a Mon Sep 17 00:00:00 2001 From: tjb Date: Wed, 1 Apr 2026 13:51:51 +0200 Subject: [PATCH 01/37] Add manual FPS benchmark Influx pipeline --- .github/workflows/fps_benchmark_influx.yaml | 91 ++++++++++++ tests/test_benchmark/conftest.py | 96 +++++++++++++ tests/test_benchmark/run_hil_tests.sh | 86 +++++++++++- .../test_benchmark_regression.py | 132 +++++++++++++++++- 4 files changed, 397 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/fps_benchmark_influx.yaml diff --git a/.github/workflows/fps_benchmark_influx.yaml b/.github/workflows/fps_benchmark_influx.yaml new file mode 100644 index 0000000..a35085f --- /dev/null +++ b/.github/workflows/fps_benchmark_influx.yaml @@ -0,0 +1,91 @@ +name: HIL FPS Benchmark Influx + +on: + workflow_dispatch: + inputs: + testbed: + description: "HIL testbed to run on" + required: true + type: string + depthai_version: + description: "DepthAI version to test against (e.g. 3.3.0)" + required: true + type: string + os_version: + description: "Optional camera OS version to install before the benchmark" + required: false + type: string + hold_reservation: + description: "Hold testbed reservation after the run" + required: false + type: boolean + +env: + TESTBED: ${{ github.event.inputs.testbed }} + DEPTHAI_VERSION: ${{ github.event.inputs.depthai_version }} + OS_VERSION: ${{ github.event.inputs.os_version }} + HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }} + HUBAI_API_KEY: ${{ secrets.HUBAI_API_KEY }} + HOLD_RESERVATION: ${{ github.event.inputs.hold_reservation }} + INFLUX_BUCKET: fps_metric + +jobs: + fps-benchmark: + runs-on: ["self-hosted", "testbed-runner"] + + steps: + - uses: actions/checkout@v4 + + - name: Run benchmark and push to Influx + run: | + set -euo pipefail + + pip install hil-framework --upgrade \ + --index-url "https://__token__:$HIL_FRAMEWORK_TOKEN@gitlab.luxonis.com/api/v4/projects/213/packages/pypi/simple" + + export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#fps-benchmark-influx" + RESERVATION_OPTION="--reservation-name $RESERVATION_NAME" + + HOLD_RESERVATION_OPTION="" + if [[ "$HOLD_RESERVATION" == "true" ]]; then + HOLD_RESERVATION_OPTION="--hold-reservation" + fi + + OS_VERSION_OPTION="" + if [[ -n "$OS_VERSION" ]]; then + OS_VERSION_OPTION="--os-version $OS_VERSION" + fi + + if [[ "$DEPTHAI_VERSION" =~ ^3\.[0-9]{1,2}\.[0-9]{1,2}$ ]]; then + DEPTHAI_VERSION_CHECKED="$DEPTHAI_VERSION" + else + echo "Invalid depthai version: $DEPTHAI_VERSION" >&2 + exit 1 + fi + + : "${INFLUX_HOST:?INFLUX_HOST is required on the runner}" + : "${INFLUX_ORG:?INFLUX_ORG is required on the runner}" + : "${INFLUX_TOKEN:?INFLUX_TOKEN is required on the runner}" + + : "${INFLUX_BUCKET:?INFLUX_BUCKET must be set by the workflow}" + + REMOTE_CMD="export HIL_RUN_ID=\"$GITHUB_RUN_ID\" \ + INFLUX_HOST=\"$INFLUX_HOST\" \ + INFLUX_ORG=\"$INFLUX_ORG\" \ + INFLUX_BUCKET=\"$INFLUX_BUCKET\" \ + INFLUX_TOKEN=\"$INFLUX_TOKEN\" && \ + cd /tmp/modelconverter && \ + ./tests/test_benchmark/run_hil_tests.sh \ + \"$HUBAI_API_KEY\" \ + \"$HIL_FRAMEWORK_TOKEN\" \ + \"$DEPTHAI_VERSION_CHECKED\" \ + \"$TESTBED\"" + + exec hil_runner \ + --testbed "$TESTBED" \ + $HOLD_RESERVATION_OPTION \ + --wait \ + $OS_VERSION_OPTION \ + $RESERVATION_OPTION \ + --sync-workspace --rsync-args="--exclude=venv" \ + --commands "$REMOTE_CMD" diff --git a/tests/test_benchmark/conftest.py b/tests/test_benchmark/conftest.py index 9d91a9c..4f4ca3f 100644 --- a/tests/test_benchmark/conftest.py +++ b/tests/test_benchmark/conftest.py @@ -1,4 +1,6 @@ import os +from datetime import datetime, timezone +from uuid import uuid4 import pytest @@ -16,6 +18,54 @@ def pytest_addoption(parser: pytest.Parser) -> None: default="rvc4", help="Target platform to benchmark (default: rvc4).", ) + parser.addoption( + "--testbed-name", + action="store", + default=None, + help="Logical HIL testbed name for Influx metadata.", + ) + parser.addoption( + "--camera-mxid", + action="store", + default=None, + help="Camera MXID for Influx metadata.", + ) + parser.addoption( + "--camera-os-version", + action="store", + default=None, + help="Camera OS version for Influx metadata.", + ) + parser.addoption( + "--camera-model", + action="store", + default=None, + help="Camera model for Influx metadata.", + ) + parser.addoption( + "--camera-agent-version", + action="store", + default=None, + help="Camera agent version for Influx metadata.", + ) + parser.addoption( + "--runner", + action="store", + default=None, + help="Runner name for Influx metadata.", + ) + parser.addoption( + "--server-os", + action="store", + default=None, + help="Server OS for Influx metadata.", + ) + parser.addoption( + "--run-id", + action="store", + default=None, + help="Optional run identifier for grouping benchmark results in Influx.", + ) def pytest_configure(config: pytest.Config) -> None: @@ -34,3 +84,49 @@ def device_ip(request: pytest.FixtureRequest) -> str | None: @pytest.fixture def benchmark_target(request: pytest.FixtureRequest) -> str: return request.config.getoption("--benchmark-target") + + +def _option_or_env( + request: pytest.FixtureRequest, + option_name: str, + env_name: str, +) -> str | None: + return request.config.getoption(option_name) or os.environ.get(env_name) + + +@pytest.fixture(scope="session") +def influx_metadata(request: pytest.FixtureRequest) -> dict[str, str | None]: + return { + "testbed_name": _option_or_env( + request, "--testbed-name", "HIL_TESTBED_NAME" + ), + "camera_mxid": _option_or_env( + request, "--camera-mxid", "HIL_CAMERA_MXID" + ), + "camera_os_version": _option_or_env( + request, "--camera-os-version", "HIL_CAMERA_OS_VERSION" + ), + "camera_model": _option_or_env( + request, "--camera-model", "HIL_CAMERA_MODEL" + ), + "camera_agent_version": _option_or_env( + request, "--camera-agent-version", "HIL_CAMERA_AGENT_VERSION" + ), + "runner": _option_or_env(request, "--runner", "HIL_RUNNER") + or os.environ.get("GITHUB_RUNNER_NAME") + or os.environ.get("HOSTNAME") + or os.environ.get("USER"), + "server_os": _option_or_env(request, "--server-os", "HIL_SERVER_OS"), + } + + +@pytest.fixture(scope="session") +def benchmark_run_id(request: pytest.FixtureRequest) -> str: + configured_run_id = ( + request.config.getoption("--run-id") or os.environ.get("HIL_RUN_ID") + ) + if configured_run_id: + return configured_run_id + + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + return f"benchmark-{timestamp}-{uuid4().hex[:8]}" diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index e0ae2f0..a5010b2 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -3,8 +3,8 @@ set -e # Exit immediately if a command fails # Check if required arguments were provided -if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then - echo "Usage: $0 " +if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ] || [ -z "$4" ]; then + echo "Usage: $0 " exit 1 fi @@ -12,6 +12,7 @@ fi export HUBAI_API_KEY="$1" export PAT_TOKEN="$2" export DEPTHAI_VERSION="$3" +export HIL_TESTBED_NAME="$4" # Navigate to project directory cd /tmp/modelconverter @@ -35,10 +36,81 @@ pip install --upgrade \ --extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-python-release-local \ "depthai==${DEPTHAI_VERSION}" -# Extract hostname of first rvc4 device -hostname=$(hil_camera -t "$HIL_TESTBED" -n test all info -j \ - | jq -r '.[] | select(.platform=="rvc4") | .hostname' \ - | head -n1) +# Cache device metadata once for the whole run using oakctl. If oakctl is not +# available, keep the benchmark runnable but disable Influx push for this run. +oakctl_output=$(oakctl list --format json 2>/dev/null || printf '') + +if [ -z "$oakctl_output" ]; then + echo "Warning: best-effort metadata lookup via oakctl failed; disabling Influx push for this run." >&2 + unset INFLUX_HOST INFLUX_ORG INFLUX_BUCKET INFLUX_TOKEN +fi + +device_hostname=$( + printf '%s' "$oakctl_output" \ + | jq -r '.items[0].ip_addresses[0] // empty' 2>/dev/null \ + | head -n1 +) +camera_mxid=$( + printf '%s' "$oakctl_output" \ + | jq -r '.items[0].serial_number // empty' 2>/dev/null \ + | head -n1 +) +camera_model=$( + printf '%s' "$oakctl_output" \ + | jq -r '.items[0].model // empty' 2>/dev/null \ + | head -n1 +) +camera_agent_version=$( + printf '%s' "$oakctl_output" \ + | jq -r '.items[0].agent_version // empty' 2>/dev/null \ + | head -n1 +) +camera_os=$( + printf '%s' "$oakctl_output" \ + | jq -r '.items[0].os // empty' 2>/dev/null \ + | head -n1 +) +runner_hostname=$(hostname 2>/dev/null || printf 'unknown') +server_os=$(uname -s 2>/dev/null | tr '[:upper:]' '[:lower:]' || printf 'unknown') + +if [ -z "$device_hostname" ]; then + device_hostname="unknown" +fi +if [ -z "$camera_mxid" ]; then + camera_mxid="unknown" +fi +if [ -z "$camera_model" ]; then + camera_model="unknown" +fi +if [ -z "$camera_agent_version" ]; then + camera_agent_version="unknown" +fi +if [ -z "$camera_os" ]; then + camera_os="unknown" +fi +if [ -z "$runner_hostname" ]; then + runner_hostname="unknown" +fi +if [ -z "$server_os" ]; then + server_os="unknown" +fi # Run tests -pytest -s -v tests/test_benchmark/ --device-ip "$hostname" \ No newline at end of file +pytest_args=( + -s + -v + tests/test_benchmark/ + --testbed-name "$HIL_TESTBED_NAME" + --camera-mxid "$camera_mxid" + --camera-os-version "$camera_os" + --camera-model "$camera_model" + --camera-agent-version "$camera_agent_version" + --runner "$runner_hostname" + --server-os "$server_os" +) + +if [ "$device_hostname" != "unknown" ]; then + pytest_args+=(--device-ip "$device_hostname") +fi + +pytest "${pytest_args[@]}" diff --git a/tests/test_benchmark/test_benchmark_regression.py b/tests/test_benchmark/test_benchmark_regression.py index bcd91b6..de83120 100644 --- a/tests/test_benchmark/test_benchmark_regression.py +++ b/tests/test_benchmark/test_benchmark_regression.py @@ -1,5 +1,7 @@ import json +import os from pathlib import Path +from urllib import error, parse, request import pytest @@ -21,6 +23,115 @@ def _model_id(slug: str) -> str: return slug.rsplit("/", 1)[-1] +def _escape_tag(value: str) -> str: + return ( + value.replace("\\", "\\\\") + .replace(",", "\\,") + .replace(" ", "\\ ") + .replace("=", "\\=") + ) + + +def _format_field(value: str | float | bool) -> str: + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, int) and not isinstance(value, bool): + return f"{value}i" + if isinstance(value, float): + return repr(value) + + escaped = str(value).replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' + + +def _write_fps_result_to_influx( + *, + model_slug: str, + benchmark_target: str, + benchmark_run_id: str, + device_ip: str | None, + actual_fps: float, + expected_fps: float, + tolerance_low: float, + tolerance_high: float, + fps_min: float, + fps_max: float, + deviation_pct: float, + success: bool, + influx_metadata: dict[str, str | None], +) -> None: + influx_url = os.environ.get("INFLUX_HOST") + influx_org = os.environ.get("INFLUX_ORG") + influx_bucket = os.environ.get("INFLUX_BUCKET") + influx_token = os.environ.get("INFLUX_TOKEN") + + if not all([influx_url, influx_org, influx_bucket, influx_token]): + return + + tags = { + "model_slug": model_slug, + "benchmark_target": benchmark_target, + "run_id": benchmark_run_id, + "status": "passed" if success else "failed", + } + optional_tags = { + "testbed_name": influx_metadata.get("testbed_name"), + "camera_mxid": influx_metadata.get("camera_mxid"), + "camera_os_version": influx_metadata.get("camera_os_version"), + "camera_model": influx_metadata.get("camera_model"), + "camera_agent_version": influx_metadata.get("camera_agent_version"), + "runner": influx_metadata.get("runner"), + "server_os": influx_metadata.get("server_os"), + "device_ip": device_ip, + } + tags.update( + { + key: value + for key, value in optional_tags.items() + if value not in (None, "") + } + ) + + fields = { + "actual_fps": actual_fps, + "expected_fps": expected_fps, + "tolerance_low": tolerance_low, + "tolerance_high": tolerance_high, + "fps_min": fps_min, + "fps_max": fps_max, + "deviation_pct": deviation_pct, + "success": success, + } + + tag_set = ",".join(f"{key}={_escape_tag(value)}" for key, value in tags.items()) + field_set = ",".join( + f"{key}={_format_field(value)}" for key, value in fields.items() + ) + line = f"fps_benchmark,{tag_set} {field_set}" + + write_url = ( + f"{influx_url.rstrip('/')}/api/v2/write?" + f"org={parse.quote(influx_org, safe='')}&" + f"bucket={parse.quote(influx_bucket, safe='')}&precision=ns" + ) + influx_request = request.Request( + write_url, + data=line.encode(), + headers={ + "Authorization": f"Token {influx_token}", + "Content-Type": "text/plain; charset=utf-8", + "Accept": "application/json", + }, + method="POST", + ) + + try: + with request.urlopen(influx_request, timeout=5): + return + except (error.URLError, TimeoutError, OSError) as exc: + print(f"Failed to write benchmark result to InfluxDB: {exc}") + + @pytest.mark.parametrize( "model_slug", _model_slugs("rvc4"), @@ -30,6 +141,8 @@ def test_benchmark_fps( model_slug: str, device_ip: str | None, benchmark_target: str, + benchmark_run_id: str, + influx_metadata: dict[str, str | None], ) -> None: model_config = _targets_data[benchmark_target][model_slug] expected_fps = model_config["expected_fps"] @@ -60,13 +173,30 @@ def test_benchmark_fps( fps_max = expected_fps * (1 + tolerance_high) deviation_pct = ((actual_fps - expected_fps) / expected_fps) * 100 + success = fps_min <= actual_fps <= fps_max print( f"Benchmark result for {model_slug}: " f"actual={actual_fps:.2f} FPS, expected={expected_fps:.2f} FPS. " ) - assert fps_min <= actual_fps <= fps_max, ( + _write_fps_result_to_influx( + model_slug=model_slug, + benchmark_target=benchmark_target, + benchmark_run_id=benchmark_run_id, + device_ip=device_ip, + actual_fps=actual_fps, + expected_fps=expected_fps, + tolerance_low=tolerance_low, + tolerance_high=tolerance_high, + fps_min=fps_min, + fps_max=fps_max, + deviation_pct=deviation_pct, + success=success, + influx_metadata=influx_metadata, + ) + + assert success, ( f"FPS regression for {model_slug}: " f"actual={actual_fps:.2f}, expected={expected_fps:.2f} " f"(deviation: {deviation_pct:+.1f}%, " From ce82fcd4568f4ad8aa5734df47984f1cd453c425 Mon Sep 17 00:00:00 2001 From: tjb Date: Thu, 2 Apr 2026 08:32:15 +0200 Subject: [PATCH 02/37] Use model-based HIL selection for FPS Influx workflow --- .github/workflows/fps_benchmark_influx.yaml | 10 ++-------- tests/test_benchmark/run_hil_tests.sh | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.github/workflows/fps_benchmark_influx.yaml b/.github/workflows/fps_benchmark_influx.yaml index a35085f..2d322a0 100644 --- a/.github/workflows/fps_benchmark_influx.yaml +++ b/.github/workflows/fps_benchmark_influx.yaml @@ -3,10 +3,6 @@ name: HIL FPS Benchmark Influx on: workflow_dispatch: inputs: - testbed: - description: "HIL testbed to run on" - required: true - type: string depthai_version: description: "DepthAI version to test against (e.g. 3.3.0)" required: true @@ -21,7 +17,6 @@ on: type: boolean env: - TESTBED: ${{ github.event.inputs.testbed }} DEPTHAI_VERSION: ${{ github.event.inputs.depthai_version }} OS_VERSION: ${{ github.event.inputs.os_version }} HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }} @@ -78,11 +73,10 @@ jobs: ./tests/test_benchmark/run_hil_tests.sh \ \"$HUBAI_API_KEY\" \ \"$HIL_FRAMEWORK_TOKEN\" \ - \"$DEPTHAI_VERSION_CHECKED\" \ - \"$TESTBED\"" + \"$DEPTHAI_VERSION_CHECKED\"" exec hil_runner \ - --testbed "$TESTBED" \ + --models "oak4_pro or oak4_d or oak4_s" \ $HOLD_RESERVATION_OPTION \ --wait \ $OS_VERSION_OPTION \ diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index a5010b2..9690ee2 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -3,8 +3,8 @@ set -e # Exit immediately if a command fails # Check if required arguments were provided -if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ] || [ -z "$4" ]; then - echo "Usage: $0 " +if [ -z "${1:-}" ] || [ -z "${2:-}" ] || [ -z "${3:-}" ]; then + echo "Usage: $0 [TESTBED_NAME]" exit 1 fi @@ -12,7 +12,7 @@ fi export HUBAI_API_KEY="$1" export PAT_TOKEN="$2" export DEPTHAI_VERSION="$3" -export HIL_TESTBED_NAME="$4" +export HIL_TESTBED_NAME="${4:-}" # Navigate to project directory cd /tmp/modelconverter @@ -70,6 +70,11 @@ camera_os=$( | jq -r '.items[0].os // empty' 2>/dev/null \ | head -n1 ) +detected_testbed_name=$( + printf '%s' "$oakctl_output" \ + | jq -r '.items[0].name // .items[0].hostname // .items[0].id // empty' 2>/dev/null \ + | head -n1 +) runner_hostname=$(hostname 2>/dev/null || printf 'unknown') server_os=$(uname -s 2>/dev/null | tr '[:upper:]' '[:lower:]' || printf 'unknown') @@ -94,6 +99,12 @@ fi if [ -z "$server_os" ]; then server_os="unknown" fi +if [ -z "$HIL_TESTBED_NAME" ]; then + HIL_TESTBED_NAME="${detected_testbed_name:-}" +fi +if [ -z "$HIL_TESTBED_NAME" ]; then + HIL_TESTBED_NAME="unknown" +fi # Run tests pytest_args=( From f14ff4241b8705053071e375c96b506fdb2ae203 Mon Sep 17 00:00:00 2001 From: tjb Date: Thu, 2 Apr 2026 08:57:04 +0200 Subject: [PATCH 03/37] Temporarily repurpose rvc4 workflow for FPS Influx dispatch --- .github/workflows/rvc4_test.yaml | 115 +++++++++++++++++++++---------- 1 file changed, 80 insertions(+), 35 deletions(-) diff --git a/.github/workflows/rvc4_test.yaml b/.github/workflows/rvc4_test.yaml index 0067be2..2d322a0 100644 --- a/.github/workflows/rvc4_test.yaml +++ b/.github/workflows/rvc4_test.yaml @@ -1,40 +1,85 @@ -name: Test - RVC4 +name: HIL FPS Benchmark Influx on: workflow_dispatch: - pull_request: - branches: [main] - paths: - - requirements.txt - - "modelconverter/__init__.py" - - "modelconverter/packages/rvc4/**" - - "modelconverter/packages/base_exporter.py" - - "modelconverter/packages/base_inferer.py" - - "tests/test_packages/test_rvc4.py" - - "docker/rvc4/Dockerfile" - - "docker/rvc4/entrypoint.sh" - - ".github/workflows/modelconverter_test.yaml" - - ".github/workflows/rvc4_test.yaml" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + inputs: + depthai_version: + description: "DepthAI version to test against (e.g. 3.3.0)" + required: true + type: string + os_version: + description: "Optional camera OS version to install before the benchmark" + required: false + type: string + hold_reservation: + description: "Hold testbed reservation after the run" + required: false + type: boolean + +env: + DEPTHAI_VERSION: ${{ github.event.inputs.depthai_version }} + OS_VERSION: ${{ github.event.inputs.os_version }} + HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }} + HUBAI_API_KEY: ${{ secrets.HUBAI_API_KEY }} + HOLD_RESERVATION: ${{ github.event.inputs.hold_reservation }} + INFLUX_BUCKET: fps_metric jobs: - rvc4-test: - strategy: - fail-fast: false - matrix: - version: - - "2.23.0" - - "2.24.0" - - "2.25.0" - - "2.26.2" - - "2.27.0" - - "2.32.6" - - uses: ./.github/workflows/modelconverter_test.yaml - secrets: inherit - with: - package: rvc4 - version: ${{ matrix.version }} + fps-benchmark: + runs-on: ["self-hosted", "testbed-runner"] + + steps: + - uses: actions/checkout@v4 + + - name: Run benchmark and push to Influx + run: | + set -euo pipefail + + pip install hil-framework --upgrade \ + --index-url "https://__token__:$HIL_FRAMEWORK_TOKEN@gitlab.luxonis.com/api/v4/projects/213/packages/pypi/simple" + + export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#fps-benchmark-influx" + RESERVATION_OPTION="--reservation-name $RESERVATION_NAME" + + HOLD_RESERVATION_OPTION="" + if [[ "$HOLD_RESERVATION" == "true" ]]; then + HOLD_RESERVATION_OPTION="--hold-reservation" + fi + + OS_VERSION_OPTION="" + if [[ -n "$OS_VERSION" ]]; then + OS_VERSION_OPTION="--os-version $OS_VERSION" + fi + + if [[ "$DEPTHAI_VERSION" =~ ^3\.[0-9]{1,2}\.[0-9]{1,2}$ ]]; then + DEPTHAI_VERSION_CHECKED="$DEPTHAI_VERSION" + else + echo "Invalid depthai version: $DEPTHAI_VERSION" >&2 + exit 1 + fi + + : "${INFLUX_HOST:?INFLUX_HOST is required on the runner}" + : "${INFLUX_ORG:?INFLUX_ORG is required on the runner}" + : "${INFLUX_TOKEN:?INFLUX_TOKEN is required on the runner}" + + : "${INFLUX_BUCKET:?INFLUX_BUCKET must be set by the workflow}" + + REMOTE_CMD="export HIL_RUN_ID=\"$GITHUB_RUN_ID\" \ + INFLUX_HOST=\"$INFLUX_HOST\" \ + INFLUX_ORG=\"$INFLUX_ORG\" \ + INFLUX_BUCKET=\"$INFLUX_BUCKET\" \ + INFLUX_TOKEN=\"$INFLUX_TOKEN\" && \ + cd /tmp/modelconverter && \ + ./tests/test_benchmark/run_hil_tests.sh \ + \"$HUBAI_API_KEY\" \ + \"$HIL_FRAMEWORK_TOKEN\" \ + \"$DEPTHAI_VERSION_CHECKED\"" + + exec hil_runner \ + --models "oak4_pro or oak4_d or oak4_s" \ + $HOLD_RESERVATION_OPTION \ + --wait \ + $OS_VERSION_OPTION \ + $RESERVATION_OPTION \ + --sync-workspace --rsync-args="--exclude=venv" \ + --commands "$REMOTE_CMD" From d0bf4d70d05ae64274619c1022a8e7c9d5c19312 Mon Sep 17 00:00:00 2001 From: tjb Date: Thu, 2 Apr 2026 09:32:22 +0200 Subject: [PATCH 04/37] Map Influx GitHub variables into FPS workflow env --- .github/workflows/rvc4_test.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/rvc4_test.yaml b/.github/workflows/rvc4_test.yaml index 2d322a0..16542a9 100644 --- a/.github/workflows/rvc4_test.yaml +++ b/.github/workflows/rvc4_test.yaml @@ -22,6 +22,9 @@ env: HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }} HUBAI_API_KEY: ${{ secrets.HUBAI_API_KEY }} HOLD_RESERVATION: ${{ github.event.inputs.hold_reservation }} + INFLUX_HOST: ${{ vars.INFLUX_HOST }} + INFLUX_ORG: ${{ vars.INFLUX_ORG }} + INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }} INFLUX_BUCKET: fps_metric jobs: From f65c94d44de69fa2317048e7aa07ed925ccc15fc Mon Sep 17 00:00:00 2001 From: tjb Date: Thu, 2 Apr 2026 09:42:28 +0200 Subject: [PATCH 05/37] Read Influx workflow settings from GitHub secrets --- .github/workflows/rvc4_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rvc4_test.yaml b/.github/workflows/rvc4_test.yaml index 16542a9..c360cff 100644 --- a/.github/workflows/rvc4_test.yaml +++ b/.github/workflows/rvc4_test.yaml @@ -22,8 +22,8 @@ env: HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }} HUBAI_API_KEY: ${{ secrets.HUBAI_API_KEY }} HOLD_RESERVATION: ${{ github.event.inputs.hold_reservation }} - INFLUX_HOST: ${{ vars.INFLUX_HOST }} - INFLUX_ORG: ${{ vars.INFLUX_ORG }} + INFLUX_HOST: ${{ secrets.INFLUX_HOST }} + INFLUX_ORG: ${{ secrets.INFLUX_ORG }} INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }} INFLUX_BUCKET: fps_metric From 5fad3b69876d9c1e40f7aa581a5210bb6ed7a5ec Mon Sep 17 00:00:00 2001 From: tjb Date: Thu, 2 Apr 2026 10:04:44 +0200 Subject: [PATCH 06/37] Add fake Influx CSV fixture for FPS metrics --- fake_fps_metric.csv | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 fake_fps_metric.csv diff --git a/fake_fps_metric.csv b/fake_fps_metric.csv new file mode 100644 index 0000000..be44f24 --- /dev/null +++ b/fake_fps_metric.csv @@ -0,0 +1,10 @@ +#datatype measurement,tag,tag,tag,tag,tag,field,double,double,double,double,double,tag,dateTime:RFC3339 +#group false,false,false,false,false,false,false,false,false,false,false,false,false +#default fps_benchmark,,,,,,,,,,,, +_measurement,model_slug,benchmark_target,run_id,status,testbed_name,success,actual_fps,expected_fps,tolerance_low,tolerance_high,deviation_pct,device_ip,_time +fps_benchmark,luxonis/yolov6-nano:r2-coco-512x288,rvc4,manual-test-001,passed,testbed-a,true,830.23,750.00,0.10,0.30,10.70,10.12.142.57,2026-04-02T07:53:02Z +fps_benchmark,luxonis/yolov8-instance-segmentation-nano:coco-512x288,rvc4,manual-test-001,passed,testbed-a,true,568.62,554.68,0.10,0.30,2.51,10.12.142.57,2026-04-02T07:53:30Z +fps_benchmark,luxonis/yolov8-nano-pose-estimation:coco-512x288,rvc4,manual-test-001,passed,testbed-a,true,600.26,609.11,0.10,0.30,-1.45,10.12.142.57,2026-04-02T07:53:57Z +fps_benchmark,luxonis/fastsam-s:512x288,rvc4,manual-test-001,passed,testbed-a,true,497.35,475.68,0.10,0.30,4.55,10.12.142.57,2026-04-02T07:54:26Z +fps_benchmark,luxonis/deeplab-v3-plus:512x288,rvc4,manual-test-001,passed,testbed-a,true,317.15,339.42,0.10,0.30,-6.56,10.12.142.57,2026-04-02T07:54:54Z +fps_benchmark,luxonis/deeplab-v3-plus:512x288,rvc4,manual-test-002,failed,testbed-a,false,250.00,339.42,0.10,0.30,-26.35,10.12.142.57,2026-04-02T08:10:00Z From c95f97828a99c9bdefd04b473279e3cb82119c46 Mon Sep 17 00:00:00 2001 From: tjb Date: Thu, 2 Apr 2026 10:06:28 +0200 Subject: [PATCH 07/37] Fix annotated CSV format for fake Influx fixture --- fake_fps_metric.csv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fake_fps_metric.csv b/fake_fps_metric.csv index be44f24..fd60727 100644 --- a/fake_fps_metric.csv +++ b/fake_fps_metric.csv @@ -1,6 +1,6 @@ -#datatype measurement,tag,tag,tag,tag,tag,field,double,double,double,double,double,tag,dateTime:RFC3339 -#group false,false,false,false,false,false,false,false,false,false,false,false,false -#default fps_benchmark,,,,,,,,,,,, +#datatype measurement,tag,tag,tag,tag,tag,boolean,double,double,double,double,double,tag,dateTime:RFC3339 +#group false,false,false,false,false,false,false,false,false,false,false,false,false,false +#default ,,,,,,,,,,,,, _measurement,model_slug,benchmark_target,run_id,status,testbed_name,success,actual_fps,expected_fps,tolerance_low,tolerance_high,deviation_pct,device_ip,_time fps_benchmark,luxonis/yolov6-nano:r2-coco-512x288,rvc4,manual-test-001,passed,testbed-a,true,830.23,750.00,0.10,0.30,10.70,10.12.142.57,2026-04-02T07:53:02Z fps_benchmark,luxonis/yolov8-instance-segmentation-nano:coco-512x288,rvc4,manual-test-001,passed,testbed-a,true,568.62,554.68,0.10,0.30,2.51,10.12.142.57,2026-04-02T07:53:30Z From d9308bdfbcab5a3a4bdec6a6e5354b617c9b1483 Mon Sep 17 00:00:00 2001 From: tjb Date: Thu, 2 Apr 2026 10:09:02 +0200 Subject: [PATCH 08/37] Add fake Influx line protocol fixture --- fake_fps_metric.lp | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 fake_fps_metric.lp diff --git a/fake_fps_metric.lp b/fake_fps_metric.lp new file mode 100644 index 0000000..f8d35d5 --- /dev/null +++ b/fake_fps_metric.lp @@ -0,0 +1,6 @@ +fps_benchmark,model_slug=luxonis/yolov6-nano:r2-coco-512x288,benchmark_target=rvc4,run_id=manual-test-001,status=passed,testbed_name=testbed-a,device_ip=10.12.142.57 success=true,actual_fps=830.23,expected_fps=750.00,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=10.70 1775116382000000000 +fps_benchmark,model_slug=luxonis/yolov8-instance-segmentation-nano:coco-512x288,benchmark_target=rvc4,run_id=manual-test-001,status=passed,testbed_name=testbed-a,device_ip=10.12.142.57 success=true,actual_fps=568.62,expected_fps=554.68,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=2.51 1775116410000000000 +fps_benchmark,model_slug=luxonis/yolov8-nano-pose-estimation:coco-512x288,benchmark_target=rvc4,run_id=manual-test-001,status=passed,testbed_name=testbed-a,device_ip=10.12.142.57 success=true,actual_fps=600.26,expected_fps=609.11,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=-1.45 1775116437000000000 +fps_benchmark,model_slug=luxonis/fastsam-s:512x288,benchmark_target=rvc4,run_id=manual-test-001,status=passed,testbed_name=testbed-a,device_ip=10.12.142.57 success=true,actual_fps=497.35,expected_fps=475.68,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=4.55 1775116466000000000 +fps_benchmark,model_slug=luxonis/deeplab-v3-plus:512x288,benchmark_target=rvc4,run_id=manual-test-001,status=passed,testbed_name=testbed-a,device_ip=10.12.142.57 success=true,actual_fps=317.15,expected_fps=339.42,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=-6.56 1775116494000000000 +fps_benchmark,model_slug=luxonis/deeplab-v3-plus:512x288,benchmark_target=rvc4,run_id=manual-test-002,status=failed,testbed_name=testbed-a,device_ip=10.12.142.57 success=false,actual_fps=250.00,expected_fps=339.42,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=-26.35 1775117400000000000 From 5e9630903fb2dbf2086511e159592be6ba47d553 Mon Sep 17 00:00:00 2001 From: tjb Date: Thu, 2 Apr 2026 10:11:37 +0200 Subject: [PATCH 09/37] Use correct Influx bucket name in FPS workflows --- .github/workflows/fps_benchmark_influx.yaml | 2 +- .github/workflows/rvc4_test.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fps_benchmark_influx.yaml b/.github/workflows/fps_benchmark_influx.yaml index 2d322a0..46b9bc8 100644 --- a/.github/workflows/fps_benchmark_influx.yaml +++ b/.github/workflows/fps_benchmark_influx.yaml @@ -22,7 +22,7 @@ env: HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }} HUBAI_API_KEY: ${{ secrets.HUBAI_API_KEY }} HOLD_RESERVATION: ${{ github.event.inputs.hold_reservation }} - INFLUX_BUCKET: fps_metric + INFLUX_BUCKET: fps_metrics jobs: fps-benchmark: diff --git a/.github/workflows/rvc4_test.yaml b/.github/workflows/rvc4_test.yaml index c360cff..15ffdb5 100644 --- a/.github/workflows/rvc4_test.yaml +++ b/.github/workflows/rvc4_test.yaml @@ -25,7 +25,7 @@ env: INFLUX_HOST: ${{ secrets.INFLUX_HOST }} INFLUX_ORG: ${{ secrets.INFLUX_ORG }} INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }} - INFLUX_BUCKET: fps_metric + INFLUX_BUCKET: fps_metrics jobs: fps-benchmark: From e7e23cd1147c899578c61107a92a5e51c6e474c8 Mon Sep 17 00:00:00 2001 From: tjb Date: Thu, 2 Apr 2026 12:38:25 +0200 Subject: [PATCH 10/37] Normalize FPS Influx tags and testbed fallback --- tests/test_benchmark/run_hil_tests.sh | 3 ++ .../test_benchmark_regression.py | 35 ++++++++++--------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index 9690ee2..e413551 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -102,6 +102,9 @@ fi if [ -z "$HIL_TESTBED_NAME" ]; then HIL_TESTBED_NAME="${detected_testbed_name:-}" fi +if [ -z "$HIL_TESTBED_NAME" ]; then + HIL_TESTBED_NAME="$(hostname 2>/dev/null || printf '')" +fi if [ -z "$HIL_TESTBED_NAME" ]; then HIL_TESTBED_NAME="unknown" fi diff --git a/tests/test_benchmark/test_benchmark_regression.py b/tests/test_benchmark/test_benchmark_regression.py index de83120..5ca88ac 100644 --- a/tests/test_benchmark/test_benchmark_regression.py +++ b/tests/test_benchmark/test_benchmark_regression.py @@ -32,6 +32,12 @@ def _escape_tag(value: str) -> str: ) +def _normalize_tag(value: str | None) -> str: + if value in (None, ""): + return "unknown" + return str(value) + + def _format_field(value: str | float | bool) -> str: if isinstance(value, bool): return "true" if value else "false" @@ -73,24 +79,19 @@ def _write_fps_result_to_influx( "benchmark_target": benchmark_target, "run_id": benchmark_run_id, "status": "passed" if success else "failed", + "testbed_name": _normalize_tag(influx_metadata.get("testbed_name")), + "camera_mxid": _normalize_tag(influx_metadata.get("camera_mxid")), + "camera_os_version": _normalize_tag( + influx_metadata.get("camera_os_version") + ), + "camera_model": _normalize_tag(influx_metadata.get("camera_model")), + "camera_agent_version": _normalize_tag( + influx_metadata.get("camera_agent_version") + ), + "runner": _normalize_tag(influx_metadata.get("runner")), + "server_os": _normalize_tag(influx_metadata.get("server_os")), + "device_ip": _normalize_tag(device_ip), } - optional_tags = { - "testbed_name": influx_metadata.get("testbed_name"), - "camera_mxid": influx_metadata.get("camera_mxid"), - "camera_os_version": influx_metadata.get("camera_os_version"), - "camera_model": influx_metadata.get("camera_model"), - "camera_agent_version": influx_metadata.get("camera_agent_version"), - "runner": influx_metadata.get("runner"), - "server_os": influx_metadata.get("server_os"), - "device_ip": device_ip, - } - tags.update( - { - key: value - for key, value in optional_tags.items() - if value not in (None, "") - } - ) fields = { "actual_fps": actual_fps, From 45768b9c2ef98c7d2a444c619da0fbe9b4335797 Mon Sep 17 00:00:00 2001 From: tjb Date: Tue, 7 Apr 2026 14:17:49 +0200 Subject: [PATCH 11/37] Temporary debugging logging --- tests/test_benchmark/conftest.py | 9 ++++ tests/test_benchmark/run_hil_tests.sh | 41 ++++++++++++++----- .../test_benchmark_regression.py | 8 ++++ 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/tests/test_benchmark/conftest.py b/tests/test_benchmark/conftest.py index 4f4ca3f..8d4484b 100644 --- a/tests/test_benchmark/conftest.py +++ b/tests/test_benchmark/conftest.py @@ -60,6 +60,12 @@ def pytest_addoption(parser: pytest.Parser) -> None: default=None, help="Server OS for Influx metadata.", ) + parser.addoption( + "--depthai-version", + action="store", + default=None, + help="DepthAI version used for the benchmark run.", + ) parser.addoption( "--run-id", action="store", @@ -117,6 +123,9 @@ def influx_metadata(request: pytest.FixtureRequest) -> dict[str, str | None]: or os.environ.get("HOSTNAME") or os.environ.get("USER"), "server_os": _option_or_env(request, "--server-os", "HIL_SERVER_OS"), + "depthai_version": _option_or_env( + request, "--depthai-version", "DEPTHAI_VERSION" + ), } diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index e413551..28f97b5 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -2,6 +2,8 @@ set -e # Exit immediately if a command fails +COULD_NOT_OBTAIN="could_not_obtain" + # Check if required arguments were provided if [ -z "${1:-}" ] || [ -z "${2:-}" ] || [ -z "${3:-}" ]; then echo "Usage: $0 [TESTBED_NAME]" @@ -37,12 +39,12 @@ pip install --upgrade \ "depthai==${DEPTHAI_VERSION}" # Cache device metadata once for the whole run using oakctl. If oakctl is not -# available, keep the benchmark runnable but disable Influx push for this run. +# available, keep the benchmark runnable and record explicit placeholder values +# for the missing oakctl-derived metadata. oakctl_output=$(oakctl list --format json 2>/dev/null || printf '') if [ -z "$oakctl_output" ]; then - echo "Warning: best-effort metadata lookup via oakctl failed; disabling Influx push for this run." >&2 - unset INFLUX_HOST INFLUX_ORG INFLUX_BUCKET INFLUX_TOKEN + echo "Warning: best-effort metadata lookup via oakctl failed; using placeholder metadata for this run." >&2 fi device_hostname=$( @@ -79,19 +81,19 @@ runner_hostname=$(hostname 2>/dev/null || printf 'unknown') server_os=$(uname -s 2>/dev/null | tr '[:upper:]' '[:lower:]' || printf 'unknown') if [ -z "$device_hostname" ]; then - device_hostname="unknown" + device_hostname="$COULD_NOT_OBTAIN" fi if [ -z "$camera_mxid" ]; then - camera_mxid="unknown" + camera_mxid="$COULD_NOT_OBTAIN" fi if [ -z "$camera_model" ]; then - camera_model="unknown" + camera_model="$COULD_NOT_OBTAIN" fi if [ -z "$camera_agent_version" ]; then - camera_agent_version="unknown" + camera_agent_version="$COULD_NOT_OBTAIN" fi if [ -z "$camera_os" ]; then - camera_os="unknown" + camera_os="$COULD_NOT_OBTAIN" fi if [ -z "$runner_hostname" ]; then runner_hostname="unknown" @@ -106,7 +108,7 @@ if [ -z "$HIL_TESTBED_NAME" ]; then HIL_TESTBED_NAME="$(hostname 2>/dev/null || printf '')" fi if [ -z "$HIL_TESTBED_NAME" ]; then - HIL_TESTBED_NAME="unknown" + HIL_TESTBED_NAME="$COULD_NOT_OBTAIN" fi # Run tests @@ -121,10 +123,29 @@ pytest_args=( --camera-agent-version "$camera_agent_version" --runner "$runner_hostname" --server-os "$server_os" + --depthai-version "$DEPTHAI_VERSION" ) -if [ "$device_hostname" != "unknown" ]; then +if [ "$device_hostname" != "$COULD_NOT_OBTAIN" ]; then pytest_args+=(--device-ip "$device_hostname") fi +echo "Influx metadata debug:" +echo " INFLUX_HOST=${INFLUX_HOST:-}" +echo " INFLUX_ORG=${INFLUX_ORG:-}" +echo " INFLUX_BUCKET=${INFLUX_BUCKET:-}" +echo " INFLUX_TOKEN=$(if [ -n "${INFLUX_TOKEN:-}" ]; then printf ''; else printf ''; fi)" +echo " DEPTHAI_VERSION=${DEPTHAI_VERSION:-}" +echo " HIL_TESTBED_NAME=${HIL_TESTBED_NAME:-}" +echo " device_ip=${device_hostname:-}" +echo " camera_mxid=${camera_mxid:-}" +echo " camera_os_version=${camera_os:-}" +echo " camera_model=${camera_model:-}" +echo " camera_agent_version=${camera_agent_version:-}" +echo " runner=${runner_hostname:-}" +echo " server_os=${server_os:-}" +printf ' pytest_args:' +printf ' %q' "${pytest_args[@]}" +printf '\n' + pytest "${pytest_args[@]}" diff --git a/tests/test_benchmark/test_benchmark_regression.py b/tests/test_benchmark/test_benchmark_regression.py index 5ca88ac..8544c7b 100644 --- a/tests/test_benchmark/test_benchmark_regression.py +++ b/tests/test_benchmark/test_benchmark_regression.py @@ -90,6 +90,9 @@ def _write_fps_result_to_influx( ), "runner": _normalize_tag(influx_metadata.get("runner")), "server_os": _normalize_tag(influx_metadata.get("server_os")), + "depthai_version": _normalize_tag( + influx_metadata.get("depthai_version") + ), "device_ip": _normalize_tag(device_ip), } @@ -109,12 +112,17 @@ def _write_fps_result_to_influx( f"{key}={_format_field(value)}" for key, value in fields.items() ) line = f"fps_benchmark,{tag_set} {field_set}" + print(f"Influx write debug: {line}") write_url = ( f"{influx_url.rstrip('/')}/api/v2/write?" f"org={parse.quote(influx_org, safe='')}&" f"bucket={parse.quote(influx_bucket, safe='')}&precision=ns" ) + print( + "Influx request debug: " + f"url={write_url}, token={'' if influx_token else ''}" + ) influx_request = request.Request( write_url, data=line.encode(), From 22fd18c2818050924169724e6b1d8d794a0fed8d Mon Sep 17 00:00:00 2001 From: tjb Date: Wed, 8 Apr 2026 13:06:43 +0200 Subject: [PATCH 12/37] Cleanup Commit --- .github/workflows/rvc4_test.yaml | 6 +++--- fake_fps_metric.csv | 10 ---------- fake_fps_metric.lp | 6 ------ 3 files changed, 3 insertions(+), 19 deletions(-) delete mode 100644 fake_fps_metric.csv delete mode 100644 fake_fps_metric.lp diff --git a/.github/workflows/rvc4_test.yaml b/.github/workflows/rvc4_test.yaml index 15ffdb5..570881c 100644 --- a/.github/workflows/rvc4_test.yaml +++ b/.github/workflows/rvc4_test.yaml @@ -1,4 +1,4 @@ -name: HIL FPS Benchmark Influx +name: HIL FPS Benchmark on: workflow_dispatch: @@ -34,14 +34,14 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Run benchmark and push to Influx + - name: Run benchmark run: | set -euo pipefail pip install hil-framework --upgrade \ --index-url "https://__token__:$HIL_FRAMEWORK_TOKEN@gitlab.luxonis.com/api/v4/projects/213/packages/pypi/simple" - export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#fps-benchmark-influx" + export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#fps-benchmark" RESERVATION_OPTION="--reservation-name $RESERVATION_NAME" HOLD_RESERVATION_OPTION="" diff --git a/fake_fps_metric.csv b/fake_fps_metric.csv deleted file mode 100644 index fd60727..0000000 --- a/fake_fps_metric.csv +++ /dev/null @@ -1,10 +0,0 @@ -#datatype measurement,tag,tag,tag,tag,tag,boolean,double,double,double,double,double,tag,dateTime:RFC3339 -#group false,false,false,false,false,false,false,false,false,false,false,false,false,false -#default ,,,,,,,,,,,,, -_measurement,model_slug,benchmark_target,run_id,status,testbed_name,success,actual_fps,expected_fps,tolerance_low,tolerance_high,deviation_pct,device_ip,_time -fps_benchmark,luxonis/yolov6-nano:r2-coco-512x288,rvc4,manual-test-001,passed,testbed-a,true,830.23,750.00,0.10,0.30,10.70,10.12.142.57,2026-04-02T07:53:02Z -fps_benchmark,luxonis/yolov8-instance-segmentation-nano:coco-512x288,rvc4,manual-test-001,passed,testbed-a,true,568.62,554.68,0.10,0.30,2.51,10.12.142.57,2026-04-02T07:53:30Z -fps_benchmark,luxonis/yolov8-nano-pose-estimation:coco-512x288,rvc4,manual-test-001,passed,testbed-a,true,600.26,609.11,0.10,0.30,-1.45,10.12.142.57,2026-04-02T07:53:57Z -fps_benchmark,luxonis/fastsam-s:512x288,rvc4,manual-test-001,passed,testbed-a,true,497.35,475.68,0.10,0.30,4.55,10.12.142.57,2026-04-02T07:54:26Z -fps_benchmark,luxonis/deeplab-v3-plus:512x288,rvc4,manual-test-001,passed,testbed-a,true,317.15,339.42,0.10,0.30,-6.56,10.12.142.57,2026-04-02T07:54:54Z -fps_benchmark,luxonis/deeplab-v3-plus:512x288,rvc4,manual-test-002,failed,testbed-a,false,250.00,339.42,0.10,0.30,-26.35,10.12.142.57,2026-04-02T08:10:00Z diff --git a/fake_fps_metric.lp b/fake_fps_metric.lp deleted file mode 100644 index f8d35d5..0000000 --- a/fake_fps_metric.lp +++ /dev/null @@ -1,6 +0,0 @@ -fps_benchmark,model_slug=luxonis/yolov6-nano:r2-coco-512x288,benchmark_target=rvc4,run_id=manual-test-001,status=passed,testbed_name=testbed-a,device_ip=10.12.142.57 success=true,actual_fps=830.23,expected_fps=750.00,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=10.70 1775116382000000000 -fps_benchmark,model_slug=luxonis/yolov8-instance-segmentation-nano:coco-512x288,benchmark_target=rvc4,run_id=manual-test-001,status=passed,testbed_name=testbed-a,device_ip=10.12.142.57 success=true,actual_fps=568.62,expected_fps=554.68,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=2.51 1775116410000000000 -fps_benchmark,model_slug=luxonis/yolov8-nano-pose-estimation:coco-512x288,benchmark_target=rvc4,run_id=manual-test-001,status=passed,testbed_name=testbed-a,device_ip=10.12.142.57 success=true,actual_fps=600.26,expected_fps=609.11,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=-1.45 1775116437000000000 -fps_benchmark,model_slug=luxonis/fastsam-s:512x288,benchmark_target=rvc4,run_id=manual-test-001,status=passed,testbed_name=testbed-a,device_ip=10.12.142.57 success=true,actual_fps=497.35,expected_fps=475.68,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=4.55 1775116466000000000 -fps_benchmark,model_slug=luxonis/deeplab-v3-plus:512x288,benchmark_target=rvc4,run_id=manual-test-001,status=passed,testbed_name=testbed-a,device_ip=10.12.142.57 success=true,actual_fps=317.15,expected_fps=339.42,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=-6.56 1775116494000000000 -fps_benchmark,model_slug=luxonis/deeplab-v3-plus:512x288,benchmark_target=rvc4,run_id=manual-test-002,status=failed,testbed_name=testbed-a,device_ip=10.12.142.57 success=false,actual_fps=250.00,expected_fps=339.42,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=-26.35 1775117400000000000 From 157906249c1db708e2ff28f28d78fb1619d7f49b Mon Sep 17 00:00:00 2001 From: tjb Date: Mon, 13 Apr 2026 07:24:26 +0200 Subject: [PATCH 13/37] Use HIL camera metadata in benchmark tests --- tests/test_benchmark/conftest.py | 9 ++++ tests/test_benchmark/run_hil_tests.sh | 45 +++++++++++-------- .../test_benchmark_regression.py | 3 ++ 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/tests/test_benchmark/conftest.py b/tests/test_benchmark/conftest.py index 8d4484b..3d76b1b 100644 --- a/tests/test_benchmark/conftest.py +++ b/tests/test_benchmark/conftest.py @@ -42,6 +42,12 @@ def pytest_addoption(parser: pytest.Parser) -> None: default=None, help="Camera model for Influx metadata.", ) + parser.addoption( + "--camera-revision", + action="store", + default=None, + help="Camera revision for Influx metadata.", + ) parser.addoption( "--camera-agent-version", action="store", @@ -115,6 +121,9 @@ def influx_metadata(request: pytest.FixtureRequest) -> dict[str, str | None]: "camera_model": _option_or_env( request, "--camera-model", "HIL_CAMERA_MODEL" ), + "camera_revision": _option_or_env( + request, "--camera-revision", "HIL_CAMERA_REVISION" + ), "camera_agent_version": _option_or_env( request, "--camera-agent-version", "HIL_CAMERA_AGENT_VERSION" ), diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index 28f97b5..df4cfd7 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -38,43 +38,45 @@ pip install --upgrade \ --extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-python-release-local \ "depthai==${DEPTHAI_VERSION}" -# Cache device metadata once for the whole run using oakctl. If oakctl is not -# available, keep the benchmark runnable and record explicit placeholder values -# for the missing oakctl-derived metadata. -oakctl_output=$(oakctl list --format json 2>/dev/null || printf '') +# Cache device metadata once for the whole run using the HIL camera CLI. If the +# lookup fails, keep the benchmark runnable and record explicit placeholder +# values for the missing camera-derived metadata. +camera_output=$( + camera -t "${HIL_TESTBED_NAME}" -n test all info -j 2>/dev/null || printf '' +) -if [ -z "$oakctl_output" ]; then - echo "Warning: best-effort metadata lookup via oakctl failed; using placeholder metadata for this run." >&2 +if [ -z "$camera_output" ]; then + echo "Warning: best-effort metadata lookup via camera info failed; using placeholder metadata for this run." >&2 fi device_hostname=$( - printf '%s' "$oakctl_output" \ - | jq -r '.items[0].ip_addresses[0] // empty' 2>/dev/null \ + printf '%s' "$camera_output" \ + | jq -r '.[0].hostname // empty' 2>/dev/null \ | head -n1 ) camera_mxid=$( - printf '%s' "$oakctl_output" \ - | jq -r '.items[0].serial_number // empty' 2>/dev/null \ + printf '%s' "$camera_output" \ + | jq -r '.[0].mxid // empty' 2>/dev/null \ | head -n1 ) camera_model=$( - printf '%s' "$oakctl_output" \ - | jq -r '.items[0].model // empty' 2>/dev/null \ + printf '%s' "$camera_output" \ + | jq -r '.[0].model // empty' 2>/dev/null \ | head -n1 ) -camera_agent_version=$( - printf '%s' "$oakctl_output" \ - | jq -r '.items[0].agent_version // empty' 2>/dev/null \ +camera_revision=$( + printf '%s' "$camera_output" \ + | jq -r '.[0].revision // empty' 2>/dev/null \ | head -n1 ) camera_os=$( - printf '%s' "$oakctl_output" \ - | jq -r '.items[0].os // empty' 2>/dev/null \ + printf '%s' "$camera_output" \ + | jq -r '.[0].os_version // empty' 2>/dev/null \ | head -n1 ) detected_testbed_name=$( - printf '%s' "$oakctl_output" \ - | jq -r '.items[0].name // .items[0].hostname // .items[0].id // empty' 2>/dev/null \ + printf '%s' "$camera_output" \ + | jq -r '.[0].name // empty' 2>/dev/null \ | head -n1 ) runner_hostname=$(hostname 2>/dev/null || printf 'unknown') @@ -92,6 +94,9 @@ fi if [ -z "$camera_agent_version" ]; then camera_agent_version="$COULD_NOT_OBTAIN" fi +if [ -z "$camera_revision" ]; then + camera_revision="$COULD_NOT_OBTAIN" +fi if [ -z "$camera_os" ]; then camera_os="$COULD_NOT_OBTAIN" fi @@ -120,6 +125,7 @@ pytest_args=( --camera-mxid "$camera_mxid" --camera-os-version "$camera_os" --camera-model "$camera_model" + --camera-revision "$camera_revision" --camera-agent-version "$camera_agent_version" --runner "$runner_hostname" --server-os "$server_os" @@ -141,6 +147,7 @@ echo " device_ip=${device_hostname:-}" echo " camera_mxid=${camera_mxid:-}" echo " camera_os_version=${camera_os:-}" echo " camera_model=${camera_model:-}" +echo " camera_revision=${camera_revision:-}" echo " camera_agent_version=${camera_agent_version:-}" echo " runner=${runner_hostname:-}" echo " server_os=${server_os:-}" diff --git a/tests/test_benchmark/test_benchmark_regression.py b/tests/test_benchmark/test_benchmark_regression.py index 8544c7b..20ec8b2 100644 --- a/tests/test_benchmark/test_benchmark_regression.py +++ b/tests/test_benchmark/test_benchmark_regression.py @@ -85,6 +85,9 @@ def _write_fps_result_to_influx( influx_metadata.get("camera_os_version") ), "camera_model": _normalize_tag(influx_metadata.get("camera_model")), + "camera_revision": _normalize_tag( + influx_metadata.get("camera_revision") + ), "camera_agent_version": _normalize_tag( influx_metadata.get("camera_agent_version") ), From be2ccf9d47bdf2717e4510d1a4eda768042c08ce Mon Sep 17 00:00:00 2001 From: tjb Date: Mon, 13 Apr 2026 07:25:37 +0200 Subject: [PATCH 14/37] Drop obsolete camera agent metadata --- tests/test_benchmark/conftest.py | 9 --------- tests/test_benchmark/run_hil_tests.sh | 5 ----- tests/test_benchmark/test_benchmark_regression.py | 3 --- 3 files changed, 17 deletions(-) diff --git a/tests/test_benchmark/conftest.py b/tests/test_benchmark/conftest.py index 3d76b1b..5dd8b46 100644 --- a/tests/test_benchmark/conftest.py +++ b/tests/test_benchmark/conftest.py @@ -48,12 +48,6 @@ def pytest_addoption(parser: pytest.Parser) -> None: default=None, help="Camera revision for Influx metadata.", ) - parser.addoption( - "--camera-agent-version", - action="store", - default=None, - help="Camera agent version for Influx metadata.", - ) parser.addoption( "--runner", action="store", @@ -124,9 +118,6 @@ def influx_metadata(request: pytest.FixtureRequest) -> dict[str, str | None]: "camera_revision": _option_or_env( request, "--camera-revision", "HIL_CAMERA_REVISION" ), - "camera_agent_version": _option_or_env( - request, "--camera-agent-version", "HIL_CAMERA_AGENT_VERSION" - ), "runner": _option_or_env(request, "--runner", "HIL_RUNNER") or os.environ.get("GITHUB_RUNNER_NAME") or os.environ.get("HOSTNAME") diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index df4cfd7..1b10a63 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -91,9 +91,6 @@ fi if [ -z "$camera_model" ]; then camera_model="$COULD_NOT_OBTAIN" fi -if [ -z "$camera_agent_version" ]; then - camera_agent_version="$COULD_NOT_OBTAIN" -fi if [ -z "$camera_revision" ]; then camera_revision="$COULD_NOT_OBTAIN" fi @@ -126,7 +123,6 @@ pytest_args=( --camera-os-version "$camera_os" --camera-model "$camera_model" --camera-revision "$camera_revision" - --camera-agent-version "$camera_agent_version" --runner "$runner_hostname" --server-os "$server_os" --depthai-version "$DEPTHAI_VERSION" @@ -148,7 +144,6 @@ echo " camera_mxid=${camera_mxid:-}" echo " camera_os_version=${camera_os:-}" echo " camera_model=${camera_model:-}" echo " camera_revision=${camera_revision:-}" -echo " camera_agent_version=${camera_agent_version:-}" echo " runner=${runner_hostname:-}" echo " server_os=${server_os:-}" printf ' pytest_args:' diff --git a/tests/test_benchmark/test_benchmark_regression.py b/tests/test_benchmark/test_benchmark_regression.py index 20ec8b2..914e9f0 100644 --- a/tests/test_benchmark/test_benchmark_regression.py +++ b/tests/test_benchmark/test_benchmark_regression.py @@ -88,9 +88,6 @@ def _write_fps_result_to_influx( "camera_revision": _normalize_tag( influx_metadata.get("camera_revision") ), - "camera_agent_version": _normalize_tag( - influx_metadata.get("camera_agent_version") - ), "runner": _normalize_tag(influx_metadata.get("runner")), "server_os": _normalize_tag(influx_metadata.get("server_os")), "depthai_version": _normalize_tag( From 63fdb794f48dddf83722ed5511a8ee6a8fea6769 Mon Sep 17 00:00:00 2001 From: tjb Date: Mon, 13 Apr 2026 07:31:59 +0200 Subject: [PATCH 15/37] Use HIL_TESTBED directly in benchmark script --- tests/test_benchmark/conftest.py | 2 +- tests/test_benchmark/run_hil_tests.sh | 21 ++++++++++----------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/test_benchmark/conftest.py b/tests/test_benchmark/conftest.py index 5dd8b46..a1be4cf 100644 --- a/tests/test_benchmark/conftest.py +++ b/tests/test_benchmark/conftest.py @@ -104,7 +104,7 @@ def _option_or_env( def influx_metadata(request: pytest.FixtureRequest) -> dict[str, str | None]: return { "testbed_name": _option_or_env( - request, "--testbed-name", "HIL_TESTBED_NAME" + request, "--testbed-name", "HIL_TESTBED" ), "camera_mxid": _option_or_env( request, "--camera-mxid", "HIL_CAMERA_MXID" diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index 1b10a63..2d1c378 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -6,7 +6,7 @@ COULD_NOT_OBTAIN="could_not_obtain" # Check if required arguments were provided if [ -z "${1:-}" ] || [ -z "${2:-}" ] || [ -z "${3:-}" ]; then - echo "Usage: $0 [TESTBED_NAME]" + echo "Usage: $0 " exit 1 fi @@ -14,7 +14,6 @@ fi export HUBAI_API_KEY="$1" export PAT_TOKEN="$2" export DEPTHAI_VERSION="$3" -export HIL_TESTBED_NAME="${4:-}" # Navigate to project directory cd /tmp/modelconverter @@ -42,7 +41,7 @@ pip install --upgrade \ # lookup fails, keep the benchmark runnable and record explicit placeholder # values for the missing camera-derived metadata. camera_output=$( - camera -t "${HIL_TESTBED_NAME}" -n test all info -j 2>/dev/null || printf '' + camera -t "${HIL_TESTBED}" -n test all info -j 2>/dev/null || printf '' ) if [ -z "$camera_output" ]; then @@ -103,14 +102,14 @@ fi if [ -z "$server_os" ]; then server_os="unknown" fi -if [ -z "$HIL_TESTBED_NAME" ]; then - HIL_TESTBED_NAME="${detected_testbed_name:-}" +if [ -z "$HIL_TESTBED" ]; then + HIL_TESTBED="${detected_testbed_name:-}" fi -if [ -z "$HIL_TESTBED_NAME" ]; then - HIL_TESTBED_NAME="$(hostname 2>/dev/null || printf '')" +if [ -z "$HIL_TESTBED" ]; then + HIL_TESTBED="$(hostname 2>/dev/null || printf '')" fi -if [ -z "$HIL_TESTBED_NAME" ]; then - HIL_TESTBED_NAME="$COULD_NOT_OBTAIN" +if [ -z "$HIL_TESTBED" ]; then + HIL_TESTBED="$COULD_NOT_OBTAIN" fi # Run tests @@ -118,7 +117,7 @@ pytest_args=( -s -v tests/test_benchmark/ - --testbed-name "$HIL_TESTBED_NAME" + --testbed-name "$HIL_TESTBED" --camera-mxid "$camera_mxid" --camera-os-version "$camera_os" --camera-model "$camera_model" @@ -138,7 +137,7 @@ echo " INFLUX_ORG=${INFLUX_ORG:-}" echo " INFLUX_BUCKET=${INFLUX_BUCKET:-}" echo " INFLUX_TOKEN=$(if [ -n "${INFLUX_TOKEN:-}" ]; then printf ''; else printf ''; fi)" echo " DEPTHAI_VERSION=${DEPTHAI_VERSION:-}" -echo " HIL_TESTBED_NAME=${HIL_TESTBED_NAME:-}" +echo " HIL_TESTBED=${HIL_TESTBED:-}" echo " device_ip=${device_hostname:-}" echo " camera_mxid=${camera_mxid:-}" echo " camera_os_version=${camera_os:-}" From a962d3b3b16ec25348bc2da4fb6ea65231264df7 Mon Sep 17 00:00:00 2001 From: tjb Date: Mon, 13 Apr 2026 07:35:19 +0200 Subject: [PATCH 16/37] Fail benchmark run on missing camera metadata --- tests/test_benchmark/run_hil_tests.sh | 34 +++++++++++++-------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index 2d1c378..43e7aa1 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -2,8 +2,6 @@ set -e # Exit immediately if a command fails -COULD_NOT_OBTAIN="could_not_obtain" - # Check if required arguments were provided if [ -z "${1:-}" ] || [ -z "${2:-}" ] || [ -z "${3:-}" ]; then echo "Usage: $0 " @@ -45,7 +43,8 @@ camera_output=$( ) if [ -z "$camera_output" ]; then - echo "Warning: best-effort metadata lookup via camera info failed; using placeholder metadata for this run." >&2 + echo "Error: failed to obtain camera metadata via camera info." >&2 + exit 1 fi device_hostname=$( @@ -78,24 +77,31 @@ detected_testbed_name=$( | jq -r '.[0].name // empty' 2>/dev/null \ | head -n1 ) -runner_hostname=$(hostname 2>/dev/null || printf 'unknown') -server_os=$(uname -s 2>/dev/null | tr '[:upper:]' '[:lower:]' || printf 'unknown') +missing_metadata=() if [ -z "$device_hostname" ]; then - device_hostname="$COULD_NOT_OBTAIN" + missing_metadata+=("hostname") fi if [ -z "$camera_mxid" ]; then - camera_mxid="$COULD_NOT_OBTAIN" + missing_metadata+=("mxid") fi if [ -z "$camera_model" ]; then - camera_model="$COULD_NOT_OBTAIN" + missing_metadata+=("model") fi if [ -z "$camera_revision" ]; then - camera_revision="$COULD_NOT_OBTAIN" + missing_metadata+=("revision") fi if [ -z "$camera_os" ]; then - camera_os="$COULD_NOT_OBTAIN" + missing_metadata+=("os_version") +fi + +if [ "${#missing_metadata[@]}" -ne 0 ]; then + echo "Error: camera metadata is incomplete; missing fields: ${missing_metadata[*]}" >&2 + exit 1 fi + +runner_hostname=$(hostname 2>/dev/null || printf 'unknown') +server_os=$(uname -s 2>/dev/null | tr '[:upper:]' '[:lower:]' || printf 'unknown') if [ -z "$runner_hostname" ]; then runner_hostname="unknown" fi @@ -108,10 +114,6 @@ fi if [ -z "$HIL_TESTBED" ]; then HIL_TESTBED="$(hostname 2>/dev/null || printf '')" fi -if [ -z "$HIL_TESTBED" ]; then - HIL_TESTBED="$COULD_NOT_OBTAIN" -fi - # Run tests pytest_args=( -s @@ -127,9 +129,7 @@ pytest_args=( --depthai-version "$DEPTHAI_VERSION" ) -if [ "$device_hostname" != "$COULD_NOT_OBTAIN" ]; then - pytest_args+=(--device-ip "$device_hostname") -fi +pytest_args+=(--device-ip "$device_hostname") echo "Influx metadata debug:" echo " INFLUX_HOST=${INFLUX_HOST:-}" From 60a4f54be0039543d7748a01cae5abfb6df616cd Mon Sep 17 00:00:00 2001 From: tjb Date: Mon, 13 Apr 2026 09:23:13 +0200 Subject: [PATCH 17/37] Use env metadata in benchmark tests --- tests/test_benchmark/conftest.py | 99 ++++----------------------- tests/test_benchmark/run_hil_tests.sh | 21 +++--- 2 files changed, 25 insertions(+), 95 deletions(-) diff --git a/tests/test_benchmark/conftest.py b/tests/test_benchmark/conftest.py index a1be4cf..c7a509a 100644 --- a/tests/test_benchmark/conftest.py +++ b/tests/test_benchmark/conftest.py @@ -1,6 +1,4 @@ import os -from datetime import datetime, timezone -from uuid import uuid4 import pytest @@ -18,60 +16,6 @@ def pytest_addoption(parser: pytest.Parser) -> None: default="rvc4", help="Target platform to benchmark (default: rvc4).", ) - parser.addoption( - "--testbed-name", - action="store", - default=None, - help="Logical HIL testbed name for Influx metadata.", - ) - parser.addoption( - "--camera-mxid", - action="store", - default=None, - help="Camera MXID for Influx metadata.", - ) - parser.addoption( - "--camera-os-version", - action="store", - default=None, - help="Camera OS version for Influx metadata.", - ) - parser.addoption( - "--camera-model", - action="store", - default=None, - help="Camera model for Influx metadata.", - ) - parser.addoption( - "--camera-revision", - action="store", - default=None, - help="Camera revision for Influx metadata.", - ) - parser.addoption( - "--runner", - action="store", - default=None, - help="Runner name for Influx metadata.", - ) - parser.addoption( - "--server-os", - action="store", - default=None, - help="Server OS for Influx metadata.", - ) - parser.addoption( - "--depthai-version", - action="store", - default=None, - help="DepthAI version used for the benchmark run.", - ) - parser.addoption( - "--run-id", - action="store", - default=None, - help="Optional run identifier for grouping benchmark results in Influx.", - ) def pytest_configure(config: pytest.Config) -> None: @@ -92,50 +36,31 @@ def benchmark_target(request: pytest.FixtureRequest) -> str: return request.config.getoption("--benchmark-target") -def _option_or_env( - request: pytest.FixtureRequest, - option_name: str, - env_name: str, -) -> str | None: - return request.config.getoption(option_name) or os.environ.get(env_name) - - @pytest.fixture(scope="session") def influx_metadata(request: pytest.FixtureRequest) -> dict[str, str | None]: return { - "testbed_name": _option_or_env( - request, "--testbed-name", "HIL_TESTBED" - ), - "camera_mxid": _option_or_env( - request, "--camera-mxid", "HIL_CAMERA_MXID" - ), - "camera_os_version": _option_or_env( - request, "--camera-os-version", "HIL_CAMERA_OS_VERSION" - ), - "camera_model": _option_or_env( - request, "--camera-model", "HIL_CAMERA_MODEL" - ), - "camera_revision": _option_or_env( - request, "--camera-revision", "HIL_CAMERA_REVISION" - ), - "runner": _option_or_env(request, "--runner", "HIL_RUNNER") + "testbed_name": os.environ.get("HIL_TESTBED"), + "camera_mxid": os.environ.get("HIL_CAMERA_MXID"), + "camera_os_version": os.environ.get("HIL_CAMERA_OS_VERSION"), + "camera_model": os.environ.get("HIL_CAMERA_MODEL"), + "camera_revision": os.environ.get("HIL_CAMERA_REVISION"), + "runner": os.environ.get("HIL_RUNNER") or os.environ.get("GITHUB_RUNNER_NAME") or os.environ.get("HOSTNAME") or os.environ.get("USER"), - "server_os": _option_or_env(request, "--server-os", "HIL_SERVER_OS"), - "depthai_version": _option_or_env( - request, "--depthai-version", "DEPTHAI_VERSION" - ), + "server_os": os.environ.get("HIL_SERVER_OS"), + "depthai_version": os.environ.get("DEPTHAI_VERSION"), } @pytest.fixture(scope="session") def benchmark_run_id(request: pytest.FixtureRequest) -> str: - configured_run_id = ( - request.config.getoption("--run-id") or os.environ.get("HIL_RUN_ID") - ) + configured_run_id = os.environ.get("HIL_RUN_ID") if configured_run_id: return configured_run_id + from datetime import datetime, timezone + from uuid import uuid4 + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") return f"benchmark-{timestamp}-{uuid4().hex[:8]}" diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index 43e7aa1..ec2f065 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -114,19 +114,19 @@ fi if [ -z "$HIL_TESTBED" ]; then HIL_TESTBED="$(hostname 2>/dev/null || printf '')" fi + +export HIL_TESTBED +export HIL_CAMERA_MXID="$camera_mxid" +export HIL_CAMERA_OS_VERSION="$camera_os" +export HIL_CAMERA_MODEL="$camera_model" +export HIL_CAMERA_REVISION="$camera_revision" +export HIL_SERVER_OS="$server_os" + # Run tests pytest_args=( -s -v tests/test_benchmark/ - --testbed-name "$HIL_TESTBED" - --camera-mxid "$camera_mxid" - --camera-os-version "$camera_os" - --camera-model "$camera_model" - --camera-revision "$camera_revision" - --runner "$runner_hostname" - --server-os "$server_os" - --depthai-version "$DEPTHAI_VERSION" ) pytest_args+=(--device-ip "$device_hostname") @@ -138,6 +138,11 @@ echo " INFLUX_BUCKET=${INFLUX_BUCKET:-}" echo " INFLUX_TOKEN=$(if [ -n "${INFLUX_TOKEN:-}" ]; then printf ''; else printf ''; fi)" echo " DEPTHAI_VERSION=${DEPTHAI_VERSION:-}" echo " HIL_TESTBED=${HIL_TESTBED:-}" +echo " HIL_CAMERA_MXID=${HIL_CAMERA_MXID:-}" +echo " HIL_CAMERA_OS_VERSION=${HIL_CAMERA_OS_VERSION:-}" +echo " HIL_CAMERA_MODEL=${HIL_CAMERA_MODEL:-}" +echo " HIL_CAMERA_REVISION=${HIL_CAMERA_REVISION:-}" +echo " HIL_SERVER_OS=${HIL_SERVER_OS:-}" echo " device_ip=${device_hostname:-}" echo " camera_mxid=${camera_mxid:-}" echo " camera_os_version=${camera_os:-}" From ba2e9469644ebc584f92092d2db8d4a9a2f7e9c8 Mon Sep 17 00:00:00 2001 From: tjb Date: Mon, 13 Apr 2026 11:33:02 +0200 Subject: [PATCH 18/37] Select rvc4 camera metadata --- tests/test_benchmark/run_hil_tests.sh | 35 ++++++++++++++++++--------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index ec2f065..b9cf0bf 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -47,34 +47,45 @@ if [ -z "$camera_output" ]; then exit 1 fi -device_hostname=$( +rvc4_camera=$( printf '%s' "$camera_output" \ - | jq -r '.[0].hostname // empty' 2>/dev/null \ + | jq -r '.[] | select(.platform == "rvc4") | @json' 2>/dev/null \ + | head -n1 +) + +if [ -z "$rvc4_camera" ]; then + echo "Error: no rvc4 camera found in camera metadata." >&2 + exit 1 +fi + +device_hostname=$( + printf '%s' "$rvc4_camera" \ + | jq -r '.hostname // empty' 2>/dev/null \ | head -n1 ) camera_mxid=$( - printf '%s' "$camera_output" \ - | jq -r '.[0].mxid // empty' 2>/dev/null \ + printf '%s' "$rvc4_camera" \ + | jq -r '.mxid // empty' 2>/dev/null \ | head -n1 ) camera_model=$( - printf '%s' "$camera_output" \ - | jq -r '.[0].model // empty' 2>/dev/null \ + printf '%s' "$rvc4_camera" \ + | jq -r '.model // empty' 2>/dev/null \ | head -n1 ) camera_revision=$( - printf '%s' "$camera_output" \ - | jq -r '.[0].revision // empty' 2>/dev/null \ + printf '%s' "$rvc4_camera" \ + | jq -r '.revision // empty' 2>/dev/null \ | head -n1 ) camera_os=$( - printf '%s' "$camera_output" \ - | jq -r '.[0].os_version // empty' 2>/dev/null \ + printf '%s' "$rvc4_camera" \ + | jq -r '.os_version // empty' 2>/dev/null \ | head -n1 ) detected_testbed_name=$( - printf '%s' "$camera_output" \ - | jq -r '.[0].name // empty' 2>/dev/null \ + printf '%s' "$rvc4_camera" \ + | jq -r '.name // empty' 2>/dev/null \ | head -n1 ) From c24c3ec67684faac7e5cd7b23e4ee6d2332ce9c2 Mon Sep 17 00:00:00 2001 From: tjb Date: Mon, 13 Apr 2026 12:11:17 +0200 Subject: [PATCH 19/37] Restore RVC4 workflow from main --- .github/workflows/rvc4_test.yaml | 118 +++++++++---------------------- 1 file changed, 35 insertions(+), 83 deletions(-) diff --git a/.github/workflows/rvc4_test.yaml b/.github/workflows/rvc4_test.yaml index 570881c..0067be2 100644 --- a/.github/workflows/rvc4_test.yaml +++ b/.github/workflows/rvc4_test.yaml @@ -1,88 +1,40 @@ -name: HIL FPS Benchmark +name: Test - RVC4 on: workflow_dispatch: - inputs: - depthai_version: - description: "DepthAI version to test against (e.g. 3.3.0)" - required: true - type: string - os_version: - description: "Optional camera OS version to install before the benchmark" - required: false - type: string - hold_reservation: - description: "Hold testbed reservation after the run" - required: false - type: boolean - -env: - DEPTHAI_VERSION: ${{ github.event.inputs.depthai_version }} - OS_VERSION: ${{ github.event.inputs.os_version }} - HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }} - HUBAI_API_KEY: ${{ secrets.HUBAI_API_KEY }} - HOLD_RESERVATION: ${{ github.event.inputs.hold_reservation }} - INFLUX_HOST: ${{ secrets.INFLUX_HOST }} - INFLUX_ORG: ${{ secrets.INFLUX_ORG }} - INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }} - INFLUX_BUCKET: fps_metrics + pull_request: + branches: [main] + paths: + - requirements.txt + - "modelconverter/__init__.py" + - "modelconverter/packages/rvc4/**" + - "modelconverter/packages/base_exporter.py" + - "modelconverter/packages/base_inferer.py" + - "tests/test_packages/test_rvc4.py" + - "docker/rvc4/Dockerfile" + - "docker/rvc4/entrypoint.sh" + - ".github/workflows/modelconverter_test.yaml" + - ".github/workflows/rvc4_test.yaml" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - fps-benchmark: - runs-on: ["self-hosted", "testbed-runner"] - - steps: - - uses: actions/checkout@v4 - - - name: Run benchmark - run: | - set -euo pipefail - - pip install hil-framework --upgrade \ - --index-url "https://__token__:$HIL_FRAMEWORK_TOKEN@gitlab.luxonis.com/api/v4/projects/213/packages/pypi/simple" - - export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#fps-benchmark" - RESERVATION_OPTION="--reservation-name $RESERVATION_NAME" - - HOLD_RESERVATION_OPTION="" - if [[ "$HOLD_RESERVATION" == "true" ]]; then - HOLD_RESERVATION_OPTION="--hold-reservation" - fi - - OS_VERSION_OPTION="" - if [[ -n "$OS_VERSION" ]]; then - OS_VERSION_OPTION="--os-version $OS_VERSION" - fi - - if [[ "$DEPTHAI_VERSION" =~ ^3\.[0-9]{1,2}\.[0-9]{1,2}$ ]]; then - DEPTHAI_VERSION_CHECKED="$DEPTHAI_VERSION" - else - echo "Invalid depthai version: $DEPTHAI_VERSION" >&2 - exit 1 - fi - - : "${INFLUX_HOST:?INFLUX_HOST is required on the runner}" - : "${INFLUX_ORG:?INFLUX_ORG is required on the runner}" - : "${INFLUX_TOKEN:?INFLUX_TOKEN is required on the runner}" - - : "${INFLUX_BUCKET:?INFLUX_BUCKET must be set by the workflow}" - - REMOTE_CMD="export HIL_RUN_ID=\"$GITHUB_RUN_ID\" \ - INFLUX_HOST=\"$INFLUX_HOST\" \ - INFLUX_ORG=\"$INFLUX_ORG\" \ - INFLUX_BUCKET=\"$INFLUX_BUCKET\" \ - INFLUX_TOKEN=\"$INFLUX_TOKEN\" && \ - cd /tmp/modelconverter && \ - ./tests/test_benchmark/run_hil_tests.sh \ - \"$HUBAI_API_KEY\" \ - \"$HIL_FRAMEWORK_TOKEN\" \ - \"$DEPTHAI_VERSION_CHECKED\"" - - exec hil_runner \ - --models "oak4_pro or oak4_d or oak4_s" \ - $HOLD_RESERVATION_OPTION \ - --wait \ - $OS_VERSION_OPTION \ - $RESERVATION_OPTION \ - --sync-workspace --rsync-args="--exclude=venv" \ - --commands "$REMOTE_CMD" + rvc4-test: + strategy: + fail-fast: false + matrix: + version: + - "2.23.0" + - "2.24.0" + - "2.25.0" + - "2.26.2" + - "2.27.0" + - "2.32.6" + + uses: ./.github/workflows/modelconverter_test.yaml + secrets: inherit + with: + package: rvc4 + version: ${{ matrix.version }} From 782600c01786fab6d369a9baac55cddf177e7670 Mon Sep 17 00:00:00 2001 From: tjb Date: Tue, 14 Apr 2026 15:17:08 +0200 Subject: [PATCH 20/37] Use shared HIL Influx writer for FPS benchmark --- TASK.md | 87 ++++++++++++ tests/test_benchmark/conftest.py | 4 - .../test_benchmark_regression.py | 125 +----------------- 3 files changed, 91 insertions(+), 125 deletions(-) create mode 100644 TASK.md diff --git a/TASK.md b/TASK.md new file mode 100644 index 0000000..21c38c0 --- /dev/null +++ b/TASK.md @@ -0,0 +1,87 @@ +# FPS Influx Split Plan + +Goal: move reusable Influx-writing behavior out of `modelconverter` and into `hil_framework`, while keeping all FPS benchmark logic and repository-specific CI glue in `modelconverter`. + +## Keep in `modelconverter` + +- `tests/test_benchmark/test_benchmark_regression.py` + - `test_benchmark_fps(...)` + - `_model_slugs(...)` + - `_model_id(...)` + - benchmark math and assertion logic + - reading `benchmark_targets.json` +- `tests/test_benchmark/conftest.py` + - `pytest_addoption(...)` + - `pytest_configure(...)` + - `device_ip(...)` + - `benchmark_target(...)` + - `benchmark_run_id(...)` unless it becomes shared HIL metadata +- `tests/test_benchmark/run_hil_tests.sh` +- `.github/workflows/fps_benchmark_influx.yaml` +- `tjb_run_fps_influx.sh` + +## Move to `hil_framework` + +### 1. Generic Influx write helper + +Replace the manual HTTP / line-protocol write path in `test_benchmark_regression.py` with the existing `hil_framework` client API. + +Preferred shape: + +- build a `Point("fps_benchmark")` +- attach tags and fields +- call `InfluxClient("hil").save_points([point])` + +This keeps benchmark data composition in `modelconverter`, but moves the transport and serialization style into the shared HIL implementation. + +Candidate logic to remove from `modelconverter` once the helper is in place: + +- `_escape_tag(...)` +- `_normalize_tag(...)` +- `_format_field(...)` +- the raw HTTP request construction +- the explicit Influx URL / token / org handling +- the timeout / `URLError` handling around `urllib.request.urlopen` + +The helper should live beside the existing `InfluxClient` / `StabilityInflux` code, but not inside the stability-specific classes. + +### 2. Shared HIL metadata helper + +If some metadata is better derived from `hil_framework` objects, move that logic out of `tests/test_benchmark/conftest.py` and into a shared helper: + +- `influx_metadata(...)` + +Derivable from `Testbed` / `camera` / `server`: + +- `HIL_TESTBED` from `testbed.config.name` or `camera.config.testbed_name` +- `HIL_CAMERA_MXID` +- `HIL_CAMERA_OS_VERSION` from `camera.get_os_version()` on RVC4 cameras +- `HIL_CAMERA_MODEL` +- `HIL_CAMERA_REVISION` +- `HIL_SERVER_OS` from `camera.server.os` + +`HIL_RUNNER` should be ignored. + +`DEPTHAI_VERSION` must stay as an explicit input parameter to the benchmark flow. +It should be passed through from the workflow into the benchmark test path and preserved in the Influx payload if needed. + +## Do not move + +- FPS benchmark model selection +- tolerance math +- pass/fail logic +- workflow dispatch scripts +- benchmark target baselines +- any modelconverter-only fixture data +- benchmark-specific selection of the RVC4 camera to inspect + +## Suggested implementation order + +1. Add or extend a helper in `hil_framework` that turns benchmark data into `Point` objects and writes them with `InfluxClient("hil").save_points`. +2. Update `modelconverter` to call that helper from the FPS benchmark test. +3. Keep the benchmark math and assertions in `modelconverter`. +4. Remove the now-unused local Influx serialization helpers from `modelconverter`. + +## Exclusions + +The removed fixture files `fake_fps_metric.csv` and `fake_fps_metric.lp` should stay deleted. They are leftovers, not part of the runtime benchmark flow. diff --git a/tests/test_benchmark/conftest.py b/tests/test_benchmark/conftest.py index c7a509a..2314a5b 100644 --- a/tests/test_benchmark/conftest.py +++ b/tests/test_benchmark/conftest.py @@ -44,10 +44,6 @@ def influx_metadata(request: pytest.FixtureRequest) -> dict[str, str | None]: "camera_os_version": os.environ.get("HIL_CAMERA_OS_VERSION"), "camera_model": os.environ.get("HIL_CAMERA_MODEL"), "camera_revision": os.environ.get("HIL_CAMERA_REVISION"), - "runner": os.environ.get("HIL_RUNNER") - or os.environ.get("GITHUB_RUNNER_NAME") - or os.environ.get("HOSTNAME") - or os.environ.get("USER"), "server_os": os.environ.get("HIL_SERVER_OS"), "depthai_version": os.environ.get("DEPTHAI_VERSION"), } diff --git a/tests/test_benchmark/test_benchmark_regression.py b/tests/test_benchmark/test_benchmark_regression.py index 914e9f0..b8f0b7d 100644 --- a/tests/test_benchmark/test_benchmark_regression.py +++ b/tests/test_benchmark/test_benchmark_regression.py @@ -1,12 +1,13 @@ import json -import os from pathlib import Path -from urllib import error, parse, request import pytest from modelconverter.packages import get_benchmark from modelconverter.utils.types import Target +from hil_framework.lib_testbed.db_source.FpsBenchmarkInflux import ( + write_fps_benchmark_result, +) TARGETS_FILE = Path(__file__).parent / "benchmark_targets.json" @@ -23,124 +24,6 @@ def _model_id(slug: str) -> str: return slug.rsplit("/", 1)[-1] -def _escape_tag(value: str) -> str: - return ( - value.replace("\\", "\\\\") - .replace(",", "\\,") - .replace(" ", "\\ ") - .replace("=", "\\=") - ) - - -def _normalize_tag(value: str | None) -> str: - if value in (None, ""): - return "unknown" - return str(value) - - -def _format_field(value: str | float | bool) -> str: - if isinstance(value, bool): - return "true" if value else "false" - if isinstance(value, int) and not isinstance(value, bool): - return f"{value}i" - if isinstance(value, float): - return repr(value) - - escaped = str(value).replace("\\", "\\\\").replace('"', '\\"') - return f'"{escaped}"' - - -def _write_fps_result_to_influx( - *, - model_slug: str, - benchmark_target: str, - benchmark_run_id: str, - device_ip: str | None, - actual_fps: float, - expected_fps: float, - tolerance_low: float, - tolerance_high: float, - fps_min: float, - fps_max: float, - deviation_pct: float, - success: bool, - influx_metadata: dict[str, str | None], -) -> None: - influx_url = os.environ.get("INFLUX_HOST") - influx_org = os.environ.get("INFLUX_ORG") - influx_bucket = os.environ.get("INFLUX_BUCKET") - influx_token = os.environ.get("INFLUX_TOKEN") - - if not all([influx_url, influx_org, influx_bucket, influx_token]): - return - - tags = { - "model_slug": model_slug, - "benchmark_target": benchmark_target, - "run_id": benchmark_run_id, - "status": "passed" if success else "failed", - "testbed_name": _normalize_tag(influx_metadata.get("testbed_name")), - "camera_mxid": _normalize_tag(influx_metadata.get("camera_mxid")), - "camera_os_version": _normalize_tag( - influx_metadata.get("camera_os_version") - ), - "camera_model": _normalize_tag(influx_metadata.get("camera_model")), - "camera_revision": _normalize_tag( - influx_metadata.get("camera_revision") - ), - "runner": _normalize_tag(influx_metadata.get("runner")), - "server_os": _normalize_tag(influx_metadata.get("server_os")), - "depthai_version": _normalize_tag( - influx_metadata.get("depthai_version") - ), - "device_ip": _normalize_tag(device_ip), - } - - fields = { - "actual_fps": actual_fps, - "expected_fps": expected_fps, - "tolerance_low": tolerance_low, - "tolerance_high": tolerance_high, - "fps_min": fps_min, - "fps_max": fps_max, - "deviation_pct": deviation_pct, - "success": success, - } - - tag_set = ",".join(f"{key}={_escape_tag(value)}" for key, value in tags.items()) - field_set = ",".join( - f"{key}={_format_field(value)}" for key, value in fields.items() - ) - line = f"fps_benchmark,{tag_set} {field_set}" - print(f"Influx write debug: {line}") - - write_url = ( - f"{influx_url.rstrip('/')}/api/v2/write?" - f"org={parse.quote(influx_org, safe='')}&" - f"bucket={parse.quote(influx_bucket, safe='')}&precision=ns" - ) - print( - "Influx request debug: " - f"url={write_url}, token={'' if influx_token else ''}" - ) - influx_request = request.Request( - write_url, - data=line.encode(), - headers={ - "Authorization": f"Token {influx_token}", - "Content-Type": "text/plain; charset=utf-8", - "Accept": "application/json", - }, - method="POST", - ) - - try: - with request.urlopen(influx_request, timeout=5): - return - except (error.URLError, TimeoutError, OSError) as exc: - print(f"Failed to write benchmark result to InfluxDB: {exc}") - - @pytest.mark.parametrize( "model_slug", _model_slugs("rvc4"), @@ -189,7 +72,7 @@ def test_benchmark_fps( f"actual={actual_fps:.2f} FPS, expected={expected_fps:.2f} FPS. " ) - _write_fps_result_to_influx( + write_fps_benchmark_result( model_slug=model_slug, benchmark_target=benchmark_target, benchmark_run_id=benchmark_run_id, From ca2df04d14cef79d888e0c518557d232dbe5d2a1 Mon Sep 17 00:00:00 2001 From: tjb Date: Wed, 15 Apr 2026 11:37:30 +0200 Subject: [PATCH 21/37] Pass explicit influx benchmark inputs --- .github/workflows/fps_benchmark_influx.yaml | 15 ++-- tests/test_benchmark/conftest.py | 86 +++++++++++++++++-- tests/test_benchmark/run_hil_tests.sh | 58 +++++++------ .../test_benchmark_regression.py | 4 + 4 files changed, 118 insertions(+), 45 deletions(-) diff --git a/.github/workflows/fps_benchmark_influx.yaml b/.github/workflows/fps_benchmark_influx.yaml index 46b9bc8..e00f978 100644 --- a/.github/workflows/fps_benchmark_influx.yaml +++ b/.github/workflows/fps_benchmark_influx.yaml @@ -58,22 +58,17 @@ jobs: exit 1 fi - : "${INFLUX_HOST:?INFLUX_HOST is required on the runner}" - : "${INFLUX_ORG:?INFLUX_ORG is required on the runner}" : "${INFLUX_TOKEN:?INFLUX_TOKEN is required on the runner}" - : "${INFLUX_BUCKET:?INFLUX_BUCKET must be set by the workflow}" - REMOTE_CMD="export HIL_RUN_ID=\"$GITHUB_RUN_ID\" \ - INFLUX_HOST=\"$INFLUX_HOST\" \ - INFLUX_ORG=\"$INFLUX_ORG\" \ - INFLUX_BUCKET=\"$INFLUX_BUCKET\" \ - INFLUX_TOKEN=\"$INFLUX_TOKEN\" && \ - cd /tmp/modelconverter && \ + REMOTE_CMD="cd /tmp/modelconverter && \ ./tests/test_benchmark/run_hil_tests.sh \ \"$HUBAI_API_KEY\" \ \"$HIL_FRAMEWORK_TOKEN\" \ - \"$DEPTHAI_VERSION_CHECKED\"" + \"$DEPTHAI_VERSION_CHECKED\" \ + \"$INFLUX_BUCKET\" \ + \"$INFLUX_TOKEN\" \ + \"$GITHUB_RUN_ID\"" exec hil_runner \ --models "oak4_pro or oak4_d or oak4_s" \ diff --git a/tests/test_benchmark/conftest.py b/tests/test_benchmark/conftest.py index 2314a5b..ce947db 100644 --- a/tests/test_benchmark/conftest.py +++ b/tests/test_benchmark/conftest.py @@ -16,6 +16,66 @@ def pytest_addoption(parser: pytest.Parser) -> None: default="rvc4", help="Target platform to benchmark (default: rvc4).", ) + parser.addoption( + "--benchmark-run-id", + action="store", + default=None, + help="Benchmark run identifier to store in InfluxDB.", + ) + parser.addoption( + "--influx-bucket", + action="store", + default=None, + help="InfluxDB bucket used for FPS benchmark writes.", + ) + parser.addoption( + "--influx-token", + action="store", + default=None, + help="InfluxDB token used for FPS benchmark writes.", + ) + parser.addoption( + "--depthai-version", + action="store", + default=None, + help="DepthAI version recorded in benchmark metadata.", + ) + parser.addoption( + "--testbed-name", + action="store", + default=None, + help="Testbed name recorded in benchmark metadata.", + ) + parser.addoption( + "--camera-mxid", + action="store", + default=None, + help="Camera MXID recorded in benchmark metadata.", + ) + parser.addoption( + "--camera-os-version", + action="store", + default=None, + help="Camera OS version recorded in benchmark metadata.", + ) + parser.addoption( + "--camera-model", + action="store", + default=None, + help="Camera model recorded in benchmark metadata.", + ) + parser.addoption( + "--camera-revision", + action="store", + default=None, + help="Camera revision recorded in benchmark metadata.", + ) + parser.addoption( + "--server-os", + action="store", + default=None, + help="Server OS recorded in benchmark metadata.", + ) def pytest_configure(config: pytest.Config) -> None: @@ -39,19 +99,19 @@ def benchmark_target(request: pytest.FixtureRequest) -> str: @pytest.fixture(scope="session") def influx_metadata(request: pytest.FixtureRequest) -> dict[str, str | None]: return { - "testbed_name": os.environ.get("HIL_TESTBED"), - "camera_mxid": os.environ.get("HIL_CAMERA_MXID"), - "camera_os_version": os.environ.get("HIL_CAMERA_OS_VERSION"), - "camera_model": os.environ.get("HIL_CAMERA_MODEL"), - "camera_revision": os.environ.get("HIL_CAMERA_REVISION"), - "server_os": os.environ.get("HIL_SERVER_OS"), - "depthai_version": os.environ.get("DEPTHAI_VERSION"), + "testbed_name": request.config.getoption("--testbed-name"), + "camera_mxid": request.config.getoption("--camera-mxid"), + "camera_os_version": request.config.getoption("--camera-os-version"), + "camera_model": request.config.getoption("--camera-model"), + "camera_revision": request.config.getoption("--camera-revision"), + "server_os": request.config.getoption("--server-os"), + "depthai_version": request.config.getoption("--depthai-version"), } @pytest.fixture(scope="session") def benchmark_run_id(request: pytest.FixtureRequest) -> str: - configured_run_id = os.environ.get("HIL_RUN_ID") + configured_run_id = request.config.getoption("--benchmark-run-id") if configured_run_id: return configured_run_id @@ -60,3 +120,13 @@ def benchmark_run_id(request: pytest.FixtureRequest) -> str: timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") return f"benchmark-{timestamp}-{uuid4().hex[:8]}" + + +@pytest.fixture(scope="session") +def influx_bucket(request: pytest.FixtureRequest) -> str | None: + return request.config.getoption("--influx-bucket") + + +@pytest.fixture(scope="session") +def influx_token(request: pytest.FixtureRequest) -> str | None: + return request.config.getoption("--influx-token") diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index b9cf0bf..1f38795 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -3,8 +3,8 @@ set -e # Exit immediately if a command fails # Check if required arguments were provided -if [ -z "${1:-}" ] || [ -z "${2:-}" ] || [ -z "${3:-}" ]; then - echo "Usage: $0 " +if [ -z "${1:-}" ] || [ -z "${2:-}" ] || [ -z "${3:-}" ] || [ -z "${4:-}" ] || [ -z "${5:-}" ]; then + echo "Usage: $0 [BENCHMARK_RUN_ID]" exit 1 fi @@ -12,6 +12,9 @@ fi export HUBAI_API_KEY="$1" export PAT_TOKEN="$2" export DEPTHAI_VERSION="$3" +INFLUX_BUCKET="$4" +INFLUX_TOKEN="$5" +BENCHMARK_RUN_ID="${6:-}" # Navigate to project directory cd /tmp/modelconverter @@ -119,46 +122,47 @@ fi if [ -z "$server_os" ]; then server_os="unknown" fi -if [ -z "$HIL_TESTBED" ]; then - HIL_TESTBED="${detected_testbed_name:-}" +testbed_name="${HIL_TESTBED:-}" +if [ -z "$testbed_name" ]; then + testbed_name="${detected_testbed_name:-}" fi -if [ -z "$HIL_TESTBED" ]; then - HIL_TESTBED="$(hostname 2>/dev/null || printf '')" +if [ -z "$testbed_name" ]; then + testbed_name="$(hostname 2>/dev/null || printf '')" fi -export HIL_TESTBED -export HIL_CAMERA_MXID="$camera_mxid" -export HIL_CAMERA_OS_VERSION="$camera_os" -export HIL_CAMERA_MODEL="$camera_model" -export HIL_CAMERA_REVISION="$camera_revision" -export HIL_SERVER_OS="$server_os" - # Run tests pytest_args=( -s -v tests/test_benchmark/ + --influx-bucket "$INFLUX_BUCKET" + --influx-token "$INFLUX_TOKEN" + --depthai-version "$DEPTHAI_VERSION" + --testbed-name "$testbed_name" + --camera-mxid "$camera_mxid" + --camera-os-version "$camera_os" + --camera-model "$camera_model" + --camera-revision "$camera_revision" + --server-os "$server_os" ) pytest_args+=(--device-ip "$device_hostname") +if [ -n "$BENCHMARK_RUN_ID" ]; then + pytest_args+=(--benchmark-run-id "$BENCHMARK_RUN_ID") +fi echo "Influx metadata debug:" -echo " INFLUX_HOST=${INFLUX_HOST:-}" -echo " INFLUX_ORG=${INFLUX_ORG:-}" -echo " INFLUX_BUCKET=${INFLUX_BUCKET:-}" -echo " INFLUX_TOKEN=$(if [ -n "${INFLUX_TOKEN:-}" ]; then printf ''; else printf ''; fi)" +echo " INFLUX_BUCKET=${INFLUX_BUCKET:-}" +echo " INFLUX_TOKEN=$(if [ -n "${INFLUX_TOKEN:-}" ]; then printf ''; else printf ''; fi)" echo " DEPTHAI_VERSION=${DEPTHAI_VERSION:-}" -echo " HIL_TESTBED=${HIL_TESTBED:-}" -echo " HIL_CAMERA_MXID=${HIL_CAMERA_MXID:-}" -echo " HIL_CAMERA_OS_VERSION=${HIL_CAMERA_OS_VERSION:-}" -echo " HIL_CAMERA_MODEL=${HIL_CAMERA_MODEL:-}" -echo " HIL_CAMERA_REVISION=${HIL_CAMERA_REVISION:-}" -echo " HIL_SERVER_OS=${HIL_SERVER_OS:-}" +echo " benchmark_run_id=${BENCHMARK_RUN_ID:-}" +echo " HIL_TESTBED=${testbed_name:-}" +echo " HIL_CAMERA_MXID=${camera_mxid:-}" +echo " HIL_CAMERA_OS_VERSION=${camera_os:-}" +echo " HIL_CAMERA_MODEL=${camera_model:-}" +echo " HIL_CAMERA_REVISION=${camera_revision:-}" +echo " HIL_SERVER_OS=${server_os:-}" echo " device_ip=${device_hostname:-}" -echo " camera_mxid=${camera_mxid:-}" -echo " camera_os_version=${camera_os:-}" -echo " camera_model=${camera_model:-}" -echo " camera_revision=${camera_revision:-}" echo " runner=${runner_hostname:-}" echo " server_os=${server_os:-}" printf ' pytest_args:' diff --git a/tests/test_benchmark/test_benchmark_regression.py b/tests/test_benchmark/test_benchmark_regression.py index b8f0b7d..7b7b278 100644 --- a/tests/test_benchmark/test_benchmark_regression.py +++ b/tests/test_benchmark/test_benchmark_regression.py @@ -34,6 +34,8 @@ def test_benchmark_fps( device_ip: str | None, benchmark_target: str, benchmark_run_id: str, + influx_bucket: str | None, + influx_token: str | None, influx_metadata: dict[str, str | None], ) -> None: model_config = _targets_data[benchmark_target][model_slug] @@ -73,6 +75,8 @@ def test_benchmark_fps( ) write_fps_benchmark_result( + bucket=influx_bucket, + token=influx_token, model_slug=model_slug, benchmark_target=benchmark_target, benchmark_run_id=benchmark_run_id, From 05a2cba90174330d42aa0e9c0b13a5d4c9524653 Mon Sep 17 00:00:00 2001 From: tjb Date: Wed, 15 Apr 2026 12:25:06 +0200 Subject: [PATCH 22/37] Use fixed benchmark bucket and env token --- .github/workflows/fps_benchmark_influx.yaml | 1 - tests/test_benchmark/conftest.py | 22 ++++++--------------- tests/test_benchmark/run_hil_tests.sh | 13 +++++------- 3 files changed, 11 insertions(+), 25 deletions(-) diff --git a/.github/workflows/fps_benchmark_influx.yaml b/.github/workflows/fps_benchmark_influx.yaml index e00f978..1c05539 100644 --- a/.github/workflows/fps_benchmark_influx.yaml +++ b/.github/workflows/fps_benchmark_influx.yaml @@ -66,7 +66,6 @@ jobs: \"$HUBAI_API_KEY\" \ \"$HIL_FRAMEWORK_TOKEN\" \ \"$DEPTHAI_VERSION_CHECKED\" \ - \"$INFLUX_BUCKET\" \ \"$INFLUX_TOKEN\" \ \"$GITHUB_RUN_ID\"" diff --git a/tests/test_benchmark/conftest.py b/tests/test_benchmark/conftest.py index ce947db..fd3e2e7 100644 --- a/tests/test_benchmark/conftest.py +++ b/tests/test_benchmark/conftest.py @@ -2,6 +2,8 @@ import pytest +INFLUX_BUCKET = "fps_metrics" + def pytest_addoption(parser: pytest.Parser) -> None: parser.addoption( @@ -22,18 +24,6 @@ def pytest_addoption(parser: pytest.Parser) -> None: default=None, help="Benchmark run identifier to store in InfluxDB.", ) - parser.addoption( - "--influx-bucket", - action="store", - default=None, - help="InfluxDB bucket used for FPS benchmark writes.", - ) - parser.addoption( - "--influx-token", - action="store", - default=None, - help="InfluxDB token used for FPS benchmark writes.", - ) parser.addoption( "--depthai-version", action="store", @@ -123,10 +113,10 @@ def benchmark_run_id(request: pytest.FixtureRequest) -> str: @pytest.fixture(scope="session") -def influx_bucket(request: pytest.FixtureRequest) -> str | None: - return request.config.getoption("--influx-bucket") +def influx_bucket() -> str: + return INFLUX_BUCKET @pytest.fixture(scope="session") -def influx_token(request: pytest.FixtureRequest) -> str | None: - return request.config.getoption("--influx-token") +def influx_token() -> str | None: + return os.environ.get("INFLUX_TOKEN") diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index 1f38795..031574f 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -3,8 +3,8 @@ set -e # Exit immediately if a command fails # Check if required arguments were provided -if [ -z "${1:-}" ] || [ -z "${2:-}" ] || [ -z "${3:-}" ] || [ -z "${4:-}" ] || [ -z "${5:-}" ]; then - echo "Usage: $0 [BENCHMARK_RUN_ID]" +if [ -z "${1:-}" ] || [ -z "${2:-}" ] || [ -z "${3:-}" ] || [ -z "${4:-}" ]; then + echo "Usage: $0 [BENCHMARK_RUN_ID]" exit 1 fi @@ -12,9 +12,8 @@ fi export HUBAI_API_KEY="$1" export PAT_TOKEN="$2" export DEPTHAI_VERSION="$3" -INFLUX_BUCKET="$4" -INFLUX_TOKEN="$5" -BENCHMARK_RUN_ID="${6:-}" +export INFLUX_TOKEN="$4" +BENCHMARK_RUN_ID="${5:-}" # Navigate to project directory cd /tmp/modelconverter @@ -135,8 +134,6 @@ pytest_args=( -s -v tests/test_benchmark/ - --influx-bucket "$INFLUX_BUCKET" - --influx-token "$INFLUX_TOKEN" --depthai-version "$DEPTHAI_VERSION" --testbed-name "$testbed_name" --camera-mxid "$camera_mxid" @@ -152,7 +149,7 @@ if [ -n "$BENCHMARK_RUN_ID" ]; then fi echo "Influx metadata debug:" -echo " INFLUX_BUCKET=${INFLUX_BUCKET:-}" +echo " INFLUX_BUCKET=fps_metrics" echo " INFLUX_TOKEN=$(if [ -n "${INFLUX_TOKEN:-}" ]; then printf ''; else printf ''; fi)" echo " DEPTHAI_VERSION=${DEPTHAI_VERSION:-}" echo " benchmark_run_id=${BENCHMARK_RUN_ID:-}" From f3550f23012cd8d8c7c70dcf877ac5649a960186 Mon Sep 17 00:00:00 2001 From: tjb Date: Wed, 15 Apr 2026 12:28:37 +0200 Subject: [PATCH 23/37] Move benchmark payload shaping into modelconverter --- .../test_benchmark_regression.py | 131 +++++++++++++++++- 1 file changed, 127 insertions(+), 4 deletions(-) diff --git a/tests/test_benchmark/test_benchmark_regression.py b/tests/test_benchmark/test_benchmark_regression.py index 7b7b278..885c348 100644 --- a/tests/test_benchmark/test_benchmark_regression.py +++ b/tests/test_benchmark/test_benchmark_regression.py @@ -1,13 +1,12 @@ import json from pathlib import Path +from typing import Any import pytest +from hil_framework.lib_testbed.db_source.InfluxClient import InfluxClient from modelconverter.packages import get_benchmark from modelconverter.utils.types import Target -from hil_framework.lib_testbed.db_source.FpsBenchmarkInflux import ( - write_fps_benchmark_result, -) TARGETS_FILE = Path(__file__).parent / "benchmark_targets.json" @@ -24,6 +23,130 @@ def _model_id(slug: str) -> str: return slug.rsplit("/", 1)[-1] +def _normalize_tag(value: Any) -> str: + if value in (None, ""): + return "unknown" + if hasattr(value, "value"): + return str(value.value) + return str(value) + + +def _build_fps_benchmark_data( + *, + bucket: str, + token: str, + model_slug: str, + benchmark_target: str, + benchmark_run_id: str, + device_ip: str | None, + actual_fps: float, + expected_fps: float, + tolerance_low: float, + tolerance_high: float, + fps_min: float, + fps_max: float, + deviation_pct: float, + success: bool, + influx_metadata: dict[str, str | None], +) -> dict[str, Any]: + return { + "name": "fps_benchmark", + "bucket": bucket, + "token": token, + "tags": [ + {"name": "model_slug", "value": _normalize_tag(model_slug)}, + {"name": "benchmark_target", "value": _normalize_tag(benchmark_target)}, + {"name": "run_id", "value": _normalize_tag(benchmark_run_id)}, + {"name": "status", "value": "passed" if success else "failed"}, + { + "name": "testbed_name", + "value": _normalize_tag(influx_metadata.get("testbed_name")), + }, + { + "name": "camera_mxid", + "value": _normalize_tag(influx_metadata.get("camera_mxid")), + }, + { + "name": "camera_os_version", + "value": _normalize_tag(influx_metadata.get("camera_os_version")), + }, + { + "name": "camera_model", + "value": _normalize_tag(influx_metadata.get("camera_model")), + }, + { + "name": "camera_revision", + "value": _normalize_tag(influx_metadata.get("camera_revision")), + }, + { + "name": "server_os", + "value": _normalize_tag(influx_metadata.get("server_os")), + }, + { + "name": "depthai_version", + "value": _normalize_tag(influx_metadata.get("depthai_version")), + }, + {"name": "device_ip", "value": _normalize_tag(device_ip)}, + ], + "fields": [ + {"name": "actual_fps", "value": actual_fps}, + {"name": "expected_fps", "value": expected_fps}, + {"name": "tolerance_low", "value": tolerance_low}, + {"name": "tolerance_high", "value": tolerance_high}, + {"name": "fps_min", "value": fps_min}, + {"name": "fps_max", "value": fps_max}, + {"name": "deviation_pct", "value": deviation_pct}, + {"name": "success", "value": success}, + ], + } + + +def _write_fps_benchmark_result( + *, + bucket: str, + token: str, + model_slug: str, + benchmark_target: str, + benchmark_run_id: str, + device_ip: str | None, + actual_fps: float, + expected_fps: float, + tolerance_low: float, + tolerance_high: float, + fps_min: float, + fps_max: float, + deviation_pct: float, + success: bool, + influx_metadata: dict[str, str | None], +) -> None: + benchmark_data = _build_fps_benchmark_data( + bucket=bucket, + token=token, + model_slug=model_slug, + benchmark_target=benchmark_target, + benchmark_run_id=benchmark_run_id, + device_ip=device_ip, + actual_fps=actual_fps, + expected_fps=expected_fps, + tolerance_low=tolerance_low, + tolerance_high=tolerance_high, + fps_min=fps_min, + fps_max=fps_max, + deviation_pct=deviation_pct, + success=success, + influx_metadata=influx_metadata, + ) + client = InfluxClient(bucket, token=token) + print( + "Writing fps_benchmark point to InfluxDB: " + f"bucket={client.INFLUXDB_BUCKET}, org={client.INFLUXDB_ORG}" + ) + try: + client.save_benchmark_data(benchmark_data) + finally: + client.close() + + @pytest.mark.parametrize( "model_slug", _model_slugs("rvc4"), @@ -74,7 +197,7 @@ def test_benchmark_fps( f"actual={actual_fps:.2f} FPS, expected={expected_fps:.2f} FPS. " ) - write_fps_benchmark_result( + _write_fps_benchmark_result( bucket=influx_bucket, token=influx_token, model_slug=model_slug, From 78b707a4ad96484c4c3636bde704c0c138aae1d4 Mon Sep 17 00:00:00 2001 From: Danilo Date: Thu, 16 Apr 2026 13:31:17 +0200 Subject: [PATCH 24/37] Bootleg test to check if it works with new branch --- .github/workflows/fps_benchmark_influx.yaml | 6 ++++-- tests/test_benchmark/run_hil_tests.sh | 7 ++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/fps_benchmark_influx.yaml b/.github/workflows/fps_benchmark_influx.yaml index 1c05539..e47e24d 100644 --- a/.github/workflows/fps_benchmark_influx.yaml +++ b/.github/workflows/fps_benchmark_influx.yaml @@ -35,8 +35,10 @@ jobs: run: | set -euo pipefail - pip install hil-framework --upgrade \ - --index-url "https://__token__:$HIL_FRAMEWORK_TOKEN@gitlab.luxonis.com/api/v4/projects/213/packages/pypi/simple" + python -m venv venv + source venv/bin/activate + git clone --recurse-submodules -b tjb_influx_pusher https://oauth2:$HIL_FRAMEWORK_TOKEN@gitlab.luxonis.com/luxonis/hil_lab/hil_framework.git + pip install ./hil_framework/ export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#fps-benchmark-influx" RESERVATION_OPTION="--reservation-name $RESERVATION_NAME" diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index 031574f..be9211f 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -28,9 +28,10 @@ source venv/bin/activate pip install -r requirements.txt pip install pytest -pip install hil-framework --upgrade \ - --index-url "https://__token__:$PAT_TOKEN@gitlab.luxonis.com/api/v4/projects/213/packages/pypi/simple" \ - > /dev/null +git clone --recurse-submodules -b tjb_influx_pusher https://oauth2:$PAT_TOKEN@gitlab.luxonis.com/luxonis/hil_lab/hil_framework.git +pip install ./hil_framework/ + +rm -rf hil_framework pip install --upgrade \ --extra-index-url "https://artifacts.luxonis.com/artifactory/luxonis-python-snapshot-local/" \ From 73e2f8817c2973b3508c94a9503da123d9e49097 Mon Sep 17 00:00:00 2001 From: Danilo Date: Thu, 16 Apr 2026 13:48:47 +0200 Subject: [PATCH 25/37] bootleg to test on bom tests --- .github/workflows/bom-test.yaml | 35 ++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/.github/workflows/bom-test.yaml b/.github/workflows/bom-test.yaml index ff66da3..545e5dd 100644 --- a/.github/workflows/bom-test.yaml +++ b/.github/workflows/bom-test.yaml @@ -10,6 +10,10 @@ on: required: false description: "Reservation Name" type: string + os_version: + description: "Optional camera OS version to install before the benchmark" + required: false + type: string hold_reservation: description: "Hold Testbed Reservation" required: false @@ -31,6 +35,7 @@ env: RESERVATION_NAME: ${{ github.event.inputs.reservation_name }} HOLD_RESERVATION: ${{ github.event.inputs.hold_reservation }} ADDITIONAL_OPTIONS: ${{ github.event.inputs.additional_options }} + INFLUX_BUCKET: fps_metrics jobs: id: @@ -47,8 +52,10 @@ jobs: - name: Run tests # yamllint disable rule:line-length run: | - pip install hil-framework --upgrade \ - --index-url "https://__token__:$HIL_FRAMEWORK_TOKEN@gitlab.luxonis.com/api/v4/projects/213/packages/pypi/simple" + python -m venv venv + source venv/bin/activate + git clone --recurse-submodules -b tjb_influx_pusher https://oauth2:$HIL_FRAMEWORK_TOKEN@gitlab.luxonis.com/luxonis/hil_lab/hil_framework.git + pip install ./hil_framework/ if [[ -n "$RESERVATION_NAME" ]]; then RESERVATION_OPTION="--reservation-name $RESERVATION_NAME" @@ -64,16 +71,30 @@ jobs: if [[ -n "$ADDITIONAL_OPTIONS" ]]; then ADDITIONAL_OPTIONS="$ADDITIONAL_OPTIONS" fi + OS_VERSION_OPTION="" + if [[ -n "$OS_VERSION" ]]; then + OS_VERSION_OPTION="--os-version $OS_VERSION" + fi if [[ "$DEPTHAI_VERSION" =~ ^3\.[0-9]{1,2}\.[0-9]{1,2}$ ]]; then DEPTHAI_VERSION_CHECKED="$DEPTHAI_VERSION" fi + : "${INFLUX_TOKEN:?INFLUX_TOKEN is required on the runner}" + : "${INFLUX_BUCKET:?INFLUX_BUCKET must be set by the workflow}" + + REMOTE_CMD="cd /tmp/modelconverter && \ + ./tests/test_benchmark/run_hil_tests.sh \ + \"$HUBAI_API_KEY\" \ + \"$HIL_FRAMEWORK_TOKEN\" \ + \"$DEPTHAI_VERSION_CHECKED\" \ + \"$INFLUX_TOKEN\" \ + \"$GITHUB_RUN_ID\"" exec hil_runner \ --models "oak4_pro or oak4_d or oak4_s" \ - $HOLD_RESERVATION \ + $HOLD_RESERVATION_OPTION \ --wait \ - $ADDITIONAL_OPTIONS \ - $RESERVATION_OPTION \ - --sync-workspace --rsync-args="--exclude=venv"\ - --commands "cd /tmp/modelconverter && ./tests/test_benchmark/run_hil_tests.sh $HUBAI_API_KEY $HIL_FRAMEWORK_TOKEN $DEPTHAI_VERSION_CHECKED" + $OS_VERSION_OPTION \ + $RESERVATION_OPTION \ + --sync-workspace --rsync-args="--exclude=venv" \ + --commands "$REMOTE_CMD" From 3edc0905e95ff55c1d04efff91ff7de3d9c06650 Mon Sep 17 00:00:00 2001 From: Danilo Date: Thu, 16 Apr 2026 13:50:34 +0200 Subject: [PATCH 26/37] python3 instead of python --- .github/workflows/bom-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bom-test.yaml b/.github/workflows/bom-test.yaml index 545e5dd..f07d708 100644 --- a/.github/workflows/bom-test.yaml +++ b/.github/workflows/bom-test.yaml @@ -52,7 +52,7 @@ jobs: - name: Run tests # yamllint disable rule:line-length run: | - python -m venv venv + python3 -m venv venv source venv/bin/activate git clone --recurse-submodules -b tjb_influx_pusher https://oauth2:$HIL_FRAMEWORK_TOKEN@gitlab.luxonis.com/luxonis/hil_lab/hil_framework.git pip install ./hil_framework/ From d22fc9e063d27f6e1dc2c34f7af2a80f52cf1989 Mon Sep 17 00:00:00 2001 From: Danilo Date: Thu, 16 Apr 2026 13:53:57 +0200 Subject: [PATCH 27/37] add token --- .github/workflows/bom-test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/bom-test.yaml b/.github/workflows/bom-test.yaml index f07d708..af5c8a0 100644 --- a/.github/workflows/bom-test.yaml +++ b/.github/workflows/bom-test.yaml @@ -36,6 +36,7 @@ env: HOLD_RESERVATION: ${{ github.event.inputs.hold_reservation }} ADDITIONAL_OPTIONS: ${{ github.event.inputs.additional_options }} INFLUX_BUCKET: fps_metrics + INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }} jobs: id: From c6f60768682c006a84fbb4a57c6d7e803bd0ef5c Mon Sep 17 00:00:00 2001 From: Danilo Date: Thu, 16 Apr 2026 14:02:47 +0200 Subject: [PATCH 28/37] remove hil frameowrk before --- tests/test_benchmark/run_hil_tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index be9211f..2090820 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -28,6 +28,8 @@ source venv/bin/activate pip install -r requirements.txt pip install pytest +rm -rf hil_framework + git clone --recurse-submodules -b tjb_influx_pusher https://oauth2:$PAT_TOKEN@gitlab.luxonis.com/luxonis/hil_lab/hil_framework.git pip install ./hil_framework/ From 191fc1f246c491fb115ae01a19eaea6bbd658004 Mon Sep 17 00:00:00 2001 From: tjb Date: Fri, 17 Apr 2026 10:00:48 +0200 Subject: [PATCH 29/37] Pass raw benchmark tags to InfluxClient --- .../test_benchmark_regression.py | 30 +++++++------------ 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/tests/test_benchmark/test_benchmark_regression.py b/tests/test_benchmark/test_benchmark_regression.py index 885c348..58e2c81 100644 --- a/tests/test_benchmark/test_benchmark_regression.py +++ b/tests/test_benchmark/test_benchmark_regression.py @@ -23,14 +23,6 @@ def _model_id(slug: str) -> str: return slug.rsplit("/", 1)[-1] -def _normalize_tag(value: Any) -> str: - if value in (None, ""): - return "unknown" - if hasattr(value, "value"): - return str(value.value) - return str(value) - - def _build_fps_benchmark_data( *, bucket: str, @@ -54,39 +46,39 @@ def _build_fps_benchmark_data( "bucket": bucket, "token": token, "tags": [ - {"name": "model_slug", "value": _normalize_tag(model_slug)}, - {"name": "benchmark_target", "value": _normalize_tag(benchmark_target)}, - {"name": "run_id", "value": _normalize_tag(benchmark_run_id)}, + {"name": "model_slug", "value": model_slug}, + {"name": "benchmark_target", "value": benchmark_target}, + {"name": "run_id", "value": benchmark_run_id}, {"name": "status", "value": "passed" if success else "failed"}, { "name": "testbed_name", - "value": _normalize_tag(influx_metadata.get("testbed_name")), + "value": influx_metadata.get("testbed_name"), }, { "name": "camera_mxid", - "value": _normalize_tag(influx_metadata.get("camera_mxid")), + "value": influx_metadata.get("camera_mxid"), }, { "name": "camera_os_version", - "value": _normalize_tag(influx_metadata.get("camera_os_version")), + "value": influx_metadata.get("camera_os_version"), }, { "name": "camera_model", - "value": _normalize_tag(influx_metadata.get("camera_model")), + "value": influx_metadata.get("camera_model"), }, { "name": "camera_revision", - "value": _normalize_tag(influx_metadata.get("camera_revision")), + "value": influx_metadata.get("camera_revision"), }, { "name": "server_os", - "value": _normalize_tag(influx_metadata.get("server_os")), + "value": influx_metadata.get("server_os"), }, { "name": "depthai_version", - "value": _normalize_tag(influx_metadata.get("depthai_version")), + "value": influx_metadata.get("depthai_version"), }, - {"name": "device_ip", "value": _normalize_tag(device_ip)}, + {"name": "device_ip", "value": device_ip}, ], "fields": [ {"name": "actual_fps", "value": actual_fps}, From 02f343f2854c9b25f374abfa6c6e03c861a97b2b Mon Sep 17 00:00:00 2001 From: tjb Date: Fri, 17 Apr 2026 11:15:20 +0200 Subject: [PATCH 30/37] Read benchmark metadata from HIL camera object --- tests/test_benchmark/conftest.py | 61 +++----- tests/test_benchmark/run_hil_tests.sh | 62 +------- .../test_benchmark_regression.py | 142 +++++++++++++----- 3 files changed, 134 insertions(+), 131 deletions(-) diff --git a/tests/test_benchmark/conftest.py b/tests/test_benchmark/conftest.py index fd3e2e7..e63c1f0 100644 --- a/tests/test_benchmark/conftest.py +++ b/tests/test_benchmark/conftest.py @@ -1,6 +1,7 @@ import os import pytest +from hil_framework.lib_testbed.utils.Testbed import Testbed INFLUX_BUCKET = "fps_metrics" @@ -36,36 +37,6 @@ def pytest_addoption(parser: pytest.Parser) -> None: default=None, help="Testbed name recorded in benchmark metadata.", ) - parser.addoption( - "--camera-mxid", - action="store", - default=None, - help="Camera MXID recorded in benchmark metadata.", - ) - parser.addoption( - "--camera-os-version", - action="store", - default=None, - help="Camera OS version recorded in benchmark metadata.", - ) - parser.addoption( - "--camera-model", - action="store", - default=None, - help="Camera model recorded in benchmark metadata.", - ) - parser.addoption( - "--camera-revision", - action="store", - default=None, - help="Camera revision recorded in benchmark metadata.", - ) - parser.addoption( - "--server-os", - action="store", - default=None, - help="Server OS recorded in benchmark metadata.", - ) def pytest_configure(config: pytest.Config) -> None: @@ -87,16 +58,26 @@ def benchmark_target(request: pytest.FixtureRequest) -> str: @pytest.fixture(scope="session") -def influx_metadata(request: pytest.FixtureRequest) -> dict[str, str | None]: - return { - "testbed_name": request.config.getoption("--testbed-name"), - "camera_mxid": request.config.getoption("--camera-mxid"), - "camera_os_version": request.config.getoption("--camera-os-version"), - "camera_model": request.config.getoption("--camera-model"), - "camera_revision": request.config.getoption("--camera-revision"), - "server_os": request.config.getoption("--server-os"), - "depthai_version": request.config.getoption("--depthai-version"), - } +def depthai_version(request: pytest.FixtureRequest) -> str | None: + return request.config.getoption("--depthai-version") + + +@pytest.fixture(scope="session") +def testbed_name(request: pytest.FixtureRequest) -> str | None: + configured_testbed_name = request.config.getoption("--testbed-name") + if configured_testbed_name: + return configured_testbed_name + return os.environ.get("HIL_TESTBED") + + +@pytest.fixture(scope="session") +def hil_testbed(testbed_name: str | None) -> Testbed: + if not testbed_name: + pytest.exit( + "HIL_TESTBED environment variable or --testbed-name must be set.", + returncode=1, + ) + return Testbed(testbed_name) @pytest.fixture(scope="session") diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index 2090820..77a4a0a 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -40,9 +40,7 @@ pip install --upgrade \ --extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-python-release-local \ "depthai==${DEPTHAI_VERSION}" -# Cache device metadata once for the whole run using the HIL camera CLI. If the -# lookup fails, keep the benchmark runnable and record explicit placeholder -# values for the missing camera-derived metadata. +# Resolve the benchmark device once for the whole run using the HIL camera CLI. camera_output=$( camera -t "${HIL_TESTBED}" -n test all info -j 2>/dev/null || printf '' ) @@ -68,62 +66,21 @@ device_hostname=$( | jq -r '.hostname // empty' 2>/dev/null \ | head -n1 ) -camera_mxid=$( - printf '%s' "$rvc4_camera" \ - | jq -r '.mxid // empty' 2>/dev/null \ - | head -n1 -) -camera_model=$( - printf '%s' "$rvc4_camera" \ - | jq -r '.model // empty' 2>/dev/null \ - | head -n1 -) -camera_revision=$( - printf '%s' "$rvc4_camera" \ - | jq -r '.revision // empty' 2>/dev/null \ - | head -n1 -) -camera_os=$( - printf '%s' "$rvc4_camera" \ - | jq -r '.os_version // empty' 2>/dev/null \ - | head -n1 -) detected_testbed_name=$( printf '%s' "$rvc4_camera" \ | jq -r '.name // empty' 2>/dev/null \ | head -n1 ) -missing_metadata=() if [ -z "$device_hostname" ]; then - missing_metadata+=("hostname") -fi -if [ -z "$camera_mxid" ]; then - missing_metadata+=("mxid") -fi -if [ -z "$camera_model" ]; then - missing_metadata+=("model") -fi -if [ -z "$camera_revision" ]; then - missing_metadata+=("revision") -fi -if [ -z "$camera_os" ]; then - missing_metadata+=("os_version") -fi - -if [ "${#missing_metadata[@]}" -ne 0 ]; then - echo "Error: camera metadata is incomplete; missing fields: ${missing_metadata[*]}" >&2 + echo "Error: camera metadata is incomplete; missing fields: hostname" >&2 exit 1 fi runner_hostname=$(hostname 2>/dev/null || printf 'unknown') -server_os=$(uname -s 2>/dev/null | tr '[:upper:]' '[:lower:]' || printf 'unknown') if [ -z "$runner_hostname" ]; then runner_hostname="unknown" fi -if [ -z "$server_os" ]; then - server_os="unknown" -fi testbed_name="${HIL_TESTBED:-}" if [ -z "$testbed_name" ]; then testbed_name="${detected_testbed_name:-}" @@ -138,14 +95,11 @@ pytest_args=( -v tests/test_benchmark/ --depthai-version "$DEPTHAI_VERSION" - --testbed-name "$testbed_name" - --camera-mxid "$camera_mxid" - --camera-os-version "$camera_os" - --camera-model "$camera_model" - --camera-revision "$camera_revision" - --server-os "$server_os" ) +if [ -n "$testbed_name" ]; then + pytest_args+=(--testbed-name "$testbed_name") +fi pytest_args+=(--device-ip "$device_hostname") if [ -n "$BENCHMARK_RUN_ID" ]; then pytest_args+=(--benchmark-run-id "$BENCHMARK_RUN_ID") @@ -157,14 +111,8 @@ echo " INFLUX_TOKEN=$(if [ -n "${INFLUX_TOKEN:-}" ]; then printf ''; else echo " DEPTHAI_VERSION=${DEPTHAI_VERSION:-}" echo " benchmark_run_id=${BENCHMARK_RUN_ID:-}" echo " HIL_TESTBED=${testbed_name:-}" -echo " HIL_CAMERA_MXID=${camera_mxid:-}" -echo " HIL_CAMERA_OS_VERSION=${camera_os:-}" -echo " HIL_CAMERA_MODEL=${camera_model:-}" -echo " HIL_CAMERA_REVISION=${camera_revision:-}" -echo " HIL_SERVER_OS=${server_os:-}" echo " device_ip=${device_hostname:-}" echo " runner=${runner_hostname:-}" -echo " server_os=${server_os:-}" printf ' pytest_args:' printf ' %q' "${pytest_args[@]}" printf '\n' diff --git a/tests/test_benchmark/test_benchmark_regression.py b/tests/test_benchmark/test_benchmark_regression.py index 58e2c81..383c569 100644 --- a/tests/test_benchmark/test_benchmark_regression.py +++ b/tests/test_benchmark/test_benchmark_regression.py @@ -1,4 +1,5 @@ import json +import platform from pathlib import Path from typing import Any @@ -31,6 +32,9 @@ def _build_fps_benchmark_data( benchmark_target: str, benchmark_run_id: str, device_ip: str | None, + benchmark_camera: Any, + testbed_name: str | None, + depthai_version: str | None, actual_fps: float, expected_fps: float, tolerance_low: float, @@ -39,8 +43,18 @@ def _build_fps_benchmark_data( fps_max: float, deviation_pct: float, success: bool, - influx_metadata: dict[str, str | None], ) -> dict[str, Any]: + camera_os_version = _get_camera_os_version(benchmark_camera) + server_os = platform.system().strip().lower() or None + _require_metadata( + { + "camera_mxid": benchmark_camera.mxid, + "camera_os_version": camera_os_version, + "camera_model": benchmark_camera.model, + "camera_revision": benchmark_camera.revision, + "server_os": server_os, + } + ) return { "name": "fps_benchmark", "bucket": bucket, @@ -50,34 +64,13 @@ def _build_fps_benchmark_data( {"name": "benchmark_target", "value": benchmark_target}, {"name": "run_id", "value": benchmark_run_id}, {"name": "status", "value": "passed" if success else "failed"}, - { - "name": "testbed_name", - "value": influx_metadata.get("testbed_name"), - }, - { - "name": "camera_mxid", - "value": influx_metadata.get("camera_mxid"), - }, - { - "name": "camera_os_version", - "value": influx_metadata.get("camera_os_version"), - }, - { - "name": "camera_model", - "value": influx_metadata.get("camera_model"), - }, - { - "name": "camera_revision", - "value": influx_metadata.get("camera_revision"), - }, - { - "name": "server_os", - "value": influx_metadata.get("server_os"), - }, - { - "name": "depthai_version", - "value": influx_metadata.get("depthai_version"), - }, + {"name": "testbed_name", "value": testbed_name}, + {"name": "camera_mxid", "value": benchmark_camera.mxid}, + {"name": "camera_os_version", "value": camera_os_version}, + {"name": "camera_model", "value": benchmark_camera.model}, + {"name": "camera_revision", "value": benchmark_camera.revision}, + {"name": "server_os", "value": server_os}, + {"name": "depthai_version", "value": depthai_version}, {"name": "device_ip", "value": device_ip}, ], "fields": [ @@ -101,6 +94,9 @@ def _write_fps_benchmark_result( benchmark_target: str, benchmark_run_id: str, device_ip: str | None, + benchmark_camera: Any, + testbed_name: str | None, + depthai_version: str | None, actual_fps: float, expected_fps: float, tolerance_low: float, @@ -109,7 +105,6 @@ def _write_fps_benchmark_result( fps_max: float, deviation_pct: float, success: bool, - influx_metadata: dict[str, str | None], ) -> None: benchmark_data = _build_fps_benchmark_data( bucket=bucket, @@ -118,6 +113,9 @@ def _write_fps_benchmark_result( benchmark_target=benchmark_target, benchmark_run_id=benchmark_run_id, device_ip=device_ip, + benchmark_camera=benchmark_camera, + testbed_name=testbed_name, + depthai_version=depthai_version, actual_fps=actual_fps, expected_fps=expected_fps, tolerance_low=tolerance_low, @@ -126,7 +124,6 @@ def _write_fps_benchmark_result( fps_max=fps_max, deviation_pct=deviation_pct, success=success, - influx_metadata=influx_metadata, ) client = InfluxClient(bucket, token=token) print( @@ -139,6 +136,74 @@ def _write_fps_benchmark_result( client.close() +def _get_camera_os_version(benchmark_camera: Any) -> str: + if not hasattr(benchmark_camera, "get_os_version"): + raise RuntimeError( + f"Camera {benchmark_camera.name} does not expose get_os_version()." + ) + try: + return benchmark_camera.get_os_version() + except Exception as exc: + raise RuntimeError( + f"Failed to read OS version for camera {benchmark_camera.name}." + ) from exc + + +def _require_metadata(metadata: dict[str, Any]) -> None: + missing_fields = [ + field_name + for field_name, value in metadata.items() + if value in (None, "") + ] + if missing_fields: + raise RuntimeError( + "Camera metadata is incomplete; missing fields: " + + ", ".join(missing_fields) + ) + + +def _select_benchmark_camera( + hil_testbed: Any, + benchmark_target: str, + device_ip: str | None, +) -> Any: + if device_ip: + device_matches = [ + camera + for camera in hil_testbed.cameras + if getattr(camera, "hostname", None) == device_ip + ] + if len(device_matches) == 1: + return device_matches[0] + if len(device_matches) > 1: + raise RuntimeError( + f"Multiple cameras matched benchmark device IP {device_ip}." + ) + + target_matches = [ + camera + for camera in hil_testbed.cameras + if str(getattr(camera, "platform", "")).lower() + == benchmark_target.lower() + ] + if len(target_matches) == 1: + return target_matches[0] + + available_cameras = ", ".join( + f"{camera.name}:{getattr(camera, 'hostname', None)}:{getattr(camera, 'platform', None)}" + for camera in hil_testbed.cameras + ) + if not target_matches: + raise RuntimeError( + f"No camera found for benchmark target {benchmark_target}. " + f"Available cameras: {available_cameras}" + ) + raise RuntimeError( + f"Unable to select a unique camera for benchmark target {benchmark_target}. " + f"Available cameras: {available_cameras}" + ) + + @pytest.mark.parametrize( "model_slug", _model_slugs("rvc4"), @@ -151,7 +216,8 @@ def test_benchmark_fps( benchmark_run_id: str, influx_bucket: str | None, influx_token: str | None, - influx_metadata: dict[str, str | None], + depthai_version: str | None, + hil_testbed: Any, ) -> None: model_config = _targets_data[benchmark_target][model_slug] expected_fps = model_config["expected_fps"] @@ -183,6 +249,12 @@ def test_benchmark_fps( deviation_pct = ((actual_fps - expected_fps) / expected_fps) * 100 success = fps_min <= actual_fps <= fps_max + actual_device_ip = configuration.get("device_ip") + benchmark_camera = _select_benchmark_camera( + hil_testbed=hil_testbed, + benchmark_target=benchmark_target, + device_ip=actual_device_ip, + ) print( f"Benchmark result for {model_slug}: " @@ -195,7 +267,10 @@ def test_benchmark_fps( model_slug=model_slug, benchmark_target=benchmark_target, benchmark_run_id=benchmark_run_id, - device_ip=device_ip, + device_ip=actual_device_ip, + benchmark_camera=benchmark_camera, + testbed_name=hil_testbed.config.name, + depthai_version=depthai_version, actual_fps=actual_fps, expected_fps=expected_fps, tolerance_low=tolerance_low, @@ -204,7 +279,6 @@ def test_benchmark_fps( fps_max=fps_max, deviation_pct=deviation_pct, success=success, - influx_metadata=influx_metadata, ) assert success, ( From 34d97fab9c8db2c3372961515c6aa8b990f277c5 Mon Sep 17 00:00:00 2001 From: tjb Date: Fri, 17 Apr 2026 11:15:41 +0200 Subject: [PATCH 31/37] Wire BOM test os_version input into env --- .github/workflows/bom-test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/bom-test.yaml b/.github/workflows/bom-test.yaml index af5c8a0..d292785 100644 --- a/.github/workflows/bom-test.yaml +++ b/.github/workflows/bom-test.yaml @@ -30,6 +30,7 @@ on: env: DISTINCT_ID: ${{ github.event.inputs.distinct_id }} DEPTHAI_VERSION: ${{ github.event.inputs.depthai_version }} + OS_VERSION: ${{ github.event.inputs.os_version }} HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }} HUBAI_API_KEY: ${{ secrets.HUBAI_API_KEY }} RESERVATION_NAME: ${{ github.event.inputs.reservation_name }} From e4858c732dabc9798f45f1204f09f49776ffe4a3 Mon Sep 17 00:00:00 2001 From: tjb Date: Fri, 17 Apr 2026 11:46:44 +0200 Subject: [PATCH 32/37] Construct benchmark HIL testbed from Config --- tests/test_benchmark/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_benchmark/conftest.py b/tests/test_benchmark/conftest.py index e63c1f0..3871613 100644 --- a/tests/test_benchmark/conftest.py +++ b/tests/test_benchmark/conftest.py @@ -1,6 +1,7 @@ import os import pytest +from hil_framework.lib_testbed.config.Config import Config from hil_framework.lib_testbed.utils.Testbed import Testbed INFLUX_BUCKET = "fps_metrics" @@ -77,7 +78,7 @@ def hil_testbed(testbed_name: str | None) -> Testbed: "HIL_TESTBED environment variable or --testbed-name must be set.", returncode=1, ) - return Testbed(testbed_name) + return Testbed(Config(testbed_name)) @pytest.fixture(scope="session") From d693c63b8a047ada113db50ba5c8bb3b7ce7a15a Mon Sep 17 00:00:00 2001 From: tjb Date: Fri, 17 Apr 2026 09:42:50 +0200 Subject: [PATCH 33/37] Remove uneeded workflow --- .github/workflows/fps_benchmark_influx.yaml | 81 --------------------- 1 file changed, 81 deletions(-) delete mode 100644 .github/workflows/fps_benchmark_influx.yaml diff --git a/.github/workflows/fps_benchmark_influx.yaml b/.github/workflows/fps_benchmark_influx.yaml deleted file mode 100644 index e47e24d..0000000 --- a/.github/workflows/fps_benchmark_influx.yaml +++ /dev/null @@ -1,81 +0,0 @@ -name: HIL FPS Benchmark Influx - -on: - workflow_dispatch: - inputs: - depthai_version: - description: "DepthAI version to test against (e.g. 3.3.0)" - required: true - type: string - os_version: - description: "Optional camera OS version to install before the benchmark" - required: false - type: string - hold_reservation: - description: "Hold testbed reservation after the run" - required: false - type: boolean - -env: - DEPTHAI_VERSION: ${{ github.event.inputs.depthai_version }} - OS_VERSION: ${{ github.event.inputs.os_version }} - HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }} - HUBAI_API_KEY: ${{ secrets.HUBAI_API_KEY }} - HOLD_RESERVATION: ${{ github.event.inputs.hold_reservation }} - INFLUX_BUCKET: fps_metrics - -jobs: - fps-benchmark: - runs-on: ["self-hosted", "testbed-runner"] - - steps: - - uses: actions/checkout@v4 - - - name: Run benchmark and push to Influx - run: | - set -euo pipefail - - python -m venv venv - source venv/bin/activate - git clone --recurse-submodules -b tjb_influx_pusher https://oauth2:$HIL_FRAMEWORK_TOKEN@gitlab.luxonis.com/luxonis/hil_lab/hil_framework.git - pip install ./hil_framework/ - - export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#fps-benchmark-influx" - RESERVATION_OPTION="--reservation-name $RESERVATION_NAME" - - HOLD_RESERVATION_OPTION="" - if [[ "$HOLD_RESERVATION" == "true" ]]; then - HOLD_RESERVATION_OPTION="--hold-reservation" - fi - - OS_VERSION_OPTION="" - if [[ -n "$OS_VERSION" ]]; then - OS_VERSION_OPTION="--os-version $OS_VERSION" - fi - - if [[ "$DEPTHAI_VERSION" =~ ^3\.[0-9]{1,2}\.[0-9]{1,2}$ ]]; then - DEPTHAI_VERSION_CHECKED="$DEPTHAI_VERSION" - else - echo "Invalid depthai version: $DEPTHAI_VERSION" >&2 - exit 1 - fi - - : "${INFLUX_TOKEN:?INFLUX_TOKEN is required on the runner}" - : "${INFLUX_BUCKET:?INFLUX_BUCKET must be set by the workflow}" - - REMOTE_CMD="cd /tmp/modelconverter && \ - ./tests/test_benchmark/run_hil_tests.sh \ - \"$HUBAI_API_KEY\" \ - \"$HIL_FRAMEWORK_TOKEN\" \ - \"$DEPTHAI_VERSION_CHECKED\" \ - \"$INFLUX_TOKEN\" \ - \"$GITHUB_RUN_ID\"" - - exec hil_runner \ - --models "oak4_pro or oak4_d or oak4_s" \ - $HOLD_RESERVATION_OPTION \ - --wait \ - $OS_VERSION_OPTION \ - $RESERVATION_OPTION \ - --sync-workspace --rsync-args="--exclude=venv" \ - --commands "$REMOTE_CMD" From b3cb11361b66bd7612f5a6f40af7e029741cb70b Mon Sep 17 00:00:00 2001 From: tjb Date: Fri, 17 Apr 2026 14:39:16 +0200 Subject: [PATCH 34/37] Removal of uneeded file --- TASK.md | 87 --------------------------------------------------------- 1 file changed, 87 deletions(-) delete mode 100644 TASK.md diff --git a/TASK.md b/TASK.md deleted file mode 100644 index 21c38c0..0000000 --- a/TASK.md +++ /dev/null @@ -1,87 +0,0 @@ -# FPS Influx Split Plan - -Goal: move reusable Influx-writing behavior out of `modelconverter` and into `hil_framework`, while keeping all FPS benchmark logic and repository-specific CI glue in `modelconverter`. - -## Keep in `modelconverter` - -- `tests/test_benchmark/test_benchmark_regression.py` - - `test_benchmark_fps(...)` - - `_model_slugs(...)` - - `_model_id(...)` - - benchmark math and assertion logic - - reading `benchmark_targets.json` -- `tests/test_benchmark/conftest.py` - - `pytest_addoption(...)` - - `pytest_configure(...)` - - `device_ip(...)` - - `benchmark_target(...)` - - `benchmark_run_id(...)` unless it becomes shared HIL metadata -- `tests/test_benchmark/run_hil_tests.sh` -- `.github/workflows/fps_benchmark_influx.yaml` -- `tjb_run_fps_influx.sh` - -## Move to `hil_framework` - -### 1. Generic Influx write helper - -Replace the manual HTTP / line-protocol write path in `test_benchmark_regression.py` with the existing `hil_framework` client API. - -Preferred shape: - -- build a `Point("fps_benchmark")` -- attach tags and fields -- call `InfluxClient("hil").save_points([point])` - -This keeps benchmark data composition in `modelconverter`, but moves the transport and serialization style into the shared HIL implementation. - -Candidate logic to remove from `modelconverter` once the helper is in place: - -- `_escape_tag(...)` -- `_normalize_tag(...)` -- `_format_field(...)` -- the raw HTTP request construction -- the explicit Influx URL / token / org handling -- the timeout / `URLError` handling around `urllib.request.urlopen` - -The helper should live beside the existing `InfluxClient` / `StabilityInflux` code, but not inside the stability-specific classes. - -### 2. Shared HIL metadata helper - -If some metadata is better derived from `hil_framework` objects, move that logic out of `tests/test_benchmark/conftest.py` and into a shared helper: - -- `influx_metadata(...)` - -Derivable from `Testbed` / `camera` / `server`: - -- `HIL_TESTBED` from `testbed.config.name` or `camera.config.testbed_name` -- `HIL_CAMERA_MXID` -- `HIL_CAMERA_OS_VERSION` from `camera.get_os_version()` on RVC4 cameras -- `HIL_CAMERA_MODEL` -- `HIL_CAMERA_REVISION` -- `HIL_SERVER_OS` from `camera.server.os` - -`HIL_RUNNER` should be ignored. - -`DEPTHAI_VERSION` must stay as an explicit input parameter to the benchmark flow. -It should be passed through from the workflow into the benchmark test path and preserved in the Influx payload if needed. - -## Do not move - -- FPS benchmark model selection -- tolerance math -- pass/fail logic -- workflow dispatch scripts -- benchmark target baselines -- any modelconverter-only fixture data -- benchmark-specific selection of the RVC4 camera to inspect - -## Suggested implementation order - -1. Add or extend a helper in `hil_framework` that turns benchmark data into `Point` objects and writes them with `InfluxClient("hil").save_points`. -2. Update `modelconverter` to call that helper from the FPS benchmark test. -3. Keep the benchmark math and assertions in `modelconverter`. -4. Remove the now-unused local Influx serialization helpers from `modelconverter`. - -## Exclusions - -The removed fixture files `fake_fps_metric.csv` and `fake_fps_metric.lp` should stay deleted. They are leftovers, not part of the runtime benchmark flow. From a496eb7530d2d94f9e2b67279d27e29f86ba8fde Mon Sep 17 00:00:00 2001 From: tjb Date: Fri, 17 Apr 2026 15:07:02 +0200 Subject: [PATCH 35/37] Remove benchmark device_ip pytest plumbing --- tests/test_benchmark/conftest.py | 11 ---- tests/test_benchmark/run_hil_tests.sh | 52 ++----------------- .../test_benchmark_regression.py | 46 +++++++--------- 3 files changed, 21 insertions(+), 88 deletions(-) diff --git a/tests/test_benchmark/conftest.py b/tests/test_benchmark/conftest.py index 3871613..bceaf3c 100644 --- a/tests/test_benchmark/conftest.py +++ b/tests/test_benchmark/conftest.py @@ -8,12 +8,6 @@ def pytest_addoption(parser: pytest.Parser) -> None: - parser.addoption( - "--device-ip", - action="store", - default=None, - help="IP address of the target device.", - ) parser.addoption( "--benchmark-target", action="store", @@ -48,11 +42,6 @@ def pytest_configure(config: pytest.Config) -> None: ) -@pytest.fixture -def device_ip(request: pytest.FixtureRequest) -> str | None: - return request.config.getoption("--device-ip") - - @pytest.fixture def benchmark_target(request: pytest.FixtureRequest) -> str: return request.config.getoption("--benchmark-target") diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index 77a4a0a..bf87af8 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -40,54 +40,10 @@ pip install --upgrade \ --extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-python-release-local \ "depthai==${DEPTHAI_VERSION}" -# Resolve the benchmark device once for the whole run using the HIL camera CLI. -camera_output=$( - camera -t "${HIL_TESTBED}" -n test all info -j 2>/dev/null || printf '' -) - -if [ -z "$camera_output" ]; then - echo "Error: failed to obtain camera metadata via camera info." >&2 - exit 1 -fi - -rvc4_camera=$( - printf '%s' "$camera_output" \ - | jq -r '.[] | select(.platform == "rvc4") | @json' 2>/dev/null \ - | head -n1 -) - -if [ -z "$rvc4_camera" ]; then - echo "Error: no rvc4 camera found in camera metadata." >&2 - exit 1 -fi - -device_hostname=$( - printf '%s' "$rvc4_camera" \ - | jq -r '.hostname // empty' 2>/dev/null \ - | head -n1 -) -detected_testbed_name=$( - printf '%s' "$rvc4_camera" \ - | jq -r '.name // empty' 2>/dev/null \ - | head -n1 -) - -if [ -z "$device_hostname" ]; then - echo "Error: camera metadata is incomplete; missing fields: hostname" >&2 - exit 1 -fi - runner_hostname=$(hostname 2>/dev/null || printf 'unknown') if [ -z "$runner_hostname" ]; then runner_hostname="unknown" fi -testbed_name="${HIL_TESTBED:-}" -if [ -z "$testbed_name" ]; then - testbed_name="${detected_testbed_name:-}" -fi -if [ -z "$testbed_name" ]; then - testbed_name="$(hostname 2>/dev/null || printf '')" -fi # Run tests pytest_args=( @@ -97,10 +53,9 @@ pytest_args=( --depthai-version "$DEPTHAI_VERSION" ) -if [ -n "$testbed_name" ]; then - pytest_args+=(--testbed-name "$testbed_name") +if [ -n "${HIL_TESTBED:-}" ]; then + pytest_args+=(--testbed-name "$HIL_TESTBED") fi -pytest_args+=(--device-ip "$device_hostname") if [ -n "$BENCHMARK_RUN_ID" ]; then pytest_args+=(--benchmark-run-id "$BENCHMARK_RUN_ID") fi @@ -110,8 +65,7 @@ echo " INFLUX_BUCKET=fps_metrics" echo " INFLUX_TOKEN=$(if [ -n "${INFLUX_TOKEN:-}" ]; then printf ''; else printf ''; fi)" echo " DEPTHAI_VERSION=${DEPTHAI_VERSION:-}" echo " benchmark_run_id=${BENCHMARK_RUN_ID:-}" -echo " HIL_TESTBED=${testbed_name:-}" -echo " device_ip=${device_hostname:-}" +echo " HIL_TESTBED=${HIL_TESTBED:-}" echo " runner=${runner_hostname:-}" printf ' pytest_args:' printf ' %q' "${pytest_args[@]}" diff --git a/tests/test_benchmark/test_benchmark_regression.py b/tests/test_benchmark/test_benchmark_regression.py index 383c569..af569a2 100644 --- a/tests/test_benchmark/test_benchmark_regression.py +++ b/tests/test_benchmark/test_benchmark_regression.py @@ -31,7 +31,6 @@ def _build_fps_benchmark_data( model_slug: str, benchmark_target: str, benchmark_run_id: str, - device_ip: str | None, benchmark_camera: Any, testbed_name: str | None, depthai_version: str | None, @@ -46,8 +45,10 @@ def _build_fps_benchmark_data( ) -> dict[str, Any]: camera_os_version = _get_camera_os_version(benchmark_camera) server_os = platform.system().strip().lower() or None + benchmark_device_ip = _get_benchmark_device_ip(benchmark_camera) _require_metadata( { + "device_ip": benchmark_device_ip, "camera_mxid": benchmark_camera.mxid, "camera_os_version": camera_os_version, "camera_model": benchmark_camera.model, @@ -71,7 +72,7 @@ def _build_fps_benchmark_data( {"name": "camera_revision", "value": benchmark_camera.revision}, {"name": "server_os", "value": server_os}, {"name": "depthai_version", "value": depthai_version}, - {"name": "device_ip", "value": device_ip}, + {"name": "device_ip", "value": benchmark_device_ip}, ], "fields": [ {"name": "actual_fps", "value": actual_fps}, @@ -93,7 +94,6 @@ def _write_fps_benchmark_result( model_slug: str, benchmark_target: str, benchmark_run_id: str, - device_ip: str | None, benchmark_camera: Any, testbed_name: str | None, depthai_version: str | None, @@ -112,7 +112,6 @@ def _write_fps_benchmark_result( model_slug=model_slug, benchmark_target=benchmark_target, benchmark_run_id=benchmark_run_id, - device_ip=device_ip, benchmark_camera=benchmark_camera, testbed_name=testbed_name, depthai_version=depthai_version, @@ -149,6 +148,15 @@ def _get_camera_os_version(benchmark_camera: Any) -> str: ) from exc +def _get_benchmark_device_ip(benchmark_camera: Any) -> str: + device_ip = getattr(benchmark_camera, "hostname", None) + if not device_ip: + raise RuntimeError( + f"Camera {benchmark_camera.name} does not expose a hostname." + ) + return device_ip + + def _require_metadata(metadata: dict[str, Any]) -> None: missing_fields = [ field_name @@ -165,21 +173,7 @@ def _require_metadata(metadata: dict[str, Any]) -> None: def _select_benchmark_camera( hil_testbed: Any, benchmark_target: str, - device_ip: str | None, ) -> Any: - if device_ip: - device_matches = [ - camera - for camera in hil_testbed.cameras - if getattr(camera, "hostname", None) == device_ip - ] - if len(device_matches) == 1: - return device_matches[0] - if len(device_matches) > 1: - raise RuntimeError( - f"Multiple cameras matched benchmark device IP {device_ip}." - ) - target_matches = [ camera for camera in hil_testbed.cameras @@ -211,7 +205,6 @@ def _select_benchmark_camera( ) def test_benchmark_fps( model_slug: str, - device_ip: str | None, benchmark_target: str, benchmark_run_id: str, influx_bucket: str | None, @@ -236,8 +229,12 @@ def test_benchmark_fps( "power_benchmark": False, "dsp_benchmark": False, } - if device_ip is not None: - configuration["device_ip"] = device_ip + benchmark_camera = _select_benchmark_camera( + hil_testbed=hil_testbed, + benchmark_target=benchmark_target, + ) + benchmark_device_ip = _get_benchmark_device_ip(benchmark_camera) + configuration["device_ip"] = benchmark_device_ip result = bench.benchmark(configuration) actual_fps = result.fps @@ -249,12 +246,6 @@ def test_benchmark_fps( deviation_pct = ((actual_fps - expected_fps) / expected_fps) * 100 success = fps_min <= actual_fps <= fps_max - actual_device_ip = configuration.get("device_ip") - benchmark_camera = _select_benchmark_camera( - hil_testbed=hil_testbed, - benchmark_target=benchmark_target, - device_ip=actual_device_ip, - ) print( f"Benchmark result for {model_slug}: " @@ -267,7 +258,6 @@ def test_benchmark_fps( model_slug=model_slug, benchmark_target=benchmark_target, benchmark_run_id=benchmark_run_id, - device_ip=actual_device_ip, benchmark_camera=benchmark_camera, testbed_name=hil_testbed.config.name, depthai_version=depthai_version, From c739d8c0465a8725e442c014ccc1e6e4cdb5c581 Mon Sep 17 00:00:00 2001 From: tjb Date: Fri, 17 Apr 2026 15:40:24 +0200 Subject: [PATCH 36/37] Remove unused runner hostname logging --- tests/test_benchmark/run_hil_tests.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index bf87af8..a8b5583 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -40,11 +40,6 @@ pip install --upgrade \ --extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-python-release-local \ "depthai==${DEPTHAI_VERSION}" -runner_hostname=$(hostname 2>/dev/null || printf 'unknown') -if [ -z "$runner_hostname" ]; then - runner_hostname="unknown" -fi - # Run tests pytest_args=( -s @@ -66,7 +61,6 @@ echo " INFLUX_TOKEN=$(if [ -n "${INFLUX_TOKEN:-}" ]; then printf ''; else echo " DEPTHAI_VERSION=${DEPTHAI_VERSION:-}" echo " benchmark_run_id=${BENCHMARK_RUN_ID:-}" echo " HIL_TESTBED=${HIL_TESTBED:-}" -echo " runner=${runner_hostname:-}" printf ' pytest_args:' printf ' %q' "${pytest_args[@]}" printf '\n' From 19f07b6a07e90093e153f1c406c8310ff37aab43 Mon Sep 17 00:00:00 2001 From: tjb Date: Fri, 17 Apr 2026 16:05:10 +0200 Subject: [PATCH 37/37] Restore packaged hil-framework install in BOM workflow --- .github/workflows/bom-test.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bom-test.yaml b/.github/workflows/bom-test.yaml index d292785..e28338a 100644 --- a/.github/workflows/bom-test.yaml +++ b/.github/workflows/bom-test.yaml @@ -54,10 +54,8 @@ jobs: - name: Run tests # yamllint disable rule:line-length run: | - python3 -m venv venv - source venv/bin/activate - git clone --recurse-submodules -b tjb_influx_pusher https://oauth2:$HIL_FRAMEWORK_TOKEN@gitlab.luxonis.com/luxonis/hil_lab/hil_framework.git - pip install ./hil_framework/ + pip install hil-framework --upgrade \ + --index-url "https://__token__:$HIL_FRAMEWORK_TOKEN@gitlab.luxonis.com/api/v4/projects/213/packages/pypi/simple" if [[ -n "$RESERVATION_NAME" ]]; then RESERVATION_OPTION="--reservation-name $RESERVATION_NAME"