Add FPS benchmark metric to influx #214

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open

tjb7 wants to merge 37 commits into main from tjb_fps_graphs

.github/workflows/bom-test.yaml

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -10,6 +10,10 @@ on:
  
            required: false

            description: "Reservation Name"

            type: string

          os_version:

            description: "Optional camera OS version to install before the benchmark"

            required: false

            type: string

          hold_reservation:

            description: "Hold Testbed Reservation"

            required: false

    @@ -26,11 +30,14 @@ on:
  
    env:

      DISTINCT_ID: ${{ github.event.inputs.distinct_id }}

      DEPTHAI_VERSION: ${{ github.event.inputs.depthai_version }}

      OS_VERSION: ${{ github.event.inputs.os_version }}

      HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }}

      HUBAI_API_KEY: ${{ secrets.HUBAI_API_KEY }}

      RESERVATION_NAME: ${{ github.event.inputs.reservation_name }}

      HOLD_RESERVATION: ${{ github.event.inputs.hold_reservation }}

      ADDITIONAL_OPTIONS: ${{ github.event.inputs.additional_options }}

      INFLUX_BUCKET: fps_metrics

      INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }}

    jobs:

      id:

    @@ -64,16 +71,30 @@ jobs:
  
              if [[ -n "$ADDITIONAL_OPTIONS" ]]; then

                ADDITIONAL_OPTIONS="$ADDITIONAL_OPTIONS"

              fi

              OS_VERSION_OPTION=""

              if [[ -n "$OS_VERSION" ]]; then

                OS_VERSION_OPTION="--os-version $OS_VERSION"

              fi

              if [[ "$DEPTHAI_VERSION" =~ ^3\.[0-9]{1,2}\.[0-9]{1,2}$ ]]; then

                DEPTHAI_VERSION_CHECKED="$DEPTHAI_VERSION"

              fi

              : "${INFLUX_TOKEN:?INFLUX_TOKEN is required on the runner}"

              : "${INFLUX_BUCKET:?INFLUX_BUCKET must be set by the workflow}"

              REMOTE_CMD="cd /tmp/modelconverter && \

                ./tests/test_benchmark/run_hil_tests.sh \

                  \"$HUBAI_API_KEY\" \

                  \"$HIL_FRAMEWORK_TOKEN\" \

                  \"$DEPTHAI_VERSION_CHECKED\" \

                  \"$INFLUX_TOKEN\" \

                  \"$GITHUB_RUN_ID\""

              exec hil_runner \

                --models "oak4_pro or oak4_d or oak4_s" \

                $HOLD_RESERVATION \

                $HOLD_RESERVATION_OPTION \

                --wait \

                $ADDITIONAL_OPTIONS \

                $RESERVATION_OPTION  \

                --sync-workspace --rsync-args="--exclude=venv"\

                --commands "cd /tmp/modelconverter &&  ./tests/test_benchmark/run_hil_tests.sh  $HUBAI_API_KEY $HIL_FRAMEWORK_TOKEN $DEPTHAI_VERSION_CHECKED"

                $OS_VERSION_OPTION \

                $RESERVATION_OPTION \

                --sync-workspace --rsync-args="--exclude=venv" \

                --commands "$REMOTE_CMD"

tests/test_benchmark/conftest.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -1,20 +1,36 @@
  
    import os

    import pytest

    from hil_framework.lib_testbed.config.Config import Config

    from hil_framework.lib_testbed.utils.Testbed import Testbed

    INFLUX_BUCKET = "fps_metrics"

    def pytest_addoption(parser: pytest.Parser) -> None:

        parser.addoption(

            "--device-ip",

            "--benchmark-target",

            action="store",

            default="rvc4",

            help="Target platform to benchmark (default: rvc4).",

        )

        parser.addoption(

            "--benchmark-run-id",

            action="store",

            default=None,

            help="IP address of the target device.",

            help="Benchmark run identifier to store in InfluxDB.",

        )

        parser.addoption(

            "--benchmark-target",

            "--depthai-version",

            action="store",

            default="rvc4",

            help="Target platform to benchmark (default: rvc4).",

            default=None,

            help="DepthAI version recorded in benchmark metadata.",

        )

        parser.addoption(

            "--testbed-name",

            action="store",

            default=None,

            help="Testbed name recorded in benchmark metadata.",

        )

    @@ -26,11 +42,52 @@ def pytest_configure(config: pytest.Config) -> None:
  
            )

    @pytest.fixture

    def device_ip(request: pytest.FixtureRequest) -> str | None:

        return request.config.getoption("--device-ip")

    @pytest.fixture

    def benchmark_target(request: pytest.FixtureRequest) -> str:

        return request.config.getoption("--benchmark-target")

    @pytest.fixture(scope="session")

    def depthai_version(request: pytest.FixtureRequest) -> str | None:

        return request.config.getoption("--depthai-version")

    @pytest.fixture(scope="session")

    def testbed_name(request: pytest.FixtureRequest) -> str | None:

        configured_testbed_name = request.config.getoption("--testbed-name")

        if configured_testbed_name:

            return configured_testbed_name

        return os.environ.get("HIL_TESTBED")

    @pytest.fixture(scope="session")

    def hil_testbed(testbed_name: str | None) -> Testbed:

        if not testbed_name:

            pytest.exit(

                "HIL_TESTBED environment variable or --testbed-name must be set.",

                returncode=1,

            )

        return Testbed(Config(testbed_name))

    @pytest.fixture(scope="session")

    def benchmark_run_id(request: pytest.FixtureRequest) -> str:

        configured_run_id = request.config.getoption("--benchmark-run-id")

        if configured_run_id:

            return configured_run_id

        from datetime import datetime, timezone

        from uuid import uuid4

        timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")

        return f"benchmark-{timestamp}-{uuid4().hex[:8]}"

    @pytest.fixture(scope="session")

    def influx_bucket() -> str:

        return INFLUX_BUCKET

    @pytest.fixture(scope="session")

    def influx_token() -> str | None:

        return os.environ.get("INFLUX_TOKEN")

tests/test_benchmark/run_hil_tests.sh

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -3,15 +3,17 @@
  
    set -e  # Exit immediately if a command fails

    # Check if required arguments were provided

    if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then

      echo "Usage: $0 <HUBAI_API_KEY> <PAT_TOKEN> <DAI_VERSION>"

    if [ -z "${1:-}" ] || [ -z "${2:-}" ] || [ -z "${3:-}" ] || [ -z "${4:-}" ]; then

      echo "Usage: $0 <HUBAI_API_KEY> <PAT_TOKEN> <DAI_VERSION> <INFLUX_TOKEN> [BENCHMARK_RUN_ID]"

      exit 1

    fi

    # Export variables from input arguments

    export HUBAI_API_KEY="$1"

    export PAT_TOKEN="$2"

    export DEPTHAI_VERSION="$3"

    export INFLUX_TOKEN="$4"

    BENCHMARK_RUN_ID="${5:-}"

    # Navigate to project directory

    cd /tmp/modelconverter

    @@ -26,19 +28,41 @@ source venv/bin/activate
  
    pip install -r requirements.txt

    pip install pytest

    pip install hil-framework --upgrade \

      --index-url "https://__token__:$PAT_TOKEN@gitlab.luxonis.com/api/v4/projects/213/packages/pypi/simple" \

      > /dev/null

    rm -rf hil_framework

    git clone --recurse-submodules -b tjb_influx_pusher https://oauth2:$PAT_TOKEN@gitlab.luxonis.com/luxonis/hil_lab/hil_framework.git

    pip install ./hil_framework/

    rm -rf hil_framework

    pip install --upgrade \

      --extra-index-url "https://artifacts.luxonis.com/artifactory/luxonis-python-snapshot-local/" \

      --extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-python-release-local \

      "depthai==${DEPTHAI_VERSION}"

    # Extract hostname of first rvc4 device

    hostname=$(hil_camera -t "$HIL_TESTBED" -n test all info -j \

      | jq -r '.[] | select(.platform=="rvc4") | .hostname' \

      | head -n1)

    # Run tests

    pytest -s -v tests/test_benchmark/ --device-ip "$hostname"
      
    pytest_args=(

      -s

      -v

      tests/test_benchmark/

      --depthai-version "$DEPTHAI_VERSION"

    )

    if [ -n "${HIL_TESTBED:-}" ]; then

      pytest_args+=(--testbed-name "$HIL_TESTBED")

    fi

    if [ -n "$BENCHMARK_RUN_ID" ]; then

      pytest_args+=(--benchmark-run-id "$BENCHMARK_RUN_ID")

    fi

    echo "Influx metadata debug:"

    echo "  INFLUX_BUCKET=fps_metrics"

    echo "  INFLUX_TOKEN=$(if [ -n "${INFLUX_TOKEN:-}" ]; then printf '<set>'; else printf '<empty>'; fi)"

    echo "  DEPTHAI_VERSION=${DEPTHAI_VERSION:-<empty>}"

    echo "  benchmark_run_id=${BENCHMARK_RUN_ID:-<generated>}"

    echo "  HIL_TESTBED=${HIL_TESTBED:-<empty>}"

    printf '  pytest_args:'

    printf ' %q' "${pytest_args[@]}"

    printf '\n'

    pytest "${pytest_args[@]}"

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add FPS benchmark metric to influx #214

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Add FPS benchmark metric to influx #214

Are you sure you want to change the base?

Uh oh!

Add FPS benchmark metric to influx #214

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!