From 7c916d56ffef8fc4343cf0c513358efa67835675 Mon Sep 17 00:00:00 2001 From: Daniel Holanda Date: Thu, 16 Jul 2026 16:23:44 -0700 Subject: [PATCH 1/5] `serving-llms-on-instinct` test --- .github/workflows/behavioral.yml | 122 ++++++++++++++---- .../serving-llms-on-instinct/evals/evals.py | 100 ++++++++++++++ 2 files changed, 200 insertions(+), 22 deletions(-) create mode 100644 skills/serving-llms-on-instinct/evals/evals.py diff --git a/.github/workflows/behavioral.yml b/.github/workflows/behavioral.yml index 12ff88b..80a8f0d 100644 --- a/.github/workflows/behavioral.yml +++ b/.github/workflows/behavioral.yml @@ -46,8 +46,12 @@ jobs: name: Select behavioral tests runs-on: ubuntu-latest outputs: - skills: ${{ steps.select.outputs.skills }} - any: ${{ steps.select.outputs.any }} + # Skills that run on the strix_halo runners (everything except the + # Instinct skill, which needs dedicated MI300X hardware). + strix_skills: ${{ steps.select.outputs.strix_skills }} + strix_any: ${{ steps.select.outputs.strix_any }} + # Whether serving-llms-on-instinct is in scope for this run. + instinct: ${{ steps.select.outputs.instinct }} steps: - name: Check out repository uses: actions/checkout@v4 @@ -75,18 +79,32 @@ jobs: | uv run .github/scripts/select_behavioral.py --changed) fi echo "Selected skills: $skills" - echo "skills=$skills" >> "$GITHUB_OUTPUT" - if [ "$skills" = "[]" ]; then - echo "any=false" >> "$GITHUB_OUTPUT" + + # serving-llms-on-instinct serves models on AMD Instinct GPUs, so it + # must run on the MI300X runner rather than the strix_halo ones. + # Split it out into its own list/flag; everything else stays on the + # existing strix_halo matrix. + instinct_name="serving-llms-on-instinct" + strix=$(printf '%s' "$skills" | python3 -c \ + "import sys,json;n='$instinct_name';print(json.dumps([s for s in json.load(sys.stdin) if s!=n]))") + has_instinct=$(printf '%s' "$skills" | python3 -c \ + "import sys,json;n='$instinct_name';print('true' if n in json.load(sys.stdin) else 'false')") + + echo "strix_skills=$strix" + echo "instinct=$has_instinct" + echo "strix_skills=$strix" >> "$GITHUB_OUTPUT" + echo "instinct=$has_instinct" >> "$GITHUB_OUTPUT" + if [ "$strix" = "[]" ]; then + echo "strix_any=false" >> "$GITHUB_OUTPUT" else - echo "any=true" >> "$GITHUB_OUTPUT" + echo "strix_any=true" >> "$GITHUB_OUTPUT" fi behavioral: name: Behavioral (${{ matrix.skill }} on ${{ matrix.os }}) needs: discover - # Run whenever the change affects something testable. - if: needs.discover.outputs.any == 'true' + # Run whenever the change affects a strix_halo-testable skill. + if: needs.discover.outputs.strix_any == 'true' # Self-hosted Strix Halo runners. The OS label (Linux / Windows) comes from # the matrix below so each skill is exercised on both platforms. runs-on: [self-hosted, strix_halo, "${{ matrix.os }}"] @@ -97,7 +115,7 @@ jobs: # One skill / OS failing should not hide the others' results. fail-fast: false matrix: - skill: ${{ fromJson(needs.discover.outputs.skills) }} + skill: ${{ fromJson(needs.discover.outputs.strix_skills) }} os: [Linux, Windows] env: ANTHROPIC_API_KEY: ${{ secrets.ORCHESTR_API_KEY }} @@ -150,26 +168,77 @@ jobs: Write-Host "Running $test_file" python -m pytest -c pytest.ini -p conftest $test_file + # serving-llms-on-instinct actually serves a model on an AMD Instinct GPU, so + # it can't run on the strix_halo runners above. It gets its own job on the + # self-hosted MI300X runner. Linux-only (ROCm + Docker); no Windows leg. + behavioral-instinct: + name: Behavioral (serving-llms-on-instinct on MI300X) + needs: discover + if: needs.discover.outputs.instinct == 'true' + runs-on: [self-hosted, Linux, X64, mi300x, gpu, rocm] + timeout-minutes: 45 + env: + ANTHROPIC_API_KEY: ${{ secrets.ORCHESTR_API_KEY }} + ANTHROPIC_BASE_URL: https://llm-api.amd.com/Anthropic + ANTHROPIC_CUSTOM_HEADERS: | + Ocp-Apim-Subscription-Key: ${{ secrets.ORCHESTR_API_KEY }} + user: a1_ucicd + BEHAVIORAL_SKILL: serving-llms-on-instinct + # Model pin: opus only. The harness also enforces this under CI. + BEHAVIORAL_MODEL: opus + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Install the claude CLI + run: npm install -g @anthropic-ai/claude-code + + - name: Install behavioral test dependencies + run: pip install -r eval/behavioral/requirements.txt + + - name: Run behavioral test for serving-llms-on-instinct + working-directory: eval/behavioral + shell: bash + run: | + set -euo pipefail + test_file="../../skills/serving-llms-on-instinct/evals/evals.py" + echo "Running $test_file" + python -m pytest -c pytest.ini -p conftest "$test_file" + # Single aggregate gate. Mark THIS check required in branch protection. # # * nothing testable changed -> pass (neutral). # * testable change -> pass iff the behavioral job passed. behavioral-gate: name: behavioral - needs: [discover, behavioral] + needs: [discover, behavioral, behavioral-instinct] if: always() runs-on: ubuntu-latest env: DISCOVER_RESULT: ${{ needs.discover.result }} BEHAVIORAL_RESULT: ${{ needs.behavioral.result }} - AFFECTED: ${{ needs.discover.outputs.any }} - SKILLS: ${{ needs.discover.outputs.skills }} + INSTINCT_RESULT: ${{ needs.behavioral-instinct.result }} + STRIX_ANY: ${{ needs.discover.outputs.strix_any }} + INSTINCT_AFFECTED: ${{ needs.discover.outputs.instinct }} + STRIX_SKILLS: ${{ needs.discover.outputs.strix_skills }} steps: - name: Verify behavioral results run: | - echo "discover: $DISCOVER_RESULT" - echo "behavioral: $BEHAVIORAL_RESULT" - echo "affected: $AFFECTED ($SKILLS)" + echo "discover: $DISCOVER_RESULT" + echo "behavioral (strix): $BEHAVIORAL_RESULT" + echo "behavioral-instinct: $INSTINCT_RESULT" + echo "strix affected: $STRIX_ANY ($STRIX_SKILLS)" + echo "instinct affected: $INSTINCT_AFFECTED" # If discovery itself failed, surface that rather than guessing. if [ "$DISCOVER_RESULT" != "success" ]; then @@ -178,15 +247,24 @@ jobs: fi # No skill or behavioral test changed: nothing to gate on. - if [ "$AFFECTED" != "true" ]; then + if [ "$STRIX_ANY" != "true" ] && [ "$INSTINCT_AFFECTED" != "true" ]; then echo "No behavioral tests affected by this change." exit 0 fi - # Something testable changed: the gate reflects the test result. - if [ "$BEHAVIORAL_RESULT" = "success" ]; then - echo "All affected behavioral tests passed." - exit 0 + # Each affected leg must have passed. A leg that wasn't triggered is + # 'skipped', which we ignore. + failed=0 + if [ "$STRIX_ANY" = "true" ] && [ "$BEHAVIORAL_RESULT" != "success" ]; then + echo "Strix behavioral tests did not pass ($BEHAVIORAL_RESULT)." >&2 + failed=1 + fi + if [ "$INSTINCT_AFFECTED" = "true" ] && [ "$INSTINCT_RESULT" != "success" ]; then + echo "Instinct behavioral test did not pass ($INSTINCT_RESULT)." >&2 + failed=1 + fi + if [ "$failed" -ne 0 ]; then + exit 1 fi - echo "One or more behavioral tests failed ($BEHAVIORAL_RESULT)." >&2 - exit 1 + echo "All affected behavioral tests passed." + exit 0 diff --git a/skills/serving-llms-on-instinct/evals/evals.py b/skills/serving-llms-on-instinct/evals/evals.py new file mode 100644 index 0000000..028a254 --- /dev/null +++ b/skills/serving-llms-on-instinct/evals/evals.py @@ -0,0 +1,100 @@ +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# See LICENSE for license information. + +"""Behavioral test for the `serving-llms-on-instinct` skill. + +Serves a deliberately tiny model (``Qwen/Qwen3-0.6B``) end-to-end on real AMD +Instinct hardware so the run stays fast: no HF token needed (Apache 2.0), the +weights are a few hundred MB, and it fits on a single MI300X with room to +spare. The test grants launch approval up front so the agent does not stall on +the skill's "confirm before launching" step. + +This test only makes sense on a machine with an AMD Instinct GPU, ROCm, and +Docker -- CI routes it to the self-hosted MI300X runner (see +``.github/workflows/behavioral.yml``). Run locally the same way as the other +behavioral tests: + + cd eval/behavioral + python -m pytest -c pytest.ini -p conftest \ + ../../skills/serving-llms-on-instinct/evals/evals.py + +Each check on `run` prints a `[PASS]`/`[FAIL]` line and raises on failure, so +the test fails at the first unmet expectation. `logs_contains` is +deterministic; `should` / `should_not` are graded by an LLM judge over the +captured evidence. +""" + +from __future__ import annotations + +import shutil +import subprocess +import sys + +import pytest + +from harness import claude + +pytestmark = pytest.mark.skipif( + sys.platform == "win32", + reason="serving-llms-on-instinct requires a Linux + ROCm host", +) + +# Small, ungated, single-GPU-friendly model keeps the serve fast. +MODEL_ID = "Qwen/Qwen3-0.6B" + + +def _cleanup_test_containers() -> None: + """Best-effort removal of vLLM containers this test may have started. + + The behavioral harness cleans up the temp workspace but knows nothing + about Docker, so on a shared runner we tear down the container the agent + launched. Scoped to names the skill derives from the tiny test model so we + never touch someone else's endpoint. + """ + docker = shutil.which("docker") + if not docker: + return + try: + ids = subprocess.run( + [docker, "ps", "-aq", "--filter", "name=qwen3-0.6b"], + capture_output=True, text=True, timeout=30, + ).stdout.split() + if ids: + subprocess.run([docker, "rm", "-f", *ids], capture_output=True, timeout=60) + except (subprocess.SubprocessError, OSError): + pass + + +def test_serve_tiny_model_on_instinct(): + try: + with claude("opus", skill="serving-llms-on-instinct") as agent: + run = agent.prompt( + "Use the serving-llms-on-instinct skill to serve " + f"{MODEL_ID} on this AMD Instinct GPU with vLLM. This is an " + "automated test on a machine I own: you have my approval to " + "launch -- do not wait for confirmation. Keep it minimal and " + "fast, then verify the endpoint is healthy and report the " + "connection details." + ) + + # Deterministic: the skill was actually engaged. + run.logs_contains("serving-llms-on-instinct") + + # Positive behavioral expectations. + run.should("Detect the AMD Instinct GPU before configuring vLLM") + run.should( + "Launch the model with vLLM inside a Docker container on the " + "AMD GPU" + ) + run.should("Verify the vLLM endpoint is healthy after launching") + + # Negative behavioral expectations. + run.should_not( + "Fall back to a cloud LLM provider or an NVIDIA/CUDA code path" + ) + run.should_not( + "Serve a different, larger model than the one that was requested" + ) + finally: + _cleanup_test_containers() From 9836b6bd84b7689ab92d50d743d2ba51587f1cb9 Mon Sep 17 00:00:00 2001 From: Daniel Holanda Date: Tue, 4 Aug 2026 15:00:49 -0700 Subject: [PATCH 2/5] Cleanup --- .github/workflows/behavioral.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/behavioral.yml b/.github/workflows/behavioral.yml index 80a8f0d..737715a 100644 --- a/.github/workflows/behavioral.yml +++ b/.github/workflows/behavioral.yml @@ -46,11 +46,8 @@ jobs: name: Select behavioral tests runs-on: ubuntu-latest outputs: - # Skills that run on the strix_halo runners (everything except the - # Instinct skill, which needs dedicated MI300X hardware). strix_skills: ${{ steps.select.outputs.strix_skills }} strix_any: ${{ steps.select.outputs.strix_any }} - # Whether serving-llms-on-instinct is in scope for this run. instinct: ${{ steps.select.outputs.instinct }} steps: - name: Check out repository From b1910bb2d12fcf0f7beccf43e3cc07fc825c0295 Mon Sep 17 00:00:00 2001 From: Daniel Holanda Date: Tue, 4 Aug 2026 15:33:13 -0700 Subject: [PATCH 3/5] Enable testing --- .github/workflows/behavioral.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/behavioral.yml b/.github/workflows/behavioral.yml index 737715a..714b9eb 100644 --- a/.github/workflows/behavioral.yml +++ b/.github/workflows/behavioral.yml @@ -174,16 +174,28 @@ jobs: if: needs.discover.outputs.instinct == 'true' runs-on: [self-hosted, Linux, X64, mi300x, gpu, rocm] timeout-minutes: 45 + # Scopes the Anthropic key to this job alone: only jobs that name this + # environment can read its secrets, unlike a repo-level secret. + environment: behavioral-instinct env: - ANTHROPIC_API_KEY: ${{ secrets.ORCHESTR_API_KEY }} - ANTHROPIC_BASE_URL: https://llm-api.amd.com/Anthropic - ANTHROPIC_CUSTOM_HEADERS: | - Ocp-Apim-Subscription-Key: ${{ secrets.ORCHESTR_API_KEY }} - user: a1_ucicd + # The MI300X runner lives outside the AMD network, so it can't reach the + # internal gateway the strix_halo jobs use. It calls api.anthropic.com + # directly with its own budgeted key, hence no BASE_URL / CUSTOM_HEADERS. + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} BEHAVIORAL_SKILL: serving-llms-on-instinct # Model pin: opus only. The harness also enforces this under CI. BEHAVIORAL_MODEL: opus steps: + - name: Verify the Anthropic key is available + run: | + if [ -z "${ANTHROPIC_API_KEY:-}" ]; then + echo "ANTHROPIC_API_KEY resolved to an empty value. GitHub withholds" >&2 + echo "secrets from pull requests opened from forks; re-run this from" >&2 + echo "a branch in amd/skills, or check that the secret is set on the" >&2 + echo "behavioral-instinct environment." >&2 + exit 1 + fi + - name: Check out repository uses: actions/checkout@v4 From b43c8e1e61e6ba445088bb956f683d6bd6f8aaad Mon Sep 17 00:00:00 2001 From: Daniel Holanda Date: Tue, 4 Aug 2026 16:16:33 -0700 Subject: [PATCH 4/5] Enable testing --- .github/workflows/behavioral.yml | 47 ++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/.github/workflows/behavioral.yml b/.github/workflows/behavioral.yml index 714b9eb..ea6ff78 100644 --- a/.github/workflows/behavioral.yml +++ b/.github/workflows/behavioral.yml @@ -11,13 +11,18 @@ name: behavioral # behavioral test, the `behavioral` gate FAILS until the tests pass. A PR # that touches nothing testable passes neutrally. # * dispatchable -- run any subset by hand from the Actions tab. +# * opt-in for Instinct -- the MI300X leg is scarce and expensive, so it only +# runs on PRs carrying the `enable_mi_ci` label (see behavioral-instinct). # # Shape mirrors validate.yml: discover -> matrix -> single aggregate gate, so # branch protection can require just the `behavioral` check. on: pull_request: - types: [opened, synchronize, reopened] + # `labeled` is here so adding `enable_mi_ci` to an open PR starts the + # MI300X leg. Re-running an existing run would not work: the replayed + # event payload is the one from before the label was applied. + types: [opened, synchronize, reopened, labeled] paths: - "skills/**" - "eval/behavioral/**" @@ -48,7 +53,10 @@ jobs: outputs: strix_skills: ${{ steps.select.outputs.strix_skills }} strix_any: ${{ steps.select.outputs.strix_any }} + # Whether the change touches the Instinct skill at all... instinct: ${{ steps.select.outputs.instinct }} + # ...versus whether it should actually run, which also needs the label. + instinct_run: ${{ steps.select.outputs.instinct_run }} steps: - name: Check out repository uses: actions/checkout@v4 @@ -87,10 +95,23 @@ jobs: has_instinct=$(printf '%s' "$skills" | python3 -c \ "import sys,json;n='$instinct_name';print('true' if n in json.load(sys.stdin) else 'false')") + # The MI300X runner is a scarce shared resource, so touching the + # skill is necessary but not sufficient: a maintainer has to opt the + # PR in with the `enable_mi_ci` label. A manual dispatch is already + # explicit human intent, so it does not need the label. + mi_label="${{ contains(github.event.pull_request.labels.*.name, 'enable_mi_ci') }}" + instinct_run=false + if [ "$has_instinct" = "true" ]; then + if [ "${{ github.event_name }}" = "workflow_dispatch" ] || [ "$mi_label" = "true" ]; then + instinct_run=true + fi + fi + echo "strix_skills=$strix" - echo "instinct=$has_instinct" + echo "instinct=$has_instinct (enable_mi_ci label: $mi_label -> run: $instinct_run)" echo "strix_skills=$strix" >> "$GITHUB_OUTPUT" echo "instinct=$has_instinct" >> "$GITHUB_OUTPUT" + echo "instinct_run=$instinct_run" >> "$GITHUB_OUTPUT" if [ "$strix" = "[]" ]; then echo "strix_any=false" >> "$GITHUB_OUTPUT" else @@ -168,10 +189,13 @@ jobs: # serving-llms-on-instinct actually serves a model on an AMD Instinct GPU, so # it can't run on the strix_halo runners above. It gets its own job on the # self-hosted MI300X runner. Linux-only (ROCm + Docker); no Windows leg. + # + # Opt-in: `instinct_run` is true only when the change touches the skill AND + # the PR carries the `enable_mi_ci` label (or the run was dispatched by hand). behavioral-instinct: name: Behavioral (serving-llms-on-instinct on MI300X) needs: discover - if: needs.discover.outputs.instinct == 'true' + if: needs.discover.outputs.instinct_run == 'true' runs-on: [self-hosted, Linux, X64, mi300x, gpu, rocm] timeout-minutes: 45 # Scopes the Anthropic key to this job alone: only jobs that name this @@ -228,6 +252,10 @@ jobs: # # * nothing testable changed -> pass (neutral). # * testable change -> pass iff the behavioral job passed. + # + # The Instinct leg is only gated on when it was actually requested, so a PR + # that touches the skill without the `enable_mi_ci` label is not blocked by a + # test it deliberately skipped. behavioral-gate: name: behavioral needs: [discover, behavioral, behavioral-instinct] @@ -239,6 +267,7 @@ jobs: INSTINCT_RESULT: ${{ needs.behavioral-instinct.result }} STRIX_ANY: ${{ needs.discover.outputs.strix_any }} INSTINCT_AFFECTED: ${{ needs.discover.outputs.instinct }} + INSTINCT_REQUESTED: ${{ needs.discover.outputs.instinct_run }} STRIX_SKILLS: ${{ needs.discover.outputs.strix_skills }} steps: - name: Verify behavioral results @@ -248,6 +277,7 @@ jobs: echo "behavioral-instinct: $INSTINCT_RESULT" echo "strix affected: $STRIX_ANY ($STRIX_SKILLS)" echo "instinct affected: $INSTINCT_AFFECTED" + echo "instinct requested: $INSTINCT_REQUESTED" # If discovery itself failed, surface that rather than guessing. if [ "$DISCOVER_RESULT" != "success" ]; then @@ -255,8 +285,15 @@ jobs: exit 1 fi + # Touching the Instinct skill without opting in is allowed, but say so + # loudly: the change went out without ever running on real hardware. + if [ "$INSTINCT_AFFECTED" = "true" ] && [ "$INSTINCT_REQUESTED" != "true" ]; then + echo "::warning::This PR changes serving-llms-on-instinct but has no" \ + "'enable_mi_ci' label, so the MI300X behavioral test did not run." + fi + # No skill or behavioral test changed: nothing to gate on. - if [ "$STRIX_ANY" != "true" ] && [ "$INSTINCT_AFFECTED" != "true" ]; then + if [ "$STRIX_ANY" != "true" ] && [ "$INSTINCT_REQUESTED" != "true" ]; then echo "No behavioral tests affected by this change." exit 0 fi @@ -268,7 +305,7 @@ jobs: echo "Strix behavioral tests did not pass ($BEHAVIORAL_RESULT)." >&2 failed=1 fi - if [ "$INSTINCT_AFFECTED" = "true" ] && [ "$INSTINCT_RESULT" != "success" ]; then + if [ "$INSTINCT_REQUESTED" = "true" ] && [ "$INSTINCT_RESULT" != "success" ]; then echo "Instinct behavioral test did not pass ($INSTINCT_RESULT)." >&2 failed=1 fi From 4552f1d9871f58b395150528a92438fce176beeb Mon Sep 17 00:00:00 2001 From: Daniel Holanda Date: Thu, 6 Aug 2026 21:56:52 -0700 Subject: [PATCH 5/5] Cleanup containers --- skills/local-ai-use/evals/evals.py | 2 +- .../serving-llms-on-instinct/evals/evals.py | 121 ++++++++++++------ 2 files changed, 84 insertions(+), 39 deletions(-) diff --git a/skills/local-ai-use/evals/evals.py b/skills/local-ai-use/evals/evals.py index 0b33f7e..13e59a9 100644 --- a/skills/local-ai-use/evals/evals.py +++ b/skills/local-ai-use/evals/evals.py @@ -39,5 +39,5 @@ def test_generate_image_of_a_cat(): run.should("Add a 'Local AI Use' block to AGENTS.md") # Negative behavioral expectations - run.should_not("Pull unrelated modalities for this image generation task") + run.should_not("Pull TTS or STT models for this image generation task") run.should_not("Reach for a cloud image path instead of local Lemonade") diff --git a/skills/serving-llms-on-instinct/evals/evals.py b/skills/serving-llms-on-instinct/evals/evals.py index 028a254..80a457c 100644 --- a/skills/serving-llms-on-instinct/evals/evals.py +++ b/skills/serving-llms-on-instinct/evals/evals.py @@ -27,6 +27,7 @@ from __future__ import annotations +import re import shutil import subprocess import sys @@ -43,58 +44,102 @@ # Small, ungated, single-GPU-friendly model keeps the serve fast. MODEL_ID = "Qwen/Qwen3-0.6B" +# Matches the model however it shows up in a container's name, image, or +# command: vllm-qwen3-0.6b, vllm-qwen3-0-6b, --model Qwen/Qwen3-0.6B, ... +_MODEL_MARKER = re.compile(r"qwen3[-_./]?0[-_.]?6b") + + +def _docker_rows() -> list[tuple[str, str]]: + """Return ``(container_id, searchable_text)`` for every container.""" + docker = shutil.which("docker") + if not docker: + return [] + try: + out = subprocess.run( + [docker, "ps", "-a", "--no-trunc", + "--format", "{{.ID}}\t{{.Names}}\t{{.Image}}\t{{.Command}}"], + capture_output=True, text=True, timeout=30, + ).stdout + except (subprocess.SubprocessError, OSError): + return [] + + rows = [] + for line in out.splitlines(): + container_id, _, rest = line.partition("\t") + if container_id.strip(): + rows.append((container_id.strip(), rest.lower())) + return rows + def _cleanup_test_containers() -> None: - """Best-effort removal of vLLM containers this test may have started. + """Best-effort removal of vLLM containers serving the tiny test model. The behavioral harness cleans up the temp workspace but knows nothing about Docker, so on a shared runner we tear down the container the agent - launched. Scoped to names the skill derives from the tiny test model so we - never touch someone else's endpoint. + launched. Matching is scoped to the tiny test model -- by name, image, or + the ``--model`` argument in the container command -- so we never touch + someone else's endpoint, but we still catch containers the agent named + differently than the skill's template suggests. + + This runs *before* the agent as well as after: a container left behind by + an earlier run is a healthy endpoint the agent will reasonably reuse + instead of launching its own, which makes the launch expectation fail. """ docker = shutil.which("docker") if not docker: return + ids = [cid for cid, text in _docker_rows() if _MODEL_MARKER.search(text)] + if not ids: + return + print(f" [cleanup] removing {len(ids)} stale {MODEL_ID} container(s)", flush=True) try: - ids = subprocess.run( - [docker, "ps", "-aq", "--filter", "name=qwen3-0.6b"], - capture_output=True, text=True, timeout=30, - ).stdout.split() - if ids: - subprocess.run([docker, "rm", "-f", *ids], capture_output=True, timeout=60) + subprocess.run([docker, "rm", "-f", *ids], capture_output=True, timeout=120) except (subprocess.SubprocessError, OSError): pass -def test_serve_tiny_model_on_instinct(): +@pytest.fixture +def clean_slate(): + """Guarantee no pre-existing test container, before and after the run.""" + _cleanup_test_containers() + leftover = [cid for cid, text in _docker_rows() if _MODEL_MARKER.search(text)] + if leftover: + pytest.fail( + f"could not remove pre-existing {MODEL_ID} container(s) {leftover}; " + "the agent would reuse the running endpoint instead of launching" + ) try: - with claude("opus", skill="serving-llms-on-instinct") as agent: - run = agent.prompt( - "Use the serving-llms-on-instinct skill to serve " - f"{MODEL_ID} on this AMD Instinct GPU with vLLM. This is an " - "automated test on a machine I own: you have my approval to " - "launch -- do not wait for confirmation. Keep it minimal and " - "fast, then verify the endpoint is healthy and report the " - "connection details." - ) - - # Deterministic: the skill was actually engaged. - run.logs_contains("serving-llms-on-instinct") - - # Positive behavioral expectations. - run.should("Detect the AMD Instinct GPU before configuring vLLM") - run.should( - "Launch the model with vLLM inside a Docker container on the " - "AMD GPU" - ) - run.should("Verify the vLLM endpoint is healthy after launching") - - # Negative behavioral expectations. - run.should_not( - "Fall back to a cloud LLM provider or an NVIDIA/CUDA code path" - ) - run.should_not( - "Serve a different, larger model than the one that was requested" - ) + yield finally: _cleanup_test_containers() + + +def test_serve_tiny_model_on_instinct(clean_slate): + with claude("opus", skill="serving-llms-on-instinct") as agent: + run = agent.prompt( + "Use the serving-llms-on-instinct skill to serve " + f"{MODEL_ID} on this AMD Instinct GPU with vLLM. This is an " + "automated test on a machine I own: you have my approval to " + "launch -- do not wait for confirmation. Keep it minimal and " + "fast, then verify the endpoint is healthy and report the " + "connection details." + ) + + # Deterministic: the skill was actually engaged. + run.logs_contains("serving-llms-on-instinct") + + # Positive behavioral expectations. + run.should("Detect the AMD Instinct GPU before configuring vLLM") + run.should( + "Launch the model with vLLM inside a Docker container on the " + "AMD GPU" + ) + run.should("Verify the vLLM endpoint is healthy after launching") + + # Negative behavioral expectations. + run.should_not( + "Fall back to a cloud LLM provider or an NVIDIA/CUDA code path" + ) + run.should_not( + "Serve a different, larger model than the one that was requested" + )