Add behavioral test to serving-llms-on-instinct

.github/workflows/behavioral.yml

@@ -11,13 +11,18 @@ name: behavioral
 #     behavioral test, the `behavioral` gate FAILS until the tests pass. A PR
 #     that touches nothing testable passes neutrally.
 #   * dispatchable -- run any subset by hand from the Actions tab.
+#   * opt-in for Instinct -- the MI300X leg is scarce and expensive, so it only
+#     runs on PRs carrying the `enable_mi_ci` label (see behavioral-instinct).
 #
 # Shape mirrors validate.yml: discover -> matrix -> single aggregate gate, so
 # branch protection can require just the `behavioral` check.
 
 on:
   pull_request:
-    types: [opened, synchronize, reopened]
+    # `labeled` is here so adding `enable_mi_ci` to an open PR starts the
+    # MI300X leg. Re-running an existing run would not work: the replayed
+    # event payload is the one from before the label was applied.
+    types: [opened, synchronize, reopened, labeled]
     paths:
       - "skills/**"
       - "eval/behavioral/**"
@@ -46,8 +51,12 @@ jobs:
     name: Select behavioral tests
     runs-on: ubuntu-latest
     outputs:
-      skills: ${{ steps.select.outputs.skills }}
-      any: ${{ steps.select.outputs.any }}
+      strix_skills: ${{ steps.select.outputs.strix_skills }}
+      strix_any: ${{ steps.select.outputs.strix_any }}
+      # Whether the change touches the Instinct skill at all...
+      instinct: ${{ steps.select.outputs.instinct }}
+      # ...versus whether it should actually run, which also needs the label.
+      instinct_run: ${{ steps.select.outputs.instinct_run }}
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
@@ -75,18 +84,45 @@ jobs:
               | uv run .github/scripts/select_behavioral.py --changed)
           fi
           echo "Selected skills: $skills"
-          echo "skills=$skills" >> "$GITHUB_OUTPUT"
-          if [ "$skills" = "[]" ]; then
-            echo "any=false" >> "$GITHUB_OUTPUT"
+
+          # serving-llms-on-instinct serves models on AMD Instinct GPUs, so it
+          # must run on the MI300X runner rather than the strix_halo ones.
+          # Split it out into its own list/flag; everything else stays on the
+          # existing strix_halo matrix.
+          instinct_name="serving-llms-on-instinct"
+          strix=$(printf '%s' "$skills" | python3 -c \
+            "import sys,json;n='$instinct_name';print(json.dumps([s for s in json.load(sys.stdin) if s!=n]))")
+          has_instinct=$(printf '%s' "$skills" | python3 -c \
+            "import sys,json;n='$instinct_name';print('true' if n in json.load(sys.stdin) else 'false')")
+
+          # The MI300X runner is a scarce shared resource, so touching the
+          # skill is necessary but not sufficient: a maintainer has to opt the
+          # PR in with the `enable_mi_ci` label. A manual dispatch is already
+          # explicit human intent, so it does not need the label.
+          mi_label="${{ contains(github.event.pull_request.labels.*.name, 'enable_mi_ci') }}"
+          instinct_run=false
+          if [ "$has_instinct" = "true" ]; then
+            if [ "${{ github.event_name }}" = "workflow_dispatch" ] || [ "$mi_label" = "true" ]; then
+              instinct_run=true
+            fi
+          fi
+
+          echo "strix_skills=$strix"
+          echo "instinct=$has_instinct (enable_mi_ci label: $mi_label -> run: $instinct_run)"
+          echo "strix_skills=$strix" >> "$GITHUB_OUTPUT"
+          echo "instinct=$has_instinct" >> "$GITHUB_OUTPUT"
+          echo "instinct_run=$instinct_run" >> "$GITHUB_OUTPUT"
+          if [ "$strix" = "[]" ]; then
+            echo "strix_any=false" >> "$GITHUB_OUTPUT"
           else
-            echo "any=true" >> "$GITHUB_OUTPUT"
+            echo "strix_any=true" >> "$GITHUB_OUTPUT"
           fi
 
   behavioral:
     name: Behavioral (${{ matrix.skill }} on ${{ matrix.os }})
     needs: discover
-    # Run whenever the change affects something testable.
-    if: needs.discover.outputs.any == 'true'
+    # Run whenever the change affects a strix_halo-testable skill.
+    if: needs.discover.outputs.strix_any == 'true'
     # Self-hosted Strix Halo runners. The OS label (Linux / Windows) comes from
     # the matrix below so each skill is exercised on both platforms.
     runs-on: [self-hosted, strix_halo, "${{ matrix.os }}"]
@@ -97,7 +133,7 @@ jobs:
       # One skill / OS failing should not hide the others' results.
       fail-fast: false
       matrix:
-        skill: ${{ fromJson(needs.discover.outputs.skills) }}
+        skill: ${{ fromJson(needs.discover.outputs.strix_skills) }}
         os: [Linux, Windows]
     env:
       ANTHROPIC_API_KEY: ${{ secrets.ORCHESTR_API_KEY }}
@@ -150,43 +186,131 @@ jobs:
           Write-Host "Running $test_file"
           python -m pytest -c pytest.ini -p conftest $test_file
 
+  # serving-llms-on-instinct actually serves a model on an AMD Instinct GPU, so
+  # it can't run on the strix_halo runners above. It gets its own job on the
+  # self-hosted MI300X runner. Linux-only (ROCm + Docker); no Windows leg.
+  #
+  # Opt-in: `instinct_run` is true only when the change touches the skill AND
+  # the PR carries the `enable_mi_ci` label (or the run was dispatched by hand).
+  behavioral-instinct:
+    name: Behavioral (serving-llms-on-instinct on MI300X)
+    needs: discover
+    if: needs.discover.outputs.instinct_run == 'true'
+    runs-on: [self-hosted, Linux, X64, mi300x, gpu, rocm]
+    timeout-minutes: 45
+    # Scopes the Anthropic key to this job alone: only jobs that name this
+    # environment can read its secrets, unlike a repo-level secret.
+    environment: behavioral-instinct
+    env:
+      # The MI300X runner lives outside the AMD network, so it can't reach the
+      # internal gateway the strix_halo jobs use. It calls api.anthropic.com
+      # directly with its own budgeted key, hence no BASE_URL / CUSTOM_HEADERS.
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      BEHAVIORAL_SKILL: serving-llms-on-instinct
+      # Model pin: opus only. The harness also enforces this under CI.
+      BEHAVIORAL_MODEL: opus
+    steps:
+      - name: Verify the Anthropic key is available
+        run: |
+          if [ -z "${ANTHROPIC_API_KEY:-}" ]; then
+            echo "ANTHROPIC_API_KEY resolved to an empty value. GitHub withholds" >&2
+            echo "secrets from pull requests opened from forks; re-run this from" >&2
+            echo "a branch in amd/skills, or check that the secret is set on the" >&2
+            echo "behavioral-instinct environment." >&2
+            exit 1
+          fi
+
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Install the claude CLI
+        run: npm install -g @anthropic-ai/claude-code
+
+      - name: Install behavioral test dependencies
+        run: pip install -r eval/behavioral/requirements.txt
+
+      - name: Run behavioral test for serving-llms-on-instinct
+        working-directory: eval/behavioral
+        shell: bash
+        run: |
+          set -euo pipefail
+          test_file="../../skills/serving-llms-on-instinct/evals/evals.py"
+          echo "Running $test_file"
+          python -m pytest -c pytest.ini -p conftest "$test_file"
+
   # Single aggregate gate. Mark THIS check required in branch protection.
   #
   #   * nothing testable changed -> pass (neutral).
   #   * testable change          -> pass iff the behavioral job passed.
+  #
+  # The Instinct leg is only gated on when it was actually requested, so a PR
+  # that touches the skill without the `enable_mi_ci` label is not blocked by a
+  # test it deliberately skipped.
   behavioral-gate:
     name: behavioral
-    needs: [discover, behavioral]
+    needs: [discover, behavioral, behavioral-instinct]
     if: always()
     runs-on: ubuntu-latest
     env:
       DISCOVER_RESULT: ${{ needs.discover.result }}
       BEHAVIORAL_RESULT: ${{ needs.behavioral.result }}
-      AFFECTED: ${{ needs.discover.outputs.any }}
-      SKILLS: ${{ needs.discover.outputs.skills }}
+      INSTINCT_RESULT: ${{ needs.behavioral-instinct.result }}
+      STRIX_ANY: ${{ needs.discover.outputs.strix_any }}
+      INSTINCT_AFFECTED: ${{ needs.discover.outputs.instinct }}
+      INSTINCT_REQUESTED: ${{ needs.discover.outputs.instinct_run }}
+      STRIX_SKILLS: ${{ needs.discover.outputs.strix_skills }}
     steps:
       - name: Verify behavioral results
         run: |
-          echo "discover:    $DISCOVER_RESULT"
-          echo "behavioral:  $BEHAVIORAL_RESULT"
-          echo "affected:    $AFFECTED ($SKILLS)"
+          echo "discover:            $DISCOVER_RESULT"
+          echo "behavioral (strix):  $BEHAVIORAL_RESULT"
+          echo "behavioral-instinct: $INSTINCT_RESULT"
+          echo "strix affected:      $STRIX_ANY ($STRIX_SKILLS)"
+          echo "instinct affected:   $INSTINCT_AFFECTED"
+          echo "instinct requested:  $INSTINCT_REQUESTED"
 
           # If discovery itself failed, surface that rather than guessing.
           if [ "$DISCOVER_RESULT" != "success" ]; then
             echo "The discover job did not succeed ($DISCOVER_RESULT)." >&2
             exit 1
           fi
 
+          # Touching the Instinct skill without opting in is allowed, but say so
+          # loudly: the change went out without ever running on real hardware.
+          if [ "$INSTINCT_AFFECTED" = "true" ] && [ "$INSTINCT_REQUESTED" != "true" ]; then
+            echo "::warning::This PR changes serving-llms-on-instinct but has no" \
+                 "'enable_mi_ci' label, so the MI300X behavioral test did not run."
+          fi
+
           # No skill or behavioral test changed: nothing to gate on.
-          if [ "$AFFECTED" != "true" ]; then
+          if [ "$STRIX_ANY" != "true" ] && [ "$INSTINCT_REQUESTED" != "true" ]; then
             echo "No behavioral tests affected by this change."
             exit 0
           fi
 
-          # Something testable changed: the gate reflects the test result.
-          if [ "$BEHAVIORAL_RESULT" = "success" ]; then
-            echo "All affected behavioral tests passed."
-            exit 0
+          # Each affected leg must have passed. A leg that wasn't triggered is
+          # 'skipped', which we ignore.
+          failed=0
+          if [ "$STRIX_ANY" = "true" ] && [ "$BEHAVIORAL_RESULT" != "success" ]; then
+            echo "Strix behavioral tests did not pass ($BEHAVIORAL_RESULT)." >&2
+            failed=1
+          fi
+          if [ "$INSTINCT_REQUESTED" = "true" ] && [ "$INSTINCT_RESULT" != "success" ]; then
+            echo "Instinct behavioral test did not pass ($INSTINCT_RESULT)." >&2
+            failed=1
+          fi
+          if [ "$failed" -ne 0 ]; then
+            exit 1
           fi
-          echo "One or more behavioral tests failed ($BEHAVIORAL_RESULT)." >&2
-          exit 1
+          echo "All affected behavioral tests passed."
+          exit 0

skills/local-ai-use/evals/evals.py

@@ -39,5 +39,5 @@ def test_generate_image_of_a_cat():
             run.should("Add a 'Local AI Use' block to AGENTS.md")
 
             # Negative behavioral expectations
-            run.should_not("Pull unrelated modalities for this image generation task")
+            run.should_not("Pull TTS or STT models for this image generation task")
             run.should_not("Reach for a cloud image path instead of local Lemonade")

skills/serving-llms-on-instinct/evals/evals.py

@@ -0,0 +1,145 @@
+# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved.
+#
+# See LICENSE for license information.
+
+"""Behavioral test for the `serving-llms-on-instinct` skill.
+
+Serves a deliberately tiny model (``Qwen/Qwen3-0.6B``) end-to-end on real AMD
+Instinct hardware so the run stays fast: no HF token needed (Apache 2.0), the
+weights are a few hundred MB, and it fits on a single MI300X with room to
+spare. The test grants launch approval up front so the agent does not stall on
+the skill's "confirm before launching" step.
+
+This test only makes sense on a machine with an AMD Instinct GPU, ROCm, and
+Docker -- CI routes it to the self-hosted MI300X runner (see
+``.github/workflows/behavioral.yml``). Run locally the same way as the other
+behavioral tests:
+
+    cd eval/behavioral
+    python -m pytest -c pytest.ini -p conftest \
+        ../../skills/serving-llms-on-instinct/evals/evals.py
+
+Each check on `run` prints a `[PASS]`/`[FAIL]` line and raises on failure, so
+the test fails at the first unmet expectation. `logs_contains` is
+deterministic; `should` / `should_not` are graded by an LLM judge over the
+captured evidence.
+"""
+
+from __future__ import annotations
+
+import re
+import shutil
+import subprocess
+import sys
+
+import pytest
+
+from harness import claude
+
+pytestmark = pytest.mark.skipif(
+    sys.platform == "win32",
+    reason="serving-llms-on-instinct requires a Linux + ROCm host",
+)
+
+# Small, ungated, single-GPU-friendly model keeps the serve fast.
+MODEL_ID = "Qwen/Qwen3-0.6B"
+
+# Matches the model however it shows up in a container's name, image, or
+# command: vllm-qwen3-0.6b, vllm-qwen3-0-6b, --model Qwen/Qwen3-0.6B, ...
+_MODEL_MARKER = re.compile(r"qwen3[-_./]?0[-_.]?6b")
+
+
+def _docker_rows() -> list[tuple[str, str]]:
+    """Return ``(container_id, searchable_text)`` for every container."""
+    docker = shutil.which("docker")
+    if not docker:
+        return []
+    try:
+        out = subprocess.run(
+            [docker, "ps", "-a", "--no-trunc",
+             "--format", "{{.ID}}\t{{.Names}}\t{{.Image}}\t{{.Command}}"],
+            capture_output=True, text=True, timeout=30,
+        ).stdout
+    except (subprocess.SubprocessError, OSError):
+        return []
+
+    rows = []
+    for line in out.splitlines():
+        container_id, _, rest = line.partition("\t")
+        if container_id.strip():
+            rows.append((container_id.strip(), rest.lower()))
+    return rows
+
+
+def _cleanup_test_containers() -> None:
+    """Best-effort removal of vLLM containers serving the tiny test model.
+
+    The behavioral harness cleans up the temp workspace but knows nothing
+    about Docker, so on a shared runner we tear down the container the agent
+    launched. Matching is scoped to the tiny test model -- by name, image, or
+    the ``--model`` argument in the container command -- so we never touch
+    someone else's endpoint, but we still catch containers the agent named
+    differently than the skill's template suggests.
+
+    This runs *before* the agent as well as after: a container left behind by
+    an earlier run is a healthy endpoint the agent will reasonably reuse
+    instead of launching its own, which makes the launch expectation fail.
+    """
+    docker = shutil.which("docker")
+    if not docker:
+        return
+    ids = [cid for cid, text in _docker_rows() if _MODEL_MARKER.search(text)]
+    if not ids:
+        return
+    print(f"  [cleanup] removing {len(ids)} stale {MODEL_ID} container(s)", flush=True)
+    try:
+        subprocess.run([docker, "rm", "-f", *ids], capture_output=True, timeout=120)
+    except (subprocess.SubprocessError, OSError):
+        pass
+
+
+@pytest.fixture
+def clean_slate():
+    """Guarantee no pre-existing test container, before and after the run."""
+    _cleanup_test_containers()
+    leftover = [cid for cid, text in _docker_rows() if _MODEL_MARKER.search(text)]
+    if leftover:
+        pytest.fail(
+            f"could not remove pre-existing {MODEL_ID} container(s) {leftover}; "
+            "the agent would reuse the running endpoint instead of launching"
+        )
+    try:
+        yield
+    finally:
+        _cleanup_test_containers()
+
+
+def test_serve_tiny_model_on_instinct(clean_slate):
+    with claude("opus", skill="serving-llms-on-instinct") as agent:
+        run = agent.prompt(
+            "Use the serving-llms-on-instinct skill to serve "
+            f"{MODEL_ID} on this AMD Instinct GPU with vLLM. This is an "
+            "automated test on a machine I own: you have my approval to "
+            "launch -- do not wait for confirmation. Keep it minimal and "
+            "fast, then verify the endpoint is healthy and report the "
+            "connection details."
+        )
+
+        # Deterministic: the skill was actually engaged.
+        run.logs_contains("serving-llms-on-instinct")
+
+        # Positive behavioral expectations.
+        run.should("Detect the AMD Instinct GPU before configuring vLLM")
+        run.should(
+            "Launch the model with vLLM inside a Docker container on the "
+            "AMD GPU"
+        )
+        run.should("Verify the vLLM endpoint is healthy after launching")
+
+        # Negative behavioral expectations.
+        run.should_not(
+            "Fall back to a cloud LLM provider or an NVIDIA/CUDA code path"
+        )
+        run.should_not(
+            "Serve a different, larger model than the one that was requested"
+        )