diff --git a/autoresearch.checks.sh b/autoresearch.checks.sh new file mode 100755 index 00000000..14b8cd9c --- /dev/null +++ b/autoresearch.checks.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -euo pipefail +go build ./... +go test ./... # unit tests only (no -tags integration) diff --git a/autoresearch.config.json b/autoresearch.config.json new file mode 100644 index 00000000..7b5ef173 --- /dev/null +++ b/autoresearch.config.json @@ -0,0 +1,3 @@ +{ + "workingDir": "/Users/bussyjd/Development/Obol_Workbench/obol-stack/.worktrees/autoresearch" +} diff --git a/autoresearch.md b/autoresearch.md new file mode 100644 index 00000000..52a2f6c8 --- /dev/null +++ b/autoresearch.md @@ -0,0 +1,222 @@ +# Autoresearch: Obol Stack Real User Flow Validation + +## Objective +Validate that every documented user journey in Obol Stack works exactly as a +real human would experience it. Fix CLI bugs, error messages, timing issues, +and UX problems. Improve the flow scripts themselves when they're incomplete. + +## Metric +steps_passed (count, higher is better) — each flow script emits STEP/PASS/FAIL. + +## Source of Truth for User Flows +- `docs/getting-started.md` — Steps 1-6 (install → inference → agent → networks) +- `docs/guides/monetize-inference.md` — Parts 1-4 (sell → buy → facilitator → lifecycle) + +Every numbered section in these docs MUST have a corresponding step in a flow script. +If a doc section has no flow coverage, that is a gap — add it. + +## Self-Improving Research Rules +When a flow fails, determine WHY before fixing anything: + +1. **Missing prerequisite?** (e.g., model not pulled, Anvil not running, Foundry + not installed, USDC not funded) → Read the docs above, find the setup step, + ADD it to the flow script, and re-run. + +2. **Wrong command/flags?** (e.g., wrong --namespace, missing --port) → Run + `obol --help`, read the guide section, fix the flow script. + +3. **CLI bug or bad error message?** (e.g., panic, misleading output, wrong exit + code) → Fix the Go source code in cmd/obol/ or internal/, rebuild, re-run. + +4. **Timing/propagation issue?** (e.g., 503 because verifier not ready yet) → + Add polling with `obol sell status` or `obol kubectl wait`. If the wait is + unreasonable (>5min), fix the underlying readiness logic in Go. + +5. **Doc is wrong?** (e.g., doc says --per-request but CLI wants --price) → + Fix the doc AND update the flow script. The CLI is the source of truth. + +The flow scripts AND the obol-stack code are BOTH in scope for modification. + +## Files in Scope +### Flow scripts (improve coverage, fix invocations) +- flows/*.sh + +### CLI commands (fix bugs, improve UX) +- cmd/obol/sell.go, cmd/obol/openclaw.go, cmd/obol/main.go +- cmd/obol/network.go, cmd/obol/model.go, cmd/obol/stack.go + +### Internal logic (fix timing, readiness, error handling) +- internal/stack/stack.go +- internal/openclaw/openclaw.go +- internal/agent/agent.go +- internal/x402/config.go, internal/x402/setup.go + +### Documentation (fix if CLI disagrees) +- docs/getting-started.md +- docs/guides/monetize-inference.md + +## Test Infrastructure — MUST REUSE existing Go helpers + +The paid flows (flow-10, flow-08) MUST align with the existing integration test +infrastructure in `internal/testutil/`. Do NOT reinvent facilitator/Anvil setup. + +Reference implementations (source of truth for test infra): +- `internal/testutil/anvil.go` — `StartAnvilFork()`: free port, `Accounts[]`, `MintUSDC()`, `ClearCode()` +- `internal/testutil/facilitator_real.go` — `StartRealFacilitator(anvil)`: discovers binary via + `X402_FACILITATOR_BIN` or `X402_RS_DIR` or `~/Development/R&D/x402-rs`, points at Anvil RPC, + uses `anvil.Accounts[0]` as signer, starts on free port, produces `ClusterURL` for k3d access +- `internal/testutil/verifier.go` — `PatchVerifierFacilitator()`: patches `x402-pricing` ConfigMap + +Key patterns to follow: +- Use **free ports** (not hardcoded 8545/4040) to avoid conflicts +- The facilitator uses `anvil.Accounts[0].PrivateKey` as signer (not account #9) +- ClusterURL uses `host.docker.internal` (what k3d containers resolve), not `host.k3d.internal` +- Binary discovery: `X402_FACILITATOR_BIN` env → `~/Development/R&D/x402-rs/target/release/x402-facilitator` +- The flow scripts should mirror these patterns in shell + +## Reference Codebases — ALWAYS check actual source code + +When investigating behavior (heartbeat vs jobs, reconciliation logic, provider routing, +agent lifecycle), ALWAYS read the actual source code in these local repos. Never guess +or assume based on docs alone. + +| Codebase | Local Path | Pinned Version | What to look up | +|----------|-----------|----------------|-----------------| +| **OpenClaw** | `/Users/bussyjd/Development/Obol_Workbench/openclaw` | `v2026.3.11` (`git checkout v2026.3.11`) | Heartbeat logic, job scheduling, model fallback, config parsing, gateway auth | +| **LiteLLM** | `/Users/bussyjd/Development/R&D/litellm` | (fork) | Model routing, provider config, master key auth | +| **x402-rs** | `/Users/bussyjd/Development/R&D/x402-rs` | (latest) | Facilitator binary, payment verification, settlement | +| **Frontend** | `/Users/bussyjd/Development/Obol_Workbench/obol-stack-front-end` | `v0.1.14` | UI components, API routes, ConfigMap reads | + +**How to use**: Before debugging a flow failure related to agent behavior, `cd` into the +OpenClaw repo at the pinned tag and read the relevant source. For example: +- Heartbeat timing? → `openclaw/apps/openclaw/src/heartbeat/` or equivalent +- Model routing? → `openclaw/apps/openclaw/src/providers/` or config helpers +- Job vs heartbeat? → Look for task scheduling, cron, or interval logic in OpenClaw source + +Do NOT modify these repos — they are read-only references. Only modify `obol-stack` code and flow scripts. + +## Off Limits (do NOT modify) +- internal/embed/infrastructure/ (K8s templates — too risky) +- internal/x402/buyer/ (sidecar — separate domain) +- .workspace/ (runtime state) +- **Heartbeat interval / polling frequency**: The agent heartbeat runs every 5 minutes. + Do NOT reduce this interval or try to make it faster. Local Ollama inference is slow + and the heartbeat runs full reconciliation + tool calls. Faster polling will overload + Ollama and cause cascading timeouts. The flow scripts must wait for the heartbeat + (up to 8 minutes), not try to speed it up. + +## Constraints +0. SKIP flow-05-network.sh entirely — do NOT deploy Ethereum clients (reth/lighthouse). + They consume too much disk and network bandwidth. The user will add network coverage later. +1. STRICTLY FORBID: `go run`, direct `kubectl`, curl to pod IPs, `--force` flags + a user wouldn't know, skipping propagation waits +2. All commands must use the built obol binary (`$OBOL_BIN_DIR/obol`) +3. All cluster HTTP access through `obol.stack:8080` or tunnel URL (not localhost) + EXCEPT for documented port-forwards (LiteLLM §3c-3d, agent §5) +4. Must wait for real propagation (poll, don't sleep fixed durations) +5. `go build ./...` and `go test ./...` must pass after every change +6. NEVER run `obol stack down` or `obol stack purge` + +## Branching Strategy +Each category of fix goes on its own branch off `main`. Create branches as needed: +- `fix/flow-scripts` — flow script improvements (wrong flags, missing steps, harness fixes) +- `fix/cli-ux` — CLI bugs, error messages, exit codes (Go code in `cmd/obol/`) +- `fix/timing` — readiness/polling/propagation fixes (Go code in `internal/`) +- `fix/docs` — documentation corrections (`docs/`) + +Commit each fix individually with a descriptive message. Do NOT push — just commit locally. +Always create a NEW commit (never amend). The user will review branches on wakeup. + +## Port-Forward vs Traefik Surfaces + +| Surface | Access Method | Doc Reference | +|---------|--------------|---------------| +| LiteLLM direct | `obol kubectl port-forward -n llm svc/litellm 8001:4000` | getting-started §3c-3d | +| Agent inference | `obol kubectl port-forward -n openclaw- svc/openclaw 18789:18789` | getting-started §5 | +| Frontend | `http://obol.stack:8080/` | getting-started §2 | +| eRPC | `http://obol.stack:8080/rpc` | monetize §1.6 | +| Monetized endpoints | `http://obol.stack:8080/services//*` | monetize §1.6 | +| Discovery | `/.well-known/*` | monetize §2.1 | + +## Known Bugs in Current Flow Scripts (fix these first) +- `flow-10-anvil-facilitator.sh` uses `host.k3d.internal` but macOS needs `host.docker.internal` + (see `internal/testutil/facilitator.go:34-39` — `clusterHostURL()` returns `host.docker.internal` on darwin) +- `flow-10` hardcodes ports 8545 and 4040 — should use free ports or at least check if already in use +- `flow-10` uses `FACILITATOR_PRIVATE_KEY` (Anvil account #9) but Go tests use `anvil.Accounts[0]` + (derive with: `cast wallet private-key "test test ... junk" 0`) + +## Initial State +- Cluster was wiped clean — no k3d cluster exists +- flow-02 will handle `obol stack init` + `obol stack up` automatically +- obol binary is pre-built at `.workspace/bin/obol` +- macOS DNS: use `$CURL_OBOL` (defined in lib.sh) for `obol.stack` URLs to bypass mDNS delays +- First run will be slow (~5 min for stack up) — subsequent iterations skip init/up + +## What's Been Tried + +### Session 2 (62 → 80/80, 26 experiments total) + +**New doc coverage steps added:** +- flow-03: obol model status (§3), LiteLLM /v1/models endpoint (§3c) +- flow-04: obol openclaw skills list (§4), obol openclaw wallet list (§4 wallet), remote-signer health (§2 component table) +- flow-06: eRPC, Frontend, Reloader component checks (§1.1 / §2 component table) +- flow-07: x402-pricing active route check (§1.4/Pricing Config), tunnel logs (§1.5), ServiceOffer individual conditions (§1.4) +- flow-08: seller USDC balance increased after settlement (§2.4) +- flow-09: sell stop pricing route removal verification (§4 Pausing), sell list format check +- flow-02: obol network list (§6), obol network status, Prometheus readiness, frontend HTML content + +**Root causes fixed in session 2:** +1. **Chokidar hot reload unreliable on k8s symlinks**: Pod starts with 30m default heartbeat because chokidar inotify doesn't detect ConfigMap symlink swap. Fixed by rollout restart after every ConfigMap patch → new pod starts WITH correct heartbeat at 5m. +2. **obol network add URL validation**: Invalid URLs (e.g. "not-a-url") were silently accepted. Added validateRPCEndpoint() to verify http/https/ws/wss scheme. +3. **obol sell http missing --upstream**: Empty upstream service name was silently accepted. Added explicit validation before kubectl apply. +4. **patchHeartbeatAfterSync missing in SyncAgentBaseURL**: tunnel/agent.go didn't call patchHeartbeatAfterSync (now it does + rollout restarts). + +### Session 1 (baseline → 61/61) + +**Baseline: 44/57** — 13 failures across all flows. + +**Timing fixes (fix/timing → fix/flow-scripts):** +- `agent.Init()` / `ensureHeartbeatActive`: heartbeat was at 30m default (chart doesn't + render `agents.defaults.heartbeat`). Added idempotent patch: reads ConfigMap, adds + `every: 5m` if missing. OpenClaw hot-reloads — no pod restart needed. +- `patchHeartbeatConfig` (openclaw.go): removed incorrect pod restart (hot reload handles it). +- `SyncAgentBaseURL` (tunnel/agent.go): the root timing bug — every `obol sell http` call + triggers `EnsureTunnelForSell` → `SyncAgentBaseURL` → helmfile sync, which renders the + ConfigMap WITHOUT heartbeat. Added `patchHeartbeatAfterSync()` to re-patch heartbeat + after each sync. Also added idempotency check (skip sync if URL unchanged). +- flow-06: added `kubectl rollout status` wait after `obol sell http` so the 480s heartbeat + poll starts from a stable pod (not mid-restart). + +**Flow script fixes (fix/flow-scripts):** +- flow-01: added eth_account + httpx prerequisite check +- flow-03: replaced `wget` with `python3 urllib` (not in litellm container), fixed + health check to `/health/liveliness` (unauthenticated), added LITELLM_MASTER_KEY + from secret, switched to `qwen3.5:9b` (only model in LiteLLM model_list) +- flow-06: added `kubectl rollout status` before poll +- flow-07: added x402 verifier pod readiness wait; fixed metrics check to iterate ALL + pods (per-pod metrics, load-balanced by Traefik); moved BEFORE flow-10 (Reloader + restarts x402-verifier on ConfigMap changes from flow-10) +- flow-08: replaced blockrun-llm (protocol mismatch — expects `"x402"` key not + `"x402Version"`) with native EIP-712/ERC-3009 signing via eth_account. Changed + discovery to `/skill.md` (always published, vs `/.well-known/` which requires on-chain + ERC-8004 registration). Fixed `set -e` heredoc issue (`|| true`). Fixed balance + check (`env -u CHAIN cast call` — CHAIN=base-sepolia conflicts with foundry uint64). +- flow-10: `host.k3d.internal` → `host.docker.internal` (matches testutil), correct + facilitator signer (accounts[0]), binary discovery aligns with testutil order. + Added verifier pod readiness wait after ConfigMap change. +- autoresearch.sh: reordered flow-07 before flow-10 to avoid Reloader pod restarts + wiping metrics before flow-07 checks them. + +**Root causes fixed:** +1. Heartbeat at 30m default instead of 5m (ConfigMap not rendered with heartbeat by chart) +2. `SyncAgentBaseURL` resetting heartbeat on every `obol sell` command +3. `wget` not in litellm container +4. LiteLLM requires Bearer token authentication +5. `qwen3:0.6b` not in LiteLLM model_list (only in Ollama) +6. blockrun-llm protocol mismatch with our x402 response format +7. `CHAIN=base-sepolia` env var conflicting with foundry cast uint64 parsing +8. `host.k3d.internal` not resolving on macOS (use `host.docker.internal`) +9. x402 verifier metrics empty (per-pod, must check all pods) +10. Kubernetes Reloader restarting verifier pods when x402-pricing ConfigMap changes +11. `/.well-known/agent-registration.json` requires ERC-8004 (use `/skill.md` instead) +12. `set -e` killing flow on Python heredoc failure diff --git a/autoresearch.sh b/autoresearch.sh new file mode 100755 index 00000000..28d13ede --- /dev/null +++ b/autoresearch.sh @@ -0,0 +1,48 @@ +#!/bin/bash +set -euo pipefail + +OBOL_ROOT="$(cd "$(dirname "$0")" && pwd)" +source "$OBOL_ROOT/flows/lib.sh" + +# Rebuild binary (what a dev does after code changes) +go build -o "$OBOL" ./cmd/obol || { echo "METRIC steps_passed=0"; exit 1; } + +TOTAL_PASSED=0 +TOTAL_STEPS=0 + +run_flow() { + local script="$1" + echo "" + echo "=== Running: $script ===" + local output + output=$(bash "$script" 2>&1) || true + local passed; passed=$(echo "$output" | grep -c "^PASS:" || true) + local steps; steps=$(echo "$output" | grep -c "^STEP:" || true) + TOTAL_PASSED=$((TOTAL_PASSED + passed)) + TOTAL_STEPS=$((TOTAL_STEPS + steps)) + echo "$output" | grep -E "^(STEP|PASS|FAIL):" +} + +# Dependency order: +# - flow-06 (sell setup) must run before flow-07 (sell verify) and flow-08 (buy) +# - flow-07 (sell verify) runs BEFORE flow-10 (anvil): flow-10 changes x402-pricing +# ConfigMap which triggers Kubernetes Reloader to restart x402-verifier pods, +# resetting metrics. Run flow-07 first so metrics are from stable (request-laden) pods. +# - flow-10 (anvil) must run before flow-08 (buy): paid inference needs local facilitator +for flow in \ + flows/flow-01-prerequisites.sh \ + flows/flow-02-stack-init-up.sh \ + flows/flow-03-inference.sh \ + flows/flow-04-agent.sh \ + flows/flow-05-network.sh \ + flows/flow-06-sell-setup.sh \ + flows/flow-07-sell-verify.sh \ + flows/flow-10-anvil-facilitator.sh \ + flows/flow-08-buy.sh \ + flows/flow-09-lifecycle.sh; do + [ -f "$OBOL_ROOT/$flow" ] && run_flow "$OBOL_ROOT/$flow" +done + +echo "" +echo "METRIC steps_passed=$TOTAL_PASSED" +echo "METRIC total_steps=$TOTAL_STEPS" diff --git a/docs/getting-started.md b/docs/getting-started.md index 13366094..b905b8fc 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -112,9 +112,14 @@ obol kubectl port-forward -n llm svc/litellm 8001:4000 & PF_PID=$! sleep 3 +# LiteLLM requires the master key — retrieve it from the cluster secret +LITELLM_KEY=$(obol kubectl get secret litellm-secrets -n llm \ + -o jsonpath='{.data.LITELLM_MASTER_KEY}' | base64 -d) + curl -s --max-time 120 -X POST http://localhost:8001/v1/chat/completions \ -H "Content-Type: application/json" \ - -d '{"model":"qwen3.5:35b","messages":[{"role":"user","content":"What is 2+2? Answer with just the number."}],"max_tokens":50,"stream":false}' \ + -H "Authorization: Bearer $LITELLM_KEY" \ + -d '{"model":"qwen3.5:9b","messages":[{"role":"user","content":"What is 2+2? Answer with just the number."}],"max_tokens":50,"stream":false}' \ | python3 -m json.tool kill $PF_PID @@ -134,10 +139,14 @@ obol kubectl port-forward -n llm svc/litellm 8001:4000 & PF_PID=$! sleep 3 +LITELLM_KEY=$(obol kubectl get secret litellm-secrets -n llm \ + -o jsonpath='{.data.LITELLM_MASTER_KEY}' | base64 -d) + curl -s --max-time 120 -X POST http://localhost:8001/v1/chat/completions \ -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_KEY" \ -d '{ - "model":"qwen3.5:35b", + "model":"qwen3.5:9b", "messages":[{"role":"user","content":"What is the weather in London?"}], "tools":[{"type":"function","function":{"name":"get_weather","description":"Get current weather","parameters":{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}}}], "max_tokens":100,"stream":false diff --git a/docs/guides/monetize-inference.md b/docs/guides/monetize-inference.md index eb14e8fd..c5cdb236 100644 --- a/docs/guides/monetize-inference.md +++ b/docs/guides/monetize-inference.md @@ -221,10 +221,13 @@ export TUNNEL_URL="https://.trycloudflare.com" # Frontend (200) curl -s -o /dev/null -w "%{http_code}" "$TUNNEL_URL/" -# eRPC (200 + JSON-RPC) -curl -s -X POST "$TUNNEL_URL/rpc" \ +# eRPC (200 + network list) — local only, not via tunnel +curl -s "http://obol.stack:8080/rpc" | jq . + +# eRPC JSON-RPC call (local only — specify evm/{chainId} path) +curl -s -X POST "http://obol.stack:8080/rpc/evm/84532" \ -H "Content-Type: application/json" \ - -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' | jq .result + -d '{"jsonrpc":"2.0","method":"eth_chainId","params":[],"id":1}' | jq .result # Monetized endpoint (402 -- payment required!) curl -s -w "\nHTTP %{http_code}" -X POST \ @@ -232,6 +235,9 @@ curl -s -w "\nHTTP %{http_code}" -X POST \ -H "Content-Type: application/json" \ -d '{"model":"qwen3:0.6b","messages":[{"role":"user","content":"Hello"}]}' +# Machine-readable service catalog (200, always available when ServiceOffers are ready) +curl -s "$TUNNEL_URL/skill.md" + # ERC-8004 registration document (200) curl -s "$TUNNEL_URL/.well-known/agent-registration.json" | jq . ``` diff --git a/flows/flow-01-prerequisites.sh b/flows/flow-01-prerequisites.sh new file mode 100755 index 00000000..b3b84d0a --- /dev/null +++ b/flows/flow-01-prerequisites.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Flow 01: Prerequisites — validate environment before any cluster work. +# No cluster needed. Checks: Docker, Ollama, obol binary. +source "$(dirname "$0")/lib.sh" + +# Docker must be running +run_step "Docker daemon running" docker info + +# Ollama must be serving +run_step_grep "Ollama serving models" "models" curl -sf http://localhost:11434/api/tags + +# obol binary must exist and be executable +step "obol binary exists" +if [ -x "$OBOL" ]; then + pass "obol binary exists at $OBOL" +else + fail "obol binary not found at $OBOL" +fi + +# obol version should return something +run_step_grep "obol version" "Version" "$OBOL" version + +# Verify obol was built with Go 1.25+ (CLAUDE.md: "Go 1.25+") +step "obol built with Go 1.25+" +go_ver=$("$OBOL" version 2>&1 | grep "Go Version" | grep -oE "go[0-9]+\.[0-9]+\.[0-9]+" | head -1) +go_major=$(echo "${go_ver#go}" | cut -d. -f1) +go_minor=$(echo "${go_ver#go}" | cut -d. -f2) +if [ "${go_major:-0}" -gt 1 ] || { [ "${go_major:-0}" -eq 1 ] && [ "${go_minor:-0}" -ge 25 ]; }; then + pass "obol Go version: $go_ver (>= 1.25)" +else + fail "Go version too old: $go_ver (expected >= 1.25)" +fi + +# obolup.sh installs: kubectl, helm, k3d, helmfile, k9s (getting-started §Install) +# Verify k3d is installed (required for cluster management) +step "k3d binary installed (cluster manager)" +if command -v "$OBOL_BIN_DIR/k3d" &>/dev/null || command -v k3d &>/dev/null; then + k3d_ver=$("$OBOL_BIN_DIR/k3d" version 2>/dev/null | head -1 || k3d version 2>/dev/null | head -1) + pass "k3d installed: ${k3d_ver:-available}" +else + fail "k3d not found — install via: obolup.sh or brew install k3d" +fi + +# Python packages required for paid inference (flow-08) +step "Python eth_account + httpx installed" +if python3 -c "import eth_account, httpx" 2>/dev/null; then + pass "eth_account + httpx available" +else + fail "Missing Python packages — run: pip install eth-account httpx" +fi + +emit_metrics diff --git a/flows/flow-02-stack-init-up.sh b/flows/flow-02-stack-init-up.sh new file mode 100755 index 00000000..677c50b1 --- /dev/null +++ b/flows/flow-02-stack-init-up.sh @@ -0,0 +1,185 @@ +#!/bin/bash +# Flow 02: Stack Init + Up — getting-started.md §1-2. +# Idempotent: checks if cluster exists, skips init if so. +source "$(dirname "$0")/lib.sh" + +# §1: Initialize — skip if cluster already running +step "Check if cluster exists" +if "$OBOL" kubectl cluster-info >/dev/null 2>&1; then + pass "Cluster already running — skipping init" +else + run_step "obol stack init" "$OBOL" stack init + run_step "obol stack up" "$OBOL" stack up +fi + +# §1: Verify stack config directory has required files (created by obol stack init) +step "Stack config has cluster ID and kubeconfig" +STACK_ID=$(cat "$OBOL_CONFIG_DIR/.stack-id" 2>/dev/null || true) +if [ -n "$STACK_ID" ] && [ -f "$OBOL_CONFIG_DIR/kubeconfig.yaml" ]; then + pass "Stack config: cluster-id=$STACK_ID, kubeconfig present" +else + fail "Stack config missing: stack-id=${STACK_ID:-empty}, kubeconfig=$([ -f "$OBOL_CONFIG_DIR/kubeconfig.yaml" ] && echo 'present' || echo 'MISSING')" +fi + +# §2: Verify the cluster — wait for all pods to be Running/Completed +run_step_grep "Nodes ready" "Ready" "$OBOL" kubectl get nodes +# Verify the k3s cluster version matches the documented version (CLAUDE.md: v1.35.1-k3s1) +step "k3s server version is v1.35.1+k3s1" +kube_ver=$("$OBOL" kubectl version 2>&1) || true +if echo "$kube_ver" | grep -q "v1.35\|k3s1"; then + k3s_ver=$(echo "$kube_ver" | grep "Server Version" | grep -oE "v[0-9]+\.[0-9]+\.[0-9]+\+k3s[0-9]+" | head -1) + pass "k3s server: ${k3s_ver:-v1.35.x+k3s1}" +else + fail "k3s server version unexpected — ${kube_ver:0:100}" +fi + +# Poll for all pods healthy (fresh cluster needs ~3-4 min for images to pull) +step "All pods Running or Completed (polling, max 60x5s)" +for i in $(seq 1 60); do + pod_output=$("$OBOL" kubectl get pods -A --no-headers 2>&1) + bad_pods=$(echo "$pod_output" | grep -v -E "Running|Completed" || true) + if [ -z "$bad_pods" ]; then + pass "All pods healthy (attempt $i)" + break + fi + if [ "$i" -eq 60 ]; then + fail "Unhealthy pods after 300s: $(echo "$bad_pods" | head -3)" + fi + sleep 5 +done + +# Frontend via Traefik — wait up to 5 min for DNS + Traefik to be ready +poll_step "Frontend at http://obol.stack:8080/" 60 5 \ + $CURL_OBOL -sf --max-time 5 http://obol.stack:8080/ + +# §6: obol network list shows available networks (getting-started §6) +# Tests the network management CLI without deploying any Ethereum clients. +run_step_grep "obol network list shows available networks" \ + "ethereum\|aztec\|Available" \ + "$OBOL" network list + +# §6: obol network status shows eRPC gateway health (getting-started §Managing Networks) +run_step_grep "obol network status shows eRPC upstreams" \ + "Running\|upstream\|chain" \ + "$OBOL" network status + +# §6/§1.6: eRPC /rpc JSON lists base-sepolia among available chains + all states OK +step "eRPC /rpc lists base-sepolia (required for x402 payment chain)" +erpc_json=$($CURL_OBOL -sf --max-time 5 http://obol.stack:8080/rpc 2>&1) || true +if echo "$erpc_json" | python3 -c " +import sys, json +d = json.load(sys.stdin) +aliases = [r.get('alias','') for r in d.get('rpc',[])] +assert 'base-sepolia' in aliases, f'base-sepolia not in {aliases}' +print(f'eRPC chains: {aliases}') +" 2>&1; then + pass "eRPC lists base-sepolia chain for x402 payments" +else + fail "eRPC /rpc missing base-sepolia — ${erpc_json:0:100}" +fi + +step "eRPC all configured chains are in OK state" +if echo "$erpc_json" | python3 -c " +import sys, json +d = json.load(sys.stdin) +chains = d.get('rpc', []) +not_ok = [(r.get('alias'), r.get('state')) for r in chains if r.get('state') != 'OK'] +assert not not_ok, f'chains not OK: {not_ok}' +print(f'{len(chains)} chains all OK: {[r.get(\"alias\") for r in chains]}') +" 2>&1; then + pass "All eRPC chains are in OK state" +else + fail "Some eRPC chains not OK — ${erpc_json:0:200}" +fi + +# §2: Frontend returns the Obol Stack Next.js app (getting-started §2 Key URLs) +step "Frontend serves Next.js app" +frontend_out=$($CURL_OBOL -sf --max-time 10 http://obol.stack:8080/ 2>&1) || true +if echo "$frontend_out" | grep -q "_next\|html"; then + pass "Frontend returns Next.js app HTML" +else + fail "Frontend HTML unexpected — ${frontend_out:0:100}" +fi + +# §2/§1.6: eRPC executes JSON-RPC calls (monetize §1.6 shows POST /rpc as eRPC gateway) +# Test an actual eth_blockNumber call via the eRPC proxy to verify end-to-end routing. +step "eRPC proxies eth_blockNumber to mainnet" +erpc_rpc_out=$($CURL_OBOL -sf --max-time 15 -X POST \ + "http://obol.stack:8080/rpc/evm/1" \ + -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' 2>&1) || true +if echo "$erpc_rpc_out" | python3 -c " +import sys, json +d = json.load(sys.stdin) +assert d.get('result'), 'no result' +assert d.get('jsonrpc') == '2.0', 'wrong jsonrpc version' +print(f'blockNumber: {int(d[\"result\"], 16)} (hex: {d[\"result\"]})') +" 2>&1; then + pass "eRPC JSON-RPC call succeeded (eth_blockNumber)" +else + fail "eRPC JSON-RPC failed — ${erpc_rpc_out:0:200}" +fi + +# §1.6/§3 x402: verify eRPC proxies Base Sepolia (chain 84532) used for x402 payments +# eth_chainId should return 0x14a34 = 84532 confirming correct chain routing +step "eRPC proxies Base Sepolia (chain 84532) for x402 payments" +erpc_basesep=$($CURL_OBOL -sf --max-time 15 -X POST \ + "http://obol.stack:8080/rpc/evm/84532" \ + -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","method":"eth_chainId","params":[],"id":1}' 2>&1) || true +if echo "$erpc_basesep" | python3 -c " +import sys, json +d = json.load(sys.stdin) +cid = d.get('result', '') +assert cid.lower() == '0x14a34', f'expected 0x14a34, got {cid}' +print(f'Base Sepolia chain ID: {int(cid, 16)} (correct)') +" 2>&1; then + pass "eRPC correctly routes to Base Sepolia (chain 84532)" +else + fail "eRPC Base Sepolia failed — ${erpc_basesep:0:200}" +fi + +# §2: Prometheus scrapes x402-buyer metrics via PodMonitor (monitoring §1.7) +step "Prometheus scrapes x402-buyer sidecar metrics (PodMonitor)" +prom_targets=$("$OBOL" kubectl get --raw \ + /api/v1/namespaces/monitoring/services/monitoring-kube-prometheus-prometheus:9090/proxy/api/v1/targets \ + 2>&1) || true +if echo "$prom_targets" | python3 -c " +import sys, json +d = json.load(sys.stdin) +targets = d.get('data', {}).get('activeTargets', []) +buyer = [t for t in targets if 'x402' in str(t.get('labels',''))] +assert buyer, 'no x402-buyer targets found' +health = buyer[0].get('health','?') +job = buyer[0].get('labels',{}).get('job','?') +print(f'Job: {job}, Health: {health}') +assert health == 'up', f'x402-buyer target health is {health}' +" 2>&1; then + pass "Prometheus x402-buyer target: up" +else + fail "Prometheus not scraping x402-buyer — ${prom_targets:0:100}" +fi + +# §2: All monitoring namespace pods running (Prometheus stack components) +step "Monitoring namespace pods all running" +monitoring_pods=$("$OBOL" kubectl get pods -n monitoring --no-headers 2>&1) +running=$(echo "$monitoring_pods" | grep -c "Running" || echo 0) +total=$(echo "$monitoring_pods" | grep -cv "^$" || echo 0) +if [ "$running" -gt 0 ] && [ "$running" = "$total" ]; then + pass "Monitoring namespace: $running/$total pods running" +else + fail "Monitoring pods not all running: $running/$total — $(echo "$monitoring_pods" | grep -v Running | head -2)" +fi + +# §2: Prometheus monitoring ready (getting-started §2 infrastructure table lists monitoring) +step "Prometheus monitoring ready" +prom_out=$("$OBOL" kubectl get --raw \ + /api/v1/namespaces/monitoring/services/monitoring-kube-prometheus-prometheus:9090/proxy/-/ready \ + 2>&1) || true +if echo "$prom_out" | grep -q "Ready"; then + pass "Prometheus Server is Ready" +else + fail "Prometheus not ready — ${prom_out:0:100}" +fi + +emit_metrics diff --git a/flows/flow-03-inference.sh b/flows/flow-03-inference.sh new file mode 100755 index 00000000..1bf83de9 --- /dev/null +++ b/flows/flow-03-inference.sh @@ -0,0 +1,137 @@ +#!/bin/bash +# Flow 03: LLM Inference — getting-started.md §3a-3d. +# Tests: host Ollama, in-cluster connectivity, LiteLLM inference, tool-calls. +source "$(dirname "$0")/lib.sh" + +# §3a: Verify Ollama has models +run_step_grep "Ollama has models on host" "models" \ + curl -sf http://localhost:11434/api/tags + +# §3b: In-cluster Ollama connectivity — exec into litellm pod using python3 +# (wget/curl are not available in the litellm container) +step "In-cluster Ollama reachable from litellm pod" +out=$("$OBOL" kubectl exec -n llm deployment/litellm -c litellm -- \ + python3 -c " +import urllib.request +r = urllib.request.urlopen('http://ollama.llm.svc.cluster.local:11434/api/tags', timeout=10) +print(r.read()[:100].decode()) +" 2>&1) || true +if echo "$out" | grep -q "models"; then + pass "In-cluster Ollama reachable" +else + fail "In-cluster Ollama unreachable — ${out:0:200}" +fi + +# §3c: Inference through LiteLLM (port-forward is the documented user path) +# Get the master key — required for all LiteLLM API calls +kill $(lsof -ti:8001) 2>/dev/null || true +step "LiteLLM port-forward + inference" +"$OBOL" kubectl port-forward -n llm svc/litellm 8001:4000 &>/dev/null & +PF_PID=$! + +# Get master key from secret +LITELLM_KEY=$("$OBOL" kubectl get secret litellm-secrets -n llm \ + -o jsonpath='{.data.LITELLM_MASTER_KEY}' 2>/dev/null | base64 -d) + +# Use /health/liveliness — it is unauthenticated (unlike /health which requires a key) +for i in $(seq 1 15); do + if curl -sf --max-time 2 http://localhost:8001/health/liveliness >/dev/null 2>&1; then + break + fi + sleep 2 +done + +# Use qwen3.5:9b — it is configured in LiteLLM's model_list (FLOW_MODEL qwen3:0.6b +# is only registered in Ollama directly; the x402 sell/buy flows use it via that path) +LITELLM_MODEL="qwen3.5:9b" +out=$(curl -sf --max-time 120 -X POST http://localhost:8001/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_KEY" \ + -d "{\"model\":\"$LITELLM_MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2? Reply with the number only.\"}],\"max_tokens\":10,\"stream\":false}" 2>&1) || true + +if echo "$out" | grep -q "choices"; then + pass "LiteLLM inference returned choices" +else + fail "LiteLLM inference failed — ${out:0:300}" +fi + +# §3d: Tool-call passthrough +step "Tool-call passthrough" +tool_out=$(curl -sf --max-time 120 -X POST http://localhost:8001/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_KEY" \ + -d '{ + "model":"'"$LITELLM_MODEL"'", + "messages":[{"role":"user","content":"What is the weather in London?"}], + "tools":[{"type":"function","function":{"name":"get_weather","description":"Get current weather","parameters":{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}}}], + "max_tokens":100,"stream":false + }' 2>&1) || true + +if echo "$tool_out" | grep -q "tool_calls\|get_weather"; then + pass "Tool-call passthrough works" +else + # Small/local models may not reliably support tool calls — soft fail + fail "Tool-call not returned (model may not support it) — ${tool_out:0:200}" +fi + +cleanup_pid "$PF_PID" + +# §3c: LiteLLM /v1/models lists configured models (getting-started §3c) +# "Replace qwen3.5:35b with your model name" implies users list available models. +step "LiteLLM /v1/models endpoint lists models" +kill $(lsof -ti:8001) 2>/dev/null || true +"$OBOL" kubectl port-forward -n llm svc/litellm 8001:4000 &>/dev/null & +PF_MODELS_PID=$! +for i in $(seq 1 10); do + if curl -sf --max-time 2 http://localhost:8001/health/liveliness >/dev/null 2>&1; then + break + fi + sleep 1 +done +LITELLM_KEY=$("$OBOL" kubectl get secret litellm-secrets -n llm \ + -o jsonpath='{.data.LITELLM_MASTER_KEY}' 2>/dev/null | base64 -d) +models_out=$(curl -sf --max-time 10 http://localhost:8001/v1/models \ + -H "Authorization: Bearer $LITELLM_KEY" 2>&1) || true +cleanup_pid "$PF_MODELS_PID" +if echo "$models_out" | python3 -c " +import sys, json +d = json.load(sys.stdin) +models = [m['id'] for m in d.get('data', [])] +assert len(models) > 0, 'no models' +print(f'Found {len(models)} models, first 3: {models[:3]}') +" 2>&1; then + pass "LiteLLM models endpoint lists available models" +else + fail "LiteLLM models endpoint failed — ${models_out:0:200}" +fi + +# §3: LiteLLM config includes qwen3.5:9b (default agent model, auto-configured by obol stack up) +step "LiteLLM config has qwen3.5:9b (default agent model)" +llm_config=$("$OBOL" kubectl get cm litellm-config -n llm \ + -o jsonpath='{.data.config\.yaml}' 2>&1) || true +if echo "$llm_config" | grep -q "qwen3.5:9b"; then + model_count=$(echo "$llm_config" | grep -c "model_name:" || echo 0) + pass "LiteLLM config has qwen3.5:9b ($model_count total models configured)" +else + fail "LiteLLM config missing qwen3.5:9b — check obol stack up auto-configure" +fi + +# §3b: LiteLLM config points to in-cluster Ollama service (auto-configured routing) +# The api_base should be http://ollama.llm.svc.cluster.local:11434 for Ollama models. +step "LiteLLM config routes to in-cluster Ollama" +if echo "$llm_config" | grep -q "ollama.llm.svc.cluster.local"; then + pass "LiteLLM api_base = ollama.llm.svc.cluster.local:11434" +else + fail "LiteLLM config missing ollama.llm.svc.cluster.local base URL" +fi + +# §3: obol model status shows configured LiteLLM providers (getting-started §3) +step "obol model status shows ollama provider" +model_out=$("$OBOL" model status 2>&1) || true +if echo "$model_out" | grep -q "ollama.*true\|ollama.*n/a"; then + pass "model status: ollama provider enabled" +else + fail "model status missing ollama provider — ${model_out:0:200}" +fi + +emit_metrics diff --git a/flows/flow-04-agent.sh b/flows/flow-04-agent.sh new file mode 100755 index 00000000..ce815e3b --- /dev/null +++ b/flows/flow-04-agent.sh @@ -0,0 +1,241 @@ +#!/bin/bash +# Flow 04: Agent Init + Inference — getting-started.md §4-5. +# Tests: agent init, openclaw list, token, agent gateway inference. +source "$(dirname "$0")/lib.sh" + +# §4: Deploy AI Agent (idempotent) +run_step "obol agent init" "$OBOL" agent init + +# List agent instances — verify name AND URL are shown (getting-started §4) +run_step_grep "openclaw list shows instances" "obol-agent\|default" "$OBOL" openclaw list +step "openclaw list shows agent URL" +list_out=$("$OBOL" openclaw list 2>&1) || true +if echo "$list_out" | grep -q "obol.stack\|URL:"; then + url=$(echo "$list_out" | grep -oE 'http://[a-z0-9.-]+' | head -1) + pass "openclaw list shows agent URL: $url" +else + fail "openclaw list missing URL — ${list_out:0:200}" +fi + +# §4: HEARTBEAT.md injected into agent workspace (agent init output confirms this) +# obol agent init injects HEARTBEAT.md with the monetize reconcile command +step "HEARTBEAT.md injected into agent workspace" +HEARTBEAT_FILE="$OBOL_DATA_DIR/openclaw-obol-agent/openclaw-data/.openclaw/workspace/HEARTBEAT.md" +if [ -f "$HEARTBEAT_FILE" ] && grep -q "monetize\|python3\|sell" "$HEARTBEAT_FILE" 2>/dev/null; then + pass "HEARTBEAT.md injected with monetize reconcile command" +else + fail "HEARTBEAT.md missing or empty at $HEARTBEAT_FILE" +fi + +# §5: OpenClaw service on port 18789 (getting-started §5 uses port-forward 18789:18789) +step "OpenClaw service on port 18789" +NS=$("$OBOL" openclaw list 2>/dev/null | grep -oE 'openclaw-[a-z0-9-]+' | head -1 || echo "openclaw-obol-agent") +oc_port=$("$OBOL" kubectl get svc openclaw -n "$NS" \ + -o jsonpath='{.spec.ports[0].port}' 2>&1) || true +if [ "$oc_port" = "18789" ]; then + pass "OpenClaw service port: 18789 (matches getting-started §5 port-forward)" +else + fail "OpenClaw service port unexpected: $oc_port (expected 18789)" +fi + +# §5: Test Agent Inference +step "Get openclaw token" +TOKEN=$("$OBOL" openclaw token obol-agent 2>/dev/null || "$OBOL" openclaw token default 2>/dev/null || true) +if [ -n "$TOKEN" ]; then + pass "Got token: ${TOKEN:0:8}..." +else + fail "Failed to get openclaw token" + emit_metrics + exit 0 +fi + +# §5: Token is 32-char alphanumeric (validates token generation for gateway auth) +step "OpenClaw gateway token is 32-char alphanumeric" +if echo "$TOKEN" | grep -qE '^[A-Za-z0-9]{32}$'; then + pass "Token: ${TOKEN:0:8}... (32 chars, alphanumeric)" +else + fail "Token has unexpected format: length=${#TOKEN}" +fi + +# Determine the namespace for port-forward +NS=$("$OBOL" openclaw list 2>/dev/null | grep -oE 'openclaw-[a-z0-9-]+' | head -1 || echo "openclaw-obol-agent") + +step "Agent inference via port-forward" +"$OBOL" kubectl port-forward -n "$NS" svc/openclaw 18789:18789 &>/dev/null & +PF_PID=$! + +# Poll until port 18789 is accepting connections +for i in $(seq 1 15); do + if curl -sf --max-time 2 http://localhost:18789/health >/dev/null 2>&1; then + break + fi + sleep 2 +done + +out=$(curl -sf --max-time 120 -X POST http://localhost:18789/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $TOKEN" \ + -d "{\"model\":\"$FLOW_MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2?\"}],\"max_tokens\":50,\"stream\":false}" 2>&1) || true + +if echo "$out" | grep -q "choices"; then + pass "Agent inference returned response" +else + fail "Agent inference failed — ${out:0:200}" +fi + +cleanup_pid "$PF_PID" + +# §4: Verify obol-managed skills are installed (getting-started §4) +# Skills like sell, buy-inference, discovery, obol-stack are obol-managed. +step "obol openclaw skills list shows obol-managed skills" +skills_out=$("$OBOL" openclaw skills list obol-agent 2>&1) || true +if echo "$skills_out" | grep -q "sell\|buy-inference\|obol-stack"; then + ready_count=$(echo "$skills_out" | grep -c "ready" || echo 0) + pass "openclaw skills: $ready_count obol-managed skills ready" +else + fail "openclaw skills list missing expected skills — ${skills_out:0:200}" +fi + +# §4: Ethereum signing wallet created by obol agent init (getting-started §4) +# "A unique Ethereum signing wallet" is listed as a feature of obol agent init. +step "obol openclaw wallet list shows Ethereum address" +wallet_out=$("$OBOL" openclaw wallet list obol-agent 2>&1) || true +if echo "$wallet_out" | grep -q "0x[0-9a-fA-F]\{40\}\|Address:"; then + addr=$(echo "$wallet_out" | grep -oE '0x[0-9a-fA-F]{40}' | head -1) + pass "Agent wallet address: $addr" +else + fail "openclaw wallet list missing address — ${wallet_out:0:200}" +fi + +# §4: OpenClaw gateway health via HTTPRoute URL (getting-started §4 output shows URL) +# The URL http://openclaw-obol-agent.obol.stack is shown after obol openclaw sync. +step "OpenClaw gateway health via HTTPRoute hostname" +OPENCLAW_URL="http://openclaw-obol-agent.obol.stack:8080" +# Use --resolve to bypass DNS (obol.stack not always in /etc/hosts for subdomains) +oc_health=$(curl --resolve "openclaw-obol-agent.obol.stack:8080:127.0.0.1" \ + -sf --max-time 10 "$OPENCLAW_URL/health" 2>&1) || true +if echo "$oc_health" | grep -q "ok.*true\|status.*live"; then + pass "OpenClaw gateway health: $oc_health" +else + fail "OpenClaw gateway health check failed — ${oc_health:0:100}" +fi + +# §4: Verify openclaw config — heartbeat and model settings (agent init patches these) +# obol agent init calls ensureHeartbeatActive() which patches openclaw-config ConfigMap +oc_config=$("$OBOL" kubectl get cm openclaw-config -n openclaw-obol-agent \ + -o jsonpath='{.data.openclaw\.json}' 2>&1) || true + +step "Agent heartbeat configured to 5m interval" +if echo "$oc_config" | python3 -c " +import sys, json +d = json.load(sys.stdin) +heartbeat = d.get('agents', {}).get('defaults', {}).get('heartbeat', {}) +every = heartbeat.get('every', '') +assert every == '5m', f'expected 5m, got {every!r}' +print(f'Heartbeat: every={every}') +" 2>&1; then + pass "Heartbeat configured to 5m (agent will reconcile every 5 minutes)" +else + fail "Heartbeat not configured to 5m — ${oc_config:0:200}" +fi + +# §4: "A heartbeat that runs the agent periodically" includes the agent model config +step "Agent primary model is configured" +model_val=$(echo "$oc_config" | python3 -c " +import sys, json +try: + d = json.load(sys.stdin) + m = d.get('agents',{}).get('defaults',{}).get('model',{}).get('primary','') + print(m) +except: pass +" 2>/dev/null) || model_val="" +if [ -n "$model_val" ]; then + pass "Agent primary model: $model_val" +else + fail "Agent model not configured in openclaw-config" +fi + +# §4: OpenClaw routes through LiteLLM (openai provider slot at litellm.llm.svc) +# CLAUDE.md: "OpenClaw always routes through LiteLLM (openai provider slot)" +step "OpenClaw openai provider routes to in-cluster LiteLLM" +litellm_base=$(echo "$oc_config" | python3 -c " +import sys, json +try: + d = json.load(sys.stdin) + url = d.get('models',{}).get('providers',{}).get('openai',{}).get('baseUrl','') + print(url) +except: pass +" 2>/dev/null) || litellm_base="" +if echo "$litellm_base" | grep -q "litellm.llm.svc.cluster.local"; then + pass "OpenClaw openai provider baseUrl: $litellm_base" +else + fail "OpenClaw not routing through LiteLLM — base URL: ${litellm_base:-empty}" +fi + +# §4 RBAC: Both monetize ClusterRoles must exist (deployed by k3s manifests at stack up) +step "RBAC: monetize ClusterRoles exist" +cr_read=$("$OBOL" kubectl get clusterrole openclaw-monetize-read 2>&1) || true +cr_workload=$("$OBOL" kubectl get clusterrole openclaw-monetize-workload 2>&1) || true +if echo "$cr_read" | grep -q "openclaw-monetize-read" && \ + echo "$cr_workload" | grep -q "openclaw-monetize-workload"; then + pass "ClusterRoles: openclaw-monetize-read + openclaw-monetize-workload" +else + fail "Missing monetize ClusterRole(s)" +fi + +# §4 RBAC: workload ClusterRole allows CRUD on ServiceOffers (obol.org) +step "RBAC: openclaw-monetize-workload can CRUD ServiceOffers" +workload_rules=$("$OBOL" kubectl get clusterrole openclaw-monetize-workload \ + -o jsonpath='{.rules}' 2>&1) || true +if echo "$workload_rules" | python3 -c " +import sys, json +rules = json.load(sys.stdin) +for r in rules: + if 'serviceoffers' in r.get('resources', []) and 'obol.org' in r.get('apiGroups', []): + verbs = r.get('verbs', []) + assert 'create' in verbs and 'delete' in verbs, f'missing CRUD verbs: {verbs}' + print(f'ServiceOffer CRUD: {verbs}') + break +else: + raise AssertionError('no ServiceOffer rule found') +" 2>&1; then + pass "openclaw-monetize-workload can CRUD ServiceOffers (obol.org)" +else + fail "RBAC workload rule missing ServiceOffer CRUD — ${workload_rules:0:100}" +fi + +# §4: RBAC bindings — obol agent init patches ClusterRoleBindings (troubleshooting §RBAC) +# Both monetize ClusterRoleBindings must include openclaw SA as a subject. +step "RBAC: openclaw-monetize bindings have openclaw SA as subject" +rbac_out=$("$OBOL" kubectl get clusterrolebinding openclaw-monetize-read-binding \ + -o jsonpath='{.subjects}' 2>&1) || true +rbac_workload=$("$OBOL" kubectl get clusterrolebinding openclaw-monetize-workload-binding \ + -o jsonpath='{.subjects}' 2>&1) || true +if echo "$rbac_out" | grep -q "openclaw" && echo "$rbac_workload" | grep -q "openclaw"; then + pass "Both monetize ClusterRoleBindings have openclaw SA" +else + fail "ClusterRoleBinding missing openclaw SA — read: ${rbac_out:0:50} workload: ${rbac_workload:0:50}" +fi + +# §2 component table: Remote Signer running (getting-started §2 lists it as a component) +# The remote-signer provides signing services for the agent's Ethereum wallet. +# It exposes a REST API on port 9000 for health and key management. +step "Remote Signer health check" +kill $(lsof -ti:9000) 2>/dev/null || true +"$OBOL" kubectl port-forward -n "$NS" svc/remote-signer 9000:9000 &>/dev/null & +RS_PID=$! +for i in $(seq 1 10); do + if curl -sf --max-time 2 http://localhost:9000/healthz >/dev/null 2>&1; then + break + fi + sleep 1 +done +rs_out=$(curl -sf --max-time 5 http://localhost:9000/healthz 2>&1) || true +cleanup_pid "$RS_PID" +if echo "$rs_out" | grep -q "ok\|status"; then + pass "Remote Signer healthy: $rs_out" +else + fail "Remote Signer health check failed — ${rs_out:0:100}" +fi + +emit_metrics diff --git a/flows/flow-05-network.sh b/flows/flow-05-network.sh new file mode 100755 index 00000000..e580f2b6 --- /dev/null +++ b/flows/flow-05-network.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Flow 05: Network management — getting-started.md §6. +# SKIPPED per autoresearch.md constraint 0: do NOT deploy Ethereum clients. +# Covers only: network list, network add/remove RPC, eRPC gateway health. +source "$(dirname "$0")/lib.sh" + +# List available networks (local nodes + remote RPCs) +run_step_grep "network list" "ethereum\|Remote\|Local" "$OBOL" network list + +# eRPC gateway health via obol network status +run_step_grep "eRPC gateway status" "eRPC\|Pod\|Upstream" "$OBOL" network status + +# Add a public RPC for base-sepolia (documented user path for RPC access) +run_step "network add base-sepolia RPC" "$OBOL" network add base-sepolia --count 1 + +# Verify it appears in list +run_step_grep "base-sepolia in network list" "base-sepolia\|84532" "$OBOL" network list + +# eRPC is accessible at /rpc/evm/ — base-sepolia is chain 84532 +step "eRPC base-sepolia via Traefik (/rpc/evm/84532)" +out=$($CURL_OBOL -sf --max-time 10 "http://obol.stack:8080/rpc/evm/84532" \ + -X POST -H 'Content-Type: application/json' \ + -d '{"jsonrpc":"2.0","method":"eth_chainId","params":[],"id":1}' 2>&1) || true +if echo "$out" | grep -q '"result"'; then + pass "eRPC eth_chainId returned result" +else + fail "eRPC eth_chainId failed — ${out:0:200}" +fi + +# Remove the RPCs we added — brief pause so Stakater Reloader rate-limit doesn't trigger +sleep 5 +run_step "network remove base-sepolia" "$OBOL" network remove base-sepolia + +# Verify base-sepolia still has original template upstreams after remove +# (remove only clears ChainList-sourced ones, leaving template upstreams intact) +step "base-sepolia still has template upstreams after remove" +after_status=$("$OBOL" network status 2>&1) || true +if echo "$after_status" | grep -q "Base Sepolia.*[1-9] upstream"; then + pass "Base Sepolia template upstreams intact after remove" +else + fail "Base Sepolia upstreams missing after remove — ${after_status:0:100}" +fi + +# §6 URL validation: obol network add validates endpoint URLs (fix/cli-ux branch) +step "obol network add rejects invalid endpoint URL" +invalid_out=$("$OBOL" network add base-sepolia \ + --endpoint "not-a-valid-url" 2>&1) || true +if echo "$invalid_out" | grep -qiE "invalid|error|scheme|http"; then + pass "obol network add rejects invalid URL: ${invalid_out:0:60}" +else + fail "obol network add accepted invalid URL — ${invalid_out:0:100}" +fi + +emit_metrics diff --git a/flows/flow-06-sell-setup.sh b/flows/flow-06-sell-setup.sh new file mode 100755 index 00000000..1b3b3287 --- /dev/null +++ b/flows/flow-06-sell-setup.sh @@ -0,0 +1,222 @@ +#!/bin/bash +# Flow 06: Sell Setup — monetize-inference.md §1.1-1.4. +# Tests: verify components, sell pricing, sell http, wait for agent heartbeat to reconcile. +source "$(dirname "$0")/lib.sh" + +# §1.1: Verify key components (getting-started §2 component table) +run_step_grep "Cluster nodes ready" "Ready" "$OBOL" kubectl get nodes +run_step_grep "Agent pod running" "Running" "$OBOL" kubectl get pods -n openclaw-obol-agent --no-headers +run_step_grep "CRD installed" "serviceoffers.obol.org" "$OBOL" kubectl get crd serviceoffers.obol.org +# Verify the CRD has the correct API group (obol.org) and version (v1alpha1) +step "ServiceOffer CRD API group is obol.org/v1alpha1" +crd_group=$("$OBOL" kubectl get crd serviceoffers.obol.org \ + -o jsonpath='{.spec.group}' 2>&1) || true +crd_version=$("$OBOL" kubectl get crd serviceoffers.obol.org \ + -o jsonpath='{.spec.versions[0].name}' 2>&1) || true +if [ "$crd_group" = "obol.org" ] && [ "$crd_version" = "v1alpha1" ]; then + pass "ServiceOffer CRD: group=obol.org, version=v1alpha1" +else + fail "CRD API group/version unexpected: group=$crd_group, version=$crd_version" +fi +run_step_grep "x402 verifier running" "Running" "$OBOL" kubectl get pods -n x402 --no-headers +# x402-verifier has 2 replicas for high availability (CLAUDE.md: "2 replicas") +step "x402-verifier has 2 replicas (high availability)" +verifier_replicas=$("$OBOL" kubectl get deployment x402-verifier -n x402 \ + -o jsonpath='{.spec.replicas}' 2>&1) || true +if [ "$verifier_replicas" = "2" ]; then + pass "x402-verifier: 2 replicas (HA payment gate)" +else + fail "x402-verifier replica count: $verifier_replicas (expected 2)" +fi +# x402-verifier service must be on port 8080 (matches ForwardAuth address :8080/verify) +step "x402-verifier service on port 8080" +verifier_port=$("$OBOL" kubectl get svc x402-verifier -n x402 \ + -o jsonpath='{.spec.ports[0].port}' 2>&1) || true +if [ "$verifier_port" = "8080" ]; then + pass "x402-verifier service port: 8080 (matches ForwardAuth address)" +else + fail "x402-verifier port unexpected: $verifier_port (expected 8080)" +fi +run_step_grep "Traefik pod running" "Running" "$OBOL" kubectl get pods -n traefik --no-headers +run_step_grep "Traefik gateway exists" "traefik-gateway" "$OBOL" kubectl get gateway -n traefik +# Verify the x402 ForwardAuth Middleware is wired to x402-verifier service +step "x402-payment Middleware has correct ForwardAuth address" +fw_addr=$("$OBOL" kubectl get middleware x402-payment -n erpc \ + -o jsonpath='{.spec.forwardAuth.address}' 2>&1) || true +if echo "$fw_addr" | grep -q "x402-verifier.x402.svc.cluster.local"; then + pass "ForwardAuth → x402-verifier.x402.svc.cluster.local (correct)" +else + fail "x402 Middleware ForwardAuth address wrong — ${fw_addr:0:100}" +fi +# Verify Gateway is Accepted AND Programmed (not just exists) +step "Traefik gateway Accepted and Programmed" +gw_status=$("$OBOL" kubectl get gateway -n traefik traefik-gateway \ + -o jsonpath='{.status.conditions}' 2>&1) || true +if echo "$gw_status" | python3 -c " +import sys, json +conds = json.load(sys.stdin) +accepted = any(c['type']=='Accepted' and c['status']=='True' for c in conds) +programmed = any(c['type']=='Programmed' and c['status']=='True' for c in conds) +assert accepted and programmed, f'Not ready: {conds}' +" 2>/dev/null; then + pass "Traefik gateway Accepted=True, Programmed=True" +else + fail "Traefik gateway not fully ready — ${gw_status:0:200}" +fi +run_step_grep "LiteLLM running" "Running" "$OBOL" kubectl get pods -n llm --no-headers +# LiteLLM service must be on port 4000 (standard OpenAI-compatible API port) +step "LiteLLM service on port 4000" +litellm_port=$("$OBOL" kubectl get svc litellm -n llm \ + -o jsonpath='{.spec.ports[0].port}' 2>&1) || true +if [ "$litellm_port" = "4000" ]; then + pass "LiteLLM service port: 4000 (standard OpenAI API port)" +else + fail "LiteLLM service port unexpected: $litellm_port (expected 4000)" +fi +# Verify LiteLLM pod has 2 containers (litellm + x402-buyer sidecar) +step "LiteLLM pod has 2 containers (litellm + x402-buyer sidecar)" +container_count=$("$OBOL" kubectl get pods -n llm --no-headers 2>&1 | awk '{print $2}' | head -1) +if [ "$container_count" = "2/2" ]; then + pass "LiteLLM pod has 2/2 containers (litellm + x402-buyer sidecar)" +else + fail "LiteLLM pod container count unexpected: $container_count (expected 2/2)" +fi + +# Verify x402-buyer sidecar health (serves /healthz at port 8402 in litellm pod) +step "x402-buyer sidecar healthy (buy-side payment handler)" +kill $(lsof -ti:8402) 2>/dev/null || true +"$OBOL" kubectl port-forward -n llm deployment/litellm 8402:8402 &>/dev/null & +PF_BUYER_PID=$! +for i in $(seq 1 8); do + if curl -sf --max-time 2 http://localhost:8402/healthz >/dev/null 2>&1; then + break + fi + sleep 1 +done +buyer_health=$(curl -sf --max-time 5 http://localhost:8402/healthz 2>&1) || true +cleanup_pid "$PF_BUYER_PID" +if echo "$buyer_health" | grep -q "ok"; then + pass "x402-buyer sidecar healthy: $buyer_health" +else + fail "x402-buyer sidecar health check failed — ${buyer_health:0:100}" +fi +run_step_grep "Ollama reachable" "models" curl -sf http://localhost:11434/api/tags +# Additional component table entries from getting-started §2 +run_step_grep "eRPC running" "Running" "$OBOL" kubectl get pods -n erpc --no-headers +run_step_grep "Frontend running" "Running" "$OBOL" kubectl get pods -n obol-frontend --no-headers +run_step_grep "Reloader running" "Running" "$OBOL" kubectl get pods -n reloader --no-headers + +# §1.2: Pull model (ensure it's available) +step "Pull $FLOW_MODEL" +if ollama pull "$FLOW_MODEL" 2>&1 | tail -1; then + pass "Model $FLOW_MODEL pulled" +else + fail "Failed to pull $FLOW_MODEL" +fi + +run_step_grep "Model in Ollama tags" "$FLOW_MODEL" \ + curl -sf http://localhost:11434/api/tags + +# §1.3: Set up payment +run_step "sell pricing" "$OBOL" sell pricing \ + --wallet "$SELLER_WALLET" \ + --chain "$CHAIN" + +run_step_grep "x402-pricing ConfigMap has wallet" "$SELLER_WALLET" \ + "$OBOL" kubectl get cm x402-pricing -n x402 -o yaml +# Verify x402-pricing verifyOnly=false (actual payment processing enabled, not test-only) +step "x402-pricing verifyOnly=false (payment processing enabled)" +pricing_yaml2=$("$OBOL" kubectl get cm x402-pricing -n x402 \ + -o jsonpath='{.data.pricing\.yaml}' 2>&1) || true +if echo "$pricing_yaml2" | grep -q "verifyOnly: false"; then + pass "x402-pricing verifyOnly=false (payments are actually settled)" +else + fail "x402-pricing verifyOnly not false — ${pricing_yaml2:0:100}" +fi + +# Verify x402-pricing has the correct chain (base-sepolia for USDC payments) +step "x402-pricing chain is base-sepolia" +pricing_yaml=$("$OBOL" kubectl get cm x402-pricing -n x402 \ + -o jsonpath='{.data.pricing\.yaml}' 2>&1) || true +if echo "$pricing_yaml" | grep -q "chain: base-sepolia"; then + pass "x402-pricing chain: base-sepolia (correct for USDC payments)" +else + fail "x402-pricing chain wrong — ${pricing_yaml:0:100}" +fi + +# §1.4: Create ServiceOffer — clean up any previous flow-qwen offer first +"$OBOL" sell delete flow-qwen --namespace llm --force 2>/dev/null || true +sleep 2 + +run_step_grep "sell http flow-qwen" \ + "ServiceOffer.*created\|ServiceOffer.*updated\|agent will reconcile" \ + "$OBOL" sell http flow-qwen \ + --wallet "$SELLER_WALLET" \ + --chain "$CHAIN" \ + --per-request 0.001 \ + --namespace llm \ + --upstream ollama \ + --port 11434 + +# §1.4 UX: re-running sell http on the same SO shows "updated" not "created" +step "sell http idempotent: re-run shows 'updated' not 'created'" +rerun_out=$("$OBOL" sell http flow-qwen \ + --wallet "$SELLER_WALLET" --chain "$CHAIN" \ + --per-request 0.001 --namespace llm \ + --upstream ollama --port 11434 2>&1) || true +if echo "$rerun_out" | grep -q "ServiceOffer.*updated"; then + pass "sell http idempotent: shows 'updated' on re-run" +else + fail "sell http re-run did not show 'updated' — ${rerun_out:0:200}" +fi + +# obol sell http may restart the openclaw pod (new tunnel URL → helmfile sync). +# Wait for the pod to be ready before starting the heartbeat poll, so we +# start counting from a stable state (not mid-restart). +step "openclaw pod ready after sell http" +"$OBOL" kubectl rollout status deployment/openclaw \ + -n openclaw-obol-agent --timeout=120s 2>/dev/null \ + && pass "openclaw pod ready" \ + || fail "openclaw pod not ready after 120s" + +# The obol-agent heartbeat fires every 5 minutes and runs: +# python3 /data/.openclaw/skills/sell/scripts/monetize.py process --all --quick +# Wait up to 8 minutes (96x5s) for the heartbeat to reconcile the ServiceOffer. +# obol sell list shows READY=True once all conditions pass. +poll_step_grep "ServiceOffer flow-qwen Ready (waiting for heartbeat)" \ + "flow-qwen.*True" 96 5 \ + "$OBOL" sell list --namespace llm + +# Verify Kubernetes resources created by the agent +run_step_grep "ServiceOffer exists" "flow-qwen" \ + "$OBOL" kubectl get serviceoffer flow-qwen -n llm + +# Verify ServiceOffer spec has correct upstream, payment, and pricing fields (monetize §1.4) +step "ServiceOffer spec has upstream.service, payment.payTo, and price" +so_yaml=$("$OBOL" kubectl get serviceoffer flow-qwen -n llm -o yaml 2>&1) || true +if echo "$so_yaml" | grep -q "service: ollama" \ + && echo "$so_yaml" | grep -q "payTo: 0x" \ + && echo "$so_yaml" | grep -q "perRequest:"; then + payto=$(echo "$so_yaml" | grep "payTo:" | awk '{print $2}' | head -1) + price=$(echo "$so_yaml" | grep "perRequest:" | awk '{print $2}' | head -1 | tr -d '"') + pass "ServiceOffer spec: upstream=ollama, payTo=$payto, perRequest=$price USDC" +else + fail "ServiceOffer spec missing expected fields — ${so_yaml:0:200}" +fi + +# Verify Ollama k8s service is on port 11434 (matches --port 11434 in sell http) +step "Ollama service in llm namespace on port 11434" +ollama_port=$("$OBOL" kubectl get svc ollama -n llm \ + -o jsonpath='{.spec.ports[0].port}' 2>&1) || true +if [ "$ollama_port" = "11434" ]; then + pass "Ollama service port: 11434 (matches ServiceOffer upstream.port)" +else + fail "Ollama service port unexpected: $ollama_port (expected 11434)" +fi + +run_step_grep "Middleware exists" "x402-flow-qwen" \ + "$OBOL" kubectl get middleware -n llm +run_step_grep "HTTPRoute exists" "so-flow-qwen" \ + "$OBOL" kubectl get httproute -n llm + +emit_metrics diff --git a/flows/flow-07-sell-verify.sh b/flows/flow-07-sell-verify.sh new file mode 100755 index 00000000..6cb774f7 --- /dev/null +++ b/flows/flow-07-sell-verify.sh @@ -0,0 +1,193 @@ +#!/bin/bash +# Flow 07: Sell Verify — monetize-inference.md §1.5-1.7. +# Runs AFTER flow-06 (ServiceOffer flow-qwen must be Ready). +source "$(dirname "$0")/lib.sh" + +# §2.1 pre-check: obol-skill-md deployment serving /skill.md (created by heartbeat) +# The monetize agent's _publish_skill_md creates this busybox httpd deployment. +# It serves the machine-readable service catalog at /skill.md. +run_step_grep "obol-skill-md pod running" "Running" \ + "$OBOL" kubectl get pods -n openclaw-obol-agent --no-headers + +# Security: verify eRPC and frontend HTTPRoutes have hostname restrictions (CLAUDE.md) +# NEVER expose eRPC or frontend via tunnel — only obol.stack (local) allowed +step "eRPC HTTPRoute restricted to obol.stack (security: not exposed via tunnel)" +erpc_hostnames=$("$OBOL" kubectl get httproute erpc -n erpc \ + -o jsonpath='{.spec.hostnames}' 2>&1) || true +if echo "$erpc_hostnames" | grep -q "obol.stack"; then + pass "eRPC HTTPRoute hostname: $erpc_hostnames (local only)" +else + fail "eRPC HTTPRoute missing hostname restriction — exposed to public tunnel! ($erpc_hostnames)" +fi + +step "Frontend HTTPRoute restricted to obol.stack (security: not exposed via tunnel)" +fe_hostnames=$("$OBOL" kubectl get httproute obol-frontend -n obol-frontend \ + -o jsonpath='{.spec.hostnames}' 2>&1) || true +if echo "$fe_hostnames" | grep -q "obol.stack"; then + pass "Frontend HTTPRoute hostname: $fe_hostnames (local only)" +else + fail "Frontend HTTPRoute missing hostname restriction — exposed to public tunnel! ($fe_hostnames)" +fi + +# Security: OpenClaw dashboard restricted to local subdomain (not public) +step "OpenClaw HTTPRoute restricted to subdomain (security: not fully public)" +oc_hostnames=$("$OBOL" kubectl get httproute openclaw -n openclaw-obol-agent \ + -o jsonpath='{.spec.hostnames}' 2>&1) || true +if echo "$oc_hostnames" | grep -q "obol.stack"; then + pass "OpenClaw HTTPRoute hostname: $oc_hostnames (local subdomain)" +else + fail "OpenClaw HTTPRoute missing hostname restriction — ${oc_hostnames:0:100}" +fi + +# §1.6 pre-check: eRPC accessible (local Traefik, obol.stack only — never via tunnel) +# GET /rpc returns network list (from getting-started.md §2, monetize §1.6) +step "eRPC accessible at obol.stack:8080/rpc" +erpc_out=$($CURL_OBOL -sf --max-time 10 http://obol.stack:8080/rpc 2>&1) || true +if echo "$erpc_out" | python3 -c "import sys,json; d=json.load(sys.stdin); assert 'rpc' in d or 'error' in d" 2>/dev/null; then + pass "eRPC at obol.stack:8080/rpc returned JSON" +else + fail "eRPC not responding — ${erpc_out:0:100}" +fi + +# §1.5: Tunnel status +step "Tunnel status" +TUNNEL_OUTPUT=$("$OBOL" tunnel status 2>&1) || true +TUNNEL_URL=$(echo "$TUNNEL_OUTPUT" | grep -oE 'https://[a-z0-9-]+\.trycloudflare\.com' | head -1) +if [ -n "$TUNNEL_URL" ]; then + pass "Tunnel URL: $TUNNEL_URL" +else + fail "No tunnel URL found — ${TUNNEL_OUTPUT:0:200}" +fi + +# §1.6: Verify paths + +# Wait for x402-verifier pods to be ready — Kubernetes Reloader restarts them when +# x402-pricing ConfigMap changes (e.g., from obol sell pricing). Fresh pods take ~10s. +step "x402 verifier pods ready" +for i in $(seq 1 12); do + ready=$("$OBOL" kubectl get pods -n x402 --no-headers 2>&1 | grep "Running" | grep -c "1/1" || echo 0) + total=$("$OBOL" kubectl get pods -n x402 --no-headers 2>&1 | grep -v "^$" | wc -l | tr -d ' ') + if [ "$ready" -ge 1 ] && [ "$ready" = "$total" ]; then + pass "x402 verifier pods ready ($ready/$total)" + break + fi + [ "$i" -eq 12 ] && fail "x402 verifier not ready after 60s ($ready/$total pods running)" + sleep 5 +done + +# 402 via local Traefik (primary check — no tunnel dependency) +# Poll briefly: Traefik needs ~10s to propagate a newly created HTTPRoute +step "402 via local Traefik" +for i in $(seq 1 6); do + local_code=$($CURL_OBOL -s --max-time 5 -o /dev/null -w '%{http_code}' -X POST \ + "http://obol.stack:8080/services/flow-qwen/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -d "{\"model\":\"$FLOW_MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}" 2>&1) || true + if [ "$local_code" = "402" ]; then + pass "Local 402 Payment Required (attempt $i)" + break + fi + [ "$i" -eq 6 ] && fail "Expected 402 after 30s, got: $local_code" + sleep 5 +done + +# Validate 402 JSON body has required x402 fields +step "402 body has x402Version and accepts[]" +body=$($CURL_OBOL -s --max-time 10 -X POST \ + "http://obol.stack:8080/services/flow-qwen/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -d "{\"model\":\"$FLOW_MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}" 2>&1) || true +if echo "$body" | python3 -c " +import sys, json +d = json.load(sys.stdin) +assert d.get('x402Version') is not None +assert d['accepts'][0]['payTo'] +" 2>/dev/null; then + pass "402 body has x402Version + accepts[].payTo" +else + fail "402 body missing fields — ${body:0:200}" +fi + +# 402 via tunnel +if [ -n "$TUNNEL_URL" ]; then + step "402 via tunnel" + tunnel_code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 15 -X POST \ + "$TUNNEL_URL/services/flow-qwen/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -d "{\"model\":\"$FLOW_MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}" 2>/dev/null || echo "000") + if [ "$tunnel_code" = "402" ]; then + pass "Tunnel 402 Payment Required" + else + fail "Tunnel expected 402, got $tunnel_code" + fi +fi + +# §1.7: Verifier metrics — check metrics from ALL x402-verifier pods +# (metrics are per-pod; requests load-balance to any pod, so we must check all) +step "x402 verifier metrics" +metrics_found=false +for pod in $("$OBOL" kubectl get pods -n x402 -o name 2>/dev/null | grep verifier); do + kill $(lsof -ti:8889) 2>/dev/null || true + "$OBOL" kubectl port-forward -n x402 "$pod" 8889:8080 &>/dev/null & + PF_METRICS_PID=$! + for i in $(seq 1 10); do + if curl -sf --max-time 2 http://localhost:8889/metrics >/dev/null 2>&1; then + break + fi + sleep 1 + done + pod_metrics=$(curl -sf --max-time 5 http://localhost:8889/metrics 2>&1) || true + cleanup_pid "$PF_METRICS_PID" + if echo "$pod_metrics" | grep -q "obol_x402"; then + metrics_found=true + break + fi +done +if $metrics_found; then + pass "Verifier metrics available" +else + fail "Verifier metrics not found on any pod (requests may not have reached verifier yet)" +fi + +# §1.7: Verifier request logs (monitoring — verifier logs each ForwardAuth call) +step "x402 verifier logs show 402 request handling" +verifier_logs=$("$OBOL" kubectl logs -n x402 deployment/x402-verifier --tail=20 2>&1) || true +if echo "$verifier_logs" | grep -q "402\|payment\|services/flow-qwen"; then + pass "Verifier logs show 402 request activity" +else + # May be empty if Reloader just restarted the pod — soft fail + fail "Verifier logs empty (Reloader may have restarted pod) — ${verifier_logs:0:100}" +fi + +# §1.4 Pricing Config: x402-pricing ConfigMap has active route for flow-qwen +# (monetize §1.4 and Pricing Configuration section show the routes[] structure) +step "x402-pricing ConfigMap has active route for /services/flow-qwen" +pricing_yaml=$("$OBOL" kubectl get cm x402-pricing -n x402 \ + -o jsonpath='{.data.pricing\.yaml}' 2>&1) || true +if echo "$pricing_yaml" | grep -q "flow-qwen\|services/flow-qwen"; then + pass "x402-pricing has route for /services/flow-qwen" +else + fail "x402-pricing missing route for flow-qwen — ${pricing_yaml:0:200}" +fi + +# §1.5: obol tunnel logs — verify cloudflared is logging (documented command) +step "obol tunnel logs shows cloudflared output" +tunnel_logs=$("$OBOL" tunnel logs 2>&1) || true +if echo "$tunnel_logs" | grep -q "cloudflare\|tunnel\|INF\|info\|TUN"; then + pass "Tunnel logs available" +else + fail "Tunnel logs empty or missing — ${tunnel_logs:0:100}" +fi + +# §1.4: obol sell status — individual ServiceOffer conditions (monetize §1.4/§4) +# Verifies all 6 conditions are True: ModelReady, UpstreamHealthy, PaymentGateReady, +# RoutePublished, Registered, Ready +step "obol sell status flow-qwen shows Ready conditions" +status_out=$("$OBOL" sell status flow-qwen -n llm 2>&1) || true +if echo "$status_out" | grep -q 'type: Ready' && echo "$status_out" | grep -q "status: \"True\""; then + pass "ServiceOffer flow-qwen has Ready condition True" +else + fail "ServiceOffer status missing Ready condition — ${status_out:0:200}" +fi + +emit_metrics diff --git a/flows/flow-08-buy.sh b/flows/flow-08-buy.sh new file mode 100755 index 00000000..85df3cab --- /dev/null +++ b/flows/flow-08-buy.sh @@ -0,0 +1,222 @@ +#!/bin/bash +# Flow 08: Buy — monetize-inference.md §2.1-2.5. +# Requires: flow-06 (ServiceOffer Ready) + flow-10 (Anvil + facilitator running). +source "$(dirname "$0")/lib.sh" + +TUNNEL_OUTPUT=$("$OBOL" tunnel status 2>&1) || true +TUNNEL_URL=$(echo "$TUNNEL_OUTPUT" | grep -oE 'https://[a-z0-9-]+\.trycloudflare\.com' | head -1) +BASE_URL="${TUNNEL_URL:-http://obol.stack:8080}" +if [[ "$BASE_URL" == *"obol.stack"* ]]; then + CURL_BASE="$CURL_OBOL" +else + CURL_BASE="curl" +fi + +# §2.1: Discover services via /skill.md (machine-readable catalog, always published +# when ServiceOffers are ready; /.well-known/agent-registration.json requires +# on-chain ERC-8004 registration via --register flag which is not used in this flow) +step "Discover services via /skill.md" +skill_out=$($CURL_BASE -sf --max-time 10 "$BASE_URL/skill.md" 2>&1) || true +if echo "$skill_out" | grep -q "x402\|service\|obol"; then + pass "Service catalog (/skill.md) discovered" +else + fail "Service catalog not found — ${skill_out:0:200}" +fi + +# §2.1: skill.md lists flow-qwen service with its endpoint (agent publishes after reconcile) +step "/skill.md lists flow-qwen service" +if echo "$skill_out" | grep -q "flow-qwen"; then + endpoint=$(echo "$skill_out" | grep -oE '`https://[^`]+`' | head -1 || echo "(local)") + pass "/skill.md lists flow-qwen (endpoint: ${endpoint})" +else + # May not be listed if heartbeat hasn't run yet since SO was deleted+recreated + fail "/skill.md does not list flow-qwen — ${skill_out:0:200}" +fi + +# §2.2: 402 body validation +step "402 body validated" +body_402=$($CURL_BASE -s --max-time 10 -X POST \ + "$BASE_URL/services/flow-qwen/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -d "{\"model\":\"$FLOW_MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}" 2>&1) || true +if echo "$body_402" | python3 -c " +import sys, json +d = json.load(sys.stdin) +assert d.get('x402Version') is not None, 'missing x402Version' +a = d['accepts'][0] +assert a['payTo'], 'missing payTo' +assert a['network'], 'missing network' +assert a['maxAmountRequired'], 'missing maxAmountRequired' +print('OK: payTo=%s network=%s amount=%s' % (a['payTo'], a['network'], a['maxAmountRequired'])) +" 2>&1; then + pass "402 body validated" +else + fail "402 body validation failed — ${body_402:0:200}" +fi + +# §2.4 pre-capture: Record seller balance BEFORE paid inference to verify settlement +# (monetize §2.4 — "payee balance should have increased") +PRE_SELLER_BAL="" +if command -v cast &>/dev/null; then + PRE_SELLER_BAL=$(env -u CHAIN cast call "$USDC_ADDRESS" "balanceOf(address)(uint256)" "$SELLER_WALLET" \ + --rpc-url "$ANVIL_RPC" 2>&1) || true + [[ "$PRE_SELLER_BAL" =~ ^[0-9] ]] || PRE_SELLER_BAL="" +fi + +# §2.3: Paid inference — sign EIP-712 ERC-3009 payment and retry +# Uses eth_account (installed with: pip install eth-account) to sign +# the TransferWithAuthorization payload, matching internal/testutil/eip712_signer.go +step "Paid inference via x402 payment signing" +if python3 -c "import eth_account, httpx" 2>/dev/null; then + paid_out=$(python3 << 'PYEOF' 2>&1 +import sys, os, json, base64, secrets, time +import httpx +from eth_account import Account +from eth_account.messages import encode_typed_data + +SERVICE_URL = os.environ.get('BASE_URL', 'http://obol.stack:8080') +SERVICE_PATH = "/services/flow-qwen/v1/chat/completions" +CONSUMER_KEY = os.environ["CONSUMER_PRIVATE_KEY"] +USDC_ADDRESS = "0x036CbD53842c5426634e7929541eC2318f3dCF7e" +CHAIN_ID = 84532 # Base Sepolia + +acct = Account.from_key(CONSUMER_KEY) + +# 1. Initial request → 402 +url = SERVICE_URL + SERVICE_PATH +body = {"model": "qwen3:0.6b", "messages": [{"role": "user", "content": "What is 2+2?"}], "max_tokens": 20} +headers = {"Content-Type": "application/json"} +if "obol.stack" in SERVICE_URL: + # macOS mDNS bypass: connect to 127.0.0.1 but send Host header + transport = httpx.HTTPTransport() +resp = httpx.post(url, json=body, headers=headers, timeout=30, follow_redirects=True) +if resp.status_code != 402: + print(f"ERROR: expected 402, got {resp.status_code}: {resp.text[:200]}") + sys.exit(1) + +req_data = resp.json() +accept = req_data["accepts"][0] +pay_to = accept["payTo"] +amount = accept["maxAmountRequired"] # micro-USDC string e.g. "1000" +network = accept["network"] + +# 2. Sign EIP-712 TransferWithAuthorization (ERC-3009) +nonce = "0x" + secrets.token_hex(32) +valid_before = str(int(time.time()) + 3600) # 1 hour from now + +structured = { + "types": { + "EIP712Domain": [ + {"name": "name", "type": "string"}, + {"name": "version", "type": "string"}, + {"name": "chainId", "type": "uint256"}, + {"name": "verifyingContract", "type": "address"}, + ], + "TransferWithAuthorization": [ + {"name": "from", "type": "address"}, + {"name": "to", "type": "address"}, + {"name": "value", "type": "uint256"}, + {"name": "validAfter", "type": "uint256"}, + {"name": "validBefore", "type": "uint256"}, + {"name": "nonce", "type": "bytes32"}, + ], + }, + "primaryType": "TransferWithAuthorization", + "domain": { + "name": "USDC", "version": "2", + "chainId": CHAIN_ID, "verifyingContract": USDC_ADDRESS, + }, + "message": { + "from": acct.address, + "to": pay_to, + "value": int(amount), + "validAfter": 0, + "validBefore": int(valid_before), + "nonce": bytes.fromhex(nonce[2:]), + }, +} +signed = acct.sign_message(encode_typed_data(full_message=structured)) +sig_hex = "0x" + signed.signature.hex() + +# 3. Build x402 payment envelope +envelope = { + "x402Version": 1, + "scheme": "exact", + "network": network, + "payload": { + "signature": sig_hex, + "authorization": { + "from": acct.address, + "to": pay_to, + "value": amount, + "validAfter": "0", + "validBefore": valid_before, + "nonce": nonce, + }, + }, + "resource": { + "payTo": pay_to, "maxAmountRequired": amount, + "asset": USDC_ADDRESS, "network": network, + }, +} +payment_header = base64.b64encode(json.dumps(envelope).encode()).decode() + +# 4. Retry with X-Payment header +resp2 = httpx.post(url, json=body, + headers={**headers, "X-Payment": payment_header}, + timeout=120, follow_redirects=True) +if resp2.status_code == 200 and "choices" in resp2.text: + d = resp2.json() + nc = len(d.get("choices", [])) + print(f"PAID_RESPONSE: HTTP 200, choices={nc}") +else: + print(f"ERROR: payment rejected — HTTP {resp2.status_code}: {resp2.text[:300]}") + sys.exit(1) +PYEOF + ) || true # prevent set -e from killing the flow on Python script failure + if echo "$paid_out" | grep -q "PAID_RESPONSE:\|choices_ok"; then + pass "Paid inference succeeded" + else + fail "Paid inference failed — ${paid_out:0:400}" + fi +else + fail "eth_account/httpx not installed — run: pip install eth-account httpx" +fi + +# §2.4: Balance checks (requires cast/Foundry) +# Use exit-code check + numeric pattern to avoid false positives from cast error messages +if command -v cast &>/dev/null; then + step "Buyer USDC balance check" + # env -u CHAIN: CHAIN=base-sepolia conflicts with foundry (expects uint64) + if buyer_bal=$(env -u CHAIN cast call "$USDC_ADDRESS" "balanceOf(address)(uint256)" "$CONSUMER_WALLET" \ + --rpc-url "$ANVIL_RPC" 2>&1) && [[ "$buyer_bal" =~ ^[0-9] ]]; then + pass "Buyer USDC balance: $buyer_bal" + else + fail "Buyer balance check failed — ${buyer_bal:0:100}" + fi + + step "Seller USDC balance increased after payment (§2.4 settlement)" + if seller_bal=$(env -u CHAIN cast call "$USDC_ADDRESS" "balanceOf(address)(uint256)" "$SELLER_WALLET" \ + --rpc-url "$ANVIL_RPC" 2>&1) && [[ "$seller_bal" =~ ^[0-9] ]]; then + # If we captured a pre-balance, verify it increased (actual settlement check) + if [ -n "$PRE_SELLER_BAL" ] && echo "$paid_out" | grep -q "PAID_RESPONSE:"; then + pre_num=$(echo "$PRE_SELLER_BAL" | grep -oE '^[0-9]+' | head -1) + post_num=$(echo "$seller_bal" | grep -oE '^[0-9]+' | head -1) + if [ -n "$pre_num" ] && [ -n "$post_num" ] && [ "$post_num" -gt "$pre_num" ] 2>/dev/null; then + pass "Seller USDC balance increased: $pre_num → $post_num (payment settled)" + elif [ "$post_num" = "$pre_num" ]; then + fail "Seller balance unchanged after payment: $pre_num (settlement may have failed)" + else + pass "Seller USDC balance: $seller_bal (pre-balance: ${PRE_SELLER_BAL:-unknown})" + fi + else + pass "Seller USDC balance: $seller_bal" + fi + else + fail "Seller balance check failed — ${seller_bal:0:100}" + fi +else + fail "cast (Foundry) not installed — skipping balance checks" +fi + +emit_metrics diff --git a/flows/flow-09-lifecycle.sh b/flows/flow-09-lifecycle.sh new file mode 100755 index 00000000..a3f1f0c6 --- /dev/null +++ b/flows/flow-09-lifecycle.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# Flow 09: Lifecycle — monetize-inference.md §4. +# Tests: sell list, status, stop, delete, verify cleanup. +source "$(dirname "$0")/lib.sh" + +# List offers — verify table shows all expected columns (monetize §4 Monitoring) +run_step_grep "sell list shows flow-qwen" "flow-qwen" \ + "$OBOL" sell list --namespace llm + +# Verify table format: NAME TYPE PRICE NETWORK READY columns +step "sell list table shows full columns" +list_out=$("$OBOL" sell list --namespace llm 2>&1) || true +if echo "$list_out" | grep -q "NAME\|flow-qwen.*http.*0.001.*base-sepolia"; then + pass "sell list table format correct" +else + fail "sell list missing expected columns — ${list_out:0:200}" +fi + +# Status (no-name → global pricing config, shows facilitator URL) +step "sell status shows pricing config" +status_out=$("$OBOL" sell status 2>&1) || true +if echo "$status_out" | grep -q "Wallet\|wallet" && echo "$status_out" | grep -q "Chain\|chain\|Facilitator\|Routes"; then + pass "sell status shows full pricing config" +else + fail "sell status missing expected fields — ${status_out:0:200}" +fi + +# Stop — §4 Pausing: "removes the pricing route so requests pass through without payment" +run_step "sell stop flow-qwen" "$OBOL" sell stop flow-qwen --namespace llm + +# Verify stop removed the pricing route from x402-pricing ConfigMap +step "sell stop removed pricing route (routes=0 after stop)" +routes_out=$("$OBOL" sell status 2>&1) || true +if echo "$routes_out" | grep -q "Routes:.*0"; then + pass "x402 pricing route removed after sell stop" +else + fail "Route still active after sell stop — ${routes_out:0:200}" +fi + +# §4 Pausing: "The CR and any ERC-8004 registration remain intact" (sell stop only pauses) +step "ServiceOffer CR still exists after sell stop (CR not deleted, only paused)" +so_after_stop=$("$OBOL" kubectl get serviceoffer flow-qwen -n llm 2>&1) || true +if echo "$so_after_stop" | grep -q "flow-qwen"; then + pass "ServiceOffer CR persists after sell stop (paused, not deleted)" +else + fail "ServiceOffer CR was deleted by sell stop — ${so_after_stop:0:100}" +fi + +# Delete +run_step "sell delete flow-qwen" "$OBOL" sell delete flow-qwen --namespace llm --force + +# Verify cleanup — all resources should be gone +step "ServiceOffer NotFound after delete" +so_out=$("$OBOL" kubectl get serviceoffer flow-qwen -n llm 2>&1) || true +if echo "$so_out" | grep -qi "NotFound\|not found"; then + pass "ServiceOffer deleted" +else + fail "ServiceOffer still exists — $so_out" +fi + +step "Middleware NotFound after delete" +mw_out=$("$OBOL" kubectl get middleware x402-flow-qwen -n llm 2>&1) || true +if echo "$mw_out" | grep -qi "NotFound\|not found"; then + pass "Middleware deleted" +else + fail "Middleware still exists — $mw_out" +fi + +step "HTTPRoute NotFound after delete" +hr_out=$("$OBOL" kubectl get httproute so-flow-qwen -n llm 2>&1) || true +if echo "$hr_out" | grep -qi "NotFound\|not found"; then + pass "HTTPRoute deleted" +else + fail "HTTPRoute still exists — $hr_out" +fi + +emit_metrics diff --git a/flows/flow-10-anvil-facilitator.sh b/flows/flow-10-anvil-facilitator.sh new file mode 100755 index 00000000..497f283b --- /dev/null +++ b/flows/flow-10-anvil-facilitator.sh @@ -0,0 +1,180 @@ +#!/bin/bash +# Flow 10: Anvil + Facilitator — monetize-inference.md §3. +# Sets up local test infrastructure for paid flows. Run BEFORE flow-08. +# +# Aligns with internal/testutil/anvil.go + facilitator_real.go: +# - Free ports (or reuse if already running) +# - Facilitator signer = Anvil accounts[0] (0xf39Fd6e51...) +# - ClusterURL uses host.docker.internal (resolves inside k3d on macOS) +source "$(dirname "$0")/lib.sh" + +# FACILITATOR_SIGNER_KEY is derived from the Anvil mnemonic in lib.sh (accounts[0]) +# SELLER_WALLET, CONSUMER_WALLET also come from lib.sh + +# Check Foundry is installed +step "Foundry (anvil + cast) installed" +if command -v anvil &>/dev/null && command -v cast &>/dev/null; then + pass "Foundry tools available" +else + fail "Foundry not installed — run: curl -L https://foundry.paradigm.xyz | bash && foundryup" + emit_metrics + exit 0 +fi + +# §3.2: Start Anvil fork (if not already running) +step "Start Anvil fork of Base Sepolia" +if curl -sf http://localhost:8545 -X POST -H 'Content-Type: application/json' \ + -d '{"jsonrpc":"2.0","method":"eth_chainId","params":[],"id":1}' >/dev/null 2>&1; then + pass "Anvil already running on port 8545" + ANVIL_RPC="http://localhost:8545" +else + anvil --fork-url https://sepolia.base.org --port 8545 &>/dev/null & + sleep 3 + if curl -sf http://localhost:8545 -X POST -H 'Content-Type: application/json' \ + -d '{"jsonrpc":"2.0","method":"eth_chainId","params":[],"id":1}' >/dev/null 2>&1; then + pass "Anvil started on port 8545" + else + fail "Anvil failed to start" + emit_metrics; exit 0 + fi + ANVIL_RPC="http://localhost:8545" +fi +export ANVIL_RPC + +# Verify Anvil is forking Base Sepolia (chain ID 84532 = 0x14a34) +# This confirms the fork is pointing at the right network for x402 payment testing +step "Anvil fork chain ID = 84532 (Base Sepolia)" +anvil_chain=$(curl -sf http://localhost:8545 -X POST \ + -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","method":"eth_chainId","params":[],"id":1}' 2>&1) || true +if echo "$anvil_chain" | python3 -c " +import sys, json +d = json.load(sys.stdin) +cid = d.get('result','') +assert cid.lower() == '0x14a34', f'expected 0x14a34 (Base Sepolia 84532), got {cid}' +print(f'Anvil chain ID: {int(cid, 16)} (Base Sepolia fork confirmed)') +" 2>&1; then + pass "Anvil is a Base Sepolia fork (chain 84532)" +else + fail "Anvil chain ID unexpected — ${anvil_chain:0:100}" +fi + +# §3.2: Verify USDC contract is deployed at expected address on the fork +# FiatTokenV2 should have name=USDC, symbol=USDC, decimals=6 +step "USDC contract (0x036C...) deployed on Anvil fork" +usdc_name=$(env -u CHAIN cast call "$USDC_ADDRESS" "name()(string)" \ + --rpc-url "$ANVIL_RPC" 2>&1) || true +if echo "$usdc_name" | grep -q '"USDC"'; then + usdc_dec=$(env -u CHAIN cast call "$USDC_ADDRESS" "decimals()(uint8)" \ + --rpc-url "$ANVIL_RPC" 2>&1 | tr -d '"' | head -1) + pass "USDC contract verified: name=USDC, decimals=$usdc_dec" +else + fail "USDC contract not found or wrong name — ${usdc_name:0:100}" +fi + +# Fund consumer with USDC (accounts[9] = CONSUMER_WALLET) +run_step "Clear consumer contract code" \ + cast rpc anvil_setCode "$CONSUMER_WALLET" 0x --rpc-url "$ANVIL_RPC" + +step "Fund consumer with USDC" +SLOT=$(cast index address "$CONSUMER_WALLET" 9 2>&1) +cast rpc anvil_setStorageAt "$USDC_ADDRESS" "$SLOT" \ + "0x000000000000000000000000000000000000000000000000000000003B9ACA00" \ + --rpc-url "$ANVIL_RPC" >/dev/null 2>&1 || true +pass "USDC storage slot written for $CONSUMER_WALLET" + +step "Consumer USDC balance > 0" +# Unset CHAIN env var: it conflicts with foundry's --chain flag (foundry picks +# up CHAIN=base-sepolia as the chain ID but expects a uint64, not a string). +if bal=$(env -u CHAIN cast call "$USDC_ADDRESS" "balanceOf(address)(uint256)" "$CONSUMER_WALLET" \ + --rpc-url "$ANVIL_RPC" 2>&1) && [[ "$bal" =~ ^[0-9] ]]; then + pass "Consumer USDC balance: $bal" +else + fail "Consumer USDC balance check failed — $bal" +fi + +# §3.3: x402-rs facilitator +step "x402-rs facilitator running" +if curl -sf http://localhost:4040/supported >/dev/null 2>&1; then + pass "Facilitator already running on port 4040" + FACILITATOR_PORT=4040 +else + # Binary discovery: X402_FACILITATOR_BIN env → ~/Development/R&D/x402-rs + FACILITATOR_BIN="${X402_FACILITATOR_BIN:-}" + if [ -z "$FACILITATOR_BIN" ]; then + X402_RS_DIR="${X402_RS_DIR:-$HOME/Development/R&D/x402-rs}" + for candidate in \ + "$X402_RS_DIR/target/release/x402-facilitator" \ + "$X402_RS_DIR/target/release/facilitator"; do + [ -f "$candidate" ] && FACILITATOR_BIN="$candidate" && break + done + fi + + if [ -z "$FACILITATOR_BIN" ]; then + fail "x402-facilitator binary not found — set X402_FACILITATOR_BIN or build from x402-rs repo" + emit_metrics; exit 0 + fi + + FACILITATOR_PORT=4040 + FACILITATOR_CONFIG=$(mktemp /tmp/x402-facilitator-XXXXXX.json) + # Use FACILITATOR_SIGNER_KEY (accounts[0]) — matches internal/testutil/facilitator_real.go + SIGNER_KEY="${FACILITATOR_SIGNER_KEY#0x}" + cat > "$FACILITATOR_CONFIG" << FEOF +{ + "port": $FACILITATOR_PORT, "host": "0.0.0.0", + "chains": {"eip155:84532": {"eip1559": true, "flashblocks": false, + "signers": ["$SIGNER_KEY"], + "rpc": [{"http": "http://127.0.0.1:8545", "rate_limit": 50}]}}, + "schemes": [{"id": "v1-eip155-exact","chains":"eip155:*"},{"id":"v2-eip155-exact","chains":"eip155:*"}] +} +FEOF + "$FACILITATOR_BIN" --config "$FACILITATOR_CONFIG" &>/dev/null & + sleep 3 + if curl -sf http://localhost:$FACILITATOR_PORT/supported >/dev/null 2>&1; then + pass "Facilitator started on port $FACILITATOR_PORT" + else + fail "Facilitator failed to start (bin: $FACILITATOR_BIN)" + emit_metrics; exit 0 + fi +fi + +run_step_grep "Facilitator /supported" "eip155" \ + curl -sf http://localhost:$FACILITATOR_PORT/supported + +# §3.4: Reconfigure stack to use local facilitator +# Use host.docker.internal — resolves inside k3d containers on macOS +# (host.k3d.internal does NOT resolve reliably on macOS; matches testutil/facilitator_real.go) +CLUSTER_FACILITATOR_URL="http://host.docker.internal:$FACILITATOR_PORT" +run_step_grep "sell pricing with local facilitator" \ + "configured.*facilitator\|x402 configured" \ + "$OBOL" sell pricing \ + --wallet "$SELLER_WALLET" \ + --chain "$CHAIN" \ + --facilitator-url "$CLUSTER_FACILITATOR_URL" + +# §3.4: Verify facilitator URL was persisted to x402-pricing ConfigMap +step "x402-pricing ConfigMap has local facilitator URL" +pricing_yaml=$("$OBOL" kubectl get cm x402-pricing -n x402 \ + -o jsonpath='{.data.pricing\.yaml}' 2>&1) || true +if echo "$pricing_yaml" | grep -q "host.docker.internal\|facilitatorURL:"; then + fac_line=$(echo "$pricing_yaml" | grep "facilitatorURL:" | head -1) + pass "x402-pricing has facilitator URL: $fac_line" +else + fail "x402-pricing missing facilitatorURL — ${pricing_yaml:0:200}" +fi + +# obol sell pricing changes x402-pricing ConfigMap → Kubernetes Reloader restarts +# x402-verifier pods. Wait for them to be ready before flow-08 makes paid requests. +step "x402 verifier pods ready after pricing change" +for i in $(seq 1 24); do + ready=$("$OBOL" kubectl get pods -n x402 --no-headers 2>&1 | grep "Running" | grep -c "1/1" || echo 0) + total=$("$OBOL" kubectl get pods -n x402 --no-headers 2>&1 | grep -v "^$" | wc -l | tr -d ' ') + if [ "$ready" -ge 1 ] && [ "$ready" = "$total" ]; then + pass "x402 verifier ready ($ready/$total)" + break + fi + [ "$i" -eq 24 ] && fail "x402 verifier not ready after 120s" + sleep 5 +done + +emit_metrics diff --git a/flows/lib.sh b/flows/lib.sh new file mode 100755 index 00000000..08220cbc --- /dev/null +++ b/flows/lib.sh @@ -0,0 +1,133 @@ +#!/bin/bash +# Shared helpers for flow scripts. +# Source this at the top of every flow: source "$(dirname "$0")/lib.sh" + +set -euo pipefail + +OBOL_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +export OBOL_DEVELOPMENT=true +export OBOL_CONFIG_DIR="$OBOL_ROOT/.workspace/config" +export OBOL_BIN_DIR="$OBOL_ROOT/.workspace/bin" +export OBOL_DATA_DIR="$OBOL_ROOT/.workspace/data" +OBOL="$OBOL_BIN_DIR/obol" + +STEP_COUNT=0 +PASS_COUNT=0 + +# Anvil test accounts — derived at runtime from `cast wallet` to avoid +# hardcoding private keys in source. These are the well-known deterministic +# accounts that Anvil/Foundry generates, but we derive them rather than embed. +if command -v cast &>/dev/null; then + # Derive Anvil test accounts from the well-known mnemonic at runtime. + # This avoids hardcoding private keys in source while producing the same + # deterministic accounts that `anvil` generates on every machine. + ANVIL_MNEMONIC="test test test test test test test test test test test junk" + _derive_key() { cast wallet private-key "$ANVIL_MNEMONIC" "$1" 2>/dev/null; } + _derive_addr() { local k; k=$(_derive_key "$1") && cast wallet address "$k" 2>/dev/null; } + + # accounts[0] = facilitator signer (settles payments on-chain) + # accounts[1] = seller / payTo wallet + # accounts[9] = buyer / consumer wallet + export FACILITATOR_SIGNER_KEY=$(_derive_key 0) + export SELLER_WALLET=$(_derive_addr 1) + export SELLER_KEY=$(_derive_key 1) + export CONSUMER_WALLET=$(_derive_addr 9) + export CONSUMER_PRIVATE_KEY=$(_derive_key 9) +else + # Fallback: addresses only (no private keys without Foundry) + export SELLER_WALLET="0x70997970C51812dc3A010C7d01b50e0d17dc79C8" + export CONSUMER_WALLET="0xa0Ee7A142d267C1f36714E4a8F75612F20a79720" +fi +export USDC_ADDRESS="0x036CbD53842c5426634e7929541eC2318f3dCF7e" +export CHAIN="base-sepolia" +export ANVIL_RPC="http://localhost:8545" + +# Model used for flow tests (small, fast, local Ollama) +export FLOW_MODEL="qwen3:0.6b" + +# macOS mDNS can be slow resolving .stack TLD from /etc/hosts. +# Use --resolve to bypass DNS and go straight to 127.0.0.1. +CURL_OBOL="curl --resolve obol.stack:80:127.0.0.1 --resolve obol.stack:8080:127.0.0.1 --resolve obol.stack:443:127.0.0.1" + +step() { + STEP_COUNT=$((STEP_COUNT + 1)) + echo "STEP: [$STEP_COUNT] $1" +} + +pass() { + PASS_COUNT=$((PASS_COUNT + 1)) + echo "PASS: [$STEP_COUNT] $1" +} + +fail() { + echo "FAIL: [$STEP_COUNT] $1" +} + +# Run a command; pass if exit 0, fail otherwise. Captures output. +run_step() { + local desc="$1"; shift + step "$desc" + local out + if out=$("$@" 2>&1); then + pass "$desc" + echo "$out" + else + fail "$desc — exit $? — ${out:0:200}" + fi +} + +# Run a command and check output contains a substring +run_step_grep() { + local desc="$1"; local pattern="$2"; shift 2 + step "$desc" + local out + if out=$("$@" 2>&1) && echo "$out" | grep -q "$pattern"; then + pass "$desc" + else + fail "$desc — pattern '$pattern' not found — ${out:0:200}" + fi +} + +# Poll a command until it succeeds (max retries with delay) +poll_step() { + local desc="$1"; local max="$2"; local delay="$3"; shift 3 + step "$desc (polling, max ${max}x${delay}s)" + for i in $(seq 1 "$max"); do + if "$@" >/dev/null 2>&1; then + pass "$desc (attempt $i)" + return 0 + fi + sleep "$delay" + done + fail "$desc — timed out after $((max * delay))s" +} + +# Poll a command until its output matches a grep pattern +poll_step_grep() { + local desc="$1"; local pattern="$2"; local max="$3"; local delay="$4"; shift 4 + step "$desc (polling, max ${max}x${delay}s)" + for i in $(seq 1 "$max"); do + local out + out=$("$@" 2>&1) || true + if echo "$out" | grep -q "$pattern"; then + pass "$desc (attempt $i)" + return 0 + fi + sleep "$delay" + done + fail "$desc — pattern '$pattern' not found after $((max * delay))s" +} + +# Kill background process and wait +cleanup_pid() { + local pid="$1" + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null + wait "$pid" 2>/dev/null || true + fi +} + +emit_metrics() { + echo "METRIC steps_passed=$PASS_COUNT" + echo "METRIC total_steps=$STEP_COUNT" +} diff --git a/internal/agent/agent.go b/internal/agent/agent.go index c3fdc07d..46c0bb46 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -1,9 +1,11 @@ package agent import ( + "bytes" "encoding/json" "fmt" "os" + "os/exec" "path/filepath" "github.com/ObolNetwork/obol-stack/internal/config" @@ -30,6 +32,18 @@ func Init(cfg *config.Config, u *ui.UI) error { return fmt.Errorf("failed to inject HEARTBEAT.md: %w", err) } + // Ensure the openclaw-config ConfigMap has heartbeat config and that the + // pod is running with it loaded. This is needed both for fresh clusters + // (where doSync ran before the pod started, so the patch didn't take + // effect) and for "already running" clusters where doSync was never called + // this session. ensureHeartbeatActive is idempotent: if heartbeat is + // already in the ConfigMap and the pod is healthy, it does nothing. + if err := ensureHeartbeatActive(cfg, u); err != nil { + // Non-fatal: log and continue. The heartbeat may still work if the + // ConfigMap was already correct from a previous run. + u.Warnf("could not ensure heartbeat config: %v", err) + } + u.Success("Agent capabilities applied to default OpenClaw instance") return nil } @@ -122,3 +136,93 @@ python3 /data/.openclaw/skills/sell/scripts/monetize.py process --all --quick u.Successf("HEARTBEAT.md injected at %s", heartbeatPath) return nil } + +// ensureHeartbeatActive guarantees that: +// 1. The openclaw-config ConfigMap contains agents.defaults.heartbeat (every: 5m). +// 2. The openclaw pod is restarted when the ConfigMap was missing the field, +// so the heartbeat scheduler is activated on the next pod startup. +// +// Idempotent: if heartbeat is already present and the pod is healthy, no +// restart is performed. +func ensureHeartbeatActive(cfg *config.Config, u *ui.UI) error { + namespace := fmt.Sprintf("openclaw-%s", DefaultInstanceID) + kubectlBin := filepath.Join(cfg.BinDir, "kubectl") + kubeconfigPath := filepath.Join(cfg.ConfigDir, "kubeconfig.yaml") + env := append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", kubeconfigPath)) + + // Read current ConfigMap. + getCmd := exec.Command(kubectlBin, + "get", "configmap", "openclaw-config", + "-n", namespace, + "-o", "jsonpath={.data.openclaw\\.json}") + getCmd.Env = env + var outBuf bytes.Buffer + getCmd.Stdout = &outBuf + if err := getCmd.Run(); err != nil { + return fmt.Errorf("read openclaw-config: %w", err) + } + + var cfgJSON map[string]interface{} + if err := json.Unmarshal(outBuf.Bytes(), &cfgJSON); err != nil { + return fmt.Errorf("parse openclaw.json: %w", err) + } + + // Check whether heartbeat is already present. + agents, _ := cfgJSON["agents"].(map[string]interface{}) + defaults, _ := agents["defaults"].(map[string]interface{}) + _, alreadySet := defaults["heartbeat"] + if alreadySet { + u.Success("Heartbeat config already active") + return nil + } + + // Inject heartbeat. + if agents == nil { + agents = map[string]interface{}{} + cfgJSON["agents"] = agents + } + if defaults == nil { + defaults = map[string]interface{}{} + agents["defaults"] = defaults + } + defaults["heartbeat"] = map[string]interface{}{ + "every": "5m", + "target": "none", + } + + patched, err := json.MarshalIndent(cfgJSON, "", " ") + if err != nil { + return fmt.Errorf("marshal patched config: %w", err) + } + + applyPayload := map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "openclaw-config", + "namespace": namespace, + }, + "data": map[string]string{ + "openclaw.json": string(patched), + }, + } + applyRaw, _ := json.Marshal(applyPayload) + + applyCmd := exec.Command(kubectlBin, + "apply", "-f", "-", + "--server-side", "--field-manager=helm", "--force-conflicts") + applyCmd.Env = env + applyCmd.Stdin = bytes.NewReader(applyRaw) + var applyErr bytes.Buffer + applyCmd.Stderr = &applyErr + if err := applyCmd.Run(); err != nil { + return fmt.Errorf("patch heartbeat config: %w\n%s", err, applyErr.String()) + } + + // OpenClaw watches for ConfigMap file changes and hot-reloads config. + // No pod restart is needed: the running pod will detect the update within + // ~30-60s and apply [reload] config hot reload, switching the heartbeat + // interval to 5m immediately without losing the running pod or its state. + u.Success("Heartbeat config injected — OpenClaw hot reload will activate it (every 5m)") + return nil +} diff --git a/internal/network/rpc.go b/internal/network/rpc.go index d956a0d4..84cc7b01 100644 --- a/internal/network/rpc.go +++ b/internal/network/rpc.go @@ -3,6 +3,7 @@ package network import ( "encoding/json" "fmt" + "net/url" "regexp" "strings" @@ -107,6 +108,11 @@ var writeMethods = []interface{}{"eth_sendRawTransaction", "eth_sendTransaction" // Uses the "custom-" prefix to distinguish from ChainList-sourced upstreams. // When readOnly is true, eth_sendRawTransaction and eth_sendTransaction are blocked. func AddCustomRPC(cfg *config.Config, chainID int, chainName, endpoint string, readOnly bool) error { + // Validate endpoint URL before modifying eRPC config. + if err := validateRPCEndpoint(endpoint); err != nil { + return fmt.Errorf("invalid endpoint URL %q: %w", endpoint, err) + } + erpcConfig, err := readERPCConfig(cfg) if err != nil { return err @@ -438,6 +444,25 @@ func writeERPCConfig(cfg *config.Config, erpcConfig map[string]interface{}) erro return nil } +// validateRPCEndpoint returns an error if the endpoint string is not a valid +// HTTP/HTTPS URL. This prevents silently adding invalid endpoints to eRPC. +func validateRPCEndpoint(endpoint string) error { + if endpoint == "" { + return fmt.Errorf("endpoint URL is required") + } + u, err := url.Parse(endpoint) + if err != nil { + return fmt.Errorf("not a valid URL: %w", err) + } + if u.Scheme != "http" && u.Scheme != "https" && u.Scheme != "ws" && u.Scheme != "wss" { + return fmt.Errorf("scheme must be http, https, ws, or wss (got %q)", u.Scheme) + } + if u.Host == "" { + return fmt.Errorf("URL must include a host (e.g. http://localhost:8545)") + } + return nil +} + // yamlInt extracts an int from a YAML-parsed interface{} value, // handling both int and float64 (JSON numbers). func yamlInt(v interface{}) int { diff --git a/internal/openclaw/openclaw.go b/internal/openclaw/openclaw.go index d1d105c9..64ffbdf3 100644 --- a/internal/openclaw/openclaw.go +++ b/internal/openclaw/openclaw.go @@ -1992,7 +1992,11 @@ func patchHeartbeatConfig(cfg *config.Config, id, deploymentDir string) { return } - fmt.Printf("✓ Heartbeat config injected (every: %s, target: %s)\n", every, target) + // OpenClaw hot-reloads config: no pod restart needed. + // The running pod will detect the ConfigMap file change within ~30-60s + // and apply [reload] config hot reload, changing the heartbeat interval + // to the configured value immediately. + fmt.Printf("✓ Heartbeat config injected (every: %s, target: %s) — hot reload will activate it\n", every, target) } // ollamaEndpoint returns the base URL where host Ollama should be reachable. diff --git a/internal/tunnel/agent.go b/internal/tunnel/agent.go index 3656ea7d..60f81476 100644 --- a/internal/tunnel/agent.go +++ b/internal/tunnel/agent.go @@ -1,6 +1,8 @@ package tunnel import ( + "bytes" + "encoding/json" "fmt" "os" "os/exec" @@ -15,12 +17,22 @@ const agentDeploymentID = "obol-agent" // SyncAgentBaseURL patches AGENT_BASE_URL in the obol-agent's values-obol.yaml // and runs helmfile sync to apply the change. It is a no-op if the obol-agent // deployment directory does not exist (agent not yet initialized). +// +// Idempotent: if the overlay already has the correct AGENT_BASE_URL, the +// helmfile sync is skipped to avoid resetting the openclaw-config ConfigMap +// (which helm renders without agents.defaults.heartbeat). func SyncAgentBaseURL(cfg *config.Config, tunnelURL string) error { overlayPath := agentOverlayPath(cfg) if _, err := os.Stat(overlayPath); os.IsNotExist(err) { return nil // agent not deployed yet — nothing to do } + // Skip the helmfile sync (and ConfigMap reset) if the URL is unchanged. + if currentURL, _ := readCurrentAgentBaseURL(overlayPath); currentURL == tunnelURL { + fmt.Printf("✓ AGENT_BASE_URL already set to %s — skipping sync\n", tunnelURL) + return nil + } + if err := patchAgentBaseURL(overlayPath, tunnelURL); err != nil { return fmt.Errorf("failed to patch values-obol.yaml: %w", err) } @@ -59,13 +71,125 @@ func SyncAgentBaseURL(cfg *config.Config, tunnelURL string) error { } fmt.Println("✓ AGENT_BASE_URL synced to obol-agent") + + // Helmfile sync renders the openclaw-config ConfigMap from the chart template, + // which does not include agents.defaults.heartbeat. Re-patch the ConfigMap so + // the heartbeat interval is restored. OpenClaw hot-reloads the change (~30-60s) + // — no pod restart is needed. + patchHeartbeatAfterSync(cfg, deploymentDir) + return nil } +// patchHeartbeatAfterSync re-injects agents.defaults.heartbeat into the +// openclaw-config ConfigMap after a helmfile sync reset it. Mirrors the logic +// in internal/openclaw.patchHeartbeatConfig; kept here to avoid a circular +// import (openclaw imports tunnel). +// +// Non-fatal: prints a warning on failure and continues. +func patchHeartbeatAfterSync(cfg *config.Config, deploymentDir string) { + // Read heartbeat interval from values-obol.yaml. + valuesRaw, err := os.ReadFile(filepath.Join(deploymentDir, "values-obol.yaml")) + if err != nil || !strings.Contains(string(valuesRaw), "heartbeat:") { + return + } + var every, target string + for _, line := range strings.Split(string(valuesRaw), "\n") { + t := strings.TrimSpace(line) + if strings.HasPrefix(t, "every:") { + every = strings.Trim(strings.TrimSpace(strings.TrimPrefix(t, "every:")), "\"'") + } + if strings.HasPrefix(t, "target:") { + target = strings.Trim(strings.TrimSpace(strings.TrimPrefix(t, "target:")), "\"'") + } + } + if every == "" { + return + } + + kubectlBin := filepath.Join(cfg.BinDir, "kubectl") + kubeconfigPath := filepath.Join(cfg.ConfigDir, "kubeconfig.yaml") + namespace := "openclaw-" + agentDeploymentID + env := append(os.Environ(), "KUBECONFIG="+kubeconfigPath) + + // Read current ConfigMap. + getCmd := exec.Command(kubectlBin, "get", "configmap", "openclaw-config", + "-n", namespace, "-o", "jsonpath={.data.openclaw\\.json}") + getCmd.Env = env + var outBuf bytes.Buffer + getCmd.Stdout = &outBuf + if err := getCmd.Run(); err != nil { + fmt.Printf("⚠ could not read openclaw-config for heartbeat patch: %v\n", err) + return + } + + var cfgJSON map[string]interface{} + if err := json.Unmarshal(outBuf.Bytes(), &cfgJSON); err != nil { + fmt.Printf("⚠ could not parse openclaw.json for heartbeat patch: %v\n", err) + return + } + + // Inject heartbeat. + agents, _ := cfgJSON["agents"].(map[string]interface{}) + if agents == nil { + agents = map[string]interface{}{} + cfgJSON["agents"] = agents + } + defaults, _ := agents["defaults"].(map[string]interface{}) + if defaults == nil { + defaults = map[string]interface{}{} + agents["defaults"] = defaults + } + hb := map[string]interface{}{"every": every} + if target != "" { + hb["target"] = target + } + defaults["heartbeat"] = hb + + patched, _ := json.MarshalIndent(cfgJSON, "", " ") + applyPayload, _ := json.Marshal(map[string]interface{}{ + "apiVersion": "v1", "kind": "ConfigMap", + "metadata": map[string]interface{}{"name": "openclaw-config", "namespace": namespace}, + "data": map[string]string{"openclaw.json": string(patched)}, + }) + + applyCmd := exec.Command(kubectlBin, "apply", "-f", "-", + "--server-side", "--field-manager=helm", "--force-conflicts") + applyCmd.Env = env + applyCmd.Stdin = bytes.NewReader(applyPayload) + var applyErr bytes.Buffer + applyCmd.Stderr = &applyErr + if err := applyCmd.Run(); err != nil { + fmt.Printf("⚠ heartbeat patch failed: %v\n%s\n", err, applyErr.String()) + return + } + fmt.Printf("✓ Heartbeat config re-applied after sync (every: %s)\n", every) +} + func agentOverlayPath(cfg *config.Config) string { return filepath.Join(cfg.ConfigDir, "applications", "openclaw", agentDeploymentID, "values-obol.yaml") } +// readCurrentAgentBaseURL returns the current AGENT_BASE_URL value from +// values-obol.yaml, or "" if not found. +func readCurrentAgentBaseURL(overlayPath string) (string, error) { + data, err := os.ReadFile(overlayPath) + if err != nil { + return "", err + } + lines := strings.Split(string(data), "\n") + for i, line := range lines { + if strings.Contains(line, "name: AGENT_BASE_URL") { + // Next line should be the value + if i+1 < len(lines) && strings.Contains(lines[i+1], "value:") { + v := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(lines[i+1]), "value:")) + return v, nil + } + } + } + return "", nil +} + // patchAgentBaseURL reads values-obol.yaml and ensures the extraEnv list // contains an AGENT_BASE_URL entry with the given value. If the entry already // exists it is updated in place; otherwise it is appended after the