feat(observability): LangGraph span panel with Postgres persistence

.env.example

@@ -117,4 +117,8 @@ NEXT_PUBLIC_WALLET_CONNECT_PROJECT_ID=
 # Run the model surfaces a graceful fallback. Free tier: 1M requests
 # / 15h CPU / month — see https://deno.com/deploy/pricing.
 DENO_DEPLOY_TOKEN=
-DENO_DEPLOY_ORG=
+DENO_DEPLOY_ORG=
+
+# Observability — how long CapturedSpan rows are kept before retention
+# deletes them. Must be a positive integer. Default 30 days.
+OBSERVABILITY_RETENTION_DAYS=30

.gitignore

@@ -40,6 +40,7 @@ yarn-error.log*
 
 # local visual verification screenshots
 /.verify
+/.observability-screens
 # ad-hoc screenshots/logs dropped at repo root during manual testing
 /*.png
 /*.jpg

.specify/feature.json

@@ -1,3 +1,3 @@
 {
-  "feature_directory": "specs/001-user-auth"
+  "feature_directory": "specs/002-observability-panel"
 }

CLAUDE.md

@@ -81,6 +81,8 @@ backend/
   prompt/system.ts        CHAT_AGENT_PROMPT, WEATHER_AGENT_PROMPT, CRYPTO_AGENT_PROMPT, ROUTER_AGENT_PROMPT, RENAME_THREAD_PROMPT
   tool/                   ask_location, geocode_location, get_weather, search_web, fetch_url
   tool/crypto/            get_crypto_price, get_fx_rate, get_token_balances, get_NFT_holdings, connect_wallet, place_crypto_order, get_order_status
+  observability/
+    callback-collector.ts CapturingHandler — BaseCallbackHandler that buffers in-flight spans per runId, persists on chainEnd via injected bulkInsert
 langgraph.json            CLI config: graph id, node version, env file
 app/                      Next.js App Router
   layout.tsx              Root layout, fonts, TooltipProvider
@@ -90,9 +92,16 @@ app/                      Next.js App Router
   api/[..._path]/route.ts Node catch-all proxy to LANGGRAPH_API_URL (withAuth-gated)
   api/alchemy/[...path]/route.ts Server-only JSON-RPC proxy to Alchemy (with key + per-network disabled list)
   api/alchemy/status/route.ts Returns Alchemy key health + disabled-network list
+  api/threads/[id]/observability/route.ts GET / DELETE thread spans (withAuth-gated; ownership check → 404)
+  api/threads/[id]/observability/[parentMessageId]/route.ts GET spans for a single turn (parent_message_id filter)
   globals.css             Tailwind v4 entry
 components/
   assistant-ui/           Chat primitives (thread, attachment, markdown, reasoning, tool-fallback, tool-group, tooltip-icon-button)
+observability/                          UI components (button / sheet / sheet-context / panel) — moved out of assistant-ui/ so the feature owns its own folder
+  observability/button.tsx                stateless icon (rule #8 exception) — opens the Sheet via context
+  observability/sheet.tsx                 Sheet wrapper (singleton, at ThreadRoot) — derives threadId via useAuiState
+  observability/sheet-context.tsx         Provider + useOpenObservabilitySheet() so per-message buttons share one Sheet
+  observability/panel.tsx                 Waterfall renderer (SpanResource from @assistant-ui/react-o11y)
   ui/                     shadcn/ui primitives (avatar, button, collapsible, dialog, tooltip) — new-york style, lucide icons
   ui/address-or-hash.tsx  Truncated address/hash with copy-to-clipboard
   tool-ui/ask-location/   Interrupt-driven or addResult-driven location picker card
@@ -102,8 +111,13 @@ lib/utils.ts              cn() = twMerge(clsx(...))
 lib/threads/              Threads module (schema, queries, adapter, validators)
 lib/wagmi.ts              wagmi/RainbowKit config (chains, connectors, WalletConnect projectId)
 lib/alchemy/              networks.ts (slug → Alchemy URL + disabled list) + portfolio.ts (RPC helpers)
+lib/observability/        schema (Drizzle table) · queries (bulkInsert / get / markFailed / delete — with FORBIDDEN regex) · validators (Zod) · config (retention days) · transform (CapturedSpan → SpanData)
 lib/prices/coingecko.ts   CoinGecko free-tier price client (60s in-memory cache)
 lib/decimal.ts            Decimal-based amount math for crypto (no native float)
+scripts/
+  cleanup-observability.ts  Physical-delete older spans: `pnpm exec tsx scripts/cleanup-observability.ts` (cron entry; not yet scheduled — see `docs/OBSERVABILITY.md`)
+docs/
+  OBSERVABILITY.md        Design doc — UI flow, storage schema, FORBIDDEN regex, retention policy, trade-offs. HTTP endpoints are in `docs/APIS.md` § Observability (rule #1).
 ```
 
 ### Backend graph (`backend/agent.ts`)
@@ -159,6 +173,22 @@ Consequences worth knowing:
 
 `app/layout.tsx` wraps the assistant tree in `<Web3Providers>` (`app/web3-providers.tsx`), which stacks `@tanstack/react-query` `QueryClientProvider`, `WagmiProvider` (configured by `lib/wagmi.ts`), and RainbowKit's `RainbowKitProvider`. Wallet state is global to the browser; the crypto cards read `address` / `chainId` from wagmi hooks directly — they never receive the wallet through tool args. The trade flow is fully SIMULATED regardless of wallet connectivity: `place_crypto_order` auto-funds Mock Coin on the first trade and synthesizes the order on click. Setting `NEXT_PUBLIC_CRYPTO_REAL_SWAP=true` is required to route through any real DEX path (currently dormant — wagmi hooks live in the React tree, so a server-side router alone can't reach them).
 
+### Observability
+
+`backend/model.ts` exports `chatModel` / `chatModelWithoutThink` as `ChatOpenAI.withConfig({ callbacks: [getCapturingHandler()] })`. The handler (`backend/observability/callback-collector.ts`) is a `BaseCallbackHandler` that buffers per-runId spans in a Map (Start hooks create them; End hooks mutate + persist). On every `handleChainEnd` it fires `bulkInsert([span])` against `lib/observability/queries.ts` — the second insert is a no-op via `ON CONFLICT DO NOTHING` so re-flushing an inner span from the outer chain end doesn't double-write.
+
+`bulkInsert` is constructor-injected so the handler stays DB-free and the unit tests run with `vi.fn()`. Wire-up is a one-line `bulkInsert: async (spans) => { await bulkInsertSpans(spans); }` in `getCapturingHandler()`.
+
+Front-end: `<ObservabilityButton>` is icon-only (rule #8 exception) and mounts inside `<AssistantActionBar>` of `components/assistant-ui/thread.tsx` — alongside Copy / Refresh / More on every assistant message. Click opens an `<ObservabilitySheet>` that fetches `GET /api/threads/<threadId>/observability` and renders the waterfall with a retention banner.
+
+The Sheet itself is rendered once at `<ThreadRoot>` (not co-mounted per message — that would pile on dialog backdrops). Per-message buttons talk to the Sheet through `ObservabilitySheetProvider` / `useOpenObservabilitySheet()` from `components/observability/sheet-context.tsx`; the Sheet derives its `threadId` from `useAuiState(... mainThreadId.externalId)` so it follows whichever thread the user is on.
+
+Endpoints live in `app/api/threads/[id]/observability/route.ts` and are wrapped in `withAuth` (rule #9). Ownership → 404 (no existence leak, spec FR-008). Both GET and DELETE shape the response via `lib/observability/validators.ts` Zod schemas.
+
+DB table `observability_spans` in `lib/observability/schema.ts`, FK to `threads(id) ON DELETE CASCADE` (delete the thread row, spans drop with it — no separate cleanup). The `FORBIDDEN` regex in `bulkInsertSpans` rejects any row whose `JSON.stringify` matches `api[_-]?key | _password | ^password$ | _secret$ | ^secret$ | baseURL | organization | bearer <token>` (FR-009 / SC-003). The regex throws — the API is fail-closed: any new provider kwarg that contains a forbidden token stops the write until it's whitelisted.
+
+Retention: env `OBSERVABILITY_RETENTION_DAYS` (default 30, positive int). Physical delete runs via `pnpm exec tsx scripts/cleanup-observability.ts` — system cron is the operator's responsibility (no MVP+scheduling, see `docs/OBSERVABILITY.md` trade-offs).
+
 ### Patches
 
 `patches/` is currently empty. `pnpm-workspace.yaml` retains the `patchedDependencies:` header as a placeholder — when you need to patch a package, add the entry there and drop the `.patch` file under `patches/`. Re-check on every package bump; drop the entry + file when upstream ships the fix.
@@ -386,7 +416,7 @@ When you add a key-needing tool, also update `.env.example` (with the sign-up UR
 
 For additional context about technologies to be used, project structure,
 shell commands, and other important information, read the current plan
-at `specs/001-user-auth/plan.md` (and `spec.md`, `research.md`,
+at `specs/002-observability-panel/plan.md` (and `spec.md`, `research.md`,
 `data-model.md`, `quickstart.md`, `contracts/`).
 
 <!-- SPECKIT END -->

README.md

@@ -13,6 +13,7 @@ A self-hostable chat app (this repo: `langgraph-app`) that streams tokens from a
 - **User accounts**: email + password (with email verification), GitHub and Google sign-in, 7-day persistent sessions, and per-user thread isolation. See [docs/AUTH.md](docs/AUTH.md) for the operator guide.
 - **Tool-using agent**: the `agent` node is bound to `search_web` (Jina Search) and `fetch_url` (Jina Reader) — the model can research topics and read pages mid-conversation. Tools run unconditionally; write-side tools added later will hang their own `interruptBefore` hook. See [docs/APIS.md](docs/APIS.md) for the contract.
 - **Crypto sub-agent**: price, NFT holdings (5-chain gallery via Alchemy Portfolio), and a simulated swap flow against an auto-funded Mock Coin balance — see [docs/TOOLS.md](docs/TOOLS.md) and [docs/INTERRUPT.md](docs/INTERRUPT.md) for the per-card contract.
+- **Observability panel**: every LLM / Tool / Chain / Node span is captured by a `BaseCallbackHandler` and persisted to a `observability_spans` Postgres table. Each assistant message shows an icon button that opens a per-turn waterfall — duration, token usage, nested parent/child spans. See [docs/OBSERVABILITY.md](docs/OBSERVABILITY.md).
 
 ## Tech stack
 
@@ -112,9 +113,11 @@ app/                          Next.js App Router
     threads/                  Thread metadata CRUD
 
 backend/
-  agent.ts                    LangGraph graph (parallel agent + renameThread + tools loop)
+  agent.ts                    LangGraph graph (parallel agent + renameThread + tools loop); callback handler wired here
   model.ts                    ChatOpenAI singletons (with / without thinking)
   checkpointer.ts             PostgresSaver (Postgres checkpoint tables)
+  observability/
+    callback-collector.ts     CapturingHandler — BaseCallbackHandler that buffers spans per runId, persists to Postgres on every End hook
   tool/                       LangChain tools bound to the agent
     web-search.ts             search_web — Jina Search (s.jina.ai/{query})
     web-fetch.ts              fetch_url — Jina Reader (r.jina.ai/{url})
@@ -125,6 +128,7 @@ backend/
 
 components/
   assistant-ui/               Chat primitives (thread, markdown, reasoning, …)
+  observability/              Observability UI (button, sheet, sheet-context, panel)
   ui/                         shadcn/ui primitives
 
 lib/
@@ -136,6 +140,12 @@ lib/
     queries.ts                CRUD (rename, archive, unarchive, delete, fetch, list)
     adapter.ts                RemoteThreadListAdapter for assistant-ui
     validators.ts             Zod API body schemas
+  observability/              Observability module
+    schema.ts                 Drizzle table (observability_spans)
+    queries.ts                bulkInsertSpans / getSpansByThreadId / markRunningAsFailed / deleteSpansByThreadId
+    transform.ts              CapturedSpan → SpanData (for @assistant-ui/react-o11y)
+    config.ts                 getRetentionDays() — reads OBSERVABILITY_RETENTION_DAYS
+    validators.ts             Zod schemas for GET / DELETE responses
 
 db/                           Database root
   schema.ts                   Aggregate re-export of all module schemas
@@ -205,26 +215,27 @@ Test database stays isolated from dev — never put production-like data in `lan
 
 ## Environment variables
 
-| Var                                  | Used by                   | Required?                                    |
-| ------------------------------------ | ------------------------- | -------------------------------------------- |
-| `OPENAI_API_KEY`                     | backend agent             | yes                                          |
-| `OPENAI_MODEL`                       | backend agent             | optional (default `gpt-4o-mini`)             |
-| `OPENAI_BASE_URL`                    | backend agent             | optional (OpenAI-compatible gateway)         |
-| `JINA_API_KEYS`                      | web-search + web-fetch    | yes (comma-separated; one per Jina account)  |
-| `ALCHEMY_API_KEY`                    | NFT gallery + portfolio   | yes (server-only; powers `get_NFT_holdings`) |
-| `LANGGRAPH_API_URL`                  | Next.js proxy             | optional (default `http://localhost:2024`)   |
-| `LANGCHAIN_API_KEY`                  | Next.js proxy → LangGraph | optional (leave blank locally)               |
-| `NEXT_PUBLIC_LANGGRAPH_ASSISTANT_ID` | browser runtime           | optional (default `agent`)                   |
-| `NEXT_PUBLIC_LANGGRAPH_API_URL`      | browser runtime           | optional (uses proxy if unset)               |
-| `DATABASE_URL`                       | drizzle-kit + backend     | yes                                          |
-| `DATABASE_URL_TEST`                  | vitest                    | yes                                          |
-| `BETTER_AUTH_SECRET`                 | session cookie signing    | yes (see [docs/AUTH.md](docs/AUTH.md))       |
-| `BETTER_AUTH_URL`                    | OAuth callback base       | yes (default `http://localhost:3000`)        |
-| `RESEND_API_KEY`                     | verification emails       | yes                                          |
-| `RESEND_FROM_EMAIL`                  | verification email sender | optional (`onboarding@resend.dev` default)   |
-| `GITHUB_CLIENT_ID` / `_SECRET`       | GitHub OAuth              | optional                                     |
-| `GOOGLE_CLIENT_ID` / `_SECRET`       | Google OAuth              | optional                                     |
-| `LANGSMITH_*`                        | tracing                   | optional                                     |
+| Var                                  | Used by                   | Required?                                           |
+| ------------------------------------ | ------------------------- | --------------------------------------------------- |
+| `OPENAI_API_KEY`                     | backend agent             | yes                                                 |
+| `OPENAI_MODEL`                       | backend agent             | optional (default `gpt-4o-mini`)                    |
+| `OPENAI_BASE_URL`                    | backend agent             | optional (OpenAI-compatible gateway)                |
+| `JINA_API_KEYS`                      | web-search + web-fetch    | yes (comma-separated; one per Jina account)         |
+| `ALCHEMY_API_KEY`                    | NFT gallery + portfolio   | yes (server-only; powers `get_NFT_holdings`)        |
+| `LANGGRAPH_API_URL`                  | Next.js proxy             | optional (default `http://localhost:2024`)          |
+| `LANGCHAIN_API_KEY`                  | Next.js proxy → LangGraph | optional (leave blank locally)                      |
+| `NEXT_PUBLIC_LANGGRAPH_ASSISTANT_ID` | browser runtime           | optional (default `agent`)                          |
+| `NEXT_PUBLIC_LANGGRAPH_API_URL`      | browser runtime           | optional (uses proxy if unset)                      |
+| `DATABASE_URL`                       | drizzle-kit + backend     | yes                                                 |
+| `DATABASE_URL_TEST`                  | vitest                    | yes                                                 |
+| `BETTER_AUTH_SECRET`                 | session cookie signing    | yes (see [docs/AUTH.md](docs/AUTH.md))              |
+| `BETTER_AUTH_URL`                    | OAuth callback base       | yes (default `http://localhost:3000`)               |
+| `RESEND_API_KEY`                     | verification emails       | yes                                                 |
+| `RESEND_FROM_EMAIL`                  | verification email sender | optional (`onboarding@resend.dev` default)          |
+| `GITHUB_CLIENT_ID` / `_SECRET`       | GitHub OAuth              | optional                                            |
+| `GOOGLE_CLIENT_ID` / `_SECRET`       | Google OAuth              | optional                                            |
+| `LANGSMITH_*`                        | tracing                   | optional                                            |
+| `OBSERVABILITY_RETENTION_DAYS`       | observability spans       | optional (default `30`; must be a positive integer) |
 
 ## Patches
 
@@ -233,6 +244,7 @@ Test database stays isolated from dev — never put production-like data in `lan
 ## Documentation
 
 - [`docs/APIS.md`](docs/APIS.md) — HTTP endpoint reference. Update whenever a route under `app/api/` changes.
+- [`docs/OBSERVABILITY.md`](docs/OBSERVABILITY.md) — observability panel design: callback handler wiring, `observability_spans` schema, security/redaction, retention config, and curl examples.
 - [`docs/TOOLS.md`](docs/TOOLS.md) — LangGraph tool inventory and frontend card wiring. Update whenever a tool or card is added/removed/rerouted.
 - [`docs/INTERRUPT.md`](docs/INTERRUPT.md) — interrupt-driven tool flows (ask_location, connect_wallet, place_crypto_order, get_order_status) — the two runtime paths the cards can take.
 - [`docs/AUTH.md`](docs/AUTH.md) — operator guide for the auth layer: env vars, OAuth app setup, Resend, troubleshooting.

app/api/threads/[id]/observability/[parentMessageId]/route.ts

@@ -0,0 +1,33 @@
+import { NextResponse } from "next/server";
+
+import { withAuth } from "@/lib/auth/with-auth";
+import { getThreadForUser } from "@/lib/threads/queries";
+import { getSpansByThreadId, markRunningAsFailed } from "@/lib/observability/queries";
+import { getRetentionDays } from "@/lib/observability/config";
+
+type Params = { id: string; parentMessageId: string };
+
+// ponytail: rule #9 — every app/api/** route goes through withAuth.
+// Companion to app/api/threads/[id]/observability/route.ts — same
+// auth + 404-on-cross-user contract; the only difference is the path
+// carries the assistant-ui human-message id so the panel renders only
+// the spans produced for THAT turn (see backend/observability/
+// callback-collector.ts `currentParentMessageId`).
+export const GET = withAuth<Params>(async (_req, { user, params }) => {
+  const thread = await getThreadForUser(params.id, user.id);
+  if (!thread) return NextResponse.json({ code: "NOT_FOUND" }, { status: 404 });
+  await markRunningAsFailed(params.id);
+  // ponytail: the path is double-encoded upstream via Next.js route
+  // params — no extra decoding needed, the value is already a clean
+  // string when it reaches us. The btree index
+  // observability_spans_thread_parent_started_idx serves this path.
+  const spans = await getSpansByThreadId(params.id, {
+    parentMessageId: params.parentMessageId,
+  });
+  return NextResponse.json({
+    thread_id: params.id,
+    retention_days: getRetentionDays(),
+    parent_message_id: params.parentMessageId,
+    spans,
+  });
+});