Make tool failures visible telemetry: span outcome attributes, derived error messages, e2e contract

.claude/skills/prod-telemetry/SKILL.md

@@ -0,0 +1,114 @@
+---
+name: prod-telemetry
+description: Query Executor's production telemetry — Axiom traces (executor-cloud dataset), prod Postgres via PlanetScale, PostHog product analytics — through the Executor MCP. Use when investigating prod errors, latency, usage, churn signals, or verifying a deploy's telemetry; includes the dataset field layout, working APL recipes, and the error-attribution join.
+---
+
+# Production telemetry access
+
+All three stores are queryable through the Executor MCP's connected
+integrations — no dashboards or credentials needed. Verify the connection
+exists with `connections.list` if a call fails.
+
+## Axiom traces (`axiom_mcp`)
+
+Tool: `axiom_mcp.user.axiomMcpOAuth.querydataset` — the argument is `apl`
+(NOT `query`). Dataset: `['executor-cloud']` (worker spans; browser spans
+join the same traces via traceparent).
+
+**Field layout (the part you'd otherwise rediscover by failed queries):**
+
+- Custom span attributes live under the JSON map `['attributes.custom']`,
+  NOT as top-level `attributes.*` columns. Read with
+  `['attributes.custom']['mcp.tool.name']`. A nonexistent top-level field is
+  a hard query error ("invalid field"), not an empty result.
+- Span status: `['status.code']` (`"OK"`/`"ERROR"`), `['status.message']`.
+- Exceptions: the `events` column carries `exception.type` /
+  `exception.stacktrace` JSON.
+- OTel basics are top-level: `name`, `trace_id`, `span_id`,
+  `parent_span_id`, `duration`, `_time`.
+
+**Span names worth querying** (and their custom attrs):
+
+- `executor.tool.execute` — `mcp.tool.name` (full address), and since
+  PR #992: `executor.tool.outcome` (`ok`/`fail`),
+  `executor.tool.error_code`, `executor.tool.error_status`,
+  `executor.tenant`, `executor.subject`.
+- `mcp.tool.dispatch` — `mcp.tool.name` (sandbox path),
+  `mcp.tool.integration`, same outcome attrs.
+- `plugin.openapi.invoke` — `plugin.openapi.method` / `path_template` /
+  `base_url`, and since PR #992 `http.status_code`.
+- `mcp.request` (outer) — `mcp.auth.organization_id`,
+  `mcp.auth.account_id`, `mcp.tool.name`, CF edge fields (`cf.country`…),
+  MCP client fingerprint (`mcp.client.name`…).
+
+**Recipe — error signatures by class (the daily-digest query):**
+
+```apl
+['executor-cloud']
+| where _time > ago(1d)
+| where ['status.code'] == "ERROR" and name == "executor.tool.execute"
+| extend msg = substring(tostring(['status.message']), 0, 120)
+| extend tool = tostring(['attributes.custom']['mcp.tool.name'])
+| summarize n = count() by msg, tool
+| sort by n desc
+```
+
+**Recipe — attribute errors to orgs.** Tool spans now carry
+`executor.tenant` directly (post-#992). For spans from BEFORE that deploy,
+join through the outer request span:
+
+```apl
+['executor-cloud']
+| where name == "mcp.request" and isnotnull(['attributes.custom']['mcp.auth.organization_id'])
+| project trace_id, org = tostring(['attributes.custom']['mcp.auth.organization_id'])
+| join kind=inner (
+    ['executor-cloud']
+    | where ['status.code'] == "ERROR" and name == "executor.tool.execute"
+    | project trace_id, msg = substring(tostring(['status.message']), 0, 60)
+  ) on trace_id
+| summarize n = count() by org, msg | sort by n desc
+```
+
+**Recipe — upstream failure rate per integration (post-#992 attrs):**
+
+```apl
+['executor-cloud']
+| where _time > ago(1d) and name == "mcp.tool.dispatch"
+| extend outcome = tostring(['attributes.custom']['executor.tool.outcome'])
+| extend integration = tostring(['attributes.custom']['mcp.tool.integration'])
+| where isnotnull(outcome)
+| summarize calls = count(), fails = countif(outcome == "fail") by integration
+| extend failRate = todouble(fails) / todouble(calls)
+| sort by fails desc
+```
+
+**Known signal caveats** (audited 2026-06-12):
+
+- Pre-#992 spans: `ToolResult.fail` outcomes (upstream 4xx/5xx, auth
+  rejections) are INVISIBLE — they rode the Effect success channel with no
+  span marker. Don't conclude "no errors" from old data.
+- Many pre-#992 ERROR spans have an EMPTY `status.message` (tagged errors
+  without a message field) — group those by `events` exception.type instead.
+- `[object Object]` status messages are the pre-#992 formatting bug.
+
+## Prod database (`planetscale_mcp`)
+
+Read tool needs `{organization: "answer-overflow", database: "executor",
+branch: "main"}`. It returns `ok: true` even when the SQL failed — check the
+result text for `Error:`. Use for tenant/integration/connection facts that
+spans don't carry (row sizes, config shapes, counts).
+
+## Product analytics (`posthog_api` / `mcp_posthog_com`)
+
+Browser-side events only (the ~60-event typed catalog, PR #987; server-side
+events not built). The org-key `posthog_api` connection covers the REST API;
+the OAuth MCP connection covers the higher-level tools.
+
+## Verifying a deploy's telemetry (Layer-0 canary)
+
+After deploying telemetry changes: run a known-failing tool call against
+prod, then assert the expected attributes arrive in Axiom within ~1 min.
+Absence of data looks identical to health — query for the NEW attribute
+explicitly rather than eyeballing dashboards. The e2e equivalent runs on
+every suite: `e2e/cloud/telemetry-contract.test.ts` via the `Telemetry`
+service (motel `/api/spans/search?attr.<key>=<value>`).

e2e/AGENTS.md

@@ -77,6 +77,31 @@ const r = yield * session.call("execute", { code: "return 1 + 1;" });
 // human-in-the-loop: session.approvePaused(r.text) resumes a paused execution
 ```
 
+## Telemetry scenarios (cloud)
+
+The suite boots a motel OTLP store and points the target's real exporter at
+it, so a scenario can assert on the spans the server ACTUALLY exported —
+the layer where "observability silently went dark" bugs live (an attribute
+stamped on a span the exporter never carries looks identical to health).
+
+```ts
+const telemetry = yield * Telemetry; // skips when motel didn't boot
+const span =
+  yield *
+  telemetry.expectSpan({
+    operation: "executor.tool.execute",
+    attributes: { "mcp.tool.name": failAddress }, // exact match, values stringified
+  });
+expect(span.span.tags["executor.tool.outcome"]).toBe("fail");
+```
+
+- `expectSpan` polls (~20s): exporters batch, so arrival is
+  eventually-consistent — "the span reaches the store, soon" IS the contract.
+- Spec gotcha for fixtures: give operations explicit `tags` — tool addresses
+  are `group.leaf`, and an untagged op derives its group from the URL path,
+  so `/fail` does NOT produce a `.fail`-suffixed address.
+- Prior art: `cloud/telemetry-contract.test.ts`.
+
 ## Running
 
 ```sh

e2e/cloud/telemetry-contract.test.ts

@@ -0,0 +1,181 @@
+// Cloud: the telemetry contract, end to end. A tool call that hits an
+// upstream error wall must be visible in the EXPORTED spans — not just
+// handled gracefully for the caller. This is the regression class where the
+// product silently goes dark to operators: `ToolResult.fail` rides the
+// Effect success channel (a healthy-looking span), and an attribute stamped
+// on the wrong span simply never arrives in the trace store, which looks
+// identical to health. So the assertion runs against the OTLP store the dev
+// stack actually exported to (the suite's motel — the same exporter layer
+// that ships prod spans to Axiom), driving the whole production topology:
+// HTTP API → execution engine → sandbox → OpenAPI invoke → a real upstream
+// returning 502 → span batch → OTLP export.
+//
+// Pins two regressions found live in prod (2026-06-12): http.status_code was
+// annotated inside the inner `OpenApi.invoke` span so the `plugin.openapi.
+// invoke` span queries target carried it on 0 of ~19.5k spans; and failed
+// tool calls were indistinguishable from successes on `executor.tool.execute`.
+import { randomBytes } from "node:crypto";
+import { createServer } from "node:http";
+
+import { expect } from "@effect/vitest";
+import { Effect } from "effect";
+import { composePluginApi } from "@executor-js/api/server";
+import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api";
+import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared";
+
+import { scenario } from "../src/scenario";
+import { Api, Target, Telemetry } from "../src/services";
+
+const api = composePluginApi([openApiHttpPlugin()] as const);
+
+/** Two operations: /ok answers 200, /fail answers 502 — the success and
+ *  expected-upstream-failure outcome classes the telemetry must separate. */
+const upstreamSpec = (baseUrl: string): string =>
+  JSON.stringify({
+    openapi: "3.0.3",
+    info: { title: "Telemetry Upstream", version: "1.0.0" },
+    servers: [{ url: baseUrl }],
+    paths: {
+      "/ok": {
+        get: {
+          operationId: "ok",
+          summary: "Succeeds",
+          tags: ["probe"],
+          responses: { "200": { description: "" } },
+        },
+      },
+      "/fail": {
+        get: {
+          operationId: "fail",
+          summary: "Always 502",
+          tags: ["probe"],
+          responses: { "200": { description: "" } },
+        },
+      },
+    },
+  });
+
+/** A real upstream on 127.0.0.1: /ok → 200 JSON, anything else → 502 JSON. */
+const serveUpstream = Effect.acquireRelease(
+  Effect.callback<{ readonly baseUrl: string; readonly close: () => void }>((resume) => {
+    const server = createServer((request, response) => {
+      const ok = request.url?.startsWith("/ok") ?? false;
+      response.writeHead(ok ? 200 : 502, { "content-type": "application/json" });
+      response.end(ok ? '{"fine":true}' : '{"error":{"message":"bad gateway"}}');
+    });
+    server.listen(0, "127.0.0.1", () => {
+      const address = server.address();
+      const port = typeof address === "object" && address ? address.port : 0;
+      resume(
+        Effect.succeed({
+          baseUrl: `http://127.0.0.1:${port}`,
+          close: () => {
+            server.close();
+            server.closeAllConnections();
+          },
+        }),
+      );
+    });
+  }),
+  (server) => Effect.sync(server.close),
+);
+
+scenario(
+  "Telemetry · a failing tool call is visible in the exported spans",
+  { timeout: 180_000 },
+  Effect.scoped(
+    Effect.gen(function* () {
+      const target = yield* Target;
+      const { client: apiClient } = yield* Api;
+      const telemetry = yield* Telemetry;
+      const identity = yield* target.newIdentity();
+      const client = yield* apiClient(api, identity);
+
+      const upstream = yield* serveUpstream;
+
+      // Identifier-safe slug: it becomes a property path in the sandbox code.
+      const slug = IntegrationSlug.make(`telscn${randomBytes(4).toString("hex")}`);
+      yield* client.openapi.addSpec({
+        payload: {
+          spec: { kind: "blob", value: upstreamSpec(upstream.baseUrl) },
+          slug,
+          baseUrl: upstream.baseUrl,
+          authenticationTemplate: [
+            {
+              slug: "apiKey",
+              type: "apiKey",
+              headers: { Authorization: ["Bearer ", { type: "variable", name: "token" }] },
+            },
+          ],
+        },
+      });
+      yield* client.connections.create({
+        payload: {
+          owner: "org",
+          name: ConnectionName.make("main"),
+          integration: slug,
+          template: AuthTemplateSlug.make("apiKey"),
+          value: "telemetry-scenario-token",
+        },
+      });
+
+      const tools = yield* client.tools.list({ query: {} });
+      const addressOf = (op: string) => {
+        const tool = tools.find(
+          (entry) =>
+            String(entry.integration) === String(slug) && String(entry.address).endsWith(`.${op}`),
+        );
+        expect(tool, `the ${op} tool is in the catalog`).toBeDefined();
+        return String(tool!.address);
+      };
+      const failAddress = addressOf("fail");
+      const okAddress = addressOf("ok");
+
+      // Drive both outcome classes through the full production path. The
+      // failing call still completes for the caller — that is exactly why
+      // the exported span is the only place an operator can see it.
+      for (const address of [okAddress, failAddress]) {
+        const execution = yield* client.executions.execute({
+          payload: { code: `return await ${address}({});` },
+        });
+        expect(execution.status, `the ${address} execution completes`).toBe("completed");
+      }
+
+      // The failure: outcome attributes on the tool span...
+      const failSpan = yield* telemetry.expectSpan({
+        operation: "executor.tool.execute",
+        attributes: { "mcp.tool.name": failAddress },
+      });
+      expect(failSpan.span.tags, "a failed tool call is marked on the exported span").toMatchObject(
+        {
+          "executor.tool.outcome": "fail",
+          "executor.tool.error_code": "upstream_http_error",
+          "executor.tool.error_status": "502",
+        },
+      );
+      expect(
+        failSpan.span.tags["executor.tenant"],
+        "the span carries tenant attribution (no trace-join needed to ask 'whose error?')",
+      ).toBeTruthy();
+
+      // ...and the upstream status on the HTTP span queries actually target.
+      const invokeSpan = yield* telemetry.expectSpan({
+        operation: "plugin.openapi.invoke",
+        attributes: { "plugin.openapi.base_url": upstream.baseUrl, "http.status_code": "502" },
+      });
+      expect(
+        invokeSpan.span.tags["plugin.openapi.method"],
+        "the invoke span names the method",
+      ).toBe("GET");
+
+      // The success is distinguishable from the failure.
+      const okSpan = yield* telemetry.expectSpan({
+        operation: "executor.tool.execute",
+        attributes: { "mcp.tool.name": okAddress },
+      });
+      expect(okSpan.span.tags["executor.tool.outcome"], "a successful call is marked ok").toBe(
+        "ok",
+      );
+    }),
+  ),
+);

e2e/setup/cloud.globalsetup.ts

@@ -28,6 +28,10 @@ export default async function setup(): Promise<(() => Promise<void>) | void> {
 
   // Suite-owned trace store — every run captures distributed traces.
   const motel = await bootMotel();
+  // Publish to the test workers (they inherit this process's env): scenarios
+  // that assert on exported spans yield the Telemetry service, which exists
+  // only when this is set. No motel → those scenarios skip, never fail.
+  if (motel) process.env.E2E_MOTEL_URL = motel.url;
 
   const publicUrl = `http://127.0.0.1:${ports.E2E_CLOUD_PORT!}`;
   let booted;

e2e/src/scenario.ts

@@ -24,6 +24,7 @@ import { makeApiSurface } from "./surfaces/api";
 import { makeBrowserSurface } from "./surfaces/browser";
 import { makeCliSurface } from "./surfaces/cli";
 import { makeMcpSurface } from "./surfaces/mcp";
+import { makeTelemetrySurface } from "./surfaces/telemetry";
 import { completeOAuthConsent, hasOpenCode, makeOpenCodeHome, warmUp } from "./clients/opencode";
 import {
   Api,
@@ -35,6 +36,7 @@ import {
   Restart,
   RunDir,
   Target,
+  Telemetry,
   TtlControl,
 } from "./services";
 import { buildManifest } from "./viewer/manifest";
@@ -62,7 +64,8 @@ type AllServices =
   | Billing
   | OpenCode
   | TtlControl
-  | Restart;
+  | Restart
+  | Telemetry;
 
 /**
  * What this target on this host can provide. Services beyond the base are
@@ -94,6 +97,9 @@ const contextFor = (target: TargetShape, dir: string): Context.Context<AllServic
   if (target.restart) {
     context = Context.add(context, Restart, target.restart);
   }
+  if (process.env.E2E_MOTEL_URL) {
+    context = Context.add(context, Telemetry, makeTelemetrySurface(process.env.E2E_MOTEL_URL));
+  }
   return context;
 };
 

e2e/src/services.ts

@@ -11,6 +11,7 @@ import type { ApiSurface } from "./surfaces/api";
 import type { BrowserSurface } from "./surfaces/browser";
 import type { CliSurface } from "./surfaces/cli";
 import type { McpSurface } from "./surfaces/mcp";
+import type { TelemetrySurface } from "./surfaces/telemetry";
 import type { completeOAuthConsent, makeOpenCodeHome, warmUp } from "./clients/opencode";
 
 /** The target under test (always provided). */
@@ -34,6 +35,10 @@ export class Mcp extends Context.Service<Mcp, McpSurface>()("e2e/mcp-oauth") {}
 /** Marker: billing limits are enforced on this target. */
 export class Billing extends Context.Service<Billing, true>()("e2e/billing") {}
 
+/** Query the suite's OTLP trace store for spans the target actually exported
+ *  (present when the suite booted motel — E2E_MOTEL_URL). */
+export class Telemetry extends Context.Service<Telemetry, TelemetrySurface>()("e2e/telemetry") {}
+
 /** The real OpenCode binary, hermetically driveable (present when installed on this host). */
 export interface OpenCodeClient {
   readonly makeHome: typeof makeOpenCodeHome;