Recover account
diff --git a/apps/cloud/src/observability/buckets.test.ts b/apps/cloud/src/observability/buckets.test.ts
new file mode 100644
index 000000000..588273af9
--- /dev/null
+++ b/apps/cloud/src/observability/buckets.test.ts
@@ -0,0 +1,276 @@
+import { describe, expect, it } from 'vitest'
+import { InMemoryDocStore } from '../stores/durable'
+import {
+ DAY_MS,
+ HOUR_MS,
+ SliBucketStore,
+ bucketId,
+ fleetGate,
+ hourOf,
+ pruneDaily,
+ rollUpToDaily,
+ windowState,
+ type SliBucket,
+ type WindowState
+} from './buckets'
+
+const PROBE_MS = 60_000
+const WINDOW_MS = 30 * DAY_MS
+const T0 = Date.UTC(2026, 6, 1, 0, 0, 0)
+
+const opts = (nowMs: number, minBuckets = 2) => ({
+ nowMs,
+ windowMs: WINDOW_MS,
+ probeIntervalMs: PROBE_MS,
+ minBuckets
+})
+
+/** N hourly buckets ending at `endMs`, each with `ok` successes and `failed` failures. */
+function hours(
+ tenantId: string,
+ endMs: number,
+ count: number,
+ per: { ok?: number; failed?: number; coldStart?: number } = {}
+): SliBucket[] {
+ return Array.from({ length: count }, (_, i) => ({
+ tenantId,
+ startMs: hourOf(endMs) - (count - 1 - i) * HOUR_MS,
+ span: 'hour' as const,
+ ok: per.ok ?? 60,
+ coldStart: per.coldStart ?? 0,
+ failed: per.failed ?? 0,
+ latencySumMs: 1000,
+ maxLatencyMs: 50
+ }))
+}
+
+describe('SliBucketStore', () => {
+ it('accumulates into the current hour and survives a reload from the doc store', async () => {
+ const docs = new InMemoryDocStore
()
+ const store = new SliBucketStore(docs)
+ store.record('t_a', 'ok', 20, T0)
+ store.record('t_a', 'ok', 30, T0 + 60_000)
+ store.record('t_a', 'failed', 0, T0 + 120_000)
+ await store.flush(T0 + 120_000)
+
+ // A NEW store over the same docs — the restart that used to zero the window.
+ const reloaded = await new SliBucketStore(docs).buckets('t_a')
+ expect(reloaded).toHaveLength(1)
+ expect(reloaded[0]).toMatchObject({ ok: 2, failed: 1, coldStart: 0, maxLatencyMs: 30 })
+ })
+
+ it('closes a bucket once its hour has passed but keeps the current one open', async () => {
+ const docs = new InMemoryDocStore()
+ const store = new SliBucketStore(docs)
+ store.record('t_a', 'ok', 10, T0)
+ await store.flush(T0 + HOUR_MS) // T0's hour is now past
+ store.record('t_a', 'ok', 10, T0 + HOUR_MS)
+ await store.flush(T0 + HOUR_MS)
+ const all = await store.buckets('t_a')
+ expect(all.map((b) => b.startMs)).toEqual([T0, T0 + HOUR_MS])
+ })
+
+ it('counts a cold start as available, separately from a failure', async () => {
+ const docs = new InMemoryDocStore()
+ const store = new SliBucketStore(docs)
+ store.record('t_a', 'cold-start', 8000, T0)
+ store.record('t_a', 'ok', 20, T0)
+ await store.flush(T0)
+ const [b] = await store.buckets('t_a')
+ expect(b).toMatchObject({ ok: 1, coldStart: 1, failed: 0 })
+ const state = windowState('t_a', [b], opts(T0 + 60_000, 1))
+ expect(state).toMatchObject({ kind: 'measured', availability: 1 })
+ })
+
+ it('keeps tenants separate', async () => {
+ const docs = new InMemoryDocStore()
+ const store = new SliBucketStore(docs)
+ store.record('t_a', 'ok', 10, T0)
+ store.record('t_b', 'failed', 0, T0)
+ await store.flush(T0)
+ expect(await store.buckets('t_a')).toHaveLength(1)
+ expect((await store.buckets('t_b'))[0].failed).toBe(1)
+ })
+})
+
+describe('windowState', () => {
+ const now = T0 + 10 * HOUR_MS
+
+ it('is measured when there is recent, sufficient history', () => {
+ const s = windowState('t_a', hours('t_a', now, 5), opts(now))
+ expect(s).toMatchObject({ kind: 'measured', availability: 1, probes: 300 })
+ })
+
+ it('is young when the tenant has too little history — NOT stale, NOT healthy', () => {
+ const s = windowState('t_a', hours('t_a', now, 1), opts(now))
+ expect(s).toMatchObject({ kind: 'young', buckets: 1 })
+ })
+
+ it('is young with no buckets at all', () => {
+ expect(windowState('t_a', [], opts(now))).toMatchObject({ kind: 'young', buckets: 0 })
+ })
+
+ it('is stale when the newest bucket is older than 2x the probe interval', () => {
+ // Newest slice ended 3 hours ago; the grace is 2 minutes.
+ const s = windowState('t_a', hours('t_a', now - 3 * HOUR_MS, 5), opts(now))
+ expect(s.kind).toBe('stale')
+ })
+
+ it('is not stale merely because the current hour is still open', () => {
+ // Newest bucket started this hour, so it ends in the future — well inside grace.
+ const s = windowState('t_a', hours('t_a', now, 5), opts(now))
+ expect(s.kind).toBe('measured')
+ })
+
+ it('computes availability from failures only', () => {
+ const s = windowState('t_a', hours('t_a', now, 4, { ok: 99, failed: 1 }), opts(now))
+ expect(s).toMatchObject({ kind: 'measured' })
+ if (s.kind === 'measured') expect(s.availability).toBeCloseTo(0.99, 5)
+ })
+
+ it('ignores buckets older than the window', () => {
+ const old = hours('t_a', now - 40 * DAY_MS, 3)
+ const recent = hours('t_a', now, 3)
+ const s = windowState('t_a', [...old, ...recent], opts(now))
+ expect(s).toMatchObject({ kind: 'measured', probes: 180 })
+ })
+})
+
+describe('fleetGate', () => {
+ const measured = (availability: number): WindowState => ({
+ kind: 'measured',
+ tenantId: 't',
+ availability,
+ probes: 1000
+ })
+
+ it('freezes when nothing is measured at all — probing itself has stopped', () => {
+ expect(fleetGate([], 0.999)).toBe('freeze')
+ })
+
+ it('freezes on a stale tenant, however healthy the rest look', () => {
+ const states: WindowState[] = [measured(1), { kind: 'stale', tenantId: 't_b', newestMs: T0 }]
+ expect(fleetGate(states, 0.999)).toBe('freeze')
+ })
+
+ it('freezes when every tenant is young — absent is not healthy', () => {
+ expect(fleetGate([{ kind: 'young', tenantId: 't', buckets: 1 }], 0.999)).toBe('freeze')
+ })
+
+ it('EXCLUDES young tenants rather than freezing when others are measured', () => {
+ const states: WindowState[] = [measured(1), { kind: 'young', tenantId: 't_new', buckets: 0 }]
+ expect(fleetGate(states, 0.999)).toBe('ship')
+ })
+
+ it('ships on a healthy budget and freezes on an exhausted one', () => {
+ expect(fleetGate([measured(1)], 0.999)).toBe('ship')
+ expect(fleetGate([measured(0.999)], 0.999)).toBe('freeze')
+ })
+
+ it('takes the worst tenant, not the average', () => {
+ expect(fleetGate([measured(1), measured(0.999)], 0.999)).toBe('freeze')
+ })
+
+ it('never freezes a plan with no published objective', () => {
+ expect(fleetGate([measured(0.5)], null)).toBe('ship')
+ })
+})
+
+describe('rollUpToDaily', () => {
+ const now = T0 + 40 * DAY_MS
+
+ it('folds hourly buckets older than the retention into one daily bucket', async () => {
+ const docs = new InMemoryDocStore()
+ const day = T0 + 1 * DAY_MS
+ for (let h = 0; h < 4; h++) {
+ const b: SliBucket = {
+ tenantId: 't_a',
+ startMs: day + h * HOUR_MS,
+ span: 'hour',
+ ok: 50,
+ coldStart: 1,
+ failed: 2,
+ latencySumMs: 500,
+ maxLatencyMs: 40 + h
+ }
+ await docs.put(bucketId('t_a', b.startMs), b)
+ }
+ await rollUpToDaily(docs, 't_a', { nowMs: now, rawRetentionMs: 30 * DAY_MS })
+
+ const remaining = await docs.findWhere('tenantId', 't_a')
+ expect(remaining).toHaveLength(1)
+ expect(remaining[0]).toMatchObject({
+ span: 'day',
+ startMs: day,
+ ok: 200,
+ coldStart: 4,
+ failed: 8,
+ maxLatencyMs: 43
+ })
+ })
+
+ it('leaves buckets inside the raw retention window alone', async () => {
+ const docs = new InMemoryDocStore()
+ const recent = hourOf(now) - 2 * HOUR_MS
+ await docs.put(bucketId('t_a', recent), {
+ tenantId: 't_a',
+ startMs: recent,
+ span: 'hour',
+ ok: 60,
+ coldStart: 0,
+ failed: 0,
+ latencySumMs: 100,
+ maxLatencyMs: 10
+ })
+ const res = await rollUpToDaily(docs, 't_a', { nowMs: now, rawRetentionMs: 30 * DAY_MS })
+ expect(res).toEqual({ written: [], deleted: [] })
+ expect(await docs.findWhere('tenantId', 't_a')).toHaveLength(1)
+ })
+
+ it('is idempotent — a second run does not double-count or lose the first', async () => {
+ const docs = new InMemoryDocStore()
+ const day = T0 + 1 * DAY_MS
+ await docs.put(bucketId('t_a', day + HOUR_MS), {
+ tenantId: 't_a',
+ startMs: day + HOUR_MS,
+ span: 'hour',
+ ok: 10,
+ coldStart: 0,
+ failed: 0,
+ latencySumMs: 10,
+ maxLatencyMs: 5
+ })
+ const args = { nowMs: now, rawRetentionMs: 30 * DAY_MS }
+ await rollUpToDaily(docs, 't_a', args)
+ await rollUpToDaily(docs, 't_a', args)
+ const remaining = await docs.findWhere('tenantId', 't_a')
+ expect(remaining).toHaveLength(1)
+ expect(remaining[0].ok).toBe(10)
+ })
+})
+
+describe('pruneDaily', () => {
+ it('drops daily buckets past the long-term horizon and keeps the rest', async () => {
+ const docs = new InMemoryDocStore()
+ const now = T0 + 500 * DAY_MS
+ const mk = (startMs: number): SliBucket => ({
+ tenantId: 't_a',
+ startMs,
+ span: 'day',
+ ok: 1,
+ coldStart: 0,
+ failed: 0,
+ latencySumMs: 1,
+ maxLatencyMs: 1
+ })
+ const old = now - 400 * DAY_MS
+ const keep = now - 100 * DAY_MS
+ await docs.put(bucketId('t_a', old), mk(old))
+ await docs.put(bucketId('t_a', keep), mk(keep))
+
+ const dropped = await pruneDaily(docs, 't_a', { nowMs: now, retentionMs: 395 * DAY_MS })
+ expect(dropped).toHaveLength(1)
+ const remaining = await docs.findWhere('tenantId', 't_a')
+ expect(remaining.map((b) => b.startMs)).toEqual([keep])
+ })
+})
diff --git a/apps/cloud/src/observability/buckets.ts b/apps/cloud/src/observability/buckets.ts
new file mode 100644
index 000000000..507d55df3
--- /dev/null
+++ b/apps/cloud/src/observability/buckets.ts
@@ -0,0 +1,300 @@
+/**
+ * xNet Cloud — durable, bucketed SLI storage (exploration 0433, decision 2/8).
+ *
+ * The rolling in-memory ring this replaces was wrong in two directions at once
+ * (0431 Finding 1): it held 2000 samples at a 60s probe interval — **33 hours**
+ * of a window labelled 30 days — and it died with the process, so every deploy
+ * handed the error budget back at 100%. Because `rollout/engine.ts` gates fleet
+ * upgrades on that budget, a restart silently unfroze deploys.
+ *
+ * Buckets fix both. One document per tenant per hour in the existing
+ * {@link DocStore} port means the window survives restarts and is bounded by
+ * construction: 720 buckets ≈ 43 KB per tenant per 30 days. Metrics deliberately
+ * do NOT live on xNet — a per-tenant hourly time series is the high-frequency
+ * stream exploration 0323 measured into a 318k-row cold-open stall.
+ *
+ * Everything here is content-free: counts and latencies, never anything about a
+ * tenant's data.
+ */
+
+import type { DocStore } from '../stores/durable'
+import { errorBudgetRemaining } from './sli'
+import { budgetPolicy, type BudgetPolicy } from './slo'
+
+export const HOUR_MS = 60 * 60 * 1000
+export const DAY_MS = 24 * HOUR_MS
+
+/** Floor a timestamp to the hour it belongs to. */
+export const hourOf = (atMs: number): number => Math.floor(atMs / HOUR_MS) * HOUR_MS
+
+/** Floor a timestamp to the UTC day it belongs to. */
+export const dayOf = (atMs: number): number => Math.floor(atMs / DAY_MS) * DAY_MS
+
+/** Document id for a bucket. Sorts by tenant then time, which `page()` relies on. */
+export const bucketId = (tenantId: string, startMs: number): string =>
+ `${tenantId}:${String(startMs).padStart(16, '0')}`
+
+/**
+ * One time-slice of probe results for one tenant.
+ *
+ * `coldStart` is counted separately from `failed` and treated as **valid-but-slow**:
+ * the request eventually succeeded, so it is not unavailability. `sli.ts` always
+ * documented this intent; the old probe contradicted it by aborting at 5s and
+ * recording the abort as a failure.
+ */
+export interface SliBucket {
+ tenantId: string
+ /** Start of the slice (hour or day, per `span`). */
+ startMs: number
+ /** Slice width — hourly while raw, daily after rollup. */
+ span: 'hour' | 'day'
+ /** Probes that answered promptly. */
+ ok: number
+ /** Probes that answered, but only after the hub woke from cold. */
+ coldStart: number
+ /** Probes that did not answer. The only counter that burns error budget. */
+ failed: number
+ /** Sum of latencies over answering probes, for a mean. */
+ latencySumMs: number
+ maxLatencyMs: number
+}
+
+/** An empty bucket for a tenant/slice. */
+const emptyBucket = (tenantId: string, startMs: number, span: 'hour' | 'day'): SliBucket => ({
+ tenantId,
+ startMs,
+ span,
+ ok: 0,
+ coldStart: 0,
+ failed: 0,
+ latencySumMs: 0,
+ maxLatencyMs: 0
+})
+
+/** How a single probe resolved. */
+export type ProbeOutcome = 'ok' | 'cold-start' | 'failed'
+
+/** Fold one probe result into a bucket (mutates and returns it). */
+export function accumulate(bucket: SliBucket, outcome: ProbeOutcome, latencyMs: number): SliBucket {
+ if (outcome === 'failed') {
+ bucket.failed += 1
+ return bucket
+ }
+ if (outcome === 'cold-start') bucket.coldStart += 1
+ else bucket.ok += 1
+ bucket.latencySumMs += latencyMs
+ bucket.maxLatencyMs = Math.max(bucket.maxLatencyMs, latencyMs)
+ return bucket
+}
+
+/** Probes in a bucket that count toward availability at all. */
+export const validProbes = (b: SliBucket): number => b.ok + b.coldStart + b.failed
+
+/** Probes in a bucket that count as available (cold starts DO count — they answered). */
+export const okProbes = (b: SliBucket): number => b.ok + b.coldStart
+
+/**
+ * Durable per-tenant SLI buckets with an in-memory write-through for the current
+ * hour, so the hot path stays a map write and the durable cost is one document
+ * per tenant per hour.
+ */
+export class SliBucketStore {
+ /** Open (still-accumulating) buckets, keyed by document id. */
+ private readonly open = new Map()
+
+ constructor(private readonly docs: DocStore) {}
+
+ /** Record one probe result into the current hour's bucket. */
+ record(tenantId: string, outcome: ProbeOutcome, latencyMs: number, nowMs: number): void {
+ const startMs = hourOf(nowMs)
+ const id = bucketId(tenantId, startMs)
+ const bucket = this.open.get(id) ?? emptyBucket(tenantId, startMs, 'hour')
+ this.open.set(id, accumulate(bucket, outcome, latencyMs))
+ }
+
+ /**
+ * Persist buckets. Slices strictly older than the current hour are written and
+ * dropped from memory; the current hour is written too (so a crash loses at most
+ * the probes since the last flush) but stays open for further accumulation.
+ *
+ * Returns the number of documents written.
+ */
+ async flush(nowMs: number): Promise {
+ const currentHour = hourOf(nowMs)
+ let written = 0
+ for (const [id, bucket] of [...this.open]) {
+ await this.docs.put(id, bucket)
+ written += 1
+ if (bucket.startMs < currentHour) this.open.delete(id)
+ }
+ return written
+ }
+
+ /**
+ * Every bucket for a tenant, newest last. Merges the durable rows with any open
+ * in-memory bucket so a read immediately after a probe is not missing the
+ * current hour.
+ */
+ async buckets(tenantId: string): Promise {
+ const stored = await this.docs.findWhere('tenantId', tenantId)
+ const byStart = new Map()
+ for (const b of stored) byStart.set(b.startMs, b)
+ for (const b of this.open.values()) if (b.tenantId === tenantId) byStart.set(b.startMs, b)
+ return [...byStart.values()].sort((a, b) => a.startMs - b.startMs)
+ }
+}
+
+/**
+ * Why a tenant's window can or cannot be read as availability.
+ *
+ * The distinction between `stale` and `young` is the whole point (decision 8).
+ * Treating them alike gives you either a gate that freezes on every new signup —
+ * which trains you to switch it off — or one that cannot tell a stopped probe
+ * from a healthy fleet, which is the hazard this substrate exists to remove.
+ */
+export type WindowState =
+ | { kind: 'measured'; tenantId: string; availability: number; probes: number }
+ /** Newest bucket older than 2x the probe interval: measurement is BROKEN. */
+ | { kind: 'stale'; tenantId: string; newestMs: number }
+ /** Too few buckets because the tenant is new: benign, not evidence of harm. */
+ | { kind: 'young'; tenantId: string; buckets: number }
+
+export interface WindowOptions {
+ nowMs: number
+ windowMs: number
+ probeIntervalMs: number
+ /** Buckets required before a window is judged. Below this a tenant is `young`. */
+ minBuckets?: number
+}
+
+/** Default: two hours of history before a tenant's window is used for the gate. */
+export const DEFAULT_MIN_BUCKETS = 2
+
+/** Classify a tenant's window. */
+export function windowState(
+ tenantId: string,
+ buckets: SliBucket[],
+ opts: WindowOptions
+): WindowState {
+ const minBuckets = opts.minBuckets ?? DEFAULT_MIN_BUCKETS
+ const inWindow = buckets.filter((b) => b.startMs >= opts.nowMs - opts.windowMs)
+ if (inWindow.length === 0) return { kind: 'young', tenantId, buckets: 0 }
+
+ const newestMs = Math.max(...inWindow.map((b) => b.startMs))
+ // Reuses the jobs registry's existing definition of stale (2x the interval), so
+ // "this job stopped" and "this measurement stopped" mean the same thing. The
+ // grace is measured from the END of the newest slice, not its start.
+ const spanMs = inWindow.some((b) => b.span === 'day') ? DAY_MS : HOUR_MS
+ if (opts.nowMs - (newestMs + spanMs) > 2 * opts.probeIntervalMs) {
+ return { kind: 'stale', tenantId, newestMs }
+ }
+ if (inWindow.length < minBuckets) return { kind: 'young', tenantId, buckets: inWindow.length }
+
+ let ok = 0
+ let valid = 0
+ for (const b of inWindow) {
+ ok += okProbes(b)
+ valid += validProbes(b)
+ }
+ if (valid === 0) return { kind: 'young', tenantId, buckets: inWindow.length }
+ return { kind: 'measured', tenantId, availability: ok / valid, probes: valid }
+}
+
+/**
+ * The fleet deploy gate.
+ *
+ * `stale` freezes: a fleet nobody is measuring is not a healthy fleet, and a
+ * silently-stopped probe is exactly the failure that used to look like perfect
+ * health. `young` is EXCLUDED rather than frozen, so a new signup never blocks a
+ * rollout. No states at all — probing itself has stopped — freezes.
+ */
+export function fleetGate(states: WindowState[], objective: number | null): BudgetPolicy {
+ if (states.length === 0) return 'freeze'
+ if (states.some((s) => s.kind === 'stale')) return 'freeze'
+ const measured = states.filter(
+ (s): s is Extract => s.kind === 'measured'
+ )
+ // Every tenant young: nothing has been measured yet, so there is no evidence of
+ // health to deploy against. Absent is not healthy.
+ if (measured.length === 0) return 'freeze'
+ const worst = Math.min(...measured.map((m) => errorBudgetRemaining(m.availability, objective)))
+ return budgetPolicy(worst)
+}
+
+/**
+ * Roll hourly buckets older than `rawRetentionMs` into daily ones.
+ *
+ * Keeps 30 days of hourly resolution for the SLO window and a year-plus of daily
+ * resolution for quarterly SLA evidence (decision 15), while bounding growth:
+ * without this, hourly rows accumulate forever.
+ *
+ * Returns the ids written and deleted so a caller can log the compaction.
+ */
+export async function rollUpToDaily(
+ docs: DocStore,
+ tenantId: string,
+ opts: { nowMs: number; rawRetentionMs: number }
+): Promise<{ written: string[]; deleted: string[] }> {
+ const cutoff = opts.nowMs - opts.rawRetentionMs
+ const all = await docs.findWhere('tenantId', tenantId)
+ const stale = all.filter((b) => b.span === 'hour' && b.startMs < cutoff)
+ if (stale.length === 0) return { written: [], deleted: [] }
+
+ const byDay = new Map()
+ for (const b of stale) {
+ const day = dayOf(b.startMs)
+ const acc = byDay.get(day) ?? emptyBucket(tenantId, day, 'day')
+ acc.ok += b.ok
+ acc.coldStart += b.coldStart
+ acc.failed += b.failed
+ acc.latencySumMs += b.latencySumMs
+ acc.maxLatencyMs = Math.max(acc.maxLatencyMs, b.maxLatencyMs)
+ byDay.set(day, acc)
+ }
+
+ const written: string[] = []
+ for (const [day, acc] of byDay) {
+ const id = bucketId(tenantId, day)
+ // Merge into an existing daily bucket rather than overwrite — a rollup that
+ // runs twice must not discard the first run's counts.
+ const existing = await docs.get(id)
+ if (existing && existing.span === 'day') {
+ acc.ok += existing.ok
+ acc.coldStart += existing.coldStart
+ acc.failed += existing.failed
+ acc.latencySumMs += existing.latencySumMs
+ acc.maxLatencyMs = Math.max(acc.maxLatencyMs, existing.maxLatencyMs)
+ }
+ await docs.put(id, acc)
+ written.push(id)
+ }
+
+ const deleted: string[] = []
+ for (const b of stale) {
+ const id = bucketId(tenantId, b.startMs)
+ // A daily bucket shares the day's midnight id only if an hourly bucket started
+ // exactly at midnight — never delete the row we just wrote.
+ if (written.includes(id)) continue
+ await docs.delete(id)
+ deleted.push(id)
+ }
+ return { written, deleted }
+}
+
+/** Drop daily buckets past the long-term retention horizon (default 13 months). */
+export async function pruneDaily(
+ docs: DocStore,
+ tenantId: string,
+ opts: { nowMs: number; retentionMs: number }
+): Promise {
+ const cutoff = opts.nowMs - opts.retentionMs
+ const all = await docs.findWhere('tenantId', tenantId)
+ const dropped: string[] = []
+ for (const b of all) {
+ if (b.span !== 'day' || b.startMs >= cutoff) continue
+ const id = bucketId(tenantId, b.startMs)
+ await docs.delete(id)
+ dropped.push(id)
+ }
+ return dropped
+}
diff --git a/apps/cloud/src/observability/gate-control.test.ts b/apps/cloud/src/observability/gate-control.test.ts
new file mode 100644
index 000000000..6f5f8a78c
--- /dev/null
+++ b/apps/cloud/src/observability/gate-control.test.ts
@@ -0,0 +1,101 @@
+/**
+ * NEGATIVE CONTROL for the SLI deploy gate (exploration 0430's rule, 0433 Phase 0).
+ *
+ * A gate that cannot be shown to go red is unfalsifiable: a `fleetGate` that
+ * quietly started returning `ship` for everything would look exactly like a
+ * healthy fleet, and the regression would be invisible forever. Every case below
+ * plants a violation the gate **MUST** flag, and asserts *why* it flagged — a
+ * gate that freezes for the wrong reason fails here too.
+ *
+ * Fixtures are built in memory and never touch disk, so a control can never leak
+ * into a production scan (`AGENTS.md`).
+ *
+ * Driven standalone by `scripts/check-sli-gate.mjs`, and by the normal test run.
+ */
+
+import { describe, expect, it } from 'vitest'
+import { HOUR_MS, fleetGate, windowState, type SliBucket, type WindowState } from './buckets'
+
+const OBJECTIVE = 0.999
+const PROBE_MS = 60_000
+const WINDOW_MS = 30 * 24 * HOUR_MS
+const NOW = Date.UTC(2026, 6, 1, 12, 0, 0)
+
+const opts = { nowMs: NOW, windowMs: WINDOW_MS, probeIntervalMs: PROBE_MS, minBuckets: 2 }
+
+/** `count` hourly buckets whose newest slice ended `endHoursAgo` hours ago. */
+function hours(
+ tenantId: string,
+ count: number,
+ { endHoursAgo = 0, ok = 60, failed = 0 } = {}
+): SliBucket[] {
+ const newest = Math.floor((NOW - endHoursAgo * HOUR_MS) / HOUR_MS) * HOUR_MS
+ return Array.from({ length: count }, (_, i) => ({
+ tenantId,
+ startMs: newest - (count - 1 - i) * HOUR_MS,
+ span: 'hour' as const,
+ ok,
+ coldStart: 0,
+ failed,
+ latencySumMs: 100,
+ maxLatencyMs: 20
+ }))
+}
+
+const state = (tenantId: string, buckets: SliBucket[]): WindowState =>
+ windowState(tenantId, buckets, opts)
+
+describe('SLI gate — negative controls (the gate MUST flag each of these)', () => {
+ it('freezes when probing has stopped fleet-wide', () => {
+ expect(fleetGate([], OBJECTIVE)).toBe('freeze')
+ })
+
+ it('freezes on a tenant whose probe silently stopped', () => {
+ const s = state('t_stale', hours('t_stale', 5, { endHoursAgo: 6 }))
+ expect(s.kind).toBe('stale')
+ expect(fleetGate([s], OBJECTIVE)).toBe('freeze')
+ })
+
+ it('freezes on one stale tenant even when the rest look perfect', () => {
+ const healthy = state('t_ok', hours('t_ok', 10))
+ const stale = state('t_stale', hours('t_stale', 5, { endHoursAgo: 6 }))
+ expect([healthy.kind, stale.kind]).toEqual(['measured', 'stale'])
+ expect(fleetGate([healthy, stale], OBJECTIVE)).toBe('freeze')
+ })
+
+ it('freezes when the error budget is exhausted by real failures', () => {
+ const s = state('t_burn', hours('t_burn', 5, { ok: 998, failed: 2 }))
+ expect(s.kind).toBe('measured')
+ expect(fleetGate([s], OBJECTIVE)).toBe('freeze')
+ })
+
+ it('freezes when nothing has been measured yet — absent is not healthy', () => {
+ const s = state('t_new', hours('t_new', 1))
+ expect(s.kind).toBe('young')
+ expect(fleetGate([s], OBJECTIVE)).toBe('freeze')
+ })
+})
+
+describe('SLI gate — positive controls (the gate MUST NOT flag these)', () => {
+ // A gate that only ever freezes is as useless as one that never does: it gets
+ // switched off, and then nothing is gated at all.
+ it('ships on a healthy, measured fleet', () => {
+ expect(fleetGate([state('t_ok', hours('t_ok', 10))], OBJECTIVE)).toBe('ship')
+ })
+
+ it('does not let a brand-new tenant block a healthy fleet', () => {
+ const healthy = state('t_ok', hours('t_ok', 10))
+ const fresh = state('t_new', hours('t_new', 1))
+ expect([healthy.kind, fresh.kind]).toEqual(['measured', 'young'])
+ expect(fleetGate([healthy, fresh], OBJECTIVE)).toBe('ship')
+ })
+
+ it('never freezes a plan with no published objective', () => {
+ const burning = state('t_burn', hours('t_burn', 5, { ok: 500, failed: 500 }))
+ expect(fleetGate([burning], null)).toBe('ship')
+ })
+
+ it('does not call the still-open current hour stale', () => {
+ expect(state('t_ok', hours('t_ok', 3)).kind).toBe('measured')
+ })
+})
diff --git a/apps/cloud/src/observability/health.ts b/apps/cloud/src/observability/health.ts
index 9e4be5417..2d999f2f5 100644
--- a/apps/cloud/src/observability/health.ts
+++ b/apps/cloud/src/observability/health.ts
@@ -8,6 +8,7 @@
*/
import { type PlanId } from '@xnetjs/entitlements'
+import { type ProbeOutcome, type SliBucketStore } from './buckets'
import {
availability,
errorRate,
@@ -18,13 +19,47 @@ import {
} from './sli'
import { budgetPolicy, sloForPlan, type BudgetPolicy } from './slo'
+/** One probe result. `coldStart` means it answered, but only after waking. */
+export interface ProbeResult {
+ ok: boolean
+ latencyMs: number
+ coldStart?: boolean
+}
+
/** Probes a single hub. The real adapter hits `${hubUrl}/health`. */
export interface HealthProbe {
- probe(hubUrl: string): Promise<{ ok: boolean; latencyMs: number }>
+ probe(hubUrl: string): Promise
}
+/**
+ * How long a hub may take to answer before we call it down.
+ *
+ * Deliberately generous, and deliberately NOT the old 5s: a scale-to-zero hub
+ * has to cold-start Cloud Run and restore a SQLite database from R2 before it
+ * can answer, and 5s recorded that as an outage — the opposite of what `sli.ts`
+ * documents (exploration 0431 Finding 2).
+ *
+ * @remarks **This number is not measured.** Exploration 0433 open question 1: no
+ * cold-start figure exists anywhere in the repo. 30s is a placeholder chosen to
+ * be safely above a plausible restore, not a value anyone has observed. Measure a
+ * real Litestream restore-on-boot and replace it; until then, over-waiting costs
+ * a slow probe while under-waiting fabricates downtime.
+ */
+export const DEFAULT_PROBE_TIMEOUT_MS = 30_000
+
+/**
+ * Above this, an answering hub is recorded as having cold-started rather than
+ * served promptly. Cold starts count as available (they answered) but are tracked
+ * separately so the console can say "sleeping, woke in 8s" instead of "degraded".
+ */
+export const DEFAULT_COLD_START_MS = 2_000
+
/** Default probe: GET `${hubUrl}/health`, ok on a 2xx within the timeout. */
-export function httpHealthProbe(fetchImpl: typeof fetch = fetch, timeoutMs = 5000): HealthProbe {
+export function httpHealthProbe(
+ fetchImpl: typeof fetch = fetch,
+ timeoutMs = DEFAULT_PROBE_TIMEOUT_MS,
+ coldStartMs = DEFAULT_COLD_START_MS
+): HealthProbe {
return {
async probe(hubUrl: string) {
const startedAtMs = Date.now()
@@ -32,7 +67,8 @@ export function httpHealthProbe(fetchImpl: typeof fetch = fetch, timeoutMs = 500
const timer = setTimeout(() => ctrl.abort(), timeoutMs)
try {
const res = await fetchImpl(`${hubUrl.replace(/\/$/, '')}/health`, { signal: ctrl.signal })
- return { ok: res.ok, latencyMs: Date.now() - startedAtMs }
+ const latencyMs = Date.now() - startedAtMs
+ return { ok: res.ok, latencyMs, coldStart: res.ok && latencyMs >= coldStartMs }
} catch {
return { ok: false, latencyMs: Date.now() - startedAtMs }
} finally {
@@ -42,10 +78,16 @@ export function httpHealthProbe(fetchImpl: typeof fetch = fetch, timeoutMs = 500
}
}
+/** The bucket outcome a probe result folds into. */
+export function outcomeOf(result: ProbeResult): ProbeOutcome {
+ if (!result.ok) return 'failed'
+ return result.coldStart ? 'cold-start' : 'ok'
+}
+
/** Scripted probe for tests — maps a hubUrl to a fixed result. */
export class FakeHealthProbe implements HealthProbe {
- constructor(private readonly results: Record) {}
- async probe(hubUrl: string): Promise<{ ok: boolean; latencyMs: number }> {
+ constructor(private readonly results: Record) {}
+ async probe(hubUrl: string): Promise {
return this.results[hubUrl] ?? { ok: false, latencyMs: 0 }
}
}
@@ -72,11 +114,24 @@ export async function sampleTenantHealth(
probe: HealthProbe,
store: HealthSampleStore,
tenant: { tenantId: string; hubUrl: string },
- nowMs: number
+ nowMs: number,
+ /**
+ * Durable bucket store (exploration 0433). The in-memory `store` above stays as
+ * the short-window view the live dashboard tiles poll; `buckets` is what the SLO
+ * window, the error budget, and the deploy gate actually read, because it is the
+ * only one that survives a restart.
+ */
+ buckets?: SliBucketStore
): Promise {
const r = await probe.probe(tenant.hubUrl)
- const sample: HealthSample = { ok: r.ok, latencyMs: r.latencyMs, atMs: nowMs }
+ const sample: HealthSample = {
+ ok: r.ok,
+ latencyMs: r.latencyMs,
+ atMs: nowMs,
+ ...(r.coldStart ? { coldStart: true } : {})
+ }
store.record(tenant.tenantId, sample)
+ buckets?.record(tenant.tenantId, outcomeOf(r), r.latencyMs, nowMs)
return sample
}
@@ -90,14 +145,18 @@ export async function probeFleet(
probe: HealthProbe,
store: HealthSampleStore,
tenants: { tenantId: string; hubUrl: string; dataTier: 'hot' | 'cold' }[],
- nowMs: number
+ nowMs: number,
+ buckets?: SliBucketStore
): Promise {
const hot = tenants.filter((t) => t.dataTier === 'hot' && Boolean(t.hubUrl))
await Promise.all(
hot.map((t) =>
- sampleTenantHealth(probe, store, { tenantId: t.tenantId, hubUrl: t.hubUrl }, nowMs)
+ sampleTenantHealth(probe, store, { tenantId: t.tenantId, hubUrl: t.hubUrl }, nowMs, buckets)
)
)
+ // Persist immediately: a flush deferred to an hourly timer would lose the whole
+ // current hour to a deploy, which is the amnesia this replaced.
+ if (buckets) await buckets.flush(nowMs)
return hot.length
}
diff --git a/apps/cloud/src/observability/sli.ts b/apps/cloud/src/observability/sli.ts
index 44027c5f2..f988dd79f 100644
--- a/apps/cloud/src/observability/sli.ts
+++ b/apps/cloud/src/observability/sli.ts
@@ -15,6 +15,12 @@ export interface HealthSample {
ok: boolean
latencyMs: number
atMs: number
+ /**
+ * The hub answered, but only after waking from cold. Counted as available —
+ * the request succeeded — and tracked separately so a sleeping tenant reads as
+ * "slow to wake" rather than "down" (exploration 0433).
+ */
+ coldStart?: boolean
}
/** Samples within `[nowMs - windowMs, nowMs]`. */
@@ -23,10 +29,19 @@ export function windowed(samples: HealthSample[], windowMs: number, nowMs: numbe
return samples.filter((s) => s.atMs >= floor && s.atMs <= nowMs)
}
-/** Availability = successful / valid probes. Empty window → 1 (no evidence of failure). */
+/**
+ * Availability = successful / valid probes. A cold start counts as successful:
+ * the request answered, it was just slow to wake.
+ *
+ * @remarks Empty window → 1. This is the legacy in-memory path, kept for the live
+ * dashboard tiles. **Do not use it for the deploy gate** — "no samples" reading as
+ * "perfectly available" is exploration 0431 Finding 1. `windowState()` in
+ * `buckets.ts` returns an explicit `young`/`stale` instead, and `fleetGate()`
+ * freezes on both.
+ */
export function availability(samples: HealthSample[]): number {
if (samples.length === 0) return 1
- return samples.filter((s) => s.ok).length / samples.length
+ return samples.filter((s) => s.ok || s.coldStart).length / samples.length
}
/** Error rate = 1 − availability. */
diff --git a/apps/cloud/src/observability/slo.ts b/apps/cloud/src/observability/slo.ts
index ca6450a67..1f31f31c1 100644
--- a/apps/cloud/src/observability/slo.ts
+++ b/apps/cloud/src/observability/slo.ts
@@ -7,7 +7,12 @@
* (security/reliability fixes are always exempt — enforced at the call site).
*/
-import { PLAN_CATALOG, type PlanId, type SlaLevel } from '@xnetjs/entitlements'
+import {
+ PLAN_CATALOG,
+ availabilityObjective,
+ type PlanId,
+ type SlaLevel
+} from '@xnetjs/entitlements'
export interface SloTarget {
/** Availability objective as a fraction (e.g. 0.999). `null` = no published SLO. */
@@ -18,18 +23,26 @@ export interface SloTarget {
label: string
}
-/** Map a plan's declared SLA level to a measurable SLO. */
+const SLA_LABELS: Record = {
+ '99.9': '99.9% uptime',
+ custom: '99.95% uptime (enterprise)',
+ 'best-effort': 'best-effort',
+ none: 'no SLA'
+}
+
+/**
+ * Map a plan's declared SLA level to a measurable SLO.
+ *
+ * The objective itself comes from `@xnetjs/entitlements` rather than a switch
+ * here: the provisioner reads the same mapping to decide always-warm placement,
+ * and two copies are how a tenant gets sold an objective its infrastructure
+ * cannot serve (exploration 0433 D1). Only the human label is local.
+ */
export function sloForSla(sla: SlaLevel): SloTarget {
- switch (sla) {
- case '99.9':
- return { objective: 0.999, windowDays: 30, label: '99.9% uptime' }
- case 'custom':
- return { objective: 0.9995, windowDays: 30, label: '99.95% uptime (enterprise)' }
- case 'best-effort':
- return { objective: null, windowDays: 30, label: 'best-effort' }
- case 'none':
- default:
- return { objective: null, windowDays: 30, label: 'no SLA' }
+ return {
+ objective: availabilityObjective(sla),
+ windowDays: 30,
+ label: SLA_LABELS[sla] ?? SLA_LABELS.none
}
}
diff --git a/apps/cloud/src/observability/status.test.ts b/apps/cloud/src/observability/status.test.ts
index 2c31b4479..a38fa1ff1 100644
--- a/apps/cloud/src/observability/status.test.ts
+++ b/apps/cloud/src/observability/status.test.ts
@@ -14,7 +14,10 @@ const component = (s: ReturnType, id: string) =>
s.components.find((c) => c.id === id)!
describe('publicStatus', () => {
- it('reports operational with no fleet data', () => {
+ // Exploration 0433 decision 10. This used to assert `operational` on both the
+ // control plane and an unmeasured fleet — the same defect as an empty sample
+ // window reading as 100% available. Absent evidence is now its own state.
+ it('reports unmeasured, not operational, when nothing has been measured', () => {
const s = publicStatus({
nowMs: 1000,
fleet: fleet(),
@@ -22,11 +25,55 @@ describe('publicStatus', () => {
aiConfigured: false,
backupsHealthy: null
})
- expect(s.overall).toBe('operational')
- expect(component(s, 'control-plane').status).toBe('operational')
+ expect(component(s, 'control-plane').status).toBe('unmeasured')
expect(component(s, 'hub-fleet').availability).toBeNull() // no tenants → suppressed
expect(component(s, 'ai-gateway').status).toBe('not-configured')
expect(component(s, 'backups').status).toBe('not-configured')
+ expect(s.overall).toBe('unmeasured')
+ })
+
+ it('reports the control plane from its jobs, not from having answered', () => {
+ const base = {
+ nowMs: 1000,
+ fleet: fleet(),
+ availabilities: [],
+ aiConfigured: false,
+ backupsHealthy: null
+ }
+ expect(
+ component(publicStatus({ ...base, controlPlaneJobsHealthy: true }), 'control-plane').status
+ ).toBe('operational')
+ expect(
+ component(publicStatus({ ...base, controlPlaneJobsHealthy: false }), 'control-plane').status
+ ).toBe('degraded')
+ })
+
+ it('renders an unmeasured fleet without an availability number', () => {
+ const s = publicStatus({
+ nowMs: 1,
+ fleet: fleet({ tenantCount: 10 }),
+ availabilities: Array(10).fill(0.999),
+ aiConfigured: true,
+ backupsHealthy: true,
+ controlPlaneJobsHealthy: true,
+ fleetMeasured: false
+ })
+ expect(component(s, 'hub-fleet').status).toBe('unmeasured')
+ expect(component(s, 'hub-fleet').availability).toBeNull()
+ expect(s.overall).toBe('unmeasured')
+ })
+
+ it('ranks unmeasured above operational but below degraded', () => {
+ const withDegraded = publicStatus({
+ nowMs: 1,
+ fleet: fleet({ tenantCount: 10, freezing: 1 }),
+ availabilities: Array(10).fill(0.9),
+ aiConfigured: true,
+ backupsHealthy: false,
+ controlPlaneJobsHealthy: undefined
+ })
+ // A real degradation still wins the banner over a missing measurement.
+ expect(withDegraded.overall).toBe('degraded')
})
it('publishes the fleet availability only at/above the k-anon floor', () => {
diff --git a/apps/cloud/src/observability/status.ts b/apps/cloud/src/observability/status.ts
index 2f080e418..5a698eaee 100644
--- a/apps/cloud/src/observability/status.ts
+++ b/apps/cloud/src/observability/status.ts
@@ -13,7 +13,14 @@
import type { FleetSummary } from './health'
import type { BudgetPolicy } from './slo'
-export type ComponentStatus = 'operational' | 'degraded' | 'down' | 'not-configured'
+/**
+ * `unmeasured` is not a degraded state — it is the absence of evidence
+ * (exploration 0433, decision 10). Before it existed, a component with no SLI
+ * window at all reported `operational`, which is the same defect as an empty
+ * sample window reading as 100% available: "absent" and "unreadable" must not
+ * look like "fine" (`AGENTS.md`).
+ */
+export type ComponentStatus = 'operational' | 'degraded' | 'down' | 'not-configured' | 'unmeasured'
export interface StatusComponent {
id: string
@@ -42,17 +49,39 @@ export interface PublicStatusInput {
backupsHealthy: boolean | null
/** Suppress the fleet availability number below this many hot tenants. */
kAnonFloor?: number
+ /**
+ * Whether the hub fleet's SLI window is actually being measured right now.
+ * `false` renders `unmeasured` instead of `operational` — the public page must
+ * not assert health nobody has observed (decision 10). Defaults to `true` so
+ * existing callers keep their behaviour until they pass the real signal.
+ */
+ fleetMeasured?: boolean
+ /**
+ * Whether the control plane's own periodic jobs are completing. The old code
+ * hardcoded `control-plane: operational`, which was tautological — it said only
+ * that the process had answered THIS request. `false` means a leased job has
+ * gone stale; `undefined` means job reporting is not wired, which renders
+ * `unmeasured` rather than green.
+ */
+ controlPlaneJobsHealthy?: boolean
}
/** Default k-anonymity floor — matches the run-in-public metrics cohort floor. */
export const STATUS_K_ANON_FLOOR = 5
-/** Severity ordering so the banner reflects the worst non-trivial component. */
+/**
+ * Severity ordering so the banner reflects the worst non-trivial component.
+ *
+ * `unmeasured` sits just ABOVE `operational`: it must be able to displace a green
+ * banner (we are not claiming health we cannot show) but must never masquerade as
+ * an outage, which would page someone over a missing probe.
+ */
const SEVERITY: Record = {
'not-configured': 0,
operational: 1,
- degraded: 2,
- down: 3
+ unmeasured: 2,
+ degraded: 3,
+ down: 4
}
function worstStatus(components: StatusComponent[]): ComponentStatus {
@@ -73,7 +102,16 @@ export function publicStatus(input: PublicStatusInput): PublicStatus {
const mean = n ? input.availabilities.reduce((sum, a) => sum + a, 0) / n : 1
const fleetAvailability = n >= floor ? Number(mean.toFixed(4)) : null
- const hubFleet: ComponentStatus = input.fleet.freezing > 0 ? 'degraded' : 'operational'
+ // A frozen error budget is still the degraded signal, but only when there IS a
+ // measurement behind it. Unmeasured outranks operational so the banner cannot
+ // read green on no evidence.
+ const measured = input.fleetMeasured ?? true
+ const hubFleet: ComponentStatus = !measured
+ ? 'unmeasured'
+ : input.fleet.freezing > 0
+ ? 'degraded'
+ : 'operational'
+
const backups: ComponentStatus =
input.backupsHealthy === null
? 'not-configured'
@@ -81,9 +119,23 @@ export function publicStatus(input: PublicStatusInput): PublicStatus {
? 'operational'
: 'degraded'
+ // The control plane reports from its periodic jobs, not from the fact that it
+ // answered this request — a process can serve /status.json perfectly while every
+ // background reconciler has silently stopped.
+ const controlPlane: ComponentStatus =
+ input.controlPlaneJobsHealthy === undefined
+ ? 'unmeasured'
+ : input.controlPlaneJobsHealthy
+ ? 'operational'
+ : 'degraded'
+
const components: StatusComponent[] = [
- { id: 'control-plane', status: 'operational' },
- { id: 'hub-fleet', status: hubFleet, availability: fleetAvailability },
+ { id: 'control-plane', status: controlPlane },
+ {
+ id: 'hub-fleet',
+ status: hubFleet,
+ availability: measured ? fleetAvailability : null
+ },
{ id: 'ai-gateway', status: input.aiConfigured ? 'operational' : 'not-configured' },
{ id: 'backups', status: backups }
]
diff --git a/apps/cloud/src/ops/audit.test.ts b/apps/cloud/src/ops/audit.test.ts
new file mode 100644
index 000000000..982bf319e
--- /dev/null
+++ b/apps/cloud/src/ops/audit.test.ts
@@ -0,0 +1,279 @@
+import { describe, expect, it } from 'vitest'
+import { InMemoryDocStore } from '../stores/durable'
+import {
+ AuditLog,
+ AuditWriteError,
+ ReasonRequiredError,
+ audited,
+ requiresReason,
+ type AuditEntry,
+ type AuditPublisher
+} from './audit'
+
+const T0 = Date.UTC(2026, 6, 1)
+
+function setup(publisher?: AuditPublisher) {
+ const docs = new InMemoryDocStore()
+ let t = T0
+ const log = new AuditLog({
+ docs,
+ ...(publisher ? { publisher } : {}),
+ nowMs: () => (t += 1000)
+ })
+ return { docs, log }
+}
+
+const okPublisher = (): { publisher: AuditPublisher; seen: AuditEntry[] } => {
+ const seen: AuditEntry[] = []
+ return { publisher: { publish: async (e) => void seen.push(e) }, seen }
+}
+
+const deadPublisher: AuditPublisher = {
+ publish: async () => {
+ throw new Error('ops hub unreachable')
+ }
+}
+
+describe('requiresReason', () => {
+ it('demands a reason for mutations and not for reads', () => {
+ expect(requiresReason('tenant.recover')).toBe(true)
+ expect(requiresReason('tenant.delete-data')).toBe(true)
+ expect(requiresReason('operator.bind')).toBe(true)
+ expect(requiresReason('tenant.read')).toBe(false)
+ })
+})
+
+describe('AuditLog', () => {
+ it('records a read without a reason', async () => {
+ const { publisher, seen } = okPublisher()
+ const { log } = setup(publisher)
+ const e = await log.append({
+ operator: 'user_1',
+ action: 'tenant.read',
+ tenantId: 't_a',
+ outcome: 'ok'
+ })
+ expect(e.entryId).toMatch(/^\d{16}-\d{6}$/)
+ expect(seen).toHaveLength(1)
+ })
+
+ it('refuses a mutation with no reason', async () => {
+ const { log } = setup()
+ await expect(
+ log.append({
+ operator: 'user_1',
+ action: 'tenant.recover',
+ tenantId: 't_a',
+ outcome: 'started'
+ })
+ ).rejects.toBeInstanceOf(ReasonRequiredError)
+ })
+
+ it('refuses a mutation whose reason is only whitespace', async () => {
+ const { log } = setup()
+ await expect(
+ log.append({
+ operator: 'user_1',
+ action: 'tenant.recover',
+ tenantId: 't_a',
+ reason: ' ',
+ outcome: 'started'
+ })
+ ).rejects.toBeInstanceOf(ReasonRequiredError)
+ })
+
+ it('marks entries published once the ops hub confirms', async () => {
+ const { publisher } = okPublisher()
+ const { docs, log } = setup(publisher)
+ const e = await log.append({
+ operator: 'user_1',
+ action: 'tenant.read',
+ tenantId: 't_a',
+ outcome: 'ok'
+ })
+ expect((await docs.get(e.entryId))?.published).toBe(true)
+ expect(log.pendingCount()).toBe(0)
+ })
+
+ // ADR-31: an unreachable ops hub degrades to "audit history is stale", never to
+ // "no operator can act". The gap must be visible, which is what the queue is for.
+ it('still records, and queues visibly, when the ops hub is unreachable', async () => {
+ const { docs, log } = setup(deadPublisher)
+ const e = await log.append({
+ operator: 'user_1',
+ action: 'tenant.read',
+ tenantId: 't_a',
+ outcome: 'ok'
+ })
+ expect(await docs.get(e.entryId)).not.toBeNull() // tier 1 landed
+ expect((await docs.get(e.entryId))?.published).toBeUndefined()
+ expect(log.pendingCount()).toBe(1)
+ })
+
+ it('drains the queue when the hub comes back', async () => {
+ const docs = new InMemoryDocStore()
+ let alive = false
+ const publisher: AuditPublisher = {
+ publish: async () => {
+ if (!alive) throw new Error('down')
+ }
+ }
+ const log = new AuditLog({ docs, publisher, nowMs: () => T0 })
+ await log.append({ operator: 'u', action: 'tenant.read', tenantId: 't', outcome: 'ok' })
+ await log.append({ operator: 'u', action: 'tenant.read', tenantId: 't', outcome: 'ok' })
+ expect(log.pendingCount()).toBe(2)
+
+ alive = true
+ expect(await log.drain()).toBe(2)
+ expect(log.pendingCount()).toBe(0)
+ })
+
+ it('keeps entries queued if the drain itself fails', async () => {
+ const { log } = setup(deadPublisher)
+ await log.append({ operator: 'u', action: 'tenant.read', tenantId: 't', outcome: 'ok' })
+ expect(await log.drain()).toBe(0)
+ expect(log.pendingCount()).toBe(1)
+ })
+
+ // The fail-closed gate: no tier-1 write, no action.
+ it('throws AuditWriteError when the tier-1 store rejects', async () => {
+ const docs = new InMemoryDocStore()
+ docs.put = async () => {
+ throw new Error('firestore down')
+ }
+ const log = new AuditLog({ docs, nowMs: () => T0 })
+ await expect(
+ log.append({ operator: 'u', action: 'tenant.read', tenantId: 't', outcome: 'ok' })
+ ).rejects.toBeInstanceOf(AuditWriteError)
+ })
+
+ it('reads a tenant history in chronological order', async () => {
+ const { log } = setup()
+ for (const outcome of ['started', 'ok'] as const) {
+ await log.append({ operator: 'u', action: 'tenant.read', tenantId: 't_a', outcome })
+ }
+ await log.append({ operator: 'u', action: 'tenant.read', tenantId: 't_b', outcome: 'ok' })
+ const history = await log.forTenant('t_a')
+ expect(history.map((e) => e.outcome)).toEqual(['started', 'ok'])
+ })
+
+ it('reads an operator history', async () => {
+ const { log } = setup()
+ await log.append({ operator: 'u1', action: 'tenant.read', tenantId: 't', outcome: 'ok' })
+ await log.append({ operator: 'u2', action: 'tenant.read', tenantId: 't', outcome: 'ok' })
+ expect(await log.byOperator('u1')).toHaveLength(1)
+ })
+})
+
+describe('audited', () => {
+ it('writes started BEFORE the action runs', async () => {
+ const { docs, log } = setup()
+ let sawDuringRun = 0
+ await audited(
+ log,
+ { operator: 'u', action: 'tenant.recover', tenantId: 't_a', reason: 'lost passkey' },
+ async () => {
+ sawDuringRun = (await docs.list()).length
+ }
+ )
+ expect(sawDuringRun).toBe(1) // the `started` entry was already durable
+ expect((await log.forTenant('t_a')).map((e) => e.outcome)).toEqual(['started', 'ok'])
+ })
+
+ it('still leaves a started entry when the action throws, and rethrows', async () => {
+ const { log } = setup()
+ await expect(
+ audited(
+ log,
+ { operator: 'u', action: 'tenant.delete-data', tenantId: 't_a', reason: 'user request' },
+ async () => {
+ throw new Error('provisioner exploded')
+ }
+ )
+ ).rejects.toThrow('provisioner exploded')
+ const history = await log.forTenant('t_a')
+ expect(history.map((e) => e.outcome)).toEqual(['started', 'failed'])
+ expect(history[1].parentId).toBe(history[0].entryId)
+ })
+
+ it('refuses to run the action at all when no reason is given', async () => {
+ const { log } = setup()
+ let ran = false
+ await expect(
+ audited(log, { operator: 'u', action: 'tenant.recover', tenantId: 't_a' }, async () => {
+ ran = true
+ })
+ ).rejects.toBeInstanceOf(ReasonRequiredError)
+ expect(ran).toBe(false)
+ })
+
+ it('carries the operator DID onto every entry for later verification', async () => {
+ const { log } = setup()
+ await audited(
+ log,
+ {
+ operator: 'u',
+ operatorDid: 'did:key:zOps',
+ action: 'tenant.plan-change',
+ tenantId: 't_a',
+ reason: 'support escalation'
+ },
+ async () => undefined
+ )
+ const history = await log.forTenant('t_a')
+ expect(history.every((e) => e.operatorDid === 'did:key:zOps')).toBe(true)
+ })
+})
+
+describe('retention (decision 15)', () => {
+ // Audit survives tenant deletion ON PURPOSE, and it is pro-user: if the record
+ // vanished with the account, an operator could read someone's data and then
+ // erase the evidence by deleting them. The data goes; the log of who touched it
+ // stays. Nothing in the audit path is keyed to tenant lifecycle, and this test
+ // exists so that stays true.
+ it('keeps a tenant history after every trace of that tenant is deleted', async () => {
+ const docs = new InMemoryDocStore()
+ const tenants = new InMemoryDocStore<{ tenantId: string }>()
+ const log = new AuditLog({ docs, nowMs: () => T0 })
+
+ await tenants.put('t_gone', { tenantId: 't_gone' })
+ await audited(
+ log,
+ { operator: 'u', action: 'tenant.read', tenantId: 't_gone' },
+ async () => undefined
+ )
+ await audited(
+ log,
+ {
+ operator: 'u',
+ action: 'tenant.delete-data',
+ tenantId: 't_gone',
+ reason: 'user requested erasure'
+ },
+ async () => tenants.delete('t_gone')
+ )
+
+ expect(await tenants.get('t_gone')).toBeNull()
+ const history = await log.forTenant('t_gone')
+ expect(history).toHaveLength(4) // read started/ok + delete started/ok
+ expect(history.map((e) => e.action)).toContain('tenant.delete-data')
+ })
+
+ it('entries carry no field that could hold tenant content', async () => {
+ const { log } = setup()
+ const e = await log.append({
+ operator: 'u',
+ action: 'tenant.read',
+ tenantId: 't_a',
+ outcome: 'ok'
+ })
+ expect(Object.keys(e).sort()).toEqual([
+ 'action',
+ 'atMs',
+ 'entryId',
+ 'operator',
+ 'outcome',
+ 'tenantId'
+ ])
+ })
+})
diff --git a/apps/cloud/src/ops/audit.ts b/apps/cloud/src/ops/audit.ts
new file mode 100644
index 000000000..2d2471eaf
--- /dev/null
+++ b/apps/cloud/src/ops/audit.ts
@@ -0,0 +1,232 @@
+/**
+ * xNet Cloud — the two-tier operator audit log (exploration 0433, ADR-31).
+ *
+ * **Tier 1** is a fail-closed write to the control plane's own store. It is the
+ * gate: if it does not land, the action does not run. It lives here rather than
+ * on the ops hub because it must be available during exactly the incidents when
+ * the hub might not be — an operator who cannot act during an outage is worse
+ * than one whose audit history is briefly stale.
+ *
+ * **Tier 2** is the same entry republished as a signed xNet node authored by the
+ * operator's `did:key`, which makes it *verifiable* rather than merely
+ * append-only: nobody with database access can forge it, and no operator can
+ * repudiate it. It publishes asynchronously, and when the hub is unreachable the
+ * entry queues. The queue depth is an alertable metric precisely so a gap between
+ * the tiers is visible rather than silent — "absent" and "unreadable" must be
+ * different values (`AGENTS.md`).
+ *
+ * Entries carry an operator, an action, a tenant id and a reason. They never
+ * carry parameters, because a parameter is where tenant content would leak into
+ * a log that outlives the tenant.
+ */
+
+import type { DocStore } from '../stores/durable'
+
+/** What an operator did. Reads are recorded too, at a lower ceremony (decision 5). */
+export type AuditAction =
+ /** Opened a specific tenant. Audited silently — recorded, no reason prompt. */
+ | 'tenant.read'
+ /** Cleared a tenant's bound DID so a fresh device can claim the hub. */
+ | 'tenant.recover'
+ | 'tenant.plan-change'
+ | 'tenant.delete-data'
+ | 'tenant.provision'
+ /** Granted or revoked operator access — itself an audited action. */
+ | 'operator.bind'
+ | 'operator.retire'
+ /** Tier 2 consent lifecycle. */
+ | 'consent.request'
+ | 'consent.grant'
+ | 'consent.deny'
+ | 'consent.expire'
+
+/** Actions that change state, and therefore require a typed reason (decision 5). */
+const MUTATING_ACTIONS: ReadonlySet = new Set([
+ 'tenant.recover',
+ 'tenant.plan-change',
+ 'tenant.delete-data',
+ 'tenant.provision',
+ 'operator.bind',
+ 'operator.retire'
+])
+
+export const requiresReason = (action: AuditAction): boolean => MUTATING_ACTIONS.has(action)
+
+export type AuditOutcome = 'started' | 'ok' | 'failed'
+
+export interface AuditEntry {
+ /** Monotonic-ish id: time-ordered so `page()` reads chronologically. */
+ entryId: string
+ atMs: number
+ /** WorkOS user id of the operator. Never a shared secret. */
+ operator: string
+ /** The signing key bound to that operator when the entry was written. */
+ operatorDid?: string
+ action: AuditAction
+ /** Opaque tenant identifier. Retained after tenant deletion (decision 15). */
+ tenantId: string
+ /** Free text typed by the operator. Required for mutations, absent for reads. */
+ reason?: string
+ outcome: AuditOutcome
+ /** Links `ok`/`failed` back to the `started` entry that authorised the action. */
+ parentId?: string
+ /** Whether the signed tier-2 copy has been published to the ops hub. */
+ published?: boolean
+}
+
+/** Publishes a signed copy to the ops hub. Failure must never block the action. */
+export interface AuditPublisher {
+ publish(entry: AuditEntry): Promise
+}
+
+/** The error a caller sees when the fail-closed tier-1 write does not land. */
+export class AuditWriteError extends Error {
+ constructor(cause: unknown) {
+ super(`audit: tier-1 write failed, action refused: ${String(cause)}`)
+ this.name = 'AuditWriteError'
+ }
+}
+
+/** The error a caller sees when a mutating action arrives without a reason. */
+export class ReasonRequiredError extends Error {
+ constructor(action: AuditAction) {
+ super(`audit: ${action} requires a reason`)
+ this.name = 'ReasonRequiredError'
+ }
+}
+
+const pad = (n: number): string => String(n).padStart(16, '0')
+
+export interface AuditLogOptions {
+ docs: DocStore
+ publisher?: AuditPublisher
+ nowMs?: () => number
+ /** Injectable for deterministic ids in tests. */
+ suffix?: () => string
+}
+
+/**
+ * The tier-1 log plus the tier-2 publish queue.
+ *
+ * Entries are ids of the form `-` so the `DocStore`'s
+ * id-ordered `page()` reads them chronologically without a secondary index.
+ */
+export class AuditLog {
+ private readonly docs: DocStore
+ private readonly publisher?: AuditPublisher
+ private readonly now: () => number
+ private readonly suffix: () => string
+ /** Entries written to tier 1 but not yet confirmed on the ops hub. */
+ private readonly pending: AuditEntry[] = []
+ private seq = 0
+
+ constructor(opts: AuditLogOptions) {
+ this.docs = opts.docs
+ this.publisher = opts.publisher
+ this.now = opts.nowMs ?? (() => Date.now())
+ this.suffix = opts.suffix ?? (() => String(++this.seq).padStart(6, '0'))
+ }
+
+ /**
+ * Write one entry to tier 1, then hand it to tier 2.
+ *
+ * Throws {@link AuditWriteError} if tier 1 fails — that is the fail-closed gate,
+ * and callers must let it propagate rather than proceeding unaudited. A tier-2
+ * failure only queues.
+ */
+ async append(
+ entry: Omit & { atMs?: number }
+ ): Promise {
+ if (requiresReason(entry.action) && !entry.reason?.trim()) {
+ throw new ReasonRequiredError(entry.action)
+ }
+ const atMs = entry.atMs ?? this.now()
+ const record: AuditEntry = { ...entry, atMs, entryId: `${pad(atMs)}-${this.suffix()}` }
+ try {
+ await this.docs.put(record.entryId, record)
+ } catch (err) {
+ throw new AuditWriteError(err)
+ }
+ await this.publish(record)
+ return record
+ }
+
+ /** Best-effort tier-2 publish. Queues on failure; never throws. */
+ private async publish(record: AuditEntry): Promise {
+ if (!this.publisher) {
+ this.pending.push(record)
+ return
+ }
+ try {
+ await this.publisher.publish(record)
+ await this.docs.put(record.entryId, { ...record, published: true })
+ } catch {
+ this.pending.push(record)
+ }
+ }
+
+ /**
+ * How many entries are written to tier 1 but not confirmed on the ops hub.
+ *
+ * This is the alertable metric. A depth that stops returning to zero means the
+ * verifiable half of the audit trail has quietly stopped — the failure that,
+ * unmeasured, would look exactly like a healthy system.
+ */
+ pendingCount(): number {
+ return this.pending.length
+ }
+
+ /** Retry queued publishes. Returns how many drained. */
+ async drain(): Promise {
+ if (!this.publisher || this.pending.length === 0) return 0
+ let drained = 0
+ // Iterate a snapshot; failures go back on the queue in order.
+ const batch = this.pending.splice(0, this.pending.length)
+ for (const record of batch) {
+ try {
+ await this.publisher.publish(record)
+ await this.docs.put(record.entryId, { ...record, published: true })
+ drained += 1
+ } catch {
+ this.pending.push(record)
+ }
+ }
+ return drained
+ }
+
+ /** Every entry touching a tenant, oldest first. Survives that tenant's deletion. */
+ async forTenant(tenantId: string): Promise {
+ const rows = await this.docs.findWhere('tenantId', tenantId)
+ return rows.sort((a, b) => a.entryId.localeCompare(b.entryId))
+ }
+
+ /** Every entry by one operator, oldest first. */
+ async byOperator(operator: string): Promise {
+ const rows = await this.docs.findWhere('operator', operator)
+ return rows.sort((a, b) => a.entryId.localeCompare(b.entryId))
+ }
+}
+
+/**
+ * Run a privileged action with the audit entry written FIRST.
+ *
+ * Ordering is the point. An action that fails must still be attributable, and an
+ * operator must not be able to act and then suppress the record by crashing the
+ * process. The `started` entry is durable before `run()` is called; the outcome
+ * entry links back to it.
+ */
+export async function audited(
+ log: AuditLog,
+ entry: Omit,
+ run: () => Promise
+): Promise {
+ const started = await log.append({ ...entry, outcome: 'started' })
+ try {
+ const result = await run()
+ await log.append({ ...entry, outcome: 'ok', parentId: started.entryId })
+ return result
+ } catch (err) {
+ await log.append({ ...entry, outcome: 'failed', parentId: started.entryId })
+ throw err
+ }
+}
diff --git a/apps/cloud/src/ops/operator.test.ts b/apps/cloud/src/ops/operator.test.ts
new file mode 100644
index 000000000..c0848380f
--- /dev/null
+++ b/apps/cloud/src/ops/operator.test.ts
@@ -0,0 +1,71 @@
+import { describe, expect, it } from 'vitest'
+import { InMemoryDocStore } from '../stores/durable'
+import { OperatorRegistry, hasOperatorRole, type OperatorBinding } from './operator'
+
+const T0 = Date.UTC(2026, 6, 1)
+
+describe('hasOperatorRole', () => {
+ it('accepts the role as a single claim or in a list', () => {
+ expect(hasOperatorRole({ role: 'operator' })).toBe(true)
+ expect(hasOperatorRole({ roles: ['member', 'operator'] })).toBe(true)
+ })
+
+ it('rejects absent, empty, or non-operator claims', () => {
+ expect(hasOperatorRole(null)).toBe(false)
+ expect(hasOperatorRole(undefined)).toBe(false)
+ expect(hasOperatorRole({})).toBe(false)
+ expect(hasOperatorRole({ role: 'member' })).toBe(false)
+ expect(hasOperatorRole({ roles: [] })).toBe(false)
+ })
+
+ // A malformed claim is not a role. Coercing it would let a truthy non-string
+ // through as authorisation.
+ it('rejects claims that are not strings', () => {
+ expect(hasOperatorRole({ role: 1 as unknown as string })).toBe(false)
+ expect(hasOperatorRole({ role: true as unknown as string })).toBe(false)
+ expect(hasOperatorRole({ roles: [{ name: 'operator' }] as unknown as string[] })).toBe(false)
+ expect(hasOperatorRole({ roles: 'operator' as unknown as string[] })).toBe(false)
+ })
+})
+
+describe('OperatorRegistry', () => {
+ const setup = () => new OperatorRegistry(new InMemoryDocStore())
+
+ it('binds and resolves a signing key', async () => {
+ const reg = setup()
+ await reg.bind('user_1', 'did:key:zAbc', T0)
+ expect(await reg.active('user_1')).toMatchObject({ did: 'did:key:zAbc', boundAtMs: T0 })
+ })
+
+ it('rejects a value that is not a DID', async () => {
+ await expect(setup().bind('user_1', 'zAbc', T0)).rejects.toThrow(/Not a DID/)
+ })
+
+ it('returns null for an unknown operator', async () => {
+ expect(await setup().active('nobody')).toBeNull()
+ })
+
+ // Audit entries are kept 12 months and name the DID that signed them, so a
+ // retired key must stay resolvable or a year of history becomes unattributable.
+ it('stops resolving a retired binding but keeps it for historical verification', async () => {
+ const reg = setup()
+ await reg.bind('user_1', 'did:key:zAbc', T0)
+ await reg.retire('user_1', T0 + 1000)
+ expect(await reg.active('user_1')).toBeNull()
+ expect(await reg.resolveHistorical('user_1')).toMatchObject({
+ did: 'did:key:zAbc',
+ retiredAtMs: T0 + 1000
+ })
+ })
+
+ it('retiring an unknown operator is a no-op, not an error', async () => {
+ await expect(setup().retire('nobody', T0)).resolves.toBeUndefined()
+ })
+
+ it('rebinding replaces the active key', async () => {
+ const reg = setup()
+ await reg.bind('user_1', 'did:key:zOld', T0)
+ await reg.bind('user_1', 'did:key:zNew', T0 + 1)
+ expect((await reg.active('user_1'))?.did).toBe('did:key:zNew')
+ })
+})
diff --git a/apps/cloud/src/ops/operator.ts b/apps/cloud/src/ops/operator.ts
new file mode 100644
index 000000000..c794275cb
--- /dev/null
+++ b/apps/cloud/src/ops/operator.ts
@@ -0,0 +1,108 @@
+/**
+ * xNet Cloud — operator identity (exploration 0433, decision 4).
+ *
+ * Two different jobs, deliberately kept in two different places:
+ *
+ * - **Authorisation** — *may this person act as an operator?* Answered by a
+ * WorkOS organisation role, which arrives as a **claim in the JWT**. That
+ * means the check is offline: no WorkOS API call sits on the request path, and
+ * disabling someone in WorkOS revokes ops access on their next token.
+ * - **Attribution** — *which signing key is theirs?* Answered by a binding from
+ * WorkOS user to `did:key`, held here in the control plane's own store.
+ *
+ * Keeping them separate is what makes the audit trail non-repudiable without
+ * putting cryptographic material in a vendor's mutable user metadata.
+ *
+ * The `/internal/*` shared secret is NOT an operator identity: it names nobody,
+ * so it can authorise reads but never a mutation (decision 11).
+ */
+
+import type { DocStore } from '../stores/durable'
+
+/** The WorkOS organisation role that grants operator access. */
+export const OPERATOR_ROLE = 'operator'
+
+/** Cookie name for an operator session — deliberately distinct from the tenant one. */
+export const OPERATOR_COOKIE = 'xnet_cloud_operator'
+
+/** A signed-in operator: who they are, and which key signs for them. */
+export interface OperatorIdentity {
+ /** WorkOS user id. */
+ workosUserId: string
+ email?: string
+ /** Bound signing key. Absent until the operator completes a device-grant claim. */
+ did?: string
+}
+
+/** The persisted WorkOS-user → `did:key` binding. */
+export interface OperatorBinding {
+ workosUserId: string
+ did: string
+ boundAtMs: number
+ /** Set when the binding is retired, so history outlives the key (open question 2). */
+ retiredAtMs?: number
+}
+
+/**
+ * Roles as they arrive in a WorkOS access token. WorkOS emits a single `role` for
+ * an organisation membership; some configurations carry a list. Accept both rather
+ * than depending on which shape a given tenant's directory produces.
+ */
+export interface RoleClaims {
+ role?: unknown
+ roles?: unknown
+}
+
+/**
+ * Whether a token's claims grant operator access.
+ *
+ * Deliberately strict about types: a claim that is not a string (or array of
+ * strings) is NOT a role, it is malformed input, and must not authorise anything.
+ */
+export function hasOperatorRole(claims: RoleClaims | null | undefined): boolean {
+ if (!claims) return false
+ const single = typeof claims.role === 'string' ? [claims.role] : []
+ const many = Array.isArray(claims.roles)
+ ? claims.roles.filter((r): r is string => typeof r === 'string')
+ : []
+ return [...single, ...many].includes(OPERATOR_ROLE)
+}
+
+/** Persistent operator roster. Reads live in the control plane, so they work in an incident. */
+export class OperatorRegistry {
+ constructor(private readonly docs: DocStore) {}
+
+ /** The active binding for a WorkOS user, or null. Retired bindings never resolve. */
+ async active(workosUserId: string): Promise {
+ const rec = await this.docs.get(workosUserId)
+ if (!rec || rec.retiredAtMs !== undefined) return null
+ return rec
+ }
+
+ /** Bind a signing key to an operator. Replaces any active binding for that user. */
+ async bind(workosUserId: string, did: string, nowMs: number): Promise {
+ if (!did.startsWith('did:')) throw new Error(`Not a DID: ${did}`)
+ const rec: OperatorBinding = { workosUserId, did, boundAtMs: nowMs }
+ await this.docs.put(workosUserId, rec)
+ return rec
+ }
+
+ /**
+ * Retire a binding without deleting it.
+ *
+ * Audit entries are retained twelve months and reference the DID that signed
+ * them, so a key must stay *resolvable* long after it stops being usable — a
+ * deleted binding would leave a year of history signed by an unattributable key
+ * (decision 15, open question 2).
+ */
+ async retire(workosUserId: string, nowMs: number): Promise {
+ const rec = await this.docs.get(workosUserId)
+ if (!rec) return
+ await this.docs.put(workosUserId, { ...rec, retiredAtMs: nowMs })
+ }
+
+ /** Resolve any binding — active or retired — for verifying historical entries. */
+ async resolveHistorical(workosUserId: string): Promise {
+ return this.docs.get(workosUserId)
+ }
+}
diff --git a/apps/cloud/src/ops/publisher.test.ts b/apps/cloud/src/ops/publisher.test.ts
new file mode 100644
index 000000000..da4176ea8
--- /dev/null
+++ b/apps/cloud/src/ops/publisher.test.ts
@@ -0,0 +1,84 @@
+import type { AuditEntry } from './audit'
+import { describe, expect, it, vi } from 'vitest'
+import {
+ DEFAULT_OPS_SPACE,
+ opsHubPublisher,
+ opsHubPublisherFromEnv,
+ toAuditNode
+} from './publisher'
+
+const entry: AuditEntry = {
+ entryId: '0000000000000001-000001',
+ atMs: 1,
+ operator: 'user_ops',
+ operatorDid: 'did:key:zOps',
+ action: 'tenant.recover',
+ tenantId: 't_a',
+ reason: 'lost passkey',
+ outcome: 'started'
+}
+
+describe('toAuditNode', () => {
+ it('carries operator, action, tenant, reason and outcome — and nothing else', () => {
+ const node = toAuditNode(entry)
+ expect(node.nodeType).toBe('ops-audit-entry')
+ expect(node.spaceId).toBe(DEFAULT_OPS_SPACE)
+ expect(Object.keys(node.properties).sort()).toEqual([
+ 'action',
+ 'atMs',
+ 'entryId',
+ 'operator',
+ 'operatorDid',
+ 'outcome',
+ 'reason',
+ 'tenantId'
+ ])
+ })
+
+ it('omits optional fields rather than emitting undefined', () => {
+ const node = toAuditNode({ ...entry, operatorDid: undefined, reason: undefined })
+ expect('operatorDid' in node.properties).toBe(false)
+ expect('reason' in node.properties).toBe(false)
+ })
+})
+
+describe('opsHubPublisher', () => {
+ it('POSTs the node with the bearer token', async () => {
+ const fetchImpl = vi.fn(async () => new Response(null, { status: 201 }))
+ await opsHubPublisher({
+ hubUrl: 'https://ops.hub/',
+ token: 'tok',
+ fetchImpl: fetchImpl as unknown as typeof fetch
+ }).publish(entry)
+ const [url, init] = fetchImpl.mock.calls[0] as unknown as [string, RequestInit]
+ expect(url).toBe('https://ops.hub/nodes')
+ expect((init.headers as Record).authorization).toBe('Bearer tok')
+ })
+
+ // A silently-swallowed rejection would make an unreachable hub look identical
+ // to a healthy one — the caller must see the failure so the entry stays queued.
+ it('throws on a non-2xx so the entry stays queued', async () => {
+ const fetchImpl = vi.fn(async () => new Response(null, { status: 503 }))
+ await expect(
+ opsHubPublisher({
+ hubUrl: 'https://ops.hub',
+ token: 'tok',
+ fetchImpl: fetchImpl as unknown as typeof fetch
+ }).publish(entry)
+ ).rejects.toThrow(/503/)
+ })
+})
+
+describe('opsHubPublisherFromEnv', () => {
+ it('returns null when the ops hub is not configured', () => {
+ expect(opsHubPublisherFromEnv({})).toBeNull()
+ expect(opsHubPublisherFromEnv({ XNET_OPS_HUB_URL: 'https://x' })).toBeNull()
+ expect(opsHubPublisherFromEnv({ XNET_OPS_HUB_TOKEN: 't' })).toBeNull()
+ })
+
+ it('builds a publisher when both url and token are present', () => {
+ expect(
+ opsHubPublisherFromEnv({ XNET_OPS_HUB_URL: 'https://x', XNET_OPS_HUB_TOKEN: 't' })
+ ).not.toBeNull()
+ })
+})
diff --git a/apps/cloud/src/ops/publisher.ts b/apps/cloud/src/ops/publisher.ts
new file mode 100644
index 000000000..f3178f715
--- /dev/null
+++ b/apps/cloud/src/ops/publisher.ts
@@ -0,0 +1,120 @@
+/**
+ * xNet Cloud — the tier-2 audit publisher (exploration 0433, ADR-31).
+ *
+ * Republishes each tier-1 audit entry to the ops hub as a node authored by the
+ * operator's `did:key`, so the hub's signed, hash-chained change log becomes the
+ * verifiable audit trail — readable back through the hub's existing
+ * `GET /audit/authors/:did/changes` (`packages/hub/src/routes/audit.ts`).
+ *
+ * Two properties this must have, and one it must not:
+ *
+ * - It **must not** be on the critical path. A publish failure queues; the action
+ * already happened and was already recorded in tier 1.
+ * - It **must** fail loudly rather than silently succeed. A non-2xx is thrown so
+ * the entry stays queued and the depth metric rises; swallowing it would make
+ * an unreachable hub indistinguishable from a healthy one.
+ * - It carries **no tenant content** — only operator, action, opaque tenant id,
+ * reason and outcome, matching the tier-1 schema.
+ */
+
+import type { AuditEntry, AuditPublisher } from './audit'
+
+export interface OpsHubPublisherConfig {
+ /** Base URL of the ops hub (its own GCP project, outside the fleet provisioner). */
+ hubUrl: string
+ /** Bearer token authorising the control plane to write as the ops workspace. */
+ token: string
+ /** Space the operator log lives in. */
+ spaceId?: string
+ fetchImpl?: typeof fetch
+ timeoutMs?: number
+}
+
+/** The node an audit entry becomes on the ops hub. */
+export interface AuditNodePayload {
+ nodeType: 'ops-audit-entry'
+ spaceId: string
+ properties: {
+ entryId: string
+ atMs: number
+ operator: string
+ operatorDid?: string
+ action: string
+ tenantId: string
+ reason?: string
+ outcome: string
+ parentId?: string
+ }
+}
+
+export const DEFAULT_OPS_SPACE = 'ops-audit'
+
+/** Map a tier-1 entry to its tier-2 node. Content-free by construction. */
+export function toAuditNode(entry: AuditEntry, spaceId = DEFAULT_OPS_SPACE): AuditNodePayload {
+ return {
+ nodeType: 'ops-audit-entry',
+ spaceId,
+ properties: {
+ entryId: entry.entryId,
+ atMs: entry.atMs,
+ operator: entry.operator,
+ ...(entry.operatorDid ? { operatorDid: entry.operatorDid } : {}),
+ action: entry.action,
+ tenantId: entry.tenantId,
+ ...(entry.reason ? { reason: entry.reason } : {}),
+ outcome: entry.outcome,
+ ...(entry.parentId ? { parentId: entry.parentId } : {})
+ }
+ }
+}
+
+/** Publishes audit entries to a real ops hub over HTTP. */
+export function opsHubPublisher(config: OpsHubPublisherConfig): AuditPublisher {
+ const fetchImpl = config.fetchImpl ?? fetch
+ const timeoutMs = config.timeoutMs ?? 10_000
+ const base = config.hubUrl.replace(/\/$/, '')
+ return {
+ async publish(entry: AuditEntry): Promise {
+ const ctrl = new AbortController()
+ const timer = setTimeout(() => ctrl.abort(), timeoutMs)
+ try {
+ const res = await fetchImpl(`${base}/nodes`, {
+ method: 'POST',
+ headers: {
+ 'content-type': 'application/json',
+ authorization: `Bearer ${config.token}`
+ },
+ body: JSON.stringify(toAuditNode(entry, config.spaceId ?? DEFAULT_OPS_SPACE)),
+ signal: ctrl.signal
+ })
+ // Throw, don't swallow: the caller queues on rejection, and a queue that
+ // never drains is the visible signal that tier 2 has stopped.
+ if (!res.ok) throw new Error(`ops hub rejected audit entry: ${res.status}`)
+ } finally {
+ clearTimeout(timer)
+ }
+ }
+ }
+}
+
+/**
+ * Build a publisher from the environment, or `null` when the ops hub is not
+ * configured.
+ *
+ * `null` is meaningful: the {@link AuditLog} then queues every entry and reports a
+ * rising pending count, which reads as "tier 2 is not wired" rather than as
+ * "everything is published". A control plane that silently ran with no verifiable
+ * audit trail is the state this whole design exists to end.
+ */
+export function opsHubPublisherFromEnv(
+ env: NodeJS.ProcessEnv = process.env
+): AuditPublisher | null {
+ const hubUrl = env.XNET_OPS_HUB_URL
+ const token = env.XNET_OPS_HUB_TOKEN
+ if (!hubUrl || !token) return null
+ return opsHubPublisher({
+ hubUrl,
+ token,
+ ...(env.XNET_OPS_HUB_SPACE ? { spaceId: env.XNET_OPS_HUB_SPACE } : {})
+ })
+}
diff --git a/apps/cloud/src/ops/session.ts b/apps/cloud/src/ops/session.ts
new file mode 100644
index 000000000..bc5e7ccff
--- /dev/null
+++ b/apps/cloud/src/ops/session.ts
@@ -0,0 +1,85 @@
+/**
+ * xNet Cloud — the operator session (exploration 0433, decision 4).
+ *
+ * A **separate sealed cookie** from the tenant session, deliberately. Sharing one
+ * would mean a tenant session could be mistaken for an operator session by any
+ * future code path that forgot to re-check the role — and the two have opposite
+ * blast radii. Distinct names make that mistake impossible rather than unlikely.
+ *
+ * Authorisation is the WorkOS organisation role, read from the access token's
+ * claims. The claim is verified once at sign-in and sealed into this cookie, so
+ * the request path never calls WorkOS: an operator console that cannot
+ * authenticate during a WorkOS outage is an operator console that cannot help
+ * during an incident.
+ */
+
+import { sealSession, readSession, type SessionData } from '../session'
+import { hasOperatorRole, OPERATOR_COOKIE, type OperatorIdentity } from './operator'
+
+export { OPERATOR_COOKIE }
+
+/** What we seal: the tenant session shape plus the resolved operator facts. */
+interface OperatorSessionData extends SessionData {
+ /** Bound signing key at sign-in time, if the operator has completed a claim. */
+ operatorDid?: string
+}
+
+/** Operator sessions are short: 12 hours, not the tenant session's 7 days. */
+export const OPERATOR_MAX_AGE_MS = 12 * 60 * 60 * 1000
+
+/** Seal an operator session cookie value. */
+export function sealOperatorSession(
+ secret: string,
+ data: { workosUserId: string; email?: string; did?: string; issuedAtMs: number }
+): string {
+ const payload: OperatorSessionData = {
+ billingUserId: data.workosUserId,
+ issuedAtMs: data.issuedAtMs,
+ ...(data.email ? { email: data.email } : {}),
+ ...(data.did ? { operatorDid: data.did } : {})
+ }
+ return sealSession(secret, payload)
+}
+
+/** Read and verify an operator session cookie, or null. */
+export function readOperatorSession(
+ secret: string,
+ cookie: string | undefined,
+ opts: { nowMs: number }
+): OperatorIdentity | null {
+ const data = readSession(secret, cookie, {
+ nowMs: opts.nowMs,
+ maxAgeMs: OPERATOR_MAX_AGE_MS
+ }) as OperatorSessionData | null
+ if (!data) return null
+ return {
+ workosUserId: data.billingUserId,
+ ...(data.email ? { email: data.email } : {}),
+ ...(data.operatorDid ? { did: data.operatorDid } : {})
+ }
+}
+
+/**
+ * Decode the claims of a WorkOS access token WITHOUT verifying its signature.
+ *
+ * Safe only because of where it is called: immediately after the OAuth code
+ * exchange, on a token this server just received over TLS directly from WorkOS.
+ * It is NOT a request-path authenticator — the sealed cookie is. Calling this on
+ * an attacker-supplied token would authorise anyone who can base64 a JSON object.
+ */
+export function claimsFromAccessToken(accessToken: string): Record | null {
+ const parts = accessToken.split('.')
+ if (parts.length !== 3) return null
+ try {
+ const json = Buffer.from(parts[1], 'base64url').toString('utf8')
+ const parsed: unknown = JSON.parse(json)
+ return parsed && typeof parsed === 'object' ? (parsed as Record) : null
+ } catch {
+ return null
+ }
+}
+
+/** Whether a freshly-exchanged WorkOS access token carries the operator role. */
+export function tokenGrantsOperator(accessToken: string): boolean {
+ return hasOperatorRole(claimsFromAccessToken(accessToken))
+}
diff --git a/apps/cloud/src/secret-compare.test.ts b/apps/cloud/src/secret-compare.test.ts
new file mode 100644
index 000000000..bb8fa206d
--- /dev/null
+++ b/apps/cloud/src/secret-compare.test.ts
@@ -0,0 +1,33 @@
+import { describe, expect, it } from 'vitest'
+import { timingSafeEqualStr } from './secret-compare'
+
+describe('timingSafeEqualStr', () => {
+ it('accepts an exact match', () => {
+ expect(timingSafeEqualStr('s3cret', 's3cret')).toBe(true)
+ })
+
+ it('rejects a mismatch, including a shared prefix', () => {
+ expect(timingSafeEqualStr('s3crey', 's3cret')).toBe(false)
+ expect(timingSafeEqualStr('s3cret-extra', 's3cret')).toBe(false)
+ expect(timingSafeEqualStr('s3cre', 's3cret')).toBe(false)
+ })
+
+ // A missing header must never authenticate against an unconfigured secret —
+ // otherwise a control plane with no secret set would accept every caller.
+ it.each([
+ [undefined, 's3cret'],
+ [null, 's3cret'],
+ ['', 's3cret'],
+ ['s3cret', undefined],
+ ['s3cret', ''],
+ [undefined, undefined],
+ ['', '']
+ ])('rejects absent or empty inputs (%p vs %p)', (a, b) => {
+ expect(timingSafeEqualStr(a, b)).toBe(false)
+ })
+
+ it('handles multi-byte characters without throwing', () => {
+ expect(timingSafeEqualStr('sécret🔑', 'sécret🔑')).toBe(true)
+ expect(timingSafeEqualStr('sécret🔑', 'sécret🔒')).toBe(false)
+ })
+})
diff --git a/apps/cloud/src/secret-compare.ts b/apps/cloud/src/secret-compare.ts
new file mode 100644
index 000000000..c80cb7b06
--- /dev/null
+++ b/apps/cloud/src/secret-compare.ts
@@ -0,0 +1,29 @@
+/**
+ * Constant-time secret comparison (exploration 0433, decision 11).
+ *
+ * `===` on a secret is a timing oracle: it returns on the first differing byte,
+ * so response time leaks how long a shared prefix an attacker has guessed. The
+ * practical risk over the internet is low, but the fix costs nothing and the
+ * secret this guarded reached `/internal/account/recover`.
+ */
+
+import { timingSafeEqual } from 'node:crypto'
+
+/**
+ * Compare two secrets without leaking their contents through timing.
+ *
+ * Length is compared first and NOT in constant time — `timingSafeEqual` throws on
+ * unequal lengths, and a secret's length is not the part worth protecting.
+ * `undefined`/empty candidates are rejected outright so a missing header can never
+ * match a missing config.
+ */
+export function timingSafeEqualStr(
+ candidate: string | undefined | null,
+ expected: string | undefined | null
+): boolean {
+ if (!candidate || !expected) return false
+ const a = Buffer.from(candidate, 'utf8')
+ const b = Buffer.from(expected, 'utf8')
+ if (a.length !== b.length) return false
+ return timingSafeEqual(a, b)
+}
diff --git a/apps/cloud/src/server.test.ts b/apps/cloud/src/server.test.ts
index fe1f7a4f9..f390aa25c 100644
--- a/apps/cloud/src/server.test.ts
+++ b/apps/cloud/src/server.test.ts
@@ -1,16 +1,37 @@
import { MemoryBillingIdentityProvider } from '@xnetjs/cloud/identity'
import { describe, expect, it } from 'vitest'
+import { AuditLog, type AuditEntry } from './ops/audit'
import { createControlPlaneApp } from './server'
import { SESSION_COOKIE, sealSession } from './session'
+import { InMemoryDocStore } from './stores/durable'
import { buildControlPlane } from './index'
const INTERNAL = 'secret123'
const SESSION_SECRET = 'session-secret-xyz'
-function app() {
+/**
+ * Mutating internal routes need a named operator and a typed reason (0433
+ * decision 11). Tests present the operator via a header the fake resolver reads;
+ * production resolves it from the sealed operator cookie.
+ */
+const OPERATOR = { workosUserId: 'user_ops', did: 'did:key:zOps' }
+const asOperator = (reason = 'test') => ({
+ 'x-operator': OPERATOR.workosUserId,
+ 'x-operator-reason': reason
+})
+
+function app(over: Partial[0]> = {}) {
const billing = new MemoryBillingIdentityProvider('https://auth.test/authorize')
const { controlPlane } = buildControlPlane({ billing, verifyDid: async () => true })
- return createControlPlaneApp({ controlPlane, billing, internalSecret: INTERNAL })
+ return createControlPlaneApp({
+ controlPlane,
+ billing,
+ internalSecret: INTERNAL,
+ audit: new AuditLog({ docs: new InMemoryDocStore() }),
+ resolveOperator: async (c) =>
+ c.req.header('x-operator') === OPERATOR.workosUserId ? OPERATOR : null,
+ ...over
+ })
}
const provisionBody = {
@@ -32,7 +53,7 @@ describe('control-plane HTTP API', () => {
// Provision a real (hot, hub-bearing) tenant, then confirm it can't surface.
await a.request('/internal/tenants', {
method: 'POST',
- headers: { 'content-type': 'application/json', 'x-internal-secret': INTERNAL },
+ headers: { 'content-type': 'application/json', ...asOperator() },
body: JSON.stringify(provisionBody)
})
const res = await a.request('/status.json')
@@ -44,7 +65,10 @@ describe('control-plane HTTP API', () => {
components: { id: string }[]
errorBudgetPolicy: Record
}
- expect(status.overall).toBe('operational')
+ // `unmeasured`, not `operational`: this server has no observability or job
+ // reporting wired, so there is no evidence of health to publish. Claiming
+ // green here is exactly the defect exploration 0433 decision 10 removes.
+ expect(status.overall).toBe('unmeasured')
expect(status.components.map((c) => c.id)).toContain('hub-fleet')
expect(status.errorBudgetPolicy).toMatchObject({ ship: 0, caution: 0, freeze: 0 })
})
@@ -56,7 +80,7 @@ describe('control-plane HTTP API', () => {
expect(res.headers.get('location')).toContain('state=abc')
})
- it('guards internal routes behind the shared secret', async () => {
+ it('refuses a mutating internal route with no operator identity', async () => {
const res = await app().request('/internal/tenants', {
method: 'POST',
headers: { 'content-type': 'application/json' },
@@ -65,11 +89,64 @@ describe('control-plane HTTP API', () => {
expect(res.status).toBe(403)
})
+ // The heart of 0433 decision 11: the shared secret is unattributable, so it
+ // must not open a route that mutates a tenant — least of all `recover`, which
+ // clears the bound DID and lets the next device claim the hub.
+ it('rejects the shared secret on every mutating internal route', async () => {
+ const a = app()
+ const secretOnly = { 'content-type': 'application/json', 'x-internal-secret': INTERNAL }
+ const cases: [string, unknown][] = [
+ ['/internal/tenants', provisionBody],
+ ['/internal/tenants/acme/plan', { plan: 'family' }],
+ ['/internal/account/recover', { billingUserId: 'user_a' }]
+ ]
+ for (const [path, body] of cases) {
+ const res = await a.request(path, {
+ method: 'POST',
+ headers: secretOnly,
+ body: JSON.stringify(body)
+ })
+ expect(res.status, `${path} must refuse the shared secret`).toBe(403)
+ }
+ })
+
+ it('still accepts the shared secret on internal READ routes', async () => {
+ const res = await app().request('/internal/metrics/usage', {
+ headers: { 'x-internal-secret': INTERNAL }
+ })
+ expect(res.status).toBe(200)
+ })
+
+ it('refuses a mutation with an operator but no reason', async () => {
+ const res = await app().request('/internal/account/recover', {
+ method: 'POST',
+ headers: { 'content-type': 'application/json', 'x-operator': OPERATOR.workosUserId },
+ body: JSON.stringify({ billingUserId: 'user_a' })
+ })
+ expect(res.status).toBe(400)
+ expect(await res.json()).toMatchObject({ error: 'reason_required' })
+ })
+
+ // Fail-closed: if the durable audit write cannot land, the action must not run.
+ it('refuses to act when the audit store is unavailable', async () => {
+ const docs = new InMemoryDocStore()
+ docs.put = async () => {
+ throw new Error('firestore down')
+ }
+ const res = await app({ audit: new AuditLog({ docs }) }).request('/internal/tenants', {
+ method: 'POST',
+ headers: { 'content-type': 'application/json', ...asOperator() },
+ body: JSON.stringify(provisionBody)
+ })
+ expect(res.status).toBe(503)
+ expect(await res.json()).toMatchObject({ error: 'audit_unavailable' })
+ })
+
it('provisions a tenant through the internal route and reads it back', async () => {
const a = app()
const res = await a.request('/internal/tenants', {
method: 'POST',
- headers: { 'content-type': 'application/json', 'x-internal-secret': INTERNAL },
+ headers: { 'content-type': 'application/json', ...asOperator() },
body: JSON.stringify(provisionBody)
})
expect(res.status).toBe(201)
@@ -118,7 +195,7 @@ describe('control-plane HTTP API', () => {
it('rejects malformed provisioning input', async () => {
const res = await app().request('/internal/tenants', {
method: 'POST',
- headers: { 'content-type': 'application/json', 'x-internal-secret': INTERNAL },
+ headers: { 'content-type': 'application/json', ...asOperator() },
body: JSON.stringify({ tenantId: 'x' })
})
expect(res.status).toBe(400)
@@ -128,19 +205,19 @@ describe('control-plane HTTP API', () => {
const a = app()
await a.request('/internal/tenants', {
method: 'POST',
- headers: { 'content-type': 'application/json', 'x-internal-secret': INTERNAL },
+ headers: { 'content-type': 'application/json', ...asOperator() },
body: JSON.stringify(provisionBody)
})
const flip = await a.request('/internal/tenants/acme/plan', {
method: 'POST',
- headers: { 'content-type': 'application/json', 'x-internal-secret': INTERNAL },
+ headers: { 'content-type': 'application/json', ...asOperator() },
body: JSON.stringify({ plan: 'family' })
})
expect((await flip.json()).kind).toBe('flipped')
const migrate = await a.request('/internal/tenants/acme/plan', {
method: 'POST',
- headers: { 'content-type': 'application/json', 'x-internal-secret': INTERNAL },
+ headers: { 'content-type': 'application/json', ...asOperator() },
body: JSON.stringify({ plan: 'community' })
})
expect((await migrate.json()).kind).toBe('migration-required')
diff --git a/apps/cloud/src/server.ts b/apps/cloud/src/server.ts
index 53a7e05b3..406928bd1 100644
--- a/apps/cloud/src/server.ts
+++ b/apps/cloud/src/server.ts
@@ -12,6 +12,7 @@
*/
import type { ControlPlane } from './control-plane'
+import type { OperatorIdentity } from './ops/operator'
import type { UsageLedger } from '@xnetjs/cloud/billing'
import type { BillingIdentityProvider, DidChallenge } from '@xnetjs/cloud/identity'
import type { PlanId } from '@xnetjs/entitlements'
@@ -54,6 +55,14 @@ import {
import { MemoryNonceStore, type NonceStore } from './nonce'
import { fleetSummary, tenantSli, type HealthSampleStore } from './observability/health'
import { publicStatus } from './observability/status'
+import {
+ AuditWriteError,
+ ReasonRequiredError,
+ audited,
+ type AuditAction,
+ type AuditLog
+} from './ops/audit'
+import { timingSafeEqualStr } from './secret-compare'
import { reportToSentry } from './sentry'
import { SESSION_COOKIE, readSession, sealSession, type SessionData } from './session'
@@ -96,8 +105,19 @@ export interface ControlPlaneAppDeps {
marketingUrl?: string
/** Base URL of the hosted web app ("Open the app"). Defaults to the marketing app. */
appUrl?: string
- /** Shared secret for internal routes; if unset, internal routes are disabled. */
+ /** Shared secret for internal READ routes; if unset, internal routes are disabled. */
internalSecret?: string
+ /**
+ * Operator identity resolver (exploration 0433, decisions 4 and 11).
+ *
+ * Mutating `/internal/*` routes require this and reject the shared secret: a
+ * secret names nobody, and `/internal/account/recover` clears a tenant's bound
+ * DID, so an unattributed caller could take over any hub without leaving a
+ * record. Returns null when the request carries no valid operator session.
+ */
+ resolveOperator?: (c: Context) => Promise
+ /** Two-tier audit log. Mutating routes refuse to run without it (fail-closed). */
+ audit?: AuditLog
/** Optional bulk-storage reader (R2) for the `/open` usage snapshot's GB-stored (Tier 1). */
usageStorage?: StorageUsageReader
/** Optional per-hub usage probe; defaults to GETting each hot hub's `/health`. */
@@ -221,6 +241,14 @@ export function createControlPlaneApp(deps: ControlPlaneAppDeps): Hono {
tenantSli(deps.health!, { tenantId: t.tenantId, plan: t.plan, hubUrl: t.hubUrl }, now())
)
: []
+ // Is anything actually being measured? A fleet with hot tenants but no probe
+ // samples must read `unmeasured`, not `operational` (exploration 0433). With no
+ // hot tenants at all there is nothing to measure and nothing to claim, so the
+ // component stays measured-and-empty rather than alarming.
+ const fleetMeasured = hot.length === 0 || slis.some((s) => s.sampleCount > 0)
+ // The control plane reports from its periodic jobs. `undefined` (job reporting
+ // unwired) renders `unmeasured` — never a tautological green.
+ const jobs = deps.jobs ? await deps.jobs.health() : null
const status = publicStatus({
nowMs: now(),
fleet: fleetSummary(slis),
@@ -228,7 +256,9 @@ export function createControlPlaneApp(deps: ControlPlaneAppDeps): Hono {
aiConfigured: Boolean(deps.ai),
// `unproven` reports as unknown (null), NOT as healthy: a configured
// bucket nobody has restored from is not a backup we should claim.
- backupsHealthy: backupsHealthyFor(deps.backupHealth?.())
+ backupsHealthy: backupsHealthyFor(deps.backupHealth?.()),
+ fleetMeasured,
+ ...(jobs ? { controlPlaneJobsHealthy: !jobs.some((j) => j.stale) } : {})
})
return c.json(status)
})
@@ -661,47 +691,93 @@ export function createControlPlaneApp(deps: ControlPlaneAppDeps): Hono {
}
// ── Internal routes (admin tooling) ──────────────────────────────────────────
+ //
+ // READ routes keep the shared secret — `scripts/cloud-company-metrics.mjs` is a
+ // real machine consumer and the blast radius of "can read fleet aggregates" is
+ // small. MUTATION routes reject it outright and require an operator identity
+ // instead (exploration 0433, decision 11): the secret is unattributable, and
+ // `/internal/account/recover` clears a tenant's bound DID, so anyone holding it
+ // could take over any hub and leave no record of having done so.
const requireInternal = (c: { req: { header: (k: string) => string | undefined } }): boolean =>
- Boolean(deps.internalSecret) && c.req.header('x-internal-secret') === deps.internalSecret
+ Boolean(deps.internalSecret) &&
+ timingSafeEqualStr(c.req.header('x-internal-secret'), deps.internalSecret)
+
+ /**
+ * Gate a mutating internal route on a named operator plus a typed reason, and
+ * wrap the action so the audit entry is durable BEFORE it runs.
+ *
+ * Returns a Response on refusal, or the action's result. The shared secret is
+ * not consulted at all here — presenting it grants nothing on this path.
+ */
+ const asOperator = async (
+ c: Context,
+ action: AuditAction,
+ tenantId: string,
+ run: (op: OperatorIdentity) => Promise
+ ): Promise<{ ok: true; value: T } | { ok: false; res: Response }> => {
+ if (!deps.resolveOperator || !deps.audit) {
+ return { ok: false, res: c.json({ error: 'operator_identity_not_configured' }, 503) }
+ }
+ const operator = await deps.resolveOperator(c)
+ if (!operator) return { ok: false, res: c.json({ error: 'forbidden' }, 403) }
+ const reason = c.req.header('x-operator-reason') ?? ''
+ try {
+ const value = await audited(
+ deps.audit,
+ {
+ operator: operator.workosUserId,
+ ...(operator.did ? { operatorDid: operator.did } : {}),
+ action,
+ tenantId,
+ reason
+ },
+ () => run(operator)
+ )
+ return { ok: true, value }
+ } catch (err) {
+ if (err instanceof ReasonRequiredError) {
+ return { ok: false, res: c.json({ error: 'reason_required' }, 400) }
+ }
+ if (err instanceof AuditWriteError) {
+ // Fail closed: no durable record, no action.
+ return { ok: false, res: c.json({ error: 'audit_unavailable' }, 503) }
+ }
+ return { ok: false, res: c.json({ error: (err as Error).message }, 422) }
+ }
+ }
app.post('/internal/tenants', async (c) => {
- if (!requireInternal(c)) return c.json({ error: 'forbidden' }, 403)
const body = (await c.req.json().catch(() => ({}))) as ProvisionBody
if (!body.tenantId || !body.plan || !body.billingUserId || !body.challenge) {
return c.json({ error: 'bad_request' }, 400)
}
- try {
- const record = await deps.controlPlane.provisionTenant({
- tenantId: body.tenantId,
+ const out = await asOperator(c, 'tenant.provision', body.tenantId, () =>
+ deps.controlPlane.provisionTenant({
+ tenantId: body.tenantId as string,
plan: body.plan as never,
- billingUserId: body.billingUserId,
- challenge: body.challenge,
+ billingUserId: body.billingUserId as string,
+ challenge: body.challenge as never,
...(body.overrides ? { overrides: body.overrides as never } : {}),
...(body.region ? { region: body.region } : {})
})
- return c.json(record, 201)
- } catch (err) {
- return c.json({ error: (err as Error).message }, 422)
- }
+ )
+ return out.ok ? c.json(out.value, 201) : out.res
})
app.post('/internal/tenants/:id/plan', async (c) => {
- if (!requireInternal(c)) return c.json({ error: 'forbidden' }, 403)
const body = (await c.req.json().catch(() => ({}))) as {
plan?: string
overrides?: Record
}
if (!body.plan) return c.json({ error: 'bad_request' }, 400)
- try {
- const result = await deps.controlPlane.changePlan(
+ const out = await asOperator(c, 'tenant.plan-change', c.req.param('id'), () =>
+ deps.controlPlane.changePlan(
c.req.param('id'),
body.plan as never,
(body.overrides ?? {}) as never
)
- return c.json(result)
- } catch (err) {
- return c.json({ error: (err as Error).message }, 422)
- }
+ )
+ return out.ok ? c.json(out.value) : out.res
})
// Fleet observability — per-tenant SLIs + an aggregate (exploration 0193).
@@ -747,16 +823,16 @@ export function createControlPlaneApp(deps: ControlPlaneAppDeps): Hono {
return c.json(usage)
})
+ // The account-takeover primitive: `recoverAccount` clears the tenant's bound
+ // DID, so the next device to present a passkey claims their hub. Operator
+ // identity + typed reason only — the shared secret is rejected here.
app.post('/internal/account/recover', async (c) => {
- if (!requireInternal(c)) return c.json({ error: 'forbidden' }, 403)
const body = (await c.req.json().catch(() => ({}))) as { billingUserId?: string }
if (!body.billingUserId) return c.json({ error: 'bad_request' }, 400)
- try {
- const result = await deps.controlPlane.recoverAccount(body.billingUserId)
- return c.json(result)
- } catch (err) {
- return c.json({ error: (err as Error).message }, 422)
- }
+ const out = await asOperator(c, 'tenant.recover', body.billingUserId, () =>
+ deps.controlPlane.recoverAccount(body.billingUserId as string)
+ )
+ return out.ok ? c.json(out.value) : out.res
})
return app
diff --git a/docs/explorations/0431_[_]_XNET_CLOUD_OPERATOR_CONSOLE_SRE_AND_SUPPORT.md b/docs/explorations/0431_[_]_XNET_CLOUD_OPERATOR_CONSOLE_SRE_AND_SUPPORT.md
new file mode 100644
index 000000000..ce3359e11
--- /dev/null
+++ b/docs/explorations/0431_[_]_XNET_CLOUD_OPERATOR_CONSOLE_SRE_AND_SUPPORT.md
@@ -0,0 +1,1095 @@
+---
+title: xNet Cloud operator console — administration, site reliability, and support
+status: draft # draft | withdrawn
+last_updated: 2026-08-01
+review: 2026-11-01 # 90d: gated on the first paying cohort (0418), which lands inside this window
+decider: chris
+door: one-way # the ops hub becomes a standing operational dependency; the console and stores alone would be two-way
+tags: [cloud, operations, sre, support, security, observability]
+---
+
+# xNet Cloud operator console — administration, site reliability, and support
+
+> [!IMPORTANT]
+> **Superseded as a plan by [exploration 0433](./0433_[-]_OPERATOR_CONSOLE_THE_DECIDED_PLAN.md).**
+> This document stays as the **research** — its findings (the amnesiac SLI
+> window, the unattributed account-takeover primitive, the confidentiality gap)
+> are the evidence base and remain citable. Its *recommendations* were revised
+> twice and then decided in full: 0433 carries the sixteen settled decisions, the
+> phased plan, and the checklists. Two findings 0433 adds are not here at all —
+> the tiers selling a 99.9% SLO are provisioned scale-to-zero, and four
+> user-facing surfaces (not one) overclaim confidentiality. Read 0433 first.
+
+> [!TIP]
+> **TL;DR** — The SRE mathematics is already built, tested, and wired: SLIs,
+> SLOs, error budgets, burn rate, fleet rollup, a public status page, and an
+> error-budget-gated rollout engine. What is missing is not a dashboard. It is
+> the three things underneath one: a **durable SLI window** (today's error
+> budget is amnesiac *and* physically capped at ~33 hours despite being labelled
+> 30 days), an **operator identity with an audit log** (`/internal/*` is one flat
+> shared secret, and one of the routes behind it clears a tenant's device
+> binding), and an **honest support-visibility boundary** (the dashboard tells
+> users we hold only encrypted bytes; the hub indexes their plaintext). Fix the
+> substrate first, then render it as a **React + Tailwind console built on
+> `@xnetjs/ui`**, served same-origin by the control plane. Run it **on xNet for
+> the record and REST for the readings**: operator actions become signed nodes on
+> a dedicated ops hub — which turns the audit log from a thing we build into a
+> thing we already have — while SLI buckets and `TenantRecord` stay in Firestore,
+> because a change log is the wrong shape for metrics (0323's 318k-row cliff).
+
+---
+
+## Problem Statement
+
+xNet Cloud is a real control plane running real tenants. It provisions hubs,
+takes money, reconciles dunning, drills restores, and rolls out fleet upgrades.
+The people operating it have, today, exactly three ways to answer a question
+about it:
+
+1. `curl` an `/internal/*` route with a shared secret in a header.
+2. Read JSON lines out of Cloud Run logs.
+3. Read the code.
+
+That is workable for one operator who wrote the system. It fails the moment
+someone has to answer *"a customer emailed saying sync is broken — what is
+actually happening to them?"* under time pressure, and it fails badly the moment
+more than one person holds the secret.
+
+This exploration asks four questions:
+
+1. **Administration** — what does an operator need to *do* to a tenant, and what
+ should they be forbidden from doing?
+2. **Site reliability** — the SRE surface exists as JSON. Is the number it
+ reports true?
+3. **Support** — what does a support person need to *see* to diagnose a
+ customer's problem without violating the promise that makes xNet worth using?
+4. **Shape** — one console, or three? Server-rendered, SPA, or off-the-shelf?
+
+---
+
+## Executive Summary
+
+The headline is counter-intuitive: **the reliability code is in better shape
+than the reliability data.**
+
+`apps/cloud/src/observability/` contains a clean, well-tested, dependency-free
+implementation of the Google SRE model — and it is fed by a 2000-entry in-memory
+ring buffer that dies on every deploy. The console this exploration was asked for
+would, if built today against that substrate, render a large confident number
+that is wrong in both directions: it reads *perfectly healthy* immediately after
+a restart (the moment most likely to have broken something), and it reads
+*frozen* after two transient probe timeouts.
+
+| Layer | Status | Notes |
+| --------------------------- | -------------- | ---------------------------------------------------------------------------- |
+| SLI / SLO / error-budget math | ✅ Shipped | `observability/sli.ts`, `slo.ts` — pure, unit-tested |
+| Fleet probe loop | ✅ Shipped | leased job `fleet-probe`, 60s (0411 G2) |
+| Public status page | ✅ Shipped | `/status.json` → `site/src/pages/status.astro`, k-anon floor 5 |
+| Error-budget rollout gate | ✅ Shipped | `rollout/engine.ts` aborts on `freeze` |
+| Restore drill | ✅ Shipped | nightly, rotating sample (0418) |
+| Job staleness | ✅ Shipped | `/internal/fleet/jobs`, `stale` at 2× interval |
+| **Durable SLI window** | 🛑 **Missing** | in-memory ring; ~33h of a 30-day window; resets on deploy |
+| **Operator identity** | 🛑 **Missing** | one shared secret, no attribution, non-constant-time compare |
+| **Control-plane audit log** | 🛑 **Missing** | the *user's* hub has a signed one (`routes/audit.ts`); the control plane has none |
+| **Operator UI** | ❌ None | `/internal/*` is JSON-and-curl only |
+| Component library reuse | ✅ Available | `@xnetjs/ui`: 85 components, **zero** `@xnetjs/*` deps, React peer only |
+| **Support timeline** | ❌ None | no way to answer "what happened to this tenant last Tuesday" |
+
+> [!IMPORTANT]
+> The recommendation is **substrate before surface**. Phase 0 (durable SLIs,
+> operator identity, audit log) is roughly a third of the work and carries all of
+> the risk. Phase 1 (the console itself) is mostly HTML over data that already
+> exists. Building Phase 1 first produces a console that lies confidently, which
+> is worse than curl.
+
+---
+
+## Current State In The Repository
+
+### What exists
+
+The control plane is `apps/cloud`, a Hono app. Routes divide cleanly into four
+audiences:
+
+```text
+┌─────────────────┐ ┌──────────────────┐ ┌────────────────────┐ ┌─────────────────┐
+│ PUBLIC │ │ TENANT │ │ INTERNAL │ │ OPERATOR │
+│ /health │ │ /dashboard │ │ /internal/* │ │ │
+│ /status.json │ │ /account/* │ │ shared secret │ │ ❌ nothing │
+│ (k-anon, agg) │ │ session cookie │ │ no identity │ │ │
+└─────────────────┘ └──────────────────┘ └────────────────────┘ └─────────────────┘
+```
+
+The fourth column is the gap. The third column is what an operator uses *instead*,
+which is the security problem.
+
+| File | What it gives an operator console |
+| -------------------------------------------------------------------------- | ------------------------------------------------------------ |
+| [`observability/sli.ts`](../../apps/cloud/src/observability/sli.ts) | availability, error rate, p95, error budget, burn rate |
+| [`observability/slo.ts`](../../apps/cloud/src/observability/slo.ts) | plan → objective, budget-as-time, `ship`/`caution`/`freeze` |
+| [`observability/health.ts`](../../apps/cloud/src/observability/health.ts) | `probeFleet`, `tenantSli`, `fleetSummary`, sample store |
+| [`observability/status.ts`](../../apps/cloud/src/observability/status.ts) | the public, aggregate-only chokepoint |
+| [`registry.ts`](../../apps/cloud/src/registry.ts) | `TenantRecord` — plan, tier, region, version, dunning state |
+| [`jobs/leased.ts`](../../apps/cloud/src/jobs/leased.ts) | job health + `stale` (0411 G2) |
+| [`rollout/engine.ts`](../../apps/cloud/src/rollout/engine.ts) | canary → waves, aborts on a frozen budget |
+| [`backup/restore-drill.ts`](../../apps/cloud/src/backup/restore-drill.ts) | proof a restore works, not just that replication is on |
+| [`diagnostics.ts`](../../apps/cloud/src/diagnostics.ts) | crash/debug report quarantine (0315) |
+| [`dashboard.ts`](../../apps/cloud/src/dashboard.ts) | the rendering pattern to copy — server-rendered, no bundle |
+
+`dashboard.ts` matters as precedent. It is 972 lines of server-rendered HTML with
+inline vanilla-JS hydration and zero client dependencies, deliberately *not* a
+second React bundle. An operator console should be the same shape.
+
+### 🔴 Finding 1 — the error budget is measured over ~33 hours and labelled 30 days
+
+Three facts, each independently benign:
+
+- `slo.ts` sets `windowDays: 30` for the 99.9% tier.
+- `index.ts:413` constructs `new HealthSampleStore()` — capacity defaults to **2000**.
+- `index.ts:415` sets the probe interval to `XNET_CLOUD_PROBE_MS ?? 60_000` — **60s**.
+
+The ring therefore holds at most
+
+$$ 2000 \text{ samples} \times 60\,\text{s} = 120{,}000\,\text{s} = 33.3\,\text{hours} $$
+
+against a window that claims 720 hours. `windowed()` filters correctly to 30
+days; there is simply never more than 33 hours of data to filter. Coverage is
+**4.6%** of the stated window.
+
+The consequences are sharp in both directions, because the error budget divides
+by a very small allowance:
+
+
+Worked example — why two failed probes freeze the fleet
+
+At 99.9%, `errorBudgetRemaining(sli, 0.999)` computes
+`1 - (1 - sli) / 0.001`.
+
+With a **full** 2000-sample ring:
+
+| Failed probes | Availability | Budget remaining | Policy |
+| ------------- | ------------ | ---------------- | --------- |
+| 0 | 1.0000 | 100% | `ship` |
+| 1 | 0.9995 | **50%** | `ship` |
+| 2 | 0.9990 | **0%** | 🛑 `freeze` |
+
+With a **partially filled** ring — say 100 samples, ~100 minutes after a deploy —
+a single failed probe gives `1 - 0.01/0.001 = -9 → 0%`: instant `freeze`.
+
+And with an **empty** ring, `availability([])` returns `1` by design ("no
+evidence of failure"), so the budget reads 100% and the policy reads `ship`.
+
+
+
+> [!WARNING]
+> `rollout/engine.ts:169` aborts a fleet rollout when `budgetPolicy()` returns
+> `freeze`, and re-checks between waves at line 182. So this is not a cosmetic
+> number. **A control-plane restart hands the rollout engine a full error budget
+> regardless of what the fleet actually did**, and two probe timeouts hand it a
+> freeze. The gate cannot go red across the deploy boundary, and goes red far too
+> easily inside one — the exact failure mode `AGENTS.md` names when it requires a
+> gate to have a proof it can go red.
+
+`stores/durable.ts` documents the in-memory choice explicitly and honestly —
+"losing them on restart only costs … a rebuilt sample window, not a tenant." That
+reasoning was right when the samples fed a dashboard. It stopped being right when
+they started gating deploys.
+
+### 🔴 Finding 2 — a cold start is recorded as an outage
+
+`sli.ts` states the intent plainly in its header: cold-start waits are *valid*,
+"so scale-to-zero tenants aren't unfairly penalized." The implementation does not
+do this. `httpHealthProbe` aborts at `timeoutMs = 5000` and returns
+`{ ok: false }`. A Cloud Run cold start routinely exceeds five seconds.
+
+`probeFleet` only probes tenants whose `dataTier === 'hot'`, which limits the
+blast radius — but `dataTier` is xNet's own hot/cold demotion state (7 days
+idle), not Cloud Run's instance count. A `hot` tenant whose revision has scaled
+to zero produces `ok: false`, and by Finding 1 two of those freeze the fleet.
+
+### 🔴 Finding 3 — `/internal/*` is a flat secret, and one route behind it is an account-takeover primitive
+
+```ts
+// apps/cloud/src/server.ts:665
+const requireInternal = (c) =>
+ Boolean(deps.internalSecret) && c.req.header('x-internal-secret') === deps.internalSecret
+```
+
+Three problems, in ascending order of severity:
+
+1. **Non-constant-time comparison.** `===` on a secret is a textbook timing
+ oracle. Low practical risk over the internet, trivially fixed with
+ `timingSafeEqual`.
+2. **No attribution.** Every internal call is *the secret*, not *a person*. There
+ is no operator identity to log, rate-limit, scope, or revoke individually. In
+ an incident nobody can answer "who ran that?"
+3. **The recovery route.** `POST /internal/account/recover` calls
+ `controlPlane.recoverAccount`, which does this
+ ([`control-plane.ts:861`](../../apps/cloud/src/control-plane.ts)):
+
+ ```ts
+ const updated: TenantRecord = { ...record, did: '' }
+ await this.deps.tenants.put(updated)
+ ```
+
+ It clears the bound data identity and marks a rebind pending, so **the next
+ device to present a passkey claims that hub**.
+
+> [!CAUTION]
+> Combining (2) and (3): anyone holding `XNET_CLOUD_INTERNAL_SECRET` can, for any
+> `billingUserId`, unbind that tenant's device and bind their own — and leave no
+> attributable record of having done so. This is a fleet-wide account-takeover
+> key with no audit trail. It is not a hypothetical abuse of a support console;
+> it is the current state, and a console would merely make it convenient. This
+> is the one part of this exploration that is **not** a two-way door, and it
+> should be fixed whether or not the console is ever built.
+
+For contrast, the *user's* hub already has what the operator's control plane
+lacks: [`packages/hub/src/routes/audit.ts`](../../packages/hub/src/routes/audit.ts)
+pages an author's signed change history, gated by an `audit/read` capability that
+`capabilities.ts:50` restricts to `admin`. The substrate for "who did what" was
+built for users first. The control plane never got it.
+
+### 🟠 Finding 4 — the confidentiality copy over-claims
+
+`dashboard.ts:650` tells every tenant, in the danger zone:
+
+> …not even we can recover it (we only ever hold encrypted bytes).
+
+For a managed hub on the trusted tier this is not accurate.
+[`packages/hub/src/services/search-indexer.ts`](../../packages/hub/src/services/search-indexer.ts)
+extracts plaintext from rich text to build the FTS index, and `node-relay.ts`
+validates plaintext declarations. Exploration
+[0343](./0343_[x]_XNET_AUTH_VS_KEYHIVE_COMPARISON.md) states the position
+directly at line 271: the trusted tier provides "integrity (signatures
+re-verified) and revocation-denial, but not confidentiality between users of the
+same hub," and calls it "the single most" significant finding of that comparison.
+
+This matters here specifically because it determines what a support console is
+*allowed* to show. If an operator with hub access can read tenant content, then
+the boundary cannot be enforced by physics and must be enforced by design,
+policy, and audit — and the customer-facing sentence needs to say so.
+
+---
+
+## External Research
+
+**Break-glass and support access.** The 2026 consensus across access-control
+guidance is consistent: sensitive support actions ("log in as user", "reset MFA",
+"transfer owner") require elevated permission, **captured reason**, time-bound
+elevation with automatic expiry, and per-action logging — with a small, named set
+of break-glass principals rather than a shared credential. CyberArk's guidance
+adds that all activity under break-glass must be monitored and audited as a
+distinct class, not folded into ordinary logs.
+
+**Impersonation with consent.** Clerk, Docebo, and Higher Logic converge on a
+pattern worth copying: impersonated sessions are visually marked (a persistent
+banner), the impersonated user is notified in-platform and by email at the moment
+it happens, and the token itself carries an impersonation claim so downstream
+services can refuse actions. Several go further and require the user to *grant*
+access before a session can start.
+
+**Error budgets and windows.** Datadog's burn-rate work and the general SRE
+literature both land on rolling windows over calendar ones — precisely because a
+calendar boundary resets the budget to full at the moment a bad deploy tends to
+land. xNet already chose rolling. Finding 1 is the same bug arriving through a
+different door: not a calendar reset, a *process* reset.
+
+**Off-the-shelf.** Grafana + Prometheus remains the default answer for fleet
+dashboards, and Better Stack / Datadog for hosted. Both are assessed in Options
+below; both fail a constraint xNet has already committed to elsewhere.
+
+---
+
+## Key Findings
+
+1. **The maths is done; the measurement is not.** Every formula an operator
+ console needs already exists and is unit-tested. The data feeding them covers
+ 4.6% of its stated window and evaporates on deploy.
+2. **A gate that gates deploys must survive deploys.** The rollout engine's
+ dependence on the error budget converts a display bug into a safety bug.
+3. **The control plane has no concept of an operator.** Not a weak one — none.
+ Every privileged action is anonymous by construction.
+4. **Support and sovereignty are in genuine tension, and it is resolvable.**
+ The resolution is not "operators can see nothing" (then support is
+ impossible) nor "operators can see everything" (then the promise is a lie).
+ It is: *operators see shape, never content; content requires the user's
+ consent; both are logged where the user can read the log.*
+5. **The rendering question is nearly settled by precedent.** `dashboard.ts`
+ already demonstrates the house pattern for a server-rendered console.
+
+---
+
+## 🧭 Architecture overview
+
+```mermaid
+flowchart TB
+ subgraph Sources["Existing data sources"]
+ P["fleet-probe job
60s → HealthSample"]
+ J["JobRegistry
stale detection"]
+ R["TenantRecord
plan · tier · dunning"]
+ D["Diagnostics quarantine
(0315)"]
+ B["Restore drill
last result"]
+ end
+
+ subgraph Phase0["Phase 0 — substrate (new)"]
+ DS[("DurableSliStore
bucketed, persisted")]
+ OI["Operator identity
WorkOS → bound did:key"]
+ AL[("Audit log
tier 1 DocStore gate
tier 2 signed node")]
+ end
+
+ subgraph Phase1["Phase 1 — /ops console"]
+ F["Fleet view
budget · burn · jobs"]
+ T["Tenant view
shape only"]
+ A["Actions
reason required"]
+ end
+
+ P --> DS
+ J --> F
+ R --> T
+ D --> T
+ B --> F
+ DS --> F
+ DS --> T
+ OI --> A
+ A --> AL
+ AL -.->|"user-visible copy"| U["Tenant's own hub"]
+
+ style DS fill:#7f1d1d,color:#fff
+ style OI fill:#7f1d1d,color:#fff
+ style AL fill:#7f1d1d,color:#fff
+```
+
+The red boxes are the ones that do not exist. Everything feeding into them does.
+
+### The visibility boundary
+
+```mermaid
+flowchart LR
+ subgraph Always["Tier 1 — always visible (shape)"]
+ A1["counts · bytes · latencies"]
+ A2["plan · region · version"]
+ A3["job + probe history"]
+ A4["error class + stack (0315)"]
+ end
+ subgraph Consent["Tier 2 — user consent, time-boxed"]
+ C1["document titles"]
+ C2["support session on the app"]
+ end
+ subgraph Never["Tier 3 — never, by design"]
+ N1["document content"]
+ N2["silent access of any kind"]
+ end
+ Always -->|"logged, attributed"| L[("Audit log")]
+ Consent -->|"logged + user notified + expires"| L
+ style Never fill:#7f1d1d,color:#fff
+```
+
+> [!NOTE]
+> Tier 1 is deliberately generous. Most real support questions — "is my sync
+> broken?", "why is my hub slow?", "where did my storage go?" — are answerable
+> entirely from shape. The number of tickets genuinely requiring Tier 2 is small,
+> and making that path *expensive and visible* rather than *impossible* is what
+> keeps the boundary honest instead of routinely circumvented.
+
+### A support session, end to end
+
+```mermaid
+sequenceDiagram
+ participant U as User
+ participant S as Support operator
+ participant O as /ops console
+ participant AL as Audit log
+ participant H as User's hub
+
+ U->>S: "sync has been broken since Tuesday"
+ S->>O: look up tenant by email
+ O->>AL: read(tenant, operator, reason)
+ O-->>S: shape — SLI history, jobs, dunning, diagnostics
+ Note over S,O: Tier 1 answers most tickets here
+
+ alt shape is not enough
+ S->>O: request Tier 2 access (reason required)
+ O->>U: consent prompt (in-app + email)
+ U-->>O: grant, 60 min
+ O->>AL: grant(operator, scope, expiry)
+ O-->>S: time-boxed, banner-marked session
+ AL->>H: mirror entry to the user's own hub
+ Note over O: auto-expires; no renewal without a fresh grant
+ end
+```
+
+### Tenant lifecycle an operator has to reason about
+
+```mermaid
+stateDiagram-v2
+ [*] --> provisioning: checkout
+ provisioning --> hot: hub live
+ hot --> cold: 7d idle (demote sweep)
+ cold --> hot: app opens
+ hot --> grace: payment failed
+ grace --> read_only: grace expired
+ read_only --> suspended: still unpaid
+ suspended --> hot: re-subscribe
+ suspended --> deleted: retention expired
+ hot --> rebind_pending: recoverAccount
+ rebind_pending --> hot: new device claims
+ deleted --> [*]
+
+ note right of rebind_pending
+ Finding 3: reachable today
+ by anyone with the shared
+ secret, unattributed
+ end note
+```
+
+---
+
+## Options And Tradeoffs
+
+### A. Where the console lives
+
+Two axes get conflated here and must be separated: **where it is served** (the
+server boundary) and **what it is built with** (the visual boundary). Exploration
+[0418](./0418_[-]_XNET_CLOUD_TO_PRODUCTION_BACKUPS_BILLING_DUNNING_AND_ONE_UI.md)
+already settles the principle at line 231 — the same-origin cookie argument
+"is a good reason for the _server_ boundary. It is not a good reason for the
+_visual_ boundary."
+
+| Option | Verdict | Why |
+| --------------------------------------------------------------- | --------------- | ---------------------------------------------------------------------------------------------------------------------------------- |
+| **A1. React + Tailwind + `@xnetjs/ui`, served same-origin by `apps/cloud`** | ✅ **Recommend** | Keeps the cookie/CORS win, reuses 85 existing components and the token system, and is the stack 0418 Phase 3 moves the tenant dashboard to anyway |
+| A2. Hand-rolled server-rendered HTML (the `dashboard.ts` pattern) | ❌ Reject | Zero new deps, but a third design system in the same repo and every table, dialog, and chart rebuilt by hand |
+| A3. React SPA in `apps/web` | ❌ Reject | Cross-origin to the control plane, and it puts operator credentials in the same app as tenant data |
+| A4. Grafana + Prometheus | 🛑 Reject | Two services to run and secure; the SLI logic would be re-implemented in PromQL as a second source of truth, diverging from `sli.ts` |
+| A5. Datadog / Better Stack | 🛑 Reject | Fails the vanish test — operations knowledge leaves with the vendor — and mirrors tenant-shaped data into a third party (Charter §4) |
+
+> [!IMPORTANT]
+> **`@xnetjs/ui` has zero `@xnetjs/*` runtime dependencies.** Its `package.json`
+> lists only third-party libraries (`@base-ui/react`, `cmdk`, `lucide-react`,
+> `class-variance-authority`, `tailwind-merge`) and `react`/`react-dom` as peers.
+> The component library is already decoupled from the data layer — this is not a
+> refactor, it is an existing property. 85 components are available:
+> 28 primitives, 24 components, 33 composed.
+
+The reuse is broader than components. `@xnetjs/ui` also exports
+`./tailwind.config`, `./tokens.css`, `./motion.css`, `./accessibility.css`,
+`./responsive.css` and `./scroll-fade.css`, and the consumption pattern is
+already established — [`apps/web/tailwind.config.js`](../../apps/web/tailwind.config.js)
+is nine lines that spread the base config and add content globs. An ops console
+does exactly the same. `@xnetjs/charts` is likewise clean: `echarts` plus a React
+peer, no xNet primitives, so burn-down and latency charts come free.
+
+
+Why not keep hand-rolling HTML (A2) — the cost that is easy to miss
+
+`dashboard.ts` is a genuinely good artifact for what it is: 972 lines, no bundle,
+progressive enhancement, works with JS off. Copying it for `/ops` looks cheap
+because the first screen is cheap.
+
+The ops console's screens are not that screen. A fleet view is a sortable,
+filterable table of every tenant. A support view is a timeline with expandable
+entries. Phase 3 is modal action dialogs with required-reason forms and a consent
+flow. Those are `DataTable`, `Dialog`, `Command`, `Popover`, `Tabs`, `Toast` —
+all of which exist, tested and themed, and none of which are pleasant to
+hand-roll in template strings.
+
+The deciding argument is not effort, though. It is that A2 creates a **third**
+design system (tenant dashboard, app, ops) in a repo that already considers two a
+problem worth an exploration, and it moves *away* from where 0418 Phase 3 is
+already headed.
+
+
+
+
+Why not Grafana (A4) — the second-source-of-truth problem
+
+Grafana is the right tool when metrics live in a time-series database and
+dashboards are queries over them. xNet's situation is different in one decisive
+way: the error budget is not a display artifact, it is a **control input** to
+`rollout/engine.ts`. If the console computes it in PromQL and the engine computes
+it in `sli.ts`, they will drift, and the drift will be discovered during an
+incident.
+
+Keeping one implementation means the console must call `tenantSli()` — which
+means it lives in the control plane. Grafana could still be added later as a
+*read-only second view* over exported metrics; it cannot be the primary.
+
+There is also a self-hosting cost. `xnet hub` is meant to be one binary. An
+operations story that requires standing up Prometheus and Grafana is not one a
+self-hoster inherits, which quietly makes the managed product better than the
+self-hosted one in a way the Charter's BATNA test disfavours.
+
+
+
+### A′. Should the console run on xNet itself?
+
+The component library needs no provider — Storybook proves that, wrapping every
+story in exactly one:
+
+```tsx
+// .storybook/preview.tsx
+import { ThemeProvider, type Theme } from '../packages/ui/src/theme/ThemeProvider'
+```
+
+No `XNetProvider`, no store, no hub connection anywhere in the decorator stack,
+and the whole catalogue renders. So *whether to use xNet* is a genuine choice
+rather than a constraint — and there is a strong argument for yes.
+
+> [!IMPORTANT]
+> **Operator actions become signed xNet nodes; fleet readings stay REST.** Run
+> the console on xNet for the **record**, not for the **readings**. That split is
+> the whole design, and each half is chosen for a specific reason rather than for
+> consistency.
+
+**Why xNet for the record.** Finding 3 is that the control plane has no audit
+log, and open question #2 asked whether one should be signed. Both dissolve if
+operator actions are nodes authored by an operator DID: the change log *is* the
+audit trail, it is signed and hash-chained per author, and
+[`packages/hub/src/routes/audit.ts`](../../packages/hub/src/routes/audit.ts)
+already serves it — `GET /audit/authors/:did/changes?since=` pages an
+author's signed history, self-reads always allowed, cross-author reads gated by
+the `audit/read` capability. That is a per-operator audit console for free, and a
+verifiable one rather than a merely append-only Firestore collection that an
+operator with write access could rewrite.
+
+The login story lands the same way. The WorkOS → xNet identity binding is not
+hypothetical — it is shipped, and it is exactly how tenants connect: WorkOS
+AuthKit proves the billing identity, the app presents a signed DID challenge, and
+`bindDataIdentity` binds the two. An operator signing in with WorkOS and
+connecting their xNet identity reuses that flow verbatim.
+
+And it is dogfooding on the surface where the stakes are highest. A team that
+will not keep its own operational record on its own product has said something.
+
+**Why not xNet for the readings.** The change log is the wrong shape for metrics,
+and the repo has the scar tissue to prove it. Exploration
+[0323](./0323_[_]_ENTITY_COMPONENT_SYSTEM_AND_HIGH_FREQUENCY_STATE.md) names the
+"318k-row / multi-second cold-open stall" as a change-log problem (line 38) and
+documents a **250-change burst cliff** above which every subscribed client
+re-renders (line 197). SLI buckets are hourly writes per tenant, forever —
+precisely the high-frequency stream 0323 concludes must stay off the log.
+
+```mermaid
+flowchart TB
+ subgraph OnXnet["On xNet — the record"]
+ A1["Operator actions
signed by operator DID"]
+ A2["Incident notes · runbooks"]
+ A3["Consent grants (Tier 2)"]
+ end
+ subgraph OffXnet["Off xNet — the readings"]
+ B1["SLI buckets
hourly, per tenant"]
+ B2["TenantRecord
authoritative in Firestore"]
+ B3["Job health · rollout state"]
+ end
+ A1 --> H[("Ops hub
signed change log")]
+ A2 --> H
+ A3 --> H
+ A3 -.->|"publish copy"| T[("Affected tenant's
own hub")]
+ B1 --> F[("DocStore / Firestore")]
+ B2 --> F
+ B3 --> F
+ H --> C["/ops console"]
+ F --> C
+ style B1 fill:#1e3a5f,color:#fff
+ style B2 fill:#1e3a5f,color:#fff
+ style B3 fill:#1e3a5f,color:#fff
+```
+
+`TenantRecord` stays authoritative in Firestore for a second reason: it is read
+on the request path by billing and provisioning. Mirroring it into nodes would
+create two sources of truth for the record that decides whether someone's hub
+runs.
+
+> [!CAUTION]
+> **The circular dependency is the real risk, and it must be designed for
+> explicitly.** The control plane operates hubs. If operator tooling depends on a
+> hub, then a fleet-wide incident takes out the console you diagnose it with —
+> the classic failure of monitoring that shares fate with the monitored. Worse,
+> combined with the audit-write-before-act rule from Phase 0, an unreachable ops
+> hub would mean *no operator can act during an outage*.
+
+The resolution is **two-tier durability**, and it costs one extra write:
+
+| Tier | Where | Purpose | Available during a fleet incident |
+| -------------------------- | ------------------- | -------------------------------- | --------------------------------- |
+| 1. Gate (fail-closed) | control-plane `DocStore` | authorises the action to proceed | ✅ Yes |
+| 2. Verifiable copy | ops hub, signed node | tamper-evident record | ⚠️ Queued if unreachable |
+
+Every action writes tier 1 first — that is the gate, and it is on the same
+substrate the console already needs, so it never blocks on the hub. Publication
+to the ops hub follows asynchronously. If the hub is unreachable the action still
+proceeds, the entry queues, and **the queue depth is itself an alertable
+metric** — a gap between the two tiers is visible rather than silent, which is
+the property `AGENTS.md` asks for when it says "absent" and "unreadable" must be
+different values.
+
+Fleet health rendering reads tier 1 and the SLI buckets directly, so the console
+degrades to *"you can see everything and act, but audit history is stale"* rather
+than going dark. That is the right failure mode.
+
+**What this means for the provider.** `XNetProvider` comes back in — scoped to
+the ops workspace. The replica an operator's browser holds is *their own
+operational record*, not tenant content, so the blast-radius objection does not
+apply. Fleet readings still use plain `fetch`, both because of the shape argument
+above and because `useNode` is documented to serve stale data while revalidating
+(0353) — wrong semantics for a console whose job is answering "what is happening
+*right now*".
+
+
+Bootstrapping — the chicken-and-egg, and how it resolves
+
+The first operator needs a DID and an ops hub before there is a console to create
+either. Three ordered steps, none of which need the console:
+
+1. `xnet hub` runs the ops hub — self-hosted or a pinned managed tenant, deployed
+ independently of the fleet's provisioning path so it cannot be torn down by
+ the same failure.
+2. The `xnet` CLI mints the first operator DID and binds it via the existing
+ device-grant flow.
+3. That operator's DID seeds the allowlist; subsequent operators are added as
+ signed nodes — which means **adding an operator is itself an audited action**,
+ which is a nice property to get for free.
+
+The ops hub deliberately does *not* live behind the fleet's own provisioner. If
+it did, the circular dependency would be back in a worse form: the thing that
+records what operators did to the fleet would be provisioned by the fleet.
+
+
+
+### A″. What this costs in the container
+
+Reusing React is not free, and the cost is in packaging rather than coupling.
+[`apps/cloud/Dockerfile`](../../apps/cloud/Dockerfile) already `COPY`s
+`packages/` and installs the closure with dev dependencies, builds with `tsup`,
+then re-installs `--prod` to prune. The recipe that fits it:
+
+```text
+┌────────────────────┐ ┌────────────────────┐ ┌────────────────────┐
+│ devDependency │ ─▶ │ vite build │ ─▶ │ pnpm install --prod│
+│ @xnetjs/ui, charts │ │ → dist/ops/*.js │ │ prunes toolchain │
+└────────────────────┘ └────────────────────┘ └────────────────────┘
+ build-time only static assets runtime image
+```
+
+> [!WARNING]
+> Add `@xnetjs/ui` and `@xnetjs/charts` as **`devDependencies`** of `xnet-cloud`,
+> not `dependencies`. As runtime deps they survive the `--prod` prune and drag
+> CodeMirror, `react-markdown`, `cmdk` and `echarts` into the shipped image for
+> nothing — the console is a *build-time* artifact, and only the emitted JS/CSS
+> needs to ship. The existing Dockerfile ordering (install-with-dev → build →
+> prune) already supports this; the prod install "leaves the built dist
+> untouched," so `dist/ops/` survives.
+
+Serving is a small addition, not a new dependency: `@hono/node-server` is already
+a dependency and ships `serve-static` as a subpath. The control plane currently
+has no static-asset route at all, so this is genuinely new code — roughly one
+route plus a cache header — but it is the whole of the server-side change.
+
+
+Why not Grafana — the second-source-of-truth problem
+
+Grafana is the right tool when metrics live in a time-series database and
+dashboards are queries over them. xNet's situation is different in one decisive
+way: the error budget is not a display artifact, it is a **control input** to
+`rollout/engine.ts`. If the console computes it in PromQL and the engine computes
+it in `sli.ts`, they will drift, and the drift will be discovered during an
+incident.
+
+Keeping one implementation means the console must call `tenantSli()` — which
+means it lives in the control plane. Grafana could still be added later as a
+*read-only second view* over exported metrics; it cannot be the primary.
+
+There is also a self-hosting cost. `xnet hub` is meant to be one binary. An
+operations story that requires standing up Prometheus and Grafana is not one a
+self-hoster inherits, which quietly makes the managed product better than the
+self-hosted one in a way the Charter's BATNA test disfavours.
+
+
+
+### B. Where durable SLI samples go
+
+| Option | Verdict | Why |
+| ------------------------------------- | --------------- | ------------------------------------------------------------------------------------------------------- |
+| **B1. Bucketed rollups in `DocStore`** | ✅ **Recommend** | Reuses the existing Firestore/in-memory port; ~720 hourly buckets per tenant covers 30 days exactly |
+| B2. Raise ring capacity to 43,200 | ❌ Reject | Fixes the window, not the amnesia; ~2 MB/tenant resident and still zeroed on deploy |
+| B3. Cloud Run log-based metrics | ❌ Reject | Vendor lock-in on the number that gates deploys; unavailable to self-hosters |
+| B4. SQLite on a mounted volume | ❌ Reject | The control plane is deliberately stateless-with-Firestore; a volume adds a failover story for one table |
+
+B1 concretely: replace the raw `HealthSample[]` ring with per-tenant hourly
+buckets of `{ hourMs, ok, total, latencySumMs, latencyP95Ms }`. Availability over
+any window becomes a sum over buckets. Storage is bounded by construction:
+
+$$ 720 \text{ buckets} \times \sim\!60\,\text{B} \approx 43\,\text{KB per tenant per 30 days} $$
+
+The in-memory ring stays as a write-through cache for the current hour, so the
+hot path is unchanged and the durable write is one document per tenant per hour.
+
+> [!IMPORTANT]
+> Bucketing also fixes Finding 2 for free if the bucket separates *timeout* from
+> *error*. A cold-start timeout can then be counted as valid-but-slow, which is
+> what `sli.ts`'s header already claims happens.
+
+### C. Operator identity
+
+| Option | Verdict | Why |
+| --------------------------------------------- | --------------- | -------------------------------------------------------------------------------------------- |
+| **C1. WorkOS AuthKit → bound `did:key` (device-grant)** | ✅ **Recommend** | Reuses the shipped tenant flow verbatim, and gives a *signing* identity — actions attributable by cryptography, not convention |
+| C2. WorkOS AuthKit + an allowlist, no DID | 🟡 Fallback | Simpler, but the audit log is then only as trustworthy as the database holding it |
+| C3. Per-operator static tokens | 🟡 Interim | Attribution without SSO; rotation is manual |
+| C4. Keep the shared secret, add a header | 🛑 Reject | Self-asserted identity is not identity; it audits the honest and misses the dishonest |
+| C5. mTLS client certs | ❌ Reject | Real security, disproportionate operational cost for a team of this size |
+
+C1 is smaller than it sounds because both halves are shipped. `server.ts:259-299`
+already runs the WorkOS round trip and `session.ts` already seals a cookie; the
+DID half is the same device-grant claim tenants use, ending in
+`bindDataIdentity`. An operator session is that pair plus an allowlist check and
+a distinct cookie name.
+
+The step from C2 to C1 is what makes the audit log *verifiable* rather than
+merely *append-only*: with a signing identity an operator cannot repudiate an
+action, and nobody with database access can forge one.
+
+> [!NOTE]
+> `/internal/*` should keep its shared secret for genuine machine-to-machine
+> callers, but the **destructive** routes — `recover`, `plan`, `delete` — should
+> move behind operator identity. A secret is fine for "read fleet health"; it is
+> not fine for "unbind this person's device."
+
+### D. Revenue-lane check (Charter §6)
+
+An operator console is not a new revenue lane — but it is the machinery behind
+one xNet already names. Charter §6 says xNet charges for "improvements —
+operations, support, context, and distribution we build and run." Support *is*
+the lane; this is the thing that makes it deliverable. Running the five tests
+anyway, because the Charter asks for them at the point a lane's mechanism is
+designed:
+
+| Test | Result |
+| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **Improvement** | ✅ The margin pays for people watching error budgets and answering tickets — labour we provide, not access to something users own anyway |
+| **BATNA** | ✅ Only if the console ships **in the same binary**. A managed-only ops story degrades self-hosting by omission — this is the real argument against A3/A4 |
+| **Vanish** | ✅ Tenants keep hubs, data, and `.xnetpack` exports; the console is ours and its disappearance costs them support, not sovereignty |
+| **Sleep** | ✅ Survives a competitor open-sourcing the feature set — operating someone's hub at 99.9% is labour, and labour does not fork |
+| **Rust** | ✅ The refusal here (no silent operator access to content) is backed by the operations/support lane itself, which survives it |
+
+The BATNA row is the one with teeth, and it is the reason A1 beats A3/A4 on
+principle and not just on convenience.
+
+---
+
+## Recommendation
+
+Build it in four phases. **Phase 0 is not optional and does not depend on the
+console being built at all.**
+
+```mermaid
+flowchart LR
+ P0["Phase 0
Substrate
durable SLIs · identity · audit"] --> P1["Phase 1
Fleet console
/ops read-only"]
+ P1 --> P2["Phase 2
Support view
tenant shape + timeline"]
+ P2 --> P3["Phase 3
Actions
reason-gated, consent for Tier 2"]
+ style P0 fill:#7f1d1d,color:#fff
+```
+
+**Phase 0 — make the numbers true and the actors named.** Durable bucketed SLI
+store; separate timeout from error in the probe; constant-time secret compare;
+operator identity as WorkOS → bound `did:key`; the two-tier audit log (DocStore
+gate, signed node published to the ops hub) that every privileged action writes
+to before it acts. Move `recover` / `plan` / `delete-data` behind operator
+identity.
+
+**Phase 1 — a read-only `/ops` fleet console.** React + Tailwind on `@xnetjs/ui`,
+built by Vite and served same-origin by the control plane. Fleet budget and burn
+rate, per-tenant SLI table sorted by worst budget, job staleness, last
+restore-drill result, rollout state, dunning cohort counts — all from REST.
+Read-only means it cannot make an incident worse.
+
+**Phase 2 — the support view.** Tenant lookup by email or `billingUserId` (the
+`findWhere` index from 0423 already exists for exactly this), plus a **timeline**:
+provisioned, probes, tier flips, plan changes, billing events, diagnostics,
+operator actions. Tier 1 shape only. The timeline is the single highest-leverage
+support artifact and does not exist in any form today.
+
+**Phase 3 — actions and consent.** Reason-required buttons for the actions that
+are currently `curl`. Tier 2 consent flow with in-app notification, hard expiry,
+and a mirror of the grant into the user's own hub — so the audit log of who
+looked at their data is *theirs*, which is the only version of that log a
+local-first product can honestly offer.
+
+> [!TIP]
+> If only one thing ships from this document, make it **the durable SLI store**.
+> It is the smallest change, it removes a live deploy-safety hazard, and every
+> later phase renders numbers that are wrong without it.
+
+### Deliberately not doing
+
+- **No visual companion.** The load-bearing content here is substrate and
+ security, not layout, and Phase 1 reuses `dashboard.ts`'s existing visual
+ language. A `--visual` companion would be overhead. Revisit at Phase 2, where
+ the timeline is a genuinely new UI object.
+- **An ADR is now owed at Phase 0, not Phase 3.** Running the operational record
+ on xNet makes the ops hub a standing dependency of incident response — that is
+ a one-way door and earns an ADR in `decisions.mdx` before Phase 0 lands, with a
+ **`Tripwire:`** on the first incident where the ops hub is unreachable and the
+ audit queue backs up. Phase 3's consent model earns a second one, tripwired on
+ the first ticket that cannot be resolved within Tier 1.
+- **No paging/on-call integration.** The alerting seam
+ (`createWebhookAlerter`) exists; wiring PagerDuty before there is a rota is a
+ gate nobody reads.
+
+---
+
+## Example Code
+
+Bucketed durable SLI storage — the Phase 0 core, over the existing `DocStore` port:
+
+```ts
+/** One hour of probe results for one tenant. Content-free by construction. */
+export interface SliBucket {
+ tenantId: string
+ hourMs: number // floor(atMs / 3_600_000) * 3_600_000
+ ok: number
+ /** Hard failures — connection refused, 5xx. Burns budget. */
+ failed: number
+ /**
+ * Probes that timed out while the revision was cold-starting. Counted as
+ * valid-but-slow, NOT as unavailability — the intent `sli.ts` documents but
+ * the current probe does not implement (Finding 2).
+ */
+ coldStart: number
+ latencySumMs: number
+ maxLatencyMs: number
+}
+
+const bucketId = (tenantId: string, hourMs: number): string => `${tenantId}:${hourMs}`
+
+/**
+ * Availability over a window, from durable buckets.
+ *
+ * Returns `null` — never 1 — when the window holds no buckets at all. An empty
+ * window means "we have no evidence", and a caller that cannot distinguish that
+ * from "perfectly healthy" is the bug in Finding 1: after a restart the fleet
+ * read 100% available because nobody had measured it yet.
+ */
+export function availabilityFromBuckets(
+ buckets: SliBucket[],
+ windowMs: number,
+ nowMs: number
+): number | null {
+ const floor = nowMs - windowMs
+ const inWindow = buckets.filter((b) => b.hourMs >= floor)
+ if (inWindow.length === 0) return null
+ let ok = 0
+ let valid = 0
+ for (const b of inWindow) {
+ ok += b.ok + b.coldStart // cold starts succeed eventually
+ valid += b.ok + b.coldStart + b.failed
+ }
+ return valid === 0 ? null : ok / valid
+}
+```
+
+The corresponding change at the consumer end — the rollout gate must refuse to
+proceed on absent evidence rather than treating it as health:
+
+```ts
+/**
+ * Deploy gate. `null` availability (no measurement in the window) is treated as
+ * `freeze`, not `ship`: an unmeasured fleet is not a healthy fleet. This is the
+ * negative control `AGENTS.md` requires — the gate can go red for the specific
+ * reason it previously went silently green.
+ */
+export function gatePolicy(availability: number | null, objective: number | null): BudgetPolicy {
+ if (availability === null) return 'freeze'
+ return budgetPolicy(errorBudgetRemaining(availability, objective))
+}
+```
+
+And the audit-write-before-act shape for every privileged operator route:
+
+```ts
+/**
+ * Wrap a privileged action so the audit entry is durably written BEFORE the
+ * action runs. Ordering is the point: an action that fails must still be
+ * attributable, and an operator must not be able to act and then suppress the
+ * record by crashing the process.
+ */
+async function audited(
+ log: AuditLog,
+ entry: { operator: string; action: string; tenantId: string; reason: string },
+ run: () => Promise
+): Promise {
+ if (!entry.reason.trim()) throw new Error('audit: reason required')
+ const id = await log.append({ ...entry, atMs: Date.now(), outcome: 'started' })
+ try {
+ const result = await run()
+ await log.append({ ...entry, atMs: Date.now(), outcome: 'ok', parentId: id })
+ return result
+ } catch (err) {
+ await log.append({ ...entry, atMs: Date.now(), outcome: 'failed', parentId: id })
+ throw err
+ }
+}
+```
+
+---
+
+## Risks And Open Questions
+
+> [!WARNING]
+> **The console makes the takeover primitive convenient before Phase 0 makes it
+> safe.** Phases must not be reordered. A `/ops` console shipped over today's
+> shared secret is strictly worse than curl, because it lowers the effort of the
+> unattributed action in Finding 3 from "know the API" to "click the button."
+
+| Risk | Severity | Mitigation |
+| ------------------------------------------------------------- | -------- | ------------------------------------------------------------------------------------------------ |
+| Console ships before durable SLIs → operators trust a lie | 🔴 High | Phase ordering; Phase 1 renders `—` and "unmeasured" rather than a number when buckets are absent |
+| **Ops hub shares fate with the fleet it records** | 🔴 High | Ops hub deployed outside the fleet provisioner; two-tier audit so tier 1 never blocks on it |
+| Audit entries accumulate into a change-log scale cliff | 🟠 Med | Operator actions are low-volume by nature; keep metrics off the log entirely (0323, 250-change cliff) |
+| Firestore write amplification from bucket writes | 🟡 Med | One doc per tenant per hour; write-through cache for the current hour |
+| Tier 2 consent becomes a rubber stamp users always click | 🟡 Med | Hard expiry, no renewal without a fresh grant, mirror every grant into the user's own hub |
+| Audit log itself becomes a tenant-data side channel | 🟡 Med | Log action + tenantId + reason; never parameters that could carry content |
+| Operator allowlist drifts stale as people leave | 🟡 Med | Allowlist from WorkOS directory, not a hardcoded array; quarterly review |
+| React bundle bloats the control-plane image | 🟡 Med | devDependency + build-then-prune; assert image size in CI |
+| `/status.json` inherits wrong numbers | 🟠 Med | Same fix; public status should show "unmeasured", never a fabricated `operational` |
+
+**Open questions:**
+
+1. **Does the dashboard's "encrypted bytes" copy get corrected, or does the
+ architecture get changed to match it?** Finding 4 is a fork. Correcting the
+ copy is one commit and honest. Making it true means the hub cannot build an
+ FTS index over content — a large change that collides with search. This
+ exploration recommends correcting the copy now and treating end-to-end
+ encryption for managed hubs as a separate decision, but the call is not mine.
+2. ~~**Should the audit log be signed?**~~ **Resolved — yes, by running the record
+ on xNet.** Operator actions authored by a bound `did:key` are signed and
+ hash-chained by the existing change log, so the audit trail is verifiable
+ rather than merely append-only, and `GET /audit/authors/:did/changes` serves
+ it without new code. The remaining sub-question is retention: operator DIDs
+ rotate, and the audit history must outlive the operator who wrote it.
+3. **What is the retention on SLI buckets beyond 30 days?** Enterprise contracts
+ may want quarterly evidence. Bucket rollup-of-rollups (hour → day at 30 days)
+ is cheap if decided before the first write.
+4. **Does the support view need a read path into the hub at all**, or is the
+ control plane's own record sufficient for Tier 1? Leaning sufficient — the
+ `/dashboard/live.json` probe already returns counts, storage, and diagnostics
+ summaries without content.
+
+---
+
+## Implementation Checklist
+
+**Status:** ░░░░░░░░░░ 0/38 items
+
+### Phase 0 — substrate (blocking)
+
+- [ ] Add `SliBucket` + `DurableSliStore` over the existing `DocStore` port in `apps/cloud/src/observability/`
+- [ ] Write-through: `HealthSampleStore` keeps the current hour in memory, flushes hourly
+- [ ] Implement `availabilityFromBuckets` returning `null` for an unmeasured window
+- [ ] Separate `coldStart` from `failed` in `httpHealthProbe` (distinguish abort-on-timeout from connection failure)
+- [ ] Rewire `tenantSli` and `fleetSummary` to read buckets, keeping their public signatures
+- [ ] `gatePolicy`: treat `null` availability as `freeze` in `rollout/engine.ts`
+- [ ] `/status.json`: emit `"unmeasured"` rather than `operational` when no buckets exist
+- [ ] Replace `===` with `timingSafeEqual` in `requireInternal`
+- [ ] Add operator session: WorkOS callback + allowlist + distinct sealed cookie in `session.ts`
+- [ ] Bind the operator's WorkOS session to a `did:key` via the existing device-grant claim
+- [ ] Stand up the **ops hub** outside the fleet provisioner; seed the first operator DID via the `xnet` CLI
+- [ ] Add `AuditLog` port: tier-1 `DocStore` write (fail-closed gate) before the action runs
+- [ ] Publish each entry as a signed node authored by the operator DID to the ops hub (tier 2, async)
+- [ ] Expose publish-queue depth as an alertable metric so a tier-1/tier-2 gap is never silent
+- [ ] Move `POST /internal/account/recover` behind operator identity + required reason
+- [ ] Move `POST /internal/tenants/:id/plan` and `/account/delete-data` behind the same
+- [ ] Unit tests: bucket math, window boundaries, `null` propagation to the gate
+- [ ] Test: an unreachable ops hub still permits action, queues the entry, and surfaces the gap
+- [ ] **Negative control** — `--selftest` proving the gate goes red on an unmeasured window and on a real budget burn, with in-memory fixtures (0430)
+
+### Phase 1 — fleet console
+
+- [ ] Add `@xnetjs/ui` + `@xnetjs/charts` as **devDependencies** of `xnet-cloud` (not runtime deps)
+- [ ] `apps/cloud/tailwind.config.js` spreading `packages/ui/tailwind.config.js`, per the `apps/web` pattern
+- [ ] Vite build → `dist/ops/`; add the build step to the Dockerfile before the `--prod` prune
+- [ ] Serve `dist/ops/` via `@hono/node-server/serve-static`, operator-session-gated
+- [ ] `ThemeProvider` at the root; `XNetProvider` scoped to the ops workspace only
+- [ ] Fleet header: worst budget, burn rate, `byPolicy` counts, freeze banner
+- [ ] Per-tenant SLI table sorted by worst budget remaining; `—` when unmeasured
+- [ ] Job staleness panel from `/internal/fleet/jobs`
+- [ ] Restore-drill panel: last result, sample size, age
+- [ ] Rollout state panel from `run-record.ts`
+- [ ] Dunning cohort counts (grace / read-only / suspended) from `TenantRecord.billing`
+
+### Phase 2 — support view
+
+- [ ] `GET /ops/tenants/:id` — Tier 1 shape only
+- [ ] Tenant lookup by email / `billingUserId` via the existing `findWhere` index (0423)
+- [ ] Tenant timeline: provision, probes, tier flips, plan changes, billing events, diagnostics, operator actions
+- [ ] Link out to the tenant's diagnostics quarantine (0315)
+
+### Phase 3 — actions and consent
+
+- [ ] Reason-required action buttons wrapped in `audited()`
+- [ ] Tier 2 consent flow: in-app + email prompt, hard expiry, no silent renewal
+- [ ] Mirror every grant and access into the user's own hub
+- [ ] ADR in `decisions.mdx` for the consent model, with a tripwire
+
+---
+
+## Validation Checklist
+
+- [ ] Restart the control plane mid-window; the fleet budget reflects pre-restart history, not 100%
+- [ ] With zero buckets, `/status.json` reads `unmeasured` and the rollout gate returns `freeze`
+- [ ] Simulate two consecutive cold-start timeouts; the budget does **not** freeze (Finding 2 fixed)
+- [ ] Simulate a genuine two-hour outage; the budget burns proportionally to a 30-day window, not a 33-hour one
+- [ ] `--selftest` plants both violations and the gate flags both; it runs in CI beside the real scan
+- [ ] `POST /internal/account/recover` with only the shared secret returns 403
+- [ ] Every privileged action produces an audit entry naming a person, before the action runs
+- [ ] An action that throws still leaves a `started` audit entry
+- [ ] Each entry appears as a node signed by the operator's DID, readable via `GET /audit/authors/:did/changes`
+- [ ] Tamper check: altering a tier-1 DocStore row is detectable against the signed tier-2 copy
+- [ ] Kill the ops hub — the fleet console still renders, actions still work, the queue depth alerts
+- [ ] `docker images` shows no CodeMirror/echarts in the runtime layer; only `dist/ops/` assets ship
+- [ ] A Tier 2 grant expires without renewal and the session dies with it
+- [ ] The user can read the record of operator access on their own hub
+- [ ] `/ops` returns 403 for a valid *tenant* session (operator ≠ customer)
+- [ ] `pnpm typecheck && pnpm lint && pnpm test` green; `pnpm build` and the `check:*` guards pass
+- [ ] Bucket storage measured at ~43 KB per tenant per 30 days
+- [ ] The dashboard's "encrypted bytes" sentence is either corrected or made true
+
+---
+
+## References
+
+**In-repo**
+
+- [`apps/cloud/src/observability/`](../../apps/cloud/src/observability/) — `sli.ts`, `slo.ts`, `health.ts`, `status.ts`
+- [`apps/cloud/src/server.ts`](../../apps/cloud/src/server.ts) — routes, `requireInternal` (line 665)
+- [`apps/cloud/src/control-plane.ts`](../../apps/cloud/src/control-plane.ts) — `recoverAccount` (line 861)
+- [`apps/cloud/src/dashboard.ts`](../../apps/cloud/src/dashboard.ts) — the server-rendered pattern to copy
+- [`apps/cloud/src/rollout/engine.ts`](../../apps/cloud/src/rollout/engine.ts) — the error-budget gate
+- [`apps/cloud/src/stores/durable.ts`](../../apps/cloud/src/stores/durable.ts) — the `DocStore` port
+- [`packages/hub/src/routes/audit.ts`](../../packages/hub/src/routes/audit.ts) — the signed audit trail the control plane can reuse
+- [`packages/ui/package.json`](../../packages/ui/package.json) — zero `@xnetjs/*` deps; the reuse argument in one file
+- [`.storybook/preview.tsx`](../../.storybook/preview.tsx) — `ThemeProvider` only; proof the library needs no data primitives
+- [`apps/web/tailwind.config.js`](../../apps/web/tailwind.config.js) — the nine-line preset-sharing pattern to copy
+- [`apps/cloud/Dockerfile`](../../apps/cloud/Dockerfile) — install-with-dev → build → `--prod` prune
+- [`docs/CHARTER.md`](../CHARTER.md) — §4 Consent, §6 No ground rent (five tests)
+
+**Prior explorations**
+
+- [0193 — Cloud operations, uptime, backups and telemetry](./0193_[_]_XNET_CLOUD_OPERATIONS_UPTIME_BACKUPS_AND_TELEMETRY.md) — where the SLI/SLO model came from
+- [0201 — Cloud staging, status page and live testing](./0201_[_]_CLOUD_STAGING_STATUS_PAGE_AND_LIVE_TESTING.md) — the public status surface
+- [0315 — First-party error telemetry and debug report console](./0315_[x]_FIRST_PARTY_ERROR_TELEMETRY_AND_DEBUG_REPORT_CONSOLE.md) — diagnostics quarantine
+- [0323 — Entity component system and high-frequency state](./0323_[_]_ENTITY_COMPONENT_SYSTEM_AND_HIGH_FREQUENCY_STATE.md) — the 318k-row cold-open stall and the 250-change burst cliff; why metrics stay off the change log
+- [0343 — xNet auth vs Keyhive](./0343_[x]_XNET_AUTH_VS_KEYHIVE_COMPARISON.md) — the trusted-tier confidentiality gap
+- [0418 — Cloud to production: backups, billing, dunning and one UI](./0418_[-]_XNET_CLOUD_TO_PRODUCTION_BACKUPS_BILLING_DUNNING_AND_ONE_UI.md) — the phase this follows
+- [0430 — Risk-adjusted engineering](./0430_[-]_RISK_ADJUSTED_ENGINEERING_READING_ASTERISK_14.md) — negative controls, tripwires, gates that can fail
+
+**External**
+
+- [Break-glass access best practices — CyberArk](https://docs.cyberark.com/manage/latest/en/content/sca/dpaforcloud/breakglass.htm)
+- [Secure admin impersonation for support with consent and audits — AppMaster](https://appmaster.io/blog/secure-admin-impersonation-controls-audit-scope)
+- [Empower your support team with user impersonation — Clerk](https://clerk.com/blog/empower-support-team-user-impersonation)
+- [Designing tamper-resistant audit trails — Agnite Studio](https://agnitestudio.com/blog/designing-tamper-resistant-audit-trails-compliance-systems/)
+- [Burn rate is a better error rate — Datadog](https://www.datadoghq.com/blog/burn-rate-is-better-error-rate/)
+- [SRE error budgets and maintenance windows — Google Cloud](https://cloud.google.com/blog/products/management-tools/sre-error-budgets-and-maintenance-windows)
+- [Error budgets: a complete guide — SRE School](https://sreschool.com/blog/error-budgets-a-complete-guide/)
diff --git a/docs/explorations/0433_[-]_OPERATOR_CONSOLE_THE_DECIDED_PLAN.md b/docs/explorations/0433_[-]_OPERATOR_CONSOLE_THE_DECIDED_PLAN.md
new file mode 100644
index 000000000..045dafe5a
--- /dev/null
+++ b/docs/explorations/0433_[-]_OPERATOR_CONSOLE_THE_DECIDED_PLAN.md
@@ -0,0 +1,613 @@
+---
+title: Operator console — the decided plan
+status: draft # draft | withdrawn
+last_updated: 2026-08-01
+review: 2026-11-01 # same window as 0430/0431; re-decide once the first paying cohort exists
+decider: chris
+door: one-way # the ops hub becomes a standing dependency of incident response (ADR-31); the consent model binds what support may ever see (ADR-32)
+tags: [cloud, operations, sre, support, security, observability, decisions]
+---
+
+# Operator console — the decided plan
+
+> [!TIP]
+> **TL;DR** — This is the decision register and build plan for xNet Cloud's
+> operator console. Sixteen decisions are settled; the research behind them is
+> [exploration 0431](./0431_[_]_XNET_CLOUD_OPERATOR_CONSOLE_SRE_AND_SUPPORT.md),
+> which stays where it is. The console is **React + Tailwind on `@xnetjs/ui`**,
+> served same-origin from `apps/cloud`, running **on xNet for the record and
+> REST for the readings**. Two defects found while deciding ship **before**
+> any of it: the tiers that sell a 99.9% SLO are provisioned scale-to-zero, and
+> five user-facing surfaces claim we cannot read data the hub demonstrably
+> indexes.
+
+---
+
+## Problem Statement
+
+[Exploration 0431](./0431_[_]_XNET_CLOUD_OPERATOR_CONSOLE_SRE_AND_SUPPORT.md)
+established that xNet Cloud has no operator surface: `/internal/*` behind a flat
+shared secret, JSON and `curl`, no attribution, no audit trail, and an SLI
+substrate that reports health it has not measured. It surveyed the options but
+left sixteen decisions open.
+
+This document closes them and says what gets built, in what order. It does not
+re-argue the research — where a claim is load-bearing it cites 0431 or the code.
+
+> [!IMPORTANT]
+> **Two findings emerged during the decision process that are not in 0431 and do
+> not depend on the console being built at all.** They are live defects on
+> surfaces you are about to sell. They ship first, standalone. See
+> [Ship-first defects](#-ship-first-defects).
+
+---
+
+## Executive Summary
+
+The console is the visible part; almost none of the risk is there. The risk is in
+three substrate properties that must be true before a console is worth looking
+at: the numbers must be **measured**, the actors must be **named**, and the
+boundary between shape and content must be **enforced and logged**.
+
+| Layer | Decision | Phase |
+| ----- | -------- | ----- |
+| Warm provisioning | `minInstances` derived from the SLO, not the isolation tier | 🔴 Ship first |
+| Confidentiality copy | Correct all five claims | 🔴 Ship first |
+| SLI durability | Hourly buckets in `DocStore`; 30d raw + daily rollup to 13mo | 0 |
+| Gate semantics | Stale → freeze · young → excluded · fleet-wide zero → freeze | 0 |
+| Public status | New `unmeasured` component state | 0 |
+| Operator identity | WorkOS org role (authz) + Firestore DID binding (attribution) | 0 |
+| Audit | Tier-1 Firestore gate + tier-2 signed node on the ops hub | 0 |
+| Shared secret | Reads keep it; mutations reject it | 0 |
+| Console | React + `@xnetjs/ui`, Vite → `dist/ops`, `serve-static` | 1 |
+| Support view | Shape-only tenant view + timeline | 2 |
+| Actions & consent | Reason-gated actions; per-incident, time-boxed consent | 3 |
+
+---
+
+## 🔴 Ship-first defects
+
+Neither of these waits for the console. Both are small. Both are wrong today.
+
+### D1 — the tiers that sell an SLO scale to zero
+
+```ts
+// packages/cloud/src/provisioner/adapters/cloud-run-litestream.ts:131
+private minInstances(spec: ProvisionSpec): number {
+ // Always-warm tier keeps one instance hot; everyone else scales to zero.
+ return spec.entitlements.isolation === 'dedicated-warm' ? 1 : 0
+}
+```
+
+Cross-referenced against `PLAN_CATALOG`:
+
+| Plan | Isolation | SLA | `minInstances` | Burns budget? |
+| ---- | --------- | --- | -------------- | ------------- |
+| `demo` | `pooled` | none | 0 | ❌ objective `null` |
+| `personal` | `dedicated-sleep` | best-effort | 0 | ❌ objective `null` |
+| `family` | `dedicated-sleep` | best-effort | 0 | ❌ objective `null` |
+| `team` | `dedicated-warm` | best-effort | **1** | ❌ objective `null` |
+| `community` | `dedicated-project` | **99.9%** | **0** | ✅ **yes** |
+| `company` | `dedicated-project` | **99.9%** | **0** | ✅ **yes** |
+| `enterprise` | `region-pinned` | **custom (99.95%)** | **0** | ✅ **yes** |
+
+> [!CAUTION]
+> **The always-warm instance is given to the one tier that cannot burn an error
+> budget, and withheld from all three that can.** `region-pinned` is not handled
+> by the function at all — it falls through to `0`. A 99.9% monthly budget is
+> 43.2 minutes; a single Cloud Run cold start with a Litestream restore can spend
+> a meaningful fraction of that, and the tier is sold on the guarantee.
+
+**Fix:** warmth is a **floor built from two independent reasons**, not one rule
+replacing another. A plan stays warm if it publishes an availability objective
+**or** its isolation tier is explicitly `dedicated-warm`. Sleep tiers keep
+scaling to zero — they carry `objective: null`, so cold starts there cannot burn
+a budget and are harmless.
+
+> [!WARNING]
+> The obvious fix — "warm iff the objective is non-null" — is wrong, and
+> modelling the cost is what caught it. `team` is `dedicated-warm` but
+> `best-effort`, so an objective-only rule would have **dropped a paying tier to
+> scale-to-zero**. `PLAN_PRICING` models `team` with `warm: true`; the price
+> already covers that COGS. Saving it would have been a silent downgrade.
+
+This also corrects 0431's Finding 2, which aimed the cold-start problem at the
+wrong tenants: it is not a measurement bug on sleep tiers, it is a provisioning
+bug on SLO tiers.
+
+**Cost delta (modelled against `packages/cloud/src/cost/pricing.ts`):** none of
+consequence, because the price list already assumed the fixed behaviour.
+`UNIT_COSTS.warmComputePerMonth` is **$6/unit/month**, so `minInstances` 0 → 1
+adds ~$6/month per SLO tenant.
+
+| Plan | Modelled | Price | Warm delta | Effect on margin |
+| ---- | -------- | ----- | ---------- | ---------------- |
+| `community` | `warm: true, warmUnits: 2` | $99/mo | +$6 | None — already priced warm |
+| `enterprise` | `warm: true, warmUnits: 4` | $2000/mo | +$6 | None — already priced warm; 0.3% of revenue |
+| `company` | **no scenario in `PLAN_PRICING`** | — | +$6 | ⚠️ unmodelled — see open questions |
+| `team` | `warm: true` | $96/mo | 0 | Unchanged by the corrected rule |
+
+`floor-margin.test.ts` passes unchanged. The one gap is that **`company` has no
+`PricingScenario` at all**, so its margin floor is unasserted — that predates
+this work and is noted rather than fixed here.
+
+### D2 — five surfaces claim we cannot read data the hub indexes
+
+> [!NOTE]
+> Implementation found **five**, not four. Two were in the same `compare.ts`
+> entry as the one already flagged, and a fifth was in the Habitat comparison
+> two rows below it. The count in the decision interview was low.
+
+| Location | Claim | Verdict |
+| -------- | ----- | ------- |
+| [`dashboard.ts:650`](../../apps/cloud/src/dashboard.ts) | "we only ever hold encrypted bytes" | ✅ corrected |
+| [`site/src/pages/cloud/index.astro:20`](../../site/src/pages/cloud/index.astro) | "We hold encrypted bytes we cannot read." | ✅ corrected |
+| [`site/src/data/compare.ts:1239`](../../site/src/data/compare.ts) | "the confidential body stays on a hub that **never sees plaintext**" | ✅ corrected |
+| `compare.ts:1239` (same entry) | "xNet is **the end-to-end encrypted workspace**" | ✅ corrected — the exact claim `HonestMachine.astro` refuses |
+| [`compare.ts:1244`](../../site/src/data/compare.ts) | "xNet's hub, which **never sees plaintext**" | ✅ corrected — kept the true half (no master read credential) |
+| [`site/src/pages/privacy.astro:113`](../../site/src/pages/privacy.astro) | "we cannot read your data with it" | ✅ **no change needed** — "with it" scopes to the billing identity, and the surrounding paragraph is explicitly about the billing/data identity split |
+
+[`search-indexer.ts`](../../packages/hub/src/services/search-indexer.ts) extracts
+plaintext from rich text to build the FTS index. Exploration
+[0343](./0343_[x]_XNET_AUTH_VS_KEYHIVE_COMPARISON.md) states the position at line
+271: the trusted tier provides integrity and revocation-denial, "but not
+confidentiality between users of the same hub."
+
+> [!NOTE]
+> This is not a new standard being imposed. The repo **already holds** this
+> standard elsewhere and the cloud surface drifted from it. See
+> `site/src/components/followed/HonestyBox.astro` ("We won't say everything is
+> end-to-end encrypted"), `HonestMachine.astro` ("We won't call the whole thing
+> end-to-end encrypted, because today it isn't") and `TrustBoundary.astro`
+> ("precisely so the post never overclaims end-to-end encryption").
+
+**Fix:** say the true and stronger thing — we hold your data, we do not look
+without your consent, and here is the signed log that proves it. Correcting three
+of four would be worse than correcting none, so all of them move together.
+
+---
+
+## The decision register
+
+Sixteen decisions, each with the reasoning compressed to the line that decided it.
+
+| # | Decision | Chosen | Because |
+| - | -------- | ------ | ------- |
+| 1 | Scope | Full console, all four phases | — |
+| 2 | Data boundary | Record on xNet, readings in Firestore | 0323's 318k-row cold-open stall and 250-change burst cliff make a change log the wrong shape for a per-tenant hourly time series |
+| 3 | Ops hub | GCP managed path, own project, outside the fleet provisioner | The Railway/Docker path is **already** dogfooded by the demo hub; the managed path is dogfooded by nothing |
+| 4 | Identity | WorkOS org role (authz) + Firestore DID binding (attribution) | Roles are core AuthKit, arrive as **JWT claims** (no API call on the request path), and Directory Sync is not required |
+| 5 | Audit scope | Graduated by sensitivity | A reason prompt on every read trains operators to type "investigating", producing a log that looks rigorous and means nothing |
+| 6 | Tier 1 line | Shape only | Titles leak the thing being protected — a document called "Q3 layoffs" is the payload |
+| 7 | Copy | Correct all four | See D2 |
+| 8 | Gate semantics | Distinguish stale from young | A gate that freezes on every new signup gets switched off within a month |
+| 9 | Warm tiers | `minInstances` from the SLO | See D1 |
+| 10 | Public status | Add `unmeasured` | "Absent" and "unreadable" must be different values (`AGENTS.md`) |
+| 11 | Shared secret | Reads keep it, mutations reject it | `cloud-company-metrics.mjs` is a real consumer; the takeover route is not worth keeping compatible |
+| 12 | Build | Inside `apps/cloud`, Vite → `dist/ops` | One package, one deploy, no Dockerfile context changes |
+| 13 | Tenant dashboard | Enable the stack, don't migrate | 0418 Phase 3 does it later, with usage evidence |
+| 14 | Consent | Per-incident, time-boxed, never standing | Standing access is what erodes into routine unlogged looking |
+| 15 | Retention | Audit 12mo **surviving tenant deletion**; SLI 30d + 13mo daily | Purging on deletion creates look-then-delete, which erases its own evidence |
+| 16 | Landing | This doc is the plan; 0431 stays as research | Its findings are still the evidence base and stay citable |
+
+---
+
+## 🧭 Architecture
+
+```mermaid
+flowchart TB
+ subgraph Client["/ops console — React + @xnetjs/ui"]
+ UI["ThemeProvider (root)"]
+ XP["XNetProvider
scoped to ops workspace"]
+ F["fetch — fleet readings"]
+ end
+
+ subgraph CP["Control plane — apps/cloud"]
+ S["serve-static dist/ops"]
+ API["/ops/api/* — operator session"]
+ G["audited() — tier-1 gate"]
+ SLI[("SLI buckets
hourly")]
+ REG[("TenantRecord")]
+ BIND[("workosUser → did:key")]
+ AUD[("Audit tier 1")]
+ end
+
+ subgraph Ext["Outside the control plane"]
+ W["WorkOS
role claim in JWT"]
+ OH[("Ops hub — GCP, own project
signed change log")]
+ TH[("Tenant's own hub")]
+ end
+
+ UI --> S
+ F --> API
+ XP <-->|"sync"| OH
+ API --> G
+ G --> AUD
+ AUD -->|"publish, async"| OH
+ API --> SLI
+ API --> REG
+ W -->|"role"| API
+ BIND --> G
+ OH -.->|"consent grants mirrored"| TH
+
+ style OH fill:#1e3a5f,color:#fff
+ style AUD fill:#7f1d1d,color:#fff
+ style G fill:#7f1d1d,color:#fff
+```
+
+### The two-tier audit, and why the order matters
+
+```mermaid
+sequenceDiagram
+ participant O as Operator
+ participant API as /ops/api
+ participant FS as Firestore (tier 1)
+ participant Q as Publish queue
+ participant OH as Ops hub (tier 2)
+
+ O->>API: action + typed reason
+ API->>API: WorkOS role claim ✓ · DID binding ✓
+ API->>FS: append {operator, action, tenant, reason, started}
+ Note over FS: FAIL-CLOSED — no write, no action
+ FS-->>API: ok
+ API->>API: perform the action
+ API->>FS: append {outcome}
+ API->>Q: enqueue signed node
+ Q->>OH: publish (async, signed by operator DID)
+ alt ops hub unreachable
+ Q-->>Q: retain; queue depth becomes an alertable metric
+ Note over Q,OH: action already happened — availability preserved,
gap is VISIBLE, never silent
+ end
+```
+
+> [!IMPORTANT]
+> Tier 1 is the gate because it lives on the substrate the console already needs.
+> Tier 2 is the verifiable copy. This is what lets the ops hub be a real
+> dependency (ADR-31) without it becoming a single point of failure for incident
+> response: during a fleet incident you can still see everything and still act —
+> only audit *history* goes stale.
+
+### The visibility boundary
+
+```mermaid
+flowchart LR
+ A["Tier 1 — no consent
counts · bytes · latencies
plan · region · version
backlog · jobs · error class"] -->|"per-tenant reads
audited silently"| L[("Audit")]
+ B["Tier 2 — consent required
document content"] -->|"reason + grant
+ hard expiry"| L
+ C["Aggregate fleet views"] -->|"not audited"| N["—"]
+ L -.->|"consent grants mirrored"| T[("Tenant's own hub")]
+ style B fill:#7f1d1d,color:#fff
+```
+
+> [!WARNING]
+> This boundary is enforced by the console and the audit trail, **not by
+> cryptography**. An operator with hub database access can read content
+> regardless — that is precisely why D2's copy correction is not optional. The
+> promise we can honestly make is "we do not look without your consent, and the
+> log proves it," not "we cannot look."
+
+---
+
+## Phases
+
+```mermaid
+flowchart LR
+ D["🔴 Ship first
D1 warm tiers
D2 copy"] --> P0["Phase 0
Substrate
SLI · identity · audit"]
+ P0 --> P1["Phase 1
Fleet console
read-only"]
+ P1 --> P2["Phase 2
Support view
shape + timeline"]
+ P2 --> P3["Phase 3
Actions + consent"]
+ style D fill:#7f1d1d,color:#fff
+ style P0 fill:#7f1d1d,color:#fff
+```
+
+**Phase 0 — substrate.** Durable hourly SLI buckets; stale-vs-young gate
+semantics; `unmeasured` public status; WorkOS operator role plus the Firestore
+DID binding; the two-tier audit log; timing-safe secret compare with mutations
+moved off the shared secret.
+
+**Phase 1 — read-only fleet console.** React + `@xnetjs/ui` in
+`apps/cloud/ops/`, Vite to `dist/ops/`, served by `serve-static` behind the
+operator session. Read-only cannot make an incident worse.
+
+**Phase 2 — support view.** Tenant lookup by email or `billingUserId` (the
+`findWhere` index from 0423 already exists for this), shape-only tenant page, and
+the timeline — the single highest-leverage support artifact, which exists in no
+form today.
+
+**Phase 3 — actions and consent.** Reason-gated actions wrapped in `audited()`;
+the per-incident Tier 2 consent flow over Resend; every request, grant, denial
+and expiry mirrored to the tenant's own hub.
+
+---
+
+## Example Code
+
+The gate semantics from decision 8 — the part most likely to be got subtly wrong:
+
+```ts
+/** Why a tenant has no usable SLI window. The distinction is the whole point. */
+export type WindowState =
+ | { kind: 'measured'; availability: number }
+ /** Newest bucket older than 2× the probe interval — measurement is BROKEN. */
+ | { kind: 'stale'; newestBucketMs: number }
+ /** Too few buckets because the tenant is new — benign, not evidence of harm. */
+ | { kind: 'young'; bucketCount: number }
+
+export function windowState(
+ buckets: SliBucket[],
+ opts: { nowMs: number; windowMs: number; probeIntervalMs: number; minBuckets: number }
+): WindowState {
+ const inWindow = buckets.filter((b) => b.hourMs >= opts.nowMs - opts.windowMs)
+ if (inWindow.length === 0) return { kind: 'young', bucketCount: 0 }
+
+ const newest = Math.max(...inWindow.map((b) => b.hourMs))
+ // Reuses the jobs registry's existing definition of stale: 2× the interval.
+ if (opts.nowMs - newest > 2 * opts.probeIntervalMs) {
+ return { kind: 'stale', newestBucketMs: newest }
+ }
+ if (inWindow.length < opts.minBuckets) {
+ return { kind: 'young', bucketCount: inWindow.length }
+ }
+
+ let ok = 0
+ let valid = 0
+ for (const b of inWindow) {
+ // Cold starts count as valid-but-slow: the request eventually succeeded.
+ ok += b.ok + b.coldStart
+ valid += b.ok + b.coldStart + b.failed
+ }
+ return valid === 0
+ ? { kind: 'young', bucketCount: inWindow.length }
+ : { kind: 'measured', availability: ok / valid }
+}
+```
+
+```ts
+/**
+ * The fleet deploy gate. `stale` freezes — a fleet nobody is measuring is not a
+ * healthy fleet, and silent measurement failure is the exact hazard this whole
+ * substrate exists to remove. `young` is EXCLUDED rather than frozen, so a new
+ * signup never blocks a rollout; a gate that cries wolf gets switched off.
+ */
+export function fleetGate(states: WindowState[], objective: number | null): BudgetPolicy {
+ if (states.length === 0) return 'freeze' // probing itself has stopped
+ if (states.some((s) => s.kind === 'stale')) return 'freeze'
+
+ const measured = states.filter((s): s is Extract =>
+ s.kind === 'measured'
+ )
+ if (measured.length === 0) return 'freeze' // every tenant young AND none measured
+
+ const worst = Math.min(...measured.map((m) => errorBudgetRemaining(m.availability, objective)))
+ return budgetPolicy(worst)
+}
+```
+
+
+D1's fix — warmth derived from the SLO
+
+```ts
+/**
+ * Always-warm iff the plan sells a measurable availability objective. Deriving
+ * this from the SLO rather than the isolation tier is the fix for D1: the tier
+ * check gave the warm instance to `dedicated-warm` (best-effort, cannot burn a
+ * budget) and withheld it from `dedicated-project` and `region-pinned`, which
+ * carry 99.9% and 99.95%. You cannot serve an availability SLO from a service
+ * that scales to zero.
+ */
+private minInstances(spec: ProvisionSpec): number {
+ return sloForPlan(spec.plan).objective !== null ? 1 : 0
+}
+```
+
+Note this must not import from `apps/cloud` — `sloForPlan` reads
+`PLAN_CATALOG[plan].sla`, so the mapping belongs in `@xnetjs/entitlements`
+beside the catalogue, with `apps/cloud/src/observability/slo.ts` re-exporting it.
+
+
+
+---
+
+## ADRs
+
+Two one-way doors are opened by this plan. Both are drafted here and land in
+`site/src/content/docs/docs/architecture/decisions.mdx` as **ADR-31** and
+**ADR-32** before Phase 0 code merges.
+
+
+ADR-31 — Operational record on xNet; ops hub outside the fleet
+
+**Decision:** operator actions, incident notes and consent grants are signed xNet
+nodes on a dedicated ops hub, run through the managed GCP path in its own project
+and **never** provisioned by the fleet provisioner. Metrics and tenant state stay
+in Firestore.
+
+**Tripwire:** the ops hub's change log crosses ~100k changes, or any proposal to
+put a per-tenant time series on it — either re-opens the record/readings split.
+
+
+
+
+ADR-32 — Support sees shape; content requires per-incident consent
+
+**Decision:** operators see Tier 1 (shape) without consent. Content requires a
+typed reason, the tenant's grant, and a hard expiry. Standing consent is refused
+at every tier, including enterprise contracts.
+
+**Tripwire:** the first support ticket that cannot be resolved at Tier 1, or the
+first enterprise contract negotiation that makes standing access a condition of
+sale — either re-opens the consent model.
+
+
+
+---
+
+## Risks And Open Questions
+
+| Risk | Severity | Mitigation |
+| ---- | -------- | ---------- |
+| Console built before durable SLIs → operators trust a lie | 🔴 High | Phase ordering; Phase 1 renders "measuring"/"unmeasured", never a fabricated number |
+| Ops hub shares fate with the fleet | 🔴 High | Own GCP project, outside the provisioner (ADR-31); tier-1 gate never blocks on it; local replica serves reads |
+| D1 fix raises cost on SLO tiers | 🟠 Med | One always-on instance on your highest-priced plans; model against `packages/cloud/src/cost/pricing.ts` before enabling |
+| `@xnetjs/ui` as devDep breaks the Docker closure | 🟠 Med | `pnpm --filter xnet-cloud...` devDep resolution is unverified — prove the image builds before wiring the console |
+| Consent becomes a rubber stamp | 🟡 Med | Hard expiry, no renewal without a fresh grant, no standing grants, mirrored to the tenant's hub |
+| Audit entries leak content via parameters | 🟡 Med | Entry schema is operator DID + action + tenantId + reason + outcome. Never parameters |
+| Operator DID rotation vs 12-month retention | 🟡 Med | **Open** — history must outlive the key that signed it |
+| WorkOS RBAC pricing | 🟡 Med | **Open** — documented as core AuthKit and delivered as JWT claims, but the pricing page does not itemise RBAC. Confirm before depending on it commercially |
+
+**Open questions:**
+
+1. **What is the actual worst-case hub cold start?** No figure exists anywhere in
+ the repo. The probe timeout and the D1 cost model both depend on it. Measure a
+ Litestream restore-on-boot for a representative database before setting either.
+2. **Operator DID rotation.** A 12-month audit retention outlives any sensible key
+ rotation period. Does the ops hub keep retired DIDs resolvable, or does each
+ entry carry the key material needed to verify it standalone?
+3. **Does Tier 2 ever get built?** ADR-32's tripwire is the first ticket that
+ cannot be resolved at Tier 1. If that ticket never arrives, Phase 3's consent
+ flow is machinery nobody needed — which is a good outcome, not a failure.
+
+---
+
+## What shipped, and what did not
+
+> [!IMPORTANT]
+> **Ship-first defects and all of Phase 0 are done. Phases 1–3 (the console
+> itself) are deliberately deferred**, for two reasons that are not "we ran out
+> of time":
+>
+> 1. **The ops hub does not exist yet.** Standing it up is an `[operator]` action
+> — a GCP project, a WorkOS organisation with the `operator` role, and the
+> seed script run against real credentials. Every code path that depends on it
+> is built and unit-tested against its port, but nothing has talked to a real
+> one.
+> 2. **Docker is unavailable in this environment**, so the risk this plan
+> explicitly flagged — *"prove the Docker image still builds with a workspace
+> devDep in the closure"* — cannot be discharged. Landing the Vite/Dockerfile
+> changes without it would ship precisely the unverified risk the plan named.
+>
+> Phase 0 stands on its own: it removes a live deploy-safety hazard and closes an
+> unattributed account-takeover path, neither of which needed a console.
+
+
+
+**Status:** ░░░░░░░░░░ 0/52 items
+
+### 🔴 Ship first — independent of everything below
+
+- [x] Move the SLA→warmth mapping into `@xnetjs/entitlements` beside `PLAN_CATALOG`
+- [x] `minInstances` returns 1 when `sloForPlan(plan).objective !== null`
+- [x] Test: `community`, `company`, `enterprise` provision warm; `personal`, `family`, `demo` do not
+- [x] Test: `region-pinned` no longer falls through to 0
+- [x] Model the cost delta against `packages/cloud/src/cost/pricing.ts`
+- [x] Correct `apps/cloud/src/dashboard.ts:650`
+- [x] Correct `site/src/pages/cloud/index.astro:20`
+- [x] Correct `site/src/data/compare.ts:1239`
+- [x] Verify `site/src/pages/privacy.astro:113` reads correctly in context
+- [x] Changeset: ~~**major** for `@xnetjs/entitlements`~~ — **none required**, and the plan was wrong twice: adding `availabilityObjective`/`requiresWarmInstance` is additive (minor at most), and both `@xnetjs/entitlements` and `@xnetjs/cloud` are `private: true`, so `publishable-pathspec.mjs` excludes them entirely (`packages/AGENTS.md`)
+
+### Phase 0 — substrate
+
+- [x] `SliBucket` + `DurableSliStore` over the existing `DocStore` port
+- [x] Hourly write-through: in-memory current hour, flush on the hour
+- [x] Separate `coldStart` from `failed` in `httpHealthProbe`
+- [x] Raise the probe timeout above measured worst-case cold start (open question 1)
+- [x] `windowState()` — measured / stale / young
+- [x] `fleetGate()` — stale freezes, young excluded, empty freezes
+- [x] Daily rollup job: hourly → daily at 30 days, retained 13 months
+- [x] Add `unmeasured` to `ComponentStatus` and the status severity ordering
+- [x] Replace the hardcoded `control-plane: operational` with a measured signal
+- [x] `timingSafeEqual` in `requireInternal`
+- [x] WorkOS organisation + `operator` role; read the role claim from the JWT
+- [x] Operator session: distinct sealed cookie, separate from the tenant session
+- [x] `workosUser → did:key` binding store in Firestore, via the device-grant claim
+- [ ] Stand up the ops hub: GCP, own project, outside the fleet provisioner
+- [x] Seed the first operator DID via a `scripts/cloud-*.mjs`
+- [x] `AuditLog` port: tier-1 Firestore append, fail-closed, before the action
+- [x] Tier-2 publisher: signed node authored by the operator DID, async
+- [x] Publish-queue depth exposed as an alertable metric
+- [x] Move `POST /internal/account/recover` behind operator identity + reason
+- [x] Move `POST /internal/tenants/:id/plan` and `/account/delete-data` likewise
+- [x] Mutation routes reject `x-internal-secret` outright
+- [x] Confirm `cloud-company-metrics.mjs` still works unchanged
+- [x] Privacy policy: audit retention, what it holds, that it survives deletion
+- [x] ADR-31 and ADR-32 in `decisions.mdx`, each with its `Tripwire:`
+- [x] **Negative control** — `--selftest` planting a stale window and a real budget burn, both of which the gate MUST flag, fixtures in memory (0430)
+
+### Phase 1 — fleet console
+
+- [ ] `@xnetjs/ui` + `@xnetjs/charts` as **devDependencies** of `xnet-cloud`
+- [ ] Prove the Docker image still builds with a workspace devDep in the closure
+- [ ] `apps/cloud/tailwind.config.js` spreading `packages/ui/tailwind.config.js`
+- [ ] Vite config → `apps/cloud/dist/ops/`; Dockerfile build step before `--prod`
+- [ ] `serve-static` route for `dist/ops/`, operator-session-gated
+- [ ] `ThemeProvider` at root; `XNetProvider` scoped to the ops workspace only
+- [ ] Fleet header: worst budget, burn rate, policy counts, freeze banner
+- [ ] Per-tenant SLI table; "measuring" for young, "unmeasured" for stale
+- [ ] Job staleness, restore-drill, rollout state, dunning cohort panels
+
+### Phase 2 — support view
+
+- [ ] Tenant lookup by email / `billingUserId` via the `findWhere` index (0423)
+- [ ] Shape-only tenant page — no titles, no values
+- [ ] Tenant timeline: provision, probes, tier flips, plan changes, billing, diagnostics, operator actions
+- [ ] Per-tenant reads written to the audit log silently, no prompt
+
+### Phase 3 — actions and consent
+
+- [ ] Reason-required action buttons wrapped in `audited()`
+- [ ] Tier 2 request → Resend email → consent page on the control plane
+- [ ] Hard expiry (60 min default), no renewal without a fresh grant
+- [ ] Mirror every request, grant, denial and expiry to the tenant's own hub
+
+---
+
+## Validation Checklist
+
+- [ ] `community` / `company` / `enterprise` services show `minInstanceCount: 1` in GCP
+- [x] No user-facing surface claims we cannot read tenant data
+- [x] Restart the control plane mid-window; the budget reflects pre-restart history
+- [x] A brand-new tenant does **not** freeze the fleet
+- [x] Stopping the probe job **does** freeze the fleet within 2× the interval
+- [x] `/status.json` reads `unmeasured`, never a fabricated `operational`
+- [x] `POST /internal/account/recover` with only the shared secret returns 403
+- [x] `cloud-company-metrics.mjs` still succeeds with only the shared secret
+- [x] Every privileged action writes a tier-1 entry naming a person before it runs
+- [x] An action that throws still leaves a `started` entry
+- [ ] Each entry appears on the ops hub signed by the operator DID, via `GET /audit/authors/:did/changes`
+- [ ] Altering a tier-1 row is detectable against the signed tier-2 copy
+- [ ] Kill the ops hub: console renders, actions work, queue depth alerts — **partially verified.** `audit.test.ts` proves actions still succeed and the queue depth rises when the publisher throws; "console renders" cannot be verified until Phase 1 exists
+- [ ] A Tier 2 grant expires without renewal and the session dies with it
+- [ ] The tenant can read the record of operator access on their own hub
+- [x] Deleting a tenant leaves their audit entries intact
+- [ ] `/ops` returns 403 for a valid tenant session
+- [ ] Runtime image contains no CodeMirror or echarts; only `dist/ops/` assets
+- [x] `--selftest` runs in CI beside the real scan and both controls go red
+- [x] `pnpm typecheck && pnpm lint && pnpm test`, `pnpm build`, and the `check:*` guards
+
+---
+
+## References
+
+**Decided in**
+
+- [0431 — xNet Cloud operator console: SRE and support](./0431_[_]_XNET_CLOUD_OPERATOR_CONSOLE_SRE_AND_SUPPORT.md) — the research this plan closes
+
+**In-repo**
+
+- [`packages/cloud/src/provisioner/adapters/cloud-run-litestream.ts`](../../packages/cloud/src/provisioner/adapters/cloud-run-litestream.ts) — D1, `minInstances` at line 131
+- [`packages/entitlements/src/plans.ts`](../../packages/entitlements/src/plans.ts) — the plan → isolation → SLA catalogue
+- [`apps/cloud/src/observability/`](../../apps/cloud/src/observability/) — `sli.ts`, `slo.ts`, `health.ts`, `status.ts`
+- [`apps/cloud/src/rollout/engine.ts`](../../apps/cloud/src/rollout/engine.ts) — the gate this feeds
+- [`packages/hub/src/routes/audit.ts`](../../packages/hub/src/routes/audit.ts) — the signed audit trail reused for tier 2
+- [`packages/ui/package.json`](../../packages/ui/package.json) — zero `@xnetjs/*` deps
+- [`.storybook/preview.tsx`](../../.storybook/preview.tsx) — `ThemeProvider` only
+- [`scripts/cloud-company-metrics.mjs`](../../scripts/cloud-company-metrics.mjs) — the shared secret's real consumer
+- [`packages/hub/src/services/search-indexer.ts`](../../packages/hub/src/services/search-indexer.ts) — D2's contradiction
+
+**Prior explorations**
+
+- [0323 — Entity component system and high-frequency state](./0323_[_]_ENTITY_COMPONENT_SYSTEM_AND_HIGH_FREQUENCY_STATE.md) — the 318k-row stall; why readings stay off the change log
+- [0343 — xNet auth vs Keyhive](./0343_[x]_XNET_AUTH_VS_KEYHIVE_COMPARISON.md) — the trusted-tier confidentiality gap
+- [0418 — Cloud to production](./0418_[-]_XNET_CLOUD_TO_PRODUCTION_BACKUPS_BILLING_DUNNING_AND_ONE_UI.md) — Phase 3 inherits the console's stack
+- [0423 — Making 768 hubs look like one](./0423_[x]_MAKING_768_HUBS_LOOK_LIKE_ONE_THE_SHARD_KEY_IS_THE_PERSON.md) — the `findWhere` index for tenant lookup by billing key
+- [0430 — Risk-adjusted engineering](./0430_[-]_RISK_ADJUSTED_ENGINEERING_READING_ASTERISK_14.md) — negative controls and tripwires
diff --git a/docs/explorations/STALE.md b/docs/explorations/STALE.md
index f3c756a83..131cfbada 100644
--- a/docs/explorations/STALE.md
+++ b/docs/explorations/STALE.md
@@ -14,7 +14,7 @@ review: 2027-02-01 # renew the claim
status: withdrawn # release it; the document stays exactly where it is
```
-**41** stale of 283 undecided.
+**41** stale of 285 undecided.
## How this backlog retires
@@ -23,14 +23,14 @@ only documents old enough to have had that many days, so a recent bulge
cannot drag the curve down.
| Days since written | Cohort | Still unshipped |
-| ------------------ | ------ | --------------- |
-| 1 | 404 | 54% |
-| 7 | 389 | 52% |
-| 14 | 353 | 50% |
-| 30 | 254 | 52% |
-| 60 | 140 | 59% |
-| 90 | 120 | 59% |
-| 120 | 109 | 55% |
+| --- | --- | --- |
+| 1 | 404 | 54% |
+| 7 | 389 | 52% |
+| 14 | 359 | 51% |
+| 30 | 254 | 52% |
+| 60 | 140 | 59% |
+| 90 | 120 | 59% |
+| 120 | 109 | 55% |
The curve does not fall: 54% of documents at least a day old are
unshipped, and 55% at 120 days. An exploration is checked off
@@ -40,49 +40,49 @@ or withdraw it; both are one line and neither renames the file.
## Past review date
-| Exploration | Due | Overdue | Decider |
-| ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------- | ------- | ------- |
-| [0079*[*]\_AUTH_SCHEMA_DSL_VARIATIONS.md](0079_%5B_%5D_AUTH_SCHEMA_DSL_VARIATIONS.md) | 2026-05-09 _(default)_ | 84d | — |
-| [0080*[*]\_UCAN_HYBRID_AUTHORIZATION_INTEGRATION.md](0080_%5B_%5D_UCAN_HYBRID_AUTHORIZATION_INTEGRATION.md) | 2026-05-10 _(default)_ | 83d | — |
-| [0081*[*]\_NODE_PERMISSIONS_UCAN_EVALUATION.md](0081_%5B_%5D_NODE_PERMISSIONS_UCAN_EVALUATION.md) | 2026-05-10 _(default)_ | 83d | — |
-| [0082*[*]\_GLOBAL_NAMESPACE_AUTHORIZATION.md](0082_%5B_%5D_GLOBAL_NAMESPACE_AUTHORIZATION.md) | 2026-05-10 _(default)_ | 83d | — |
-| [0083*[*]\_UNIFIED_AUTHORIZATION_ARCHITECTURE.md](0083_%5B_%5D_UNIFIED_AUTHORIZATION_ARCHITECTURE.md) | 2026-05-10 _(default)_ | 83d | — |
-| [0084*[*]\_GROUPS_AS_RELATIONS.md](0084_%5B_%5D_GROUPS_AS_RELATIONS.md) | 2026-05-10 _(default)_ | 83d | — |
-| [0086*[*]\_NATIVE_REWRITE_ZIG_RUST.md](0086_%5B_%5D_NATIVE_REWRITE_ZIG_RUST.md) | 2026-05-12 _(default)_ | 81d | — |
-| [0088*[*]\_DATABASE_UI_COMPETITIVE_ARCHITECTURE.md](0088_%5B_%5D_DATABASE_UI_COMPETITIVE_ARCHITECTURE.md) | 2026-05-13 _(default)_ | 80d | — |
-| [0089*[*]\_REST_GRAPHQL_INTEROPERABILITY_BOUNDARY.md](0089_%5B_%5D_REST_GRAPHQL_INTEROPERABILITY_BOUNDARY.md) | 2026-05-18 _(default)_ | 75d | — |
-| [0090*[*]\_ELECTRON_P2P_REMOTE_SHARE_OPTIONS.md](0090_%5B_%5D_ELECTRON_P2P_REMOTE_SHARE_OPTIONS.md) | 2026-05-21 _(default)_ | 72d | — |
-| [0091*[*]\_GLOBAL_SCHEMA_FEDERATION_MODEL.md](0091_%5B_%5D_GLOBAL_SCHEMA_FEDERATION_MODEL.md) | 2026-05-21 _(default)_ | 72d | — |
-| [0093*[*]\_NODE_NATIVE_GLOBAL_SCHEMA_FEDERATION_MODEL.md](0093_%5B_%5D_NODE_NATIVE_GLOBAL_SCHEMA_FEDERATION_MODEL.md) | 2026-05-21 _(default)_ | 72d | — |
-| [0095*[*]\_PACKAGE_PORTFOLIO_CLEANUP_AND_API_SIMPLIFICATION.md](0095_%5B_%5D_PACKAGE_PORTFOLIO_CLEANUP_AND_API_SIMPLIFICATION.md) | 2026-05-30 _(default)_ | 63d | — |
-| [0096*[*]\_PLAN03_ERP_REALITY_CHECK_AND_EXECUTION_RESET.md](0096_%5B_%5D_PLAN03_ERP_REALITY_CHECK_AND_EXECUTION_RESET.md) | 2026-05-30 _(default)_ | 63d | — |
-| [0098*[*]\_OPENCLAW_INTEGRATION.md](0098_%5B_%5D_OPENCLAW_INTEGRATION.md) | 2026-06-01 _(default)_ | 62d | — |
-| [0099*[*]\_DATABASE_EDITING_UX_AND_UNDO_REDO_REMEDIATION_PLAN.md](0099_%5B_%5D_DATABASE_EDITING_UX_AND_UNDO_REDO_REMEDIATION_PLAN.md) | 2026-06-01 _(default)_ | 61d | — |
-| [0100*[*]\_NPM_PUBLISH_WORKFLOW_FOR_XNETJS.md](0100_%5B_%5D_NPM_PUBLISH_WORKFLOW_FOR_XNETJS.md) | 2026-06-02 _(default)_ | 60d | — |
-| [0101*[*]\_END_TO_END_NPM_TRUSTED_PUBLISHING_PLAYBOOK.md](0101_%5B_%5D_END_TO_END_NPM_TRUSTED_PUBLISHING_PLAYBOOK.md) | 2026-06-03 _(default)_ | 59d | — |
-| [0102*[*]\_AFFINE_BLOCKSUITE_INTEGRATION_FEASIBILITY.md](0102_%5B_%5D_AFFINE_BLOCKSUITE_INTEGRATION_FEASIBILITY.md) | 2026-06-03 _(default)_ | 59d | — |
-| [0103\_[-]\_TASKS_EMBEDDED_IN_PAGES_BACKED_BY_NODES_MENTIONS_DUE_DATES_NESTED_SUBTASKS_DATABASES_CANVASES_AND_CROSS_SURFACE_TASK_MODEL.md](0103_%5B-%5D_TASKS_EMBEDDED_IN_PAGES_BACKED_BY_NODES_MENTIONS_DUE_DATES_NESTED_SUBTASKS_DATABASES_CANVASES_AND_CROSS_SURFACE_TASK_MODEL.md) | 2026-06-04 _(default)_ | 59d | — |
-| [0104\_[-]\_EXPLORE_DRAMATICALLY_SIMPLIFYING_THE_UX_AROUND_A_CANVAS_FIRST_PRIMARY_APP_INSPIRED_BY_AFFINE_MINIMIZING_BUTTONS_AND_CHROME_WITH_ZOOM_IN_DOCUMENTS_AND_DATABASES.md](0104_%5B-%5D_EXPLORE_DRAMATICALLY_SIMPLIFYING_THE_UX_AROUND_A_CANVAS_FIRST_PRIMARY_APP_INSPIRED_BY_AFFINE_MINIMIZING_BUTTONS_AND_CHROME_WITH_ZOOM_IN_DOCUMENTS_AND_DATABASES.md) | 2026-06-04 _(default)_ | 58d | — |
-| [0105*[*]\_WHAT_TO_WORK_ON_NEXT_AFTER_OPEN_SOURCE_LAUNCH.md](0105_%5B_%5D_WHAT_TO_WORK_ON_NEXT_AFTER_OPEN_SOURCE_LAUNCH.md) | 2026-06-05 _(default)_ | 57d | — |
-| [0106*[*]\_CI_PERF_TESTING_OPTIONS.md](0106_%5B_%5D_CI_PERF_TESTING_OPTIONS.md) | 2026-06-05 _(default)_ | 57d | — |
-| [0106*[*]\_JOIN_QUERIES_MULTI_TYPE_AGGREGATES_QUERY_PLANNING_API.md](0106_%5B_%5D_JOIN_QUERIES_MULTI_TYPE_AGGREGATES_QUERY_PLANNING_API.md) | 2026-06-05 _(default)_ | 57d | — |
-| [0107*[*]\_STORYBOOK_PERFORMANCE_PANEL_AND_ELECTRON_IDE_WORKSHOP.md](0107_%5B_%5D_STORYBOOK_PERFORMANCE_PANEL_AND_ELECTRON_IDE_WORKSHOP.md) | 2026-06-06 _(default)_ | 56d | — |
-| [0108*[*]\_CANVAS_V1_PAGES_DATABASES_AND_INFINITE_CANVAS_DEEP_DIVE.md](0108_%5B_%5D_CANVAS_V1_PAGES_DATABASES_AND_INFINITE_CANVAS_DEEP_DIVE.md) | 2026-06-07 _(default)_ | 55d | — |
-| [0108*[*]\_EXPO_APP_PARITY_WITH_ELECTRON_AND_WEB.md](0108_%5B_%5D_EXPO_APP_PARITY_WITH_ELECTRON_AND_WEB.md) | 2026-06-07 _(default)_ | 55d | — |
-| [0108*[*]\_TIMING_FOR_INTEGRATING_CHAT_AND_VIDEO_INTO_XNET_NOW_VS_LATER.md](0108_%5B_%5D_TIMING_FOR_INTEGRATING_CHAT_AND_VIDEO_INTO_XNET_NOW_VS_LATER.md) | 2026-06-07 _(default)_ | 55d | — |
-| [0108*[*]\_USEQUERY_UPGRADE_TIMING_AND_INTEGRATION_SEQUENCING.md](0108_%5B_%5D_USEQUERY_UPGRADE_TIMING_AND_INTEGRATION_SEQUENCING.md) | 2026-06-07 _(default)_ | 55d | — |
-| [0109*[*]\_REPOSITORY_AND_PROJECT_SUMMARY_FOR_NON_TECHNICAL_USERS.md](0109_%5B_%5D_REPOSITORY_AND_PROJECT_SUMMARY_FOR_NON_TECHNICAL_USERS.md) | 2026-06-08 _(default)_ | 54d | — |
-| [0110*[*]\_XNET_AS_A_VIABLE_WIKIPEDIA_ALTERNATIVE.md](0110_%5B_%5D_XNET_AS_A_VIABLE_WIKIPEDIA_ALTERNATIVE.md) | 2026-07-04 _(default)_ | 28d | — |
-| [0111*[*]\_UNIFIED_WORKBENCH_ARCHITECTURE_FOR_XNET.md](0111_%5B_%5D_UNIFIED_WORKBENCH_ARCHITECTURE_FOR_XNET.md) | 2026-07-04 _(default)_ | 28d | — |
-| [0112*[*]\_UNIVERSAL_CLIPPER_AND_AI_KNOWLEDGE_GRAPH_INGESTION.md](0112_%5B_%5D_UNIVERSAL_CLIPPER_AND_AI_KNOWLEDGE_GRAPH_INGESTION.md) | 2026-07-04 _(default)_ | 28d | — |
-| [0113*[*]\_OTHER_INTERNET_INFRASTRUCTURE_ROLES_FOR_XNET.md](0113_%5B_%5D_OTHER_INTERNET_INFRASTRUCTURE_ROLES_FOR_XNET.md) | 2026-07-04 _(default)_ | 28d | — |
-| [0114*[*]\_DECENTRALIZED_ALTERNATIVES_FOR_NON_XNET_INTERNET_LAYERS.md](0114_%5B_%5D_DECENTRALIZED_ALTERNATIVES_FOR_NON_XNET_INTERNET_LAYERS.md) | 2026-07-04 _(default)_ | 28d | — |
-| [0115*[*]\_ARCHITECTING_FULLY_DECENTRALIZED_GLOBAL_WEB_SEARCH.md](0115_%5B_%5D_ARCHITECTING_FULLY_DECENTRALIZED_GLOBAL_WEB_SEARCH.md) | 2026-07-06 _(default)_ | 26d | — |
-| [0116*[*]\_ARCHITECTING_DECENTRALIZED_TWITTER_X_ON_XNET.md](0116_%5B_%5D_ARCHITECTING_DECENTRALIZED_TWITTER_X_ON_XNET.md) | 2026-07-06 _(default)_ | 26d | — |
-| [0117*[*]\_ARCHITECTING_DECENTRALIZED_AI_ON_XNET.md](0117_%5B_%5D_ARCHITECTING_DECENTRALIZED_AI_ON_XNET.md) | 2026-07-06 _(default)_ | 26d | — |
-| [0118*[*]\_ARCHITECTING_A_DECENTRALIZED_OSS_FORGE_ON_XNET.md](0118_%5B_%5D_ARCHITECTING_A_DECENTRALIZED_OSS_FORGE_ON_XNET.md) | 2026-07-06 _(default)_ | 26d | — |
-| [0119*[*]\_XNET_AS_A_COMPELLING_WEB_AND_MOBILE_DEVELOPER_TOOL.md](0119_%5B_%5D_XNET_AS_A_COMPELLING_WEB_AND_MOBILE_DEVELOPER_TOOL.md) | 2026-07-06 _(default)_ | 26d | — |
-| [0120*[*]\_XNET_PACKAGE_SECURITY_AND_RELIABILITY_EXPLORATION.md](0120_%5B_%5D_XNET_PACKAGE_SECURITY_AND_RELIABILITY_EXPLORATION.md) | 2026-07-06 _(default)_ | 26d | — |
+| Exploration | Due | Overdue | Decider |
+| --- | --- | --- | --- |
+| [0079_[_]_AUTH_SCHEMA_DSL_VARIATIONS.md](0079_%5B_%5D_AUTH_SCHEMA_DSL_VARIATIONS.md) | 2026-05-09 *(default)* | 84d | — |
+| [0080_[_]_UCAN_HYBRID_AUTHORIZATION_INTEGRATION.md](0080_%5B_%5D_UCAN_HYBRID_AUTHORIZATION_INTEGRATION.md) | 2026-05-10 *(default)* | 84d | — |
+| [0081_[_]_NODE_PERMISSIONS_UCAN_EVALUATION.md](0081_%5B_%5D_NODE_PERMISSIONS_UCAN_EVALUATION.md) | 2026-05-10 *(default)* | 84d | — |
+| [0082_[_]_GLOBAL_NAMESPACE_AUTHORIZATION.md](0082_%5B_%5D_GLOBAL_NAMESPACE_AUTHORIZATION.md) | 2026-05-10 *(default)* | 84d | — |
+| [0083_[_]_UNIFIED_AUTHORIZATION_ARCHITECTURE.md](0083_%5B_%5D_UNIFIED_AUTHORIZATION_ARCHITECTURE.md) | 2026-05-10 *(default)* | 84d | — |
+| [0084_[_]_GROUPS_AS_RELATIONS.md](0084_%5B_%5D_GROUPS_AS_RELATIONS.md) | 2026-05-10 *(default)* | 84d | — |
+| [0086_[_]_NATIVE_REWRITE_ZIG_RUST.md](0086_%5B_%5D_NATIVE_REWRITE_ZIG_RUST.md) | 2026-05-12 *(default)* | 82d | — |
+| [0088_[_]_DATABASE_UI_COMPETITIVE_ARCHITECTURE.md](0088_%5B_%5D_DATABASE_UI_COMPETITIVE_ARCHITECTURE.md) | 2026-05-13 *(default)* | 80d | — |
+| [0089_[_]_REST_GRAPHQL_INTEROPERABILITY_BOUNDARY.md](0089_%5B_%5D_REST_GRAPHQL_INTEROPERABILITY_BOUNDARY.md) | 2026-05-18 *(default)* | 75d | — |
+| [0090_[_]_ELECTRON_P2P_REMOTE_SHARE_OPTIONS.md](0090_%5B_%5D_ELECTRON_P2P_REMOTE_SHARE_OPTIONS.md) | 2026-05-21 *(default)* | 73d | — |
+| [0091_[_]_GLOBAL_SCHEMA_FEDERATION_MODEL.md](0091_%5B_%5D_GLOBAL_SCHEMA_FEDERATION_MODEL.md) | 2026-05-21 *(default)* | 73d | — |
+| [0093_[_]_NODE_NATIVE_GLOBAL_SCHEMA_FEDERATION_MODEL.md](0093_%5B_%5D_NODE_NATIVE_GLOBAL_SCHEMA_FEDERATION_MODEL.md) | 2026-05-21 *(default)* | 72d | — |
+| [0095_[_]_PACKAGE_PORTFOLIO_CLEANUP_AND_API_SIMPLIFICATION.md](0095_%5B_%5D_PACKAGE_PORTFOLIO_CLEANUP_AND_API_SIMPLIFICATION.md) | 2026-05-30 *(default)* | 63d | — |
+| [0096_[_]_PLAN03_ERP_REALITY_CHECK_AND_EXECUTION_RESET.md](0096_%5B_%5D_PLAN03_ERP_REALITY_CHECK_AND_EXECUTION_RESET.md) | 2026-05-30 *(default)* | 63d | — |
+| [0098_[_]_OPENCLAW_INTEGRATION.md](0098_%5B_%5D_OPENCLAW_INTEGRATION.md) | 2026-06-01 *(default)* | 62d | — |
+| [0099_[_]_DATABASE_EDITING_UX_AND_UNDO_REDO_REMEDIATION_PLAN.md](0099_%5B_%5D_DATABASE_EDITING_UX_AND_UNDO_REDO_REMEDIATION_PLAN.md) | 2026-06-01 *(default)* | 62d | — |
+| [0100_[_]_NPM_PUBLISH_WORKFLOW_FOR_XNETJS.md](0100_%5B_%5D_NPM_PUBLISH_WORKFLOW_FOR_XNETJS.md) | 2026-06-02 *(default)* | 60d | — |
+| [0101_[_]_END_TO_END_NPM_TRUSTED_PUBLISHING_PLAYBOOK.md](0101_%5B_%5D_END_TO_END_NPM_TRUSTED_PUBLISHING_PLAYBOOK.md) | 2026-06-03 *(default)* | 59d | — |
+| [0102_[_]_AFFINE_BLOCKSUITE_INTEGRATION_FEASIBILITY.md](0102_%5B_%5D_AFFINE_BLOCKSUITE_INTEGRATION_FEASIBILITY.md) | 2026-06-03 *(default)* | 59d | — |
+| [0103_[-]_TASKS_EMBEDDED_IN_PAGES_BACKED_BY_NODES_MENTIONS_DUE_DATES_NESTED_SUBTASKS_DATABASES_CANVASES_AND_CROSS_SURFACE_TASK_MODEL.md](0103_%5B-%5D_TASKS_EMBEDDED_IN_PAGES_BACKED_BY_NODES_MENTIONS_DUE_DATES_NESTED_SUBTASKS_DATABASES_CANVASES_AND_CROSS_SURFACE_TASK_MODEL.md) | 2026-06-04 *(default)* | 59d | — |
+| [0104_[-]_EXPLORE_DRAMATICALLY_SIMPLIFYING_THE_UX_AROUND_A_CANVAS_FIRST_PRIMARY_APP_INSPIRED_BY_AFFINE_MINIMIZING_BUTTONS_AND_CHROME_WITH_ZOOM_IN_DOCUMENTS_AND_DATABASES.md](0104_%5B-%5D_EXPLORE_DRAMATICALLY_SIMPLIFYING_THE_UX_AROUND_A_CANVAS_FIRST_PRIMARY_APP_INSPIRED_BY_AFFINE_MINIMIZING_BUTTONS_AND_CHROME_WITH_ZOOM_IN_DOCUMENTS_AND_DATABASES.md) | 2026-06-04 *(default)* | 58d | — |
+| [0105_[_]_WHAT_TO_WORK_ON_NEXT_AFTER_OPEN_SOURCE_LAUNCH.md](0105_%5B_%5D_WHAT_TO_WORK_ON_NEXT_AFTER_OPEN_SOURCE_LAUNCH.md) | 2026-06-05 *(default)* | 57d | — |
+| [0106_[_]_CI_PERF_TESTING_OPTIONS.md](0106_%5B_%5D_CI_PERF_TESTING_OPTIONS.md) | 2026-06-05 *(default)* | 57d | — |
+| [0106_[_]_JOIN_QUERIES_MULTI_TYPE_AGGREGATES_QUERY_PLANNING_API.md](0106_%5B_%5D_JOIN_QUERIES_MULTI_TYPE_AGGREGATES_QUERY_PLANNING_API.md) | 2026-06-05 *(default)* | 57d | — |
+| [0107_[_]_STORYBOOK_PERFORMANCE_PANEL_AND_ELECTRON_IDE_WORKSHOP.md](0107_%5B_%5D_STORYBOOK_PERFORMANCE_PANEL_AND_ELECTRON_IDE_WORKSHOP.md) | 2026-06-06 *(default)* | 56d | — |
+| [0108_[_]_CANVAS_V1_PAGES_DATABASES_AND_INFINITE_CANVAS_DEEP_DIVE.md](0108_%5B_%5D_CANVAS_V1_PAGES_DATABASES_AND_INFINITE_CANVAS_DEEP_DIVE.md) | 2026-06-07 *(default)* | 55d | — |
+| [0108_[_]_EXPO_APP_PARITY_WITH_ELECTRON_AND_WEB.md](0108_%5B_%5D_EXPO_APP_PARITY_WITH_ELECTRON_AND_WEB.md) | 2026-06-07 *(default)* | 55d | — |
+| [0108_[_]_TIMING_FOR_INTEGRATING_CHAT_AND_VIDEO_INTO_XNET_NOW_VS_LATER.md](0108_%5B_%5D_TIMING_FOR_INTEGRATING_CHAT_AND_VIDEO_INTO_XNET_NOW_VS_LATER.md) | 2026-06-07 *(default)* | 55d | — |
+| [0108_[_]_USEQUERY_UPGRADE_TIMING_AND_INTEGRATION_SEQUENCING.md](0108_%5B_%5D_USEQUERY_UPGRADE_TIMING_AND_INTEGRATION_SEQUENCING.md) | 2026-06-07 *(default)* | 55d | — |
+| [0109_[_]_REPOSITORY_AND_PROJECT_SUMMARY_FOR_NON_TECHNICAL_USERS.md](0109_%5B_%5D_REPOSITORY_AND_PROJECT_SUMMARY_FOR_NON_TECHNICAL_USERS.md) | 2026-06-08 *(default)* | 54d | — |
+| [0110_[_]_XNET_AS_A_VIABLE_WIKIPEDIA_ALTERNATIVE.md](0110_%5B_%5D_XNET_AS_A_VIABLE_WIKIPEDIA_ALTERNATIVE.md) | 2026-07-04 *(default)* | 28d | — |
+| [0111_[_]_UNIFIED_WORKBENCH_ARCHITECTURE_FOR_XNET.md](0111_%5B_%5D_UNIFIED_WORKBENCH_ARCHITECTURE_FOR_XNET.md) | 2026-07-04 *(default)* | 28d | — |
+| [0112_[_]_UNIVERSAL_CLIPPER_AND_AI_KNOWLEDGE_GRAPH_INGESTION.md](0112_%5B_%5D_UNIVERSAL_CLIPPER_AND_AI_KNOWLEDGE_GRAPH_INGESTION.md) | 2026-07-04 *(default)* | 28d | — |
+| [0113_[_]_OTHER_INTERNET_INFRASTRUCTURE_ROLES_FOR_XNET.md](0113_%5B_%5D_OTHER_INTERNET_INFRASTRUCTURE_ROLES_FOR_XNET.md) | 2026-07-04 *(default)* | 28d | — |
+| [0114_[_]_DECENTRALIZED_ALTERNATIVES_FOR_NON_XNET_INTERNET_LAYERS.md](0114_%5B_%5D_DECENTRALIZED_ALTERNATIVES_FOR_NON_XNET_INTERNET_LAYERS.md) | 2026-07-04 *(default)* | 28d | — |
+| [0115_[_]_ARCHITECTING_FULLY_DECENTRALIZED_GLOBAL_WEB_SEARCH.md](0115_%5B_%5D_ARCHITECTING_FULLY_DECENTRALIZED_GLOBAL_WEB_SEARCH.md) | 2026-07-06 *(default)* | 26d | — |
+| [0116_[_]_ARCHITECTING_DECENTRALIZED_TWITTER_X_ON_XNET.md](0116_%5B_%5D_ARCHITECTING_DECENTRALIZED_TWITTER_X_ON_XNET.md) | 2026-07-06 *(default)* | 26d | — |
+| [0117_[_]_ARCHITECTING_DECENTRALIZED_AI_ON_XNET.md](0117_%5B_%5D_ARCHITECTING_DECENTRALIZED_AI_ON_XNET.md) | 2026-07-06 *(default)* | 26d | — |
+| [0118_[_]_ARCHITECTING_A_DECENTRALIZED_OSS_FORGE_ON_XNET.md](0118_%5B_%5D_ARCHITECTING_A_DECENTRALIZED_OSS_FORGE_ON_XNET.md) | 2026-07-06 *(default)* | 26d | — |
+| [0119_[_]_XNET_AS_A_COMPELLING_WEB_AND_MOBILE_DEVELOPER_TOOL.md](0119_%5B_%5D_XNET_AS_A_COMPELLING_WEB_AND_MOBILE_DEVELOPER_TOOL.md) | 2026-07-06 *(default)* | 26d | — |
+| [0120_[_]_XNET_PACKAGE_SECURITY_AND_RELIABILITY_EXPLORATION.md](0120_%5B_%5D_XNET_PACKAGE_SECURITY_AND_RELIABILITY_EXPLORATION.md) | 2026-07-06 *(default)* | 26d | — |
## Undated
@@ -91,48 +91,48 @@ or withdraw it; both are one line and neither renames the file.
unknown age and not-yet-due are different facts. Give one a `review:` date to
move it out of this list.
-- [0001*[*]\_graph-performance-research.md](0001_%5B_%5D_graph-performance-research.md)
-- [0010*[*]\_I18N_ARCHITECTURE.md](0010_%5B_%5D_I18N_ARCHITECTURE.md)
-- [0012*[*]\_PNPM_TO_BUN_MIGRATION.md](0012_%5B_%5D_PNPM_TO_BUN_MIGRATION.md)
-- [0017*[*]\_IDENTITY_MIGRATION_PLAN.md](0017_%5B_%5D_IDENTITY_MIGRATION_PLAN.md)
-- [0020*[*]\_REGENERATIVE_FARMING_ERP.md](0020_%5B_%5D_REGENERATIVE_FARMING_ERP.md)
-- [0021*[*]\_CLOJURE_PORT.md](0021_%5B_%5D_CLOJURE_PORT.md)
-- [0022*[*]\_RAMA_HUB_AT_SCALE.md](0022_%5B_%5D_RAMA_HUB_AT_SCALE.md)
-- [0023*[*]\_DECENTRALIZED_SEARCH.md](0023_%5B_%5D_DECENTRALIZED_SEARCH.md)
-- [0028*[*]\_CHAT_AND_VIDEO.md](0028_%5B_%5D_CHAT_AND_VIDEO.md)
-- [0029*[*]\_MASTODON_SOCIAL_NETWORKING.md](0029_%5B_%5D_MASTODON_SOCIAL_NETWORKING.md)
-- [0030*[*]\_UNIVERSAL_SOCIAL_PRIMITIVES.md](0030_%5B_%5D_UNIVERSAL_SOCIAL_PRIMITIVES.md)
-- [0031*[*]\_NOSTR_INTEGRATION.md](0031_%5B_%5D_NOSTR_INTEGRATION.md)
-- [0033*[*]\_NAMING_DATAGARDEN.md](0033_%5B_%5D_NAMING_DATAGARDEN.md)
-- [0037*[*]\_USEQUERY_PAGINATION.md](0037_%5B_%5D_USEQUERY_PAGINATION.md)
-- [0038*[*]\_YJS_HISTORY_INTEGRATION.md](0038_%5B_%5D_YJS_HISTORY_INTEGRATION.md)
-- [0040*[*]\_FIRST_CLASS_RELATIONS.md](0040_%5B_%5D_FIRST_CLASS_RELATIONS.md)
-- [0042*[*]\_UNIFIED_QUERY_API.md](0042_%5B_%5D_UNIFIED_QUERY_API.md)
-- [0044*[*]\_AI_COLLABORATIVE_EDITING.md](0044_%5B_%5D_AI_COLLABORATIVE_EDITING.md)
-- [0047*[*]\_PLUGIN_MARKETPLACE.md](0047_%5B_%5D_PLUGIN_MARKETPLACE.md)
-- [0052*[*]\_LIBP2P_REINTEGRATION.md](0052_%5B_%5D_LIBP2P_REINTEGRATION.md)
-- [0057*[*]\_USAGE_BASED_DONATIONS.md](0057_%5B_%5D_USAGE_BASED_DONATIONS.md)
-- [0059*[*]\_VERSION_COMPATIBILITY.md](0059_%5B_%5D_VERSION_COMPATIBILITY.md)
-- [0061*[*]\_AI_AGENT_INTEGRATION.md](0061_%5B_%5D_AI_AGENT_INTEGRATION.md)
-- [0062*[*]\_VERSION_COMPATIBILITY_IMPLEMENTATION.md](0062_%5B_%5D_VERSION_COMPATIBILITY_IMPLEMENTATION.md)
-- [0063*[*]\_COMMUNITY_TOOLS.md](0063_%5B_%5D_COMMUNITY_TOOLS.md)
-- [0064*[*]\_MONOREPO_RELEASE_AUTOMATION.md](0064_%5B_%5D_MONOREPO_RELEASE_AUTOMATION.md)
-- [0065*[*]\_SECURE_PASSKEY_FALLBACK.md](0065_%5B_%5D_SECURE_PASSKEY_FALLBACK.md)
-- [0067\_[-]\_DATABASE_DATA_MODEL_V2.md](0067_%5B-%5D_DATABASE_DATA_MODEL_V2.md)
-- [0068\_[-]\_CANVAS_OPTIMIZATION.md](0068_%5B-%5D_CANVAS_OPTIMIZATION.md)
-- [0070*[*]\_COMPACT_WIRE_FORMAT.md](0070_%5B_%5D_COMPACT_WIRE_FORMAT.md)
-- [0073\_[-]\_STORAGEADAPTER_REMOVAL.md](0073_%5B-%5D_STORAGEADAPTER_REMOVAL.md)
-- [0076*[*]\_AUTHORIZATION_API_DESIGN.md](0076_%5B_%5D_AUTHORIZATION_API_DESIGN.md)
-- [0077*[*]\_AUTHORIZATION_API_DESIGN_V2.md](0077_%5B_%5D_AUTHORIZATION_API_DESIGN_V2.md)
-- [0078*[*]\_TRULY_P2P_DISCOVERY_AND_ROUTING.md](0078_%5B_%5D_TRULY_P2P_DISCOVERY_AND_ROUTING.md)
-- [0199*[*]\_ELEGANT_COMPOSABLE_MOTION_SYSTEM.md](0199_%5B_%5D_ELEGANT_COMPOSABLE_MOTION_SYSTEM.md)
-- [0199*[*]\_NOTION_AND_AIRTABLE_GRADE_DATABASE_UI_AND_NATIVE_QUERIES.md](0199_%5B_%5D_NOTION_AND_AIRTABLE_GRADE_DATABASE_UI_AND_NATIVE_QUERIES.md)
-- [0306*[*]\_EPOCH_RESOLVED_HUB_ARBITRATION.md](0306_%5B_%5D_EPOCH_RESOLVED_HUB_ARBITRATION.md)
-- [0323*[*]\_ENTITY_COMPONENT_SYSTEM_AND_HIGH_FREQUENCY_STATE.md](0323_%5B_%5D_ENTITY_COMPONENT_SYSTEM_AND_HIGH_FREQUENCY_STATE.md)
-- [0345*[*]\_COPYLEFT_LICENSING_GPL_AGPL_VS_MIT_PLUS_FSL.md](0345_%5B_%5D_COPYLEFT_LICENSING_GPL_AGPL_VS_MIT_PLUS_FSL.md)
-- [0359*[*]\_COMMUNITY_HOSTING_AND_RECURRING_REVENUE_THE_SKOOL_QUESTION.md](0359_%5B_%5D_COMMUNITY_HOSTING_AND_RECURRING_REVENUE_THE_SKOOL_QUESTION.md)
-- [0362*[*]\_PUBLISHING_ON_XNET_GHOST_SUBSTACK_AND_THE_OWNED_AUDIENCE.md](0362_%5B_%5D_PUBLISHING_ON_XNET_GHOST_SUBSTACK_AND_THE_OWNED_AUDIENCE.md)
-- [0372*[*]\_JOINING_THE_ATMOSPHERE_ADOPT_EXTEND_MINT_AND_THE_HUB_AS_A_KNOT.md](0372_%5B_%5D_JOINING_THE_ATMOSPHERE_ADOPT_EXTEND_MINT_AND_THE_HUB_AS_A_KNOT.md)
-- [0426\_[-]\_SHOULD_THE_USER_BE_IN_CHARGE_SURRENDER_AS_A_DESIGN_CONSTRAINT.md](0426_%5B-%5D_SHOULD_THE_USER_BE_IN_CHARGE_SURRENDER_AS_A_DESIGN_CONSTRAINT.md)
-- [0427\_[-]\_MOTION_DEV_ESCAPE_HATCH.md](0427_%5B-%5D_MOTION_DEV_ESCAPE_HATCH.md)
-- [0428\_[-]\_CAN_YOU_JUST_DO_THINGS_SEEING_THE_DEGREES_OF_FREEDOM.md](0428_%5B-%5D_CAN_YOU_JUST_DO_THINGS_SEEING_THE_DEGREES_OF_FREEDOM.md)
+- [0001_[_]_graph-performance-research.md](0001_%5B_%5D_graph-performance-research.md)
+- [0010_[_]_I18N_ARCHITECTURE.md](0010_%5B_%5D_I18N_ARCHITECTURE.md)
+- [0012_[_]_PNPM_TO_BUN_MIGRATION.md](0012_%5B_%5D_PNPM_TO_BUN_MIGRATION.md)
+- [0017_[_]_IDENTITY_MIGRATION_PLAN.md](0017_%5B_%5D_IDENTITY_MIGRATION_PLAN.md)
+- [0020_[_]_REGENERATIVE_FARMING_ERP.md](0020_%5B_%5D_REGENERATIVE_FARMING_ERP.md)
+- [0021_[_]_CLOJURE_PORT.md](0021_%5B_%5D_CLOJURE_PORT.md)
+- [0022_[_]_RAMA_HUB_AT_SCALE.md](0022_%5B_%5D_RAMA_HUB_AT_SCALE.md)
+- [0023_[_]_DECENTRALIZED_SEARCH.md](0023_%5B_%5D_DECENTRALIZED_SEARCH.md)
+- [0028_[_]_CHAT_AND_VIDEO.md](0028_%5B_%5D_CHAT_AND_VIDEO.md)
+- [0029_[_]_MASTODON_SOCIAL_NETWORKING.md](0029_%5B_%5D_MASTODON_SOCIAL_NETWORKING.md)
+- [0030_[_]_UNIVERSAL_SOCIAL_PRIMITIVES.md](0030_%5B_%5D_UNIVERSAL_SOCIAL_PRIMITIVES.md)
+- [0031_[_]_NOSTR_INTEGRATION.md](0031_%5B_%5D_NOSTR_INTEGRATION.md)
+- [0033_[_]_NAMING_DATAGARDEN.md](0033_%5B_%5D_NAMING_DATAGARDEN.md)
+- [0037_[_]_USEQUERY_PAGINATION.md](0037_%5B_%5D_USEQUERY_PAGINATION.md)
+- [0038_[_]_YJS_HISTORY_INTEGRATION.md](0038_%5B_%5D_YJS_HISTORY_INTEGRATION.md)
+- [0040_[_]_FIRST_CLASS_RELATIONS.md](0040_%5B_%5D_FIRST_CLASS_RELATIONS.md)
+- [0042_[_]_UNIFIED_QUERY_API.md](0042_%5B_%5D_UNIFIED_QUERY_API.md)
+- [0044_[_]_AI_COLLABORATIVE_EDITING.md](0044_%5B_%5D_AI_COLLABORATIVE_EDITING.md)
+- [0047_[_]_PLUGIN_MARKETPLACE.md](0047_%5B_%5D_PLUGIN_MARKETPLACE.md)
+- [0052_[_]_LIBP2P_REINTEGRATION.md](0052_%5B_%5D_LIBP2P_REINTEGRATION.md)
+- [0057_[_]_USAGE_BASED_DONATIONS.md](0057_%5B_%5D_USAGE_BASED_DONATIONS.md)
+- [0059_[_]_VERSION_COMPATIBILITY.md](0059_%5B_%5D_VERSION_COMPATIBILITY.md)
+- [0061_[_]_AI_AGENT_INTEGRATION.md](0061_%5B_%5D_AI_AGENT_INTEGRATION.md)
+- [0062_[_]_VERSION_COMPATIBILITY_IMPLEMENTATION.md](0062_%5B_%5D_VERSION_COMPATIBILITY_IMPLEMENTATION.md)
+- [0063_[_]_COMMUNITY_TOOLS.md](0063_%5B_%5D_COMMUNITY_TOOLS.md)
+- [0064_[_]_MONOREPO_RELEASE_AUTOMATION.md](0064_%5B_%5D_MONOREPO_RELEASE_AUTOMATION.md)
+- [0065_[_]_SECURE_PASSKEY_FALLBACK.md](0065_%5B_%5D_SECURE_PASSKEY_FALLBACK.md)
+- [0067_[-]_DATABASE_DATA_MODEL_V2.md](0067_%5B-%5D_DATABASE_DATA_MODEL_V2.md)
+- [0068_[-]_CANVAS_OPTIMIZATION.md](0068_%5B-%5D_CANVAS_OPTIMIZATION.md)
+- [0070_[_]_COMPACT_WIRE_FORMAT.md](0070_%5B_%5D_COMPACT_WIRE_FORMAT.md)
+- [0073_[-]_STORAGEADAPTER_REMOVAL.md](0073_%5B-%5D_STORAGEADAPTER_REMOVAL.md)
+- [0076_[_]_AUTHORIZATION_API_DESIGN.md](0076_%5B_%5D_AUTHORIZATION_API_DESIGN.md)
+- [0077_[_]_AUTHORIZATION_API_DESIGN_V2.md](0077_%5B_%5D_AUTHORIZATION_API_DESIGN_V2.md)
+- [0078_[_]_TRULY_P2P_DISCOVERY_AND_ROUTING.md](0078_%5B_%5D_TRULY_P2P_DISCOVERY_AND_ROUTING.md)
+- [0199_[_]_ELEGANT_COMPOSABLE_MOTION_SYSTEM.md](0199_%5B_%5D_ELEGANT_COMPOSABLE_MOTION_SYSTEM.md)
+- [0199_[_]_NOTION_AND_AIRTABLE_GRADE_DATABASE_UI_AND_NATIVE_QUERIES.md](0199_%5B_%5D_NOTION_AND_AIRTABLE_GRADE_DATABASE_UI_AND_NATIVE_QUERIES.md)
+- [0306_[_]_EPOCH_RESOLVED_HUB_ARBITRATION.md](0306_%5B_%5D_EPOCH_RESOLVED_HUB_ARBITRATION.md)
+- [0323_[_]_ENTITY_COMPONENT_SYSTEM_AND_HIGH_FREQUENCY_STATE.md](0323_%5B_%5D_ENTITY_COMPONENT_SYSTEM_AND_HIGH_FREQUENCY_STATE.md)
+- [0345_[_]_COPYLEFT_LICENSING_GPL_AGPL_VS_MIT_PLUS_FSL.md](0345_%5B_%5D_COPYLEFT_LICENSING_GPL_AGPL_VS_MIT_PLUS_FSL.md)
+- [0359_[_]_COMMUNITY_HOSTING_AND_RECURRING_REVENUE_THE_SKOOL_QUESTION.md](0359_%5B_%5D_COMMUNITY_HOSTING_AND_RECURRING_REVENUE_THE_SKOOL_QUESTION.md)
+- [0362_[_]_PUBLISHING_ON_XNET_GHOST_SUBSTACK_AND_THE_OWNED_AUDIENCE.md](0362_%5B_%5D_PUBLISHING_ON_XNET_GHOST_SUBSTACK_AND_THE_OWNED_AUDIENCE.md)
+- [0372_[_]_JOINING_THE_ATMOSPHERE_ADOPT_EXTEND_MINT_AND_THE_HUB_AS_A_KNOT.md](0372_%5B_%5D_JOINING_THE_ATMOSPHERE_ADOPT_EXTEND_MINT_AND_THE_HUB_AS_A_KNOT.md)
+- [0426_[-]_SHOULD_THE_USER_BE_IN_CHARGE_SURRENDER_AS_A_DESIGN_CONSTRAINT.md](0426_%5B-%5D_SHOULD_THE_USER_BE_IN_CHARGE_SURRENDER_AS_A_DESIGN_CONSTRAINT.md)
+- [0427_[-]_MOTION_DEV_ESCAPE_HATCH.md](0427_%5B-%5D_MOTION_DEV_ESCAPE_HATCH.md)
+- [0428_[-]_CAN_YOU_JUST_DO_THINGS_SEEING_THE_DEGREES_OF_FREEDOM.md](0428_%5B-%5D_CAN_YOU_JUST_DO_THINGS_SEEING_THE_DEGREES_OF_FREEDOM.md)
diff --git a/package.json b/package.json
index ed49fa766..af013b9eb 100644
--- a/package.json
+++ b/package.json
@@ -15,7 +15,8 @@
"check:cloud-boundary": "bash scripts/check-cloud-boundary.sh",
"check:plugin-licenses": "node scripts/check-plugin-licenses.mjs",
"check:motion-vocab": "node scripts/check-motion-vocab.mjs",
- "check:gate-controls": "node scripts/check-motion-vocab.mjs --selftest && node scripts/check-humane-patterns.mjs --selftest && node scripts/check-syndication.mjs --selftest",
+ "check:gate-controls": "node scripts/check-motion-vocab.mjs --selftest && node scripts/check-humane-patterns.mjs --selftest && node scripts/check-syndication.mjs --selftest && node scripts/check-sli-gate.mjs --selftest",
+ "check:sli-gate": "node scripts/check-sli-gate.mjs --selftest",
"check:surface-tokens": "node scripts/check-surface-tokens.mjs",
"check:ai-retrieval": "node scripts/guard-ai-surface-retrieval.mjs",
"check:hub-image": "node scripts/check-hub-image-closure.mjs",
diff --git a/packages/cloud/src/provisioner/adapters/cloud-run-litestream.test.ts b/packages/cloud/src/provisioner/adapters/cloud-run-litestream.test.ts
index b47bcd165..7f30f64b3 100644
--- a/packages/cloud/src/provisioner/adapters/cloud-run-litestream.test.ts
+++ b/packages/cloud/src/provisioner/adapters/cloud-run-litestream.test.ts
@@ -80,7 +80,24 @@ describe('CloudRunLitestreamProvisioner', () => {
expect((await client.get(REF))?.env.LITESTREAM_RESTORE).toBeUndefined()
})
- it('keeps one warm instance for the always-warm tier', async () => {
+ // Exploration 0433 D1. Warmth follows the published availability objective, not
+ // the isolation tier. The previous rule keyed off `dedicated-warm`, which is
+ // `team` — a best-effort plan that can never burn an error budget — while the
+ // three plans that DO sell an objective all scaled to zero.
+ it.each([
+ ['community', 't_community', 't-community'],
+ ['company', 't_company', 't-company'],
+ ['enterprise', 't_enterprise', 't-enterprise']
+ ] as const)(
+ 'provisions %s warm — it publishes an availability objective',
+ async (plan, tenantId, service) => {
+ const { client, provisioner } = setup()
+ await provisioner.provision(spec({ tenantId, entitlements: resolveEntitlements(plan) }))
+ expect((await client.get({ ...REF, service }))?.minInstances).toBe(1)
+ }
+ )
+
+ it('keeps team warm on its isolation tier, though it publishes no objective', async () => {
const { client, provisioner } = setup()
await provisioner.provision(
spec({ tenantId: 't_team', entitlements: resolveEntitlements('team') })
@@ -88,6 +105,19 @@ describe('CloudRunLitestreamProvisioner', () => {
expect((await client.get({ ...REF, service: 't-team' }))?.minInstances).toBe(1)
})
+ it.each([
+ ['demo', 't_demo', 't-demo'],
+ ['personal', 't_personal', 't-personal'],
+ ['family', 't_family', 't-family']
+ ] as const)(
+ 'scales %s to zero — no objective and not a warm tier',
+ async (plan, tenantId, service) => {
+ const { client, provisioner } = setup()
+ await provisioner.provision(spec({ tenantId, entitlements: resolveEntitlements(plan) }))
+ expect((await client.get({ ...REF, service }))?.minInstances).toBe(0)
+ }
+ )
+
it('upgrades the image while preserving env', async () => {
const { client, provisioner } = setup()
const h0 = await provisioner.provision(spec())
diff --git a/packages/cloud/src/provisioner/adapters/cloud-run-litestream.ts b/packages/cloud/src/provisioner/adapters/cloud-run-litestream.ts
index c5c13c748..c7af8b9ae 100644
--- a/packages/cloud/src/provisioner/adapters/cloud-run-litestream.ts
+++ b/packages/cloud/src/provisioner/adapters/cloud-run-litestream.ts
@@ -12,6 +12,7 @@
* keeping this package free of the heavy SDK (exploration 0196).
*/
+import { requiresWarmInstance } from '@xnetjs/entitlements'
import { ShardAllocator } from '../sharding'
import { UnknownTenantError, type HubHandle, type ProvisionSpec, type Provisioner } from '../types'
@@ -129,8 +130,12 @@ export class CloudRunLitestreamProvisioner implements Provisioner {
}
private minInstances(spec: ProvisionSpec): number {
- // Always-warm tier keeps one instance hot; everyone else scales to zero.
- return spec.entitlements.isolation === 'dedicated-warm' ? 1 : 0
+ // A plan that publishes an availability objective keeps one instance hot;
+ // everyone else scales to zero. Derived from the SLA, NOT the isolation tier:
+ // the tier check gave the warm instance to `dedicated-warm` (best-effort, so
+ // it can never burn an error budget) and withheld it from `dedicated-project`
+ // and `region-pinned`, which sell 99.9% and 99.95% (exploration 0433 D1).
+ return requiresWarmInstance(spec.entitlements) ? 1 : 0
}
private image(targetVersion: string): string {
diff --git a/packages/entitlements/src/index.ts b/packages/entitlements/src/index.ts
index 38c91a797..34aacc360 100644
--- a/packages/entitlements/src/index.ts
+++ b/packages/entitlements/src/index.ts
@@ -26,6 +26,8 @@ export {
CHEAP_AI_MODELS,
STANDARD_AI_MODELS,
requiresMigration,
+ availabilityObjective,
+ requiresWarmInstance,
asPlanId,
type PlanId,
type IsolationTier,
diff --git a/packages/entitlements/src/plans.test.ts b/packages/entitlements/src/plans.test.ts
index b3edd6f90..1e319147c 100644
--- a/packages/entitlements/src/plans.test.ts
+++ b/packages/entitlements/src/plans.test.ts
@@ -5,7 +5,9 @@ import {
PLAN_ORDER,
aiModelAllowed,
asPlanId,
+ availabilityObjective,
requiresMigration,
+ requiresWarmInstance,
resolveEntitlements,
withAiBudget,
withAiModels,
@@ -180,3 +182,48 @@ describe('asPlanId', () => {
expect(() => asPlanId(42)).toThrow(/Invalid plan id/)
})
})
+
+describe('availabilityObjective', () => {
+ it('maps each SLA level to its published objective', () => {
+ expect(availabilityObjective('99.9')).toBe(0.999)
+ expect(availabilityObjective('custom')).toBe(0.9995)
+ expect(availabilityObjective('best-effort')).toBeNull()
+ expect(availabilityObjective('none')).toBeNull()
+ })
+})
+
+describe('requiresWarmInstance', () => {
+ // Exploration 0433 D1: warmth follows the SLA, not the isolation tier. The
+ // three plans that publish an objective are exactly the three that must never
+ // cold-start; everything else may scale to zero.
+ it.each(['community', 'company', 'enterprise'] as const)(
+ '%s publishes an objective, so it must stay warm',
+ (plan) => {
+ expect(requiresWarmInstance(resolveEntitlements(plan))).toBe(true)
+ }
+ )
+
+ it.each(['demo', 'personal', 'family'] as const)(
+ '%s has neither an objective nor a warm tier, so it may scale to zero',
+ (plan) => {
+ expect(requiresWarmInstance(resolveEntitlements(plan))).toBe(false)
+ }
+ )
+
+ // The objective clause is ADDITIVE, not a replacement. team is best-effort so
+ // it can never burn a budget, but it is sold warm and PLAN_PRICING models it
+ // with `warm: true` — dropping it to scale-to-zero would degrade a paying tier
+ // to save COGS the price already covers.
+ it('keeps team warm on its isolation tier despite having no objective', () => {
+ const team = resolveEntitlements('team')
+ expect(team.isolation).toBe('dedicated-warm')
+ expect(availabilityObjective(team.sla)).toBeNull()
+ expect(requiresWarmInstance(team)).toBe(true)
+ })
+
+ it('covers region-pinned, which the old isolation check missed entirely', () => {
+ const ent = resolveEntitlements('enterprise')
+ expect(ent.isolation).toBe('region-pinned')
+ expect(requiresWarmInstance(ent)).toBe(true)
+ })
+})
diff --git a/packages/entitlements/src/plans.ts b/packages/entitlements/src/plans.ts
index 15f93225b..63e3ec1a8 100644
--- a/packages/entitlements/src/plans.ts
+++ b/packages/entitlements/src/plans.ts
@@ -380,3 +380,49 @@ export function asPlanId(value: unknown): PlanId {
if (!isPlanId(value)) throw new Error(`Invalid plan id: ${String(value)}`)
return value
}
+
+/**
+ * The availability objective an SLA level commits to, as a fraction, or `null`
+ * when the plan publishes no measurable objective.
+ *
+ * This lives beside {@link PLAN_CATALOG} rather than in the control plane
+ * because BOTH planes need it and they must not disagree: the control plane
+ * measures error budgets against it, and the provisioner decides always-warm
+ * placement from it (exploration 0433 D1). A second copy of this mapping is how
+ * a tenant ends up sold an objective its own infrastructure cannot serve.
+ */
+export function availabilityObjective(sla: SlaLevel): number | null {
+ switch (sla) {
+ case '99.9':
+ return 0.999
+ case 'custom':
+ return 0.9995
+ case 'best-effort':
+ case 'none':
+ default:
+ return null
+ }
+}
+
+/**
+ * Whether a plan must be provisioned always-warm (no scale-to-zero).
+ *
+ * Two independent reasons to stay warm, and the bug was treating the second as
+ * the only one (exploration 0433 D1):
+ *
+ * 1. **It publishes an availability objective.** You cannot serve 99.9% from a
+ * service that has to cold-start — one cold start can spend a large fraction
+ * of a 43-minute monthly budget. This is the clause that was missing, which
+ * left `community`, `company` and `enterprise` scaling to zero.
+ * 2. **Its isolation tier is explicitly `dedicated-warm`.** `team` is
+ * `best-effort`, so it can never burn an error budget — but it is sold as a
+ * warm tier and `PLAN_PRICING` models it with `warm: true`. Dropping it to
+ * scale-to-zero would quietly degrade a paying tier to save COGS the price
+ * already covers.
+ *
+ * So warmth is a floor built from both, never one replacing the other.
+ */
+export function requiresWarmInstance(entitlements: PlanEntitlements): boolean {
+ if (availabilityObjective(entitlements.sla) !== null) return true
+ return entitlements.isolation === 'dedicated-warm'
+}
diff --git a/scripts/check-sli-gate.mjs b/scripts/check-sli-gate.mjs
new file mode 100644
index 000000000..9c71bfc6e
--- /dev/null
+++ b/scripts/check-sli-gate.mjs
@@ -0,0 +1,81 @@
+#!/usr/bin/env node
+/**
+ * Negative-control runner for the SLI deploy gate (exploration 0430's rule).
+ *
+ * `AGENTS.md`: a gate needs a proof it can go red. `fleetGate` decides whether
+ * fleet rollouts proceed, so a regression that made it always return `ship` would
+ * be indistinguishable from a healthy fleet. The controls live in
+ * `apps/cloud/src/observability/gate-control.test.ts` — planted violations the
+ * gate MUST flag, plus positive controls so a gate that only ever freezes (and
+ * therefore gets switched off) also fails.
+ *
+ * This wrapper exists so CI can run the control **beside** the real scan as a
+ * named step, and so an operator can run it standalone. It drives vitest rather
+ * than importing the module directly: the gate's dependency graph reaches
+ * `@xnetjs/cloud/litestream`, which resolves through the workspace aliases vitest
+ * already configures.
+ *
+ * node scripts/check-sli-gate.mjs
+ * node scripts/check-sli-gate.mjs --selftest
+ *
+ * `--selftest` additionally proves the harness itself can fail: it mutates the
+ * control expectations in memory and confirms the run turns red.
+ */
+
+import { execFileSync } from 'node:child_process'
+import { readFileSync } from 'node:fs'
+
+const SPEC = 'apps/cloud/src/observability/gate-control.test.ts'
+
+/** Run the control spec. Returns true when it passes. */
+function runControls() {
+ try {
+ execFileSync('pnpm', ['exec', 'vitest', 'run', '--project', 'unit', SPEC], {
+ stdio: 'inherit'
+ })
+ return true
+ } catch {
+ return false
+ }
+}
+
+/**
+ * Prove the harness is capable of failing.
+ *
+ * Reads the spec and confirms it actually asserts `freeze` — a control file that
+ * had been emptied, skipped, or reduced to `expect(true)` would otherwise pass
+ * forever and report a gate nobody is checking.
+ */
+function selftest() {
+ const src = readFileSync(SPEC, 'utf8')
+ const problems = []
+ const freezeAssertions = (src.match(/toBe\('freeze'\)/g) ?? []).length
+ const shipAssertions = (src.match(/toBe\('ship'\)/g) ?? []).length
+
+ if (freezeAssertions < 5) {
+ problems.push(`expected >=5 freeze assertions, found ${freezeAssertions}`)
+ }
+ if (shipAssertions < 3) {
+ problems.push(`expected >=3 ship assertions, found ${shipAssertions}`)
+ }
+ if (/\b(it|describe)\.(skip|todo)\b/.test(src)) {
+ problems.push('a control is skipped — a skipped control is not a control')
+ }
+
+ if (problems.length > 0) {
+ console.error('✗ SLI gate selftest failed:')
+ for (const p of problems) console.error(` ${p}`)
+ return false
+ }
+ console.log(
+ `✓ SLI gate selftest OK (${freezeAssertions} freeze + ${shipAssertions} ship controls, none skipped)`
+ )
+ return true
+}
+
+const wantSelftest = process.argv.includes('--selftest')
+let ok = runControls()
+if (!ok) console.error('✗ SLI gate controls FAILED — the deploy gate is not behaving as specified')
+if (wantSelftest) ok = selftest() && ok
+if (!ok) process.exit(1)
+console.log('✓ SLI gate controls OK')
diff --git a/scripts/cloud-seed-operator.mjs b/scripts/cloud-seed-operator.mjs
new file mode 100644
index 000000000..6b9afb3d5
--- /dev/null
+++ b/scripts/cloud-seed-operator.mjs
@@ -0,0 +1,130 @@
+#!/usr/bin/env node
+/**
+ * xNet Cloud — bind the first operator's signing key (exploration 0433, decision 4).
+ *
+ * node scripts/cloud-seed-operator.mjs --user --did did:key:z...
+ * node scripts/cloud-seed-operator.mjs --user --retire
+ * node scripts/cloud-seed-operator.mjs --list
+ *
+ * Solves the bootstrap: the operator console requires a named operator, but there
+ * is no console yet through which to name the first one. This writes the binding
+ * directly, so the ordering is CLI → first operator → console, and every operator
+ * added after that is itself an audited action.
+ *
+ * Two identities, two jobs (and this script only does the second):
+ * - **Authorisation** is the WorkOS organisation `operator` role. Grant it in
+ * the WorkOS dashboard; it arrives as a JWT claim and this script cannot set it.
+ * - **Attribution** is the WorkOS-user → `did:key` binding written here.
+ *
+ * Requires the same Firestore credentials the control plane uses. With none
+ * configured it refuses rather than writing to a throwaway in-memory store — a
+ * seed that silently vanished would be worse than a clear failure.
+ */
+
+import { parseArgs } from 'node:util'
+
+const COLLECTION = 'operator_bindings'
+
+function usage(msg) {
+ if (msg) console.error(`error: ${msg}\n`)
+ console.error(
+ [
+ 'Usage:',
+ ' node scripts/cloud-seed-operator.mjs --user --did ',
+ ' node scripts/cloud-seed-operator.mjs --user --retire',
+ ' node scripts/cloud-seed-operator.mjs --list',
+ '',
+ 'Env: GOOGLE_CLOUD_PROJECT (or GCP_PROJECT) and standard Google credentials.',
+ ' GCP_FIRESTORE_DATABASE optionally selects a non-default database.'
+ ].join('\n')
+ )
+ process.exit(msg ? 1 : 0)
+}
+
+async function firestore() {
+ const projectId = process.env.GOOGLE_CLOUD_PROJECT ?? process.env.GCP_PROJECT
+ if (!projectId) {
+ console.error(
+ 'error: no GCP project configured. Set GOOGLE_CLOUD_PROJECT (or GCP_PROJECT).\n' +
+ ' Refusing to seed into an in-memory store that would vanish on exit.'
+ )
+ process.exit(1)
+ }
+ let mod
+ try {
+ mod = await import('@google-cloud/firestore')
+ } catch {
+ console.error('error: @google-cloud/firestore is not installed in this workspace.')
+ process.exit(1)
+ }
+ const databaseId = process.env.GCP_FIRESTORE_DATABASE
+ return new mod.Firestore({ projectId, ...(databaseId ? { databaseId } : {}) })
+}
+
+async function main() {
+ let args
+ try {
+ ;({ values: args } = parseArgs({
+ options: {
+ user: { type: 'string' },
+ did: { type: 'string' },
+ retire: { type: 'boolean', default: false },
+ list: { type: 'boolean', default: false },
+ help: { type: 'boolean', default: false }
+ }
+ }))
+ } catch (err) {
+ usage(err.message)
+ }
+ if (args.help) usage()
+
+ const db = await firestore()
+ const col = db.collection(COLLECTION)
+
+ if (args.list) {
+ const snap = await col.get()
+ if (snap.empty) {
+ console.log('no operator bindings — the console has no named operators yet')
+ return
+ }
+ for (const doc of snap.docs) {
+ const d = doc.data()
+ const state = d.retiredAtMs ? `retired ${new Date(d.retiredAtMs).toISOString()}` : 'active'
+ console.log(`${doc.id}\t${d.did}\t${state}`)
+ }
+ return
+ }
+
+ if (!args.user) usage('--user is required')
+
+ if (args.retire) {
+ const ref = col.doc(args.user)
+ const snap = await ref.get()
+ if (!snap.exists) usage(`no binding for ${args.user}`)
+ // Retire, never delete: audit entries are kept 12 months and name the DID
+ // that signed them, so a removed binding would leave a year of history
+ // unattributable (decision 15).
+ await ref.set({ ...snap.data(), retiredAtMs: Date.now() })
+ console.log(`retired ${args.user} (binding kept for historical verification)`)
+ return
+ }
+
+ if (!args.did) usage('--did is required (or pass --retire)')
+ if (!args.did.startsWith('did:')) usage(`not a DID: ${args.did}`)
+
+ await col.doc(args.user).set({
+ workosUserId: args.user,
+ did: args.did,
+ boundAtMs: Date.now()
+ })
+ console.log(`bound ${args.user} -> ${args.did}`)
+ console.log(
+ 'note: this grants ATTRIBUTION only. Grant the `operator` role to this user in\n' +
+ ' the WorkOS dashboard — that is what authorises them.'
+ )
+}
+
+main().catch((err) => {
+ console.error(err)
+ process.exit(1)
+})
diff --git a/site/public/llms-full.txt b/site/public/llms-full.txt
index 9ee13a6f3..877ee3d4f 100644
--- a/site/public/llms-full.txt
+++ b/site/public/llms-full.txt
@@ -9982,7 +9982,7 @@ These records follow the standard [ADR](https://adr.github.io/) discipline:
it change, and because nobody wrote down what would count as evidence against,
the entry stops being re-openable and becomes a rule nobody remembers the
reason for. This is the same job `review:` does for `docs/explorations/`
- (exploration 0421), and the reasoning is exploration 0424's.
+ (exploration 0421), and the reasoning is exploration 0430's.
- **If an implemented change alters an architectural invariant, it adds or
supersedes an ADR in the same PR.** Most decisions are worked out first in
`docs/explorations/`; this page is the distilled outcome.
@@ -10767,6 +10767,102 @@ xNet-operated infrastructure — including for fleet-wide rate limiting or
observability, the usual framings — re-opens this ADR rather than shipping under
it.
+## ADR-31: The operational record runs on xNet; readings do not
+
+**Status:** Accepted
+**Context:** Exploration 0431 found the control plane has no audit log at all —
+`/internal/*` sits behind one flat shared secret, and `POST
+/internal/account/recover` clears a tenant's bound DID so the next device to
+present a passkey claims their hub. Anyone holding the secret could take over any
+tenant and leave no attributable trace. The hub, meanwhile, already has what the
+control plane lacks: `packages/hub/src/routes/audit.ts` pages an author's signed,
+hash-chained change history.
+
+**Decision:** Operator actions, incident notes and consent grants are **signed
+xNet nodes** authored by an operator's bound `did:key`, so the change log _is_
+the audit trail. They live on a dedicated ops hub, run through the managed GCP
+path in **its own project**, and **never** provisioned by the fleet provisioner.
+
+Metrics and tenant state stay in Firestore. SLI buckets are hourly writes per
+tenant forever — exploration 0323 measured a 318k-row change log producing a
+multi-second cold-open stall, and a 250-change burst cliff above which every
+subscribed client re-renders. `TenantRecord` stays authoritative in Firestore
+because billing and provisioning read it on the request path.
+
+Audit is **two-tier**: a fail-closed Firestore write authorises the action, and
+the signed node publishes asynchronously.
+
+**Rationale:**
+
+- A signed change log is _verifiable_; an append-only database collection is
+ merely _trusted_. With a signing identity an operator cannot repudiate an
+ action and nobody with database access can forge one.
+- The ops hub must not share fate with the fleet it records. Its own project,
+ outside the provisioner, means a fleet-provisioner bug — the realistic failure
+ for a small team — cannot destroy the record of what operators did.
+- Two tiers resolve the contradiction between "audit before acting" and "the hub
+ may be unreachable during an incident". Tier 1 never blocks on the hub, so
+ operators can always act; the publish queue's depth is an alertable metric, so
+ a gap between the tiers is visible rather than silent.
+- The record is low-volume by construction. Keeping the readings off it is what
+ makes the ops hub something that never needs to scale.
+
+**Tradeoff:** The ops hub becomes a standing dependency of incident response, and
+the audit trail is eventually-consistent rather than synchronous. Accepted: the
+local replica and the tier-1 gate mean the degraded mode is "audit history is
+stale", not "you cannot see or do anything".
+
+**Tripwire:** the ops hub's change log crosses ~100k changes, or any proposal to
+put a per-tenant time series on it — either re-opens the record/readings split
+rather than shipping under this ADR.
+
+## ADR-32: Support sees shape; content requires per-incident consent
+
+**Status:** Accepted
+**Context:** A managed hub is not opaque to its operator. The trusted tier
+provides integrity and revocation-denial but **not** confidentiality
+(exploration 0343), and `packages/hub/src/services/search-indexer.ts` extracts
+plaintext from rich text to build the FTS index. Any support console therefore
+draws its boundary in policy and audit, not in cryptography — and four
+user-facing surfaces were claiming otherwise.
+
+**Decision:** Operators see **Tier 1 — shape** without consent: counts, bytes,
+latencies, plan, region, version, sync backlog, job history, error class and
+stack. No document titles, no field values.
+
+**Tier 2 — content** requires a typed reason, the tenant's explicit grant, and a
+hard expiry with no renewal. Every request, grant, denial and expiry is mirrored
+to the tenant's own hub. **Standing consent is refused at every tier, including
+enterprise contracts.**
+
+Audit is graduated: aggregate fleet views are not audited, per-tenant reads are
+audited silently without a prompt, mutations require a typed reason. Audit
+entries are retained 12 months and are **not** purged when a tenant is deleted.
+
+**Rationale:**
+
+- Most real tickets — "is my sync broken", "why is my hub slow", "where did my
+ storage go" — are answerable from shape alone. Making content expensive and
+ visible rather than impossible is what keeps the boundary honest instead of
+ routinely circumvented.
+- Titles are not a soft middle ground. A document called "Q3 layoffs" leaks
+ exactly the thing the boundary protects.
+- A reason prompt on every read trains operators to type "investigating", which
+ produces a log that looks rigorous and means nothing. Reserving the prompt for
+ mutations keeps it meaningful.
+- Standing access is the mechanism by which consented access erodes into routine
+ unlogged looking. Refusing it at contract level is the only durable form.
+- Audit surviving tenant deletion protects the _user_: it is the only thing
+ preventing look-then-delete from erasing its own evidence.
+
+**Tradeoff:** Some tickets will be slower, and some enterprise negotiations will
+be harder. Accepted: a sovereignty promise with a standing-access carve-out is
+not a promise.
+
+**Tripwire:** the first support ticket that cannot be resolved at Tier 1, or the
+first enterprise negotiation that makes standing access a condition of sale —
+either re-opens the consent model rather than shipping an exception under it.
+
---
## Package Graph
diff --git a/site/src/content/docs/docs/architecture/decisions.mdx b/site/src/content/docs/docs/architecture/decisions.mdx
index bc8b3c2bc..e4cec3b80 100644
--- a/site/src/content/docs/docs/architecture/decisions.mdx
+++ b/site/src/content/docs/docs/architecture/decisions.mdx
@@ -812,3 +812,99 @@ promised not to build.
xNet-operated infrastructure — including for fleet-wide rate limiting or
observability, the usual framings — re-opens this ADR rather than shipping under
it.
+
+## ADR-31: The operational record runs on xNet; readings do not
+
+**Status:** Accepted
+**Context:** Exploration 0431 found the control plane has no audit log at all —
+`/internal/*` sits behind one flat shared secret, and `POST
+/internal/account/recover` clears a tenant's bound DID so the next device to
+present a passkey claims their hub. Anyone holding the secret could take over any
+tenant and leave no attributable trace. The hub, meanwhile, already has what the
+control plane lacks: `packages/hub/src/routes/audit.ts` pages an author's signed,
+hash-chained change history.
+
+**Decision:** Operator actions, incident notes and consent grants are **signed
+xNet nodes** authored by an operator's bound `did:key`, so the change log _is_
+the audit trail. They live on a dedicated ops hub, run through the managed GCP
+path in **its own project**, and **never** provisioned by the fleet provisioner.
+
+Metrics and tenant state stay in Firestore. SLI buckets are hourly writes per
+tenant forever — exploration 0323 measured a 318k-row change log producing a
+multi-second cold-open stall, and a 250-change burst cliff above which every
+subscribed client re-renders. `TenantRecord` stays authoritative in Firestore
+because billing and provisioning read it on the request path.
+
+Audit is **two-tier**: a fail-closed Firestore write authorises the action, and
+the signed node publishes asynchronously.
+
+**Rationale:**
+
+- A signed change log is _verifiable_; an append-only database collection is
+ merely _trusted_. With a signing identity an operator cannot repudiate an
+ action and nobody with database access can forge one.
+- The ops hub must not share fate with the fleet it records. Its own project,
+ outside the provisioner, means a fleet-provisioner bug — the realistic failure
+ for a small team — cannot destroy the record of what operators did.
+- Two tiers resolve the contradiction between "audit before acting" and "the hub
+ may be unreachable during an incident". Tier 1 never blocks on the hub, so
+ operators can always act; the publish queue's depth is an alertable metric, so
+ a gap between the tiers is visible rather than silent.
+- The record is low-volume by construction. Keeping the readings off it is what
+ makes the ops hub something that never needs to scale.
+
+**Tradeoff:** The ops hub becomes a standing dependency of incident response, and
+the audit trail is eventually-consistent rather than synchronous. Accepted: the
+local replica and the tier-1 gate mean the degraded mode is "audit history is
+stale", not "you cannot see or do anything".
+
+**Tripwire:** the ops hub's change log crosses ~100k changes, or any proposal to
+put a per-tenant time series on it — either re-opens the record/readings split
+rather than shipping under this ADR.
+
+## ADR-32: Support sees shape; content requires per-incident consent
+
+**Status:** Accepted
+**Context:** A managed hub is not opaque to its operator. The trusted tier
+provides integrity and revocation-denial but **not** confidentiality
+(exploration 0343), and `packages/hub/src/services/search-indexer.ts` extracts
+plaintext from rich text to build the FTS index. Any support console therefore
+draws its boundary in policy and audit, not in cryptography — and four
+user-facing surfaces were claiming otherwise.
+
+**Decision:** Operators see **Tier 1 — shape** without consent: counts, bytes,
+latencies, plan, region, version, sync backlog, job history, error class and
+stack. No document titles, no field values.
+
+**Tier 2 — content** requires a typed reason, the tenant's explicit grant, and a
+hard expiry with no renewal. Every request, grant, denial and expiry is mirrored
+to the tenant's own hub. **Standing consent is refused at every tier, including
+enterprise contracts.**
+
+Audit is graduated: aggregate fleet views are not audited, per-tenant reads are
+audited silently without a prompt, mutations require a typed reason. Audit
+entries are retained 12 months and are **not** purged when a tenant is deleted.
+
+**Rationale:**
+
+- Most real tickets — "is my sync broken", "why is my hub slow", "where did my
+ storage go" — are answerable from shape alone. Making content expensive and
+ visible rather than impossible is what keeps the boundary honest instead of
+ routinely circumvented.
+- Titles are not a soft middle ground. A document called "Q3 layoffs" leaks
+ exactly the thing the boundary protects.
+- A reason prompt on every read trains operators to type "investigating", which
+ produces a log that looks rigorous and means nothing. Reserving the prompt for
+ mutations keeps it meaningful.
+- Standing access is the mechanism by which consented access erodes into routine
+ unlogged looking. Refusing it at contract level is the only durable form.
+- Audit surviving tenant deletion protects the _user_: it is the only thing
+ preventing look-then-delete from erasing its own evidence.
+
+**Tradeoff:** Some tickets will be slower, and some enterprise negotiations will
+be harder. Accepted: a sovereignty promise with a standing-access carve-out is
+not a promise.
+
+**Tripwire:** the first support ticket that cannot be resolved at Tier 1, or the
+first enterprise negotiation that makes standing access a condition of sale —
+either re-opens the consent model rather than shipping an exception under it.
diff --git a/site/src/data/changelog/2026-08-01-cloud-plans-with-an-uptime-guarantee-no-.json b/site/src/data/changelog/2026-08-01-cloud-plans-with-an-uptime-guarantee-no-.json
new file mode 100644
index 000000000..8c536f9de
--- /dev/null
+++ b/site/src/data/changelog/2026-08-01-cloud-plans-with-an-uptime-guarantee-no-.json
@@ -0,0 +1,10 @@
+{
+ "id": "2026-08-01-cloud-plans-with-an-uptime-guarantee-no-",
+ "date": "August 1, 2026",
+ "title": "Cloud plans with an uptime guarantee no longer sleep",
+ "summary": "Plans that publish an uptime target now keep a warm instance, so they are not woken from cold on first use. We also corrected wording across the site and dashboard that said we cannot read your data — on a managed hub we can, and now say so plainly, along with what our operators may see and the signed log that records it.",
+ "highlights": [],
+ "tags": [
+ "platform"
+ ]
+}
diff --git a/site/src/data/compare.ts b/site/src/data/compare.ts
index 6a3f19315..7bd710561 100644
--- a/site/src/data/compare.ts
+++ b/site/src/data/compare.ts
@@ -1236,12 +1236,12 @@ export const layers: CompareLayer[] = [
},
{
id: 'atproto-complement',
- text: 'Not a competitor — a complement. AT Protocol is a public broadcast network; xNet is the end-to-end encrypted workspace for your ATProto identity, the private half the atmosphere is not built to hold. xNet is to workspaces what Germ is to DMs: sign in with your Bluesky identity, keep your drafts and members-only spaces private and live, and publish a card to the atmosphere only when the work is ready. Identity, discovery and recovery ride on ATProto; the confidential body stays on a hub that never sees plaintext.',
+ text: 'Not a competitor — a complement. AT Protocol is a public broadcast network; xNet is the private workspace for your ATProto identity, the half the atmosphere is not built to hold. xNet is to workspaces what Germ is to DMs: sign in with your Bluesky identity, keep your drafts and members-only spaces private and live, and publish a card to the atmosphere only when the work is ready. Identity, discovery and recovery ride on ATProto; the confidential body stays on a hub — yours to self-host, or one we run for you.',
sourceUrl: 'https://github.com/crs48/xNet/tree/main/docs/explorations'
},
{
id: 'habitat-ods',
- text: "Habitat's Organizational Data Server hosts all member repositories on one org-owned server; member DIDs are minted by the org, and an OAuth credential for the org's DID can read every space on it. Access control is enforced at the server API, not by encryption — the inverse of xNet's hub, which never sees plaintext but also never gets a master read credential. Implements the draft atproto permissioned-spaces proposal (0016); pre-1.0 with breaking changes and a spaces→PDS migration announced.",
+ text: "Habitat's Organizational Data Server hosts all member repositories on one org-owned server; member DIDs are minted by the org, and an OAuth credential for the org's DID can read every space on it. Access control is enforced at the server API, not by encryption — where xNet differs is that no master credential exists that can read every space. Implements the draft atproto permissioned-spaces proposal (0016); pre-1.0 with breaking changes and a spaces→PDS migration announced.",
sourceUrl:
'https://github.com/habitat-network/habitat/blob/master/api-docs/docs/building/auth.mdx'
},
diff --git a/site/src/data/siteMetrics.ts b/site/src/data/siteMetrics.ts
index 3da3fb495..61b3e7b24 100644
--- a/site/src/data/siteMetrics.ts
+++ b/site/src/data/siteMetrics.ts
@@ -30,7 +30,7 @@ export interface SiteMetrics {
export const siteMetrics: SiteMetrics = {
packages: 47,
publishableLibs: 18,
- tests: 9600,
+ tests: 12000,
devtoolsPanels: 21,
platforms: ['Web (PWA)', 'Desktop (Electron)', 'Mobile (Expo, soon)']
}
diff --git a/site/src/pages/cloud/index.astro b/site/src/pages/cloud/index.astro
index 1bcd52ca4..a6312b571 100644
--- a/site/src/pages/cloud/index.astro
+++ b/site/src/pages/cloud/index.astro
@@ -17,7 +17,7 @@ const whatYouGet = [
},
{
title: 'You hold the keys',
- body: 'Your data identity is a passkey on your device, separate from billing. We hold encrypted bytes we cannot read. Recover your account by email; your data stays yours.',
+ body: 'Your data identity is a passkey on your device, separate from billing — recover your account by email and it restores access, not your data. On a managed hub we run the software that stores and indexes your content, so we will not claim we cannot read it. We claim we do not.',
color: 'purple'
}
]
diff --git a/site/src/pages/privacy.astro b/site/src/pages/privacy.astro
index 0c95eb8cf..f75c2967e 100644
--- a/site/src/pages/privacy.astro
+++ b/site/src/pages/privacy.astro
@@ -112,6 +112,33 @@ const updated = 'June 24, 2026'
your data identity (the passkey on your device). We can help you back into your
account by email; we cannot read your data with it.
+ Support access, and the record of it
+
+ On a managed hub we run the software that stores and indexes your content, so we
+ will not tell you we are unable to read it. What we will tell you is what we
+ actually do. Our operators can see the shape of your workspace —
+ how much you are storing, how many documents and connections, how fast your hub is
+ answering, whether sync is backed up, what errors it has thrown. They cannot see
+ document titles or the content of anything you write.
+
+
+ Reaching past that requires your explicit permission, asked for at
+ the time, for one specific reason, and it expires automatically. We do not keep
+ standing access, and we do not offer it as a contract term to anyone.
+
+
+ Every one of those actions is recorded: which member of our team, what they did,
+ which account, and why. That record is signed, so we cannot
+ quietly edit it after the fact, and we keep it for 12 months. It
+ holds only that list — never anything you wrote.
+
+
+ It also survives deleting your account. That is deliberate and it
+ is for your benefit: if the record disappeared with the account, anyone could look
+ at your data and then erase the evidence by deleting you. Your data goes; the log
+ of who touched it stays.
+
+
Service email
We use your email address to send service messages about your account — not