feat: xNet Cloud operations — SLIs/SLOs, error-budget rollouts, restore drill (0193)

apps/cloud/src/backup/restore-drill.test.ts

@@ -0,0 +1,95 @@
+import type { TenantRecord } from '../registry'
+import { MemoryProvisioner } from '@xnetjs/cloud/provisioner'
+import { resolveEntitlements } from '@xnetjs/entitlements'
+import { describe, expect, it } from 'vitest'
+import {
+  pickDrillSample,
+  runRestoreDrills,
+  verifyRestore,
+  type RestoreProbe
+} from './restore-drill'
+
+const tenant = (id: string): TenantRecord => ({
+  tenantId: id,
+  plan: 'personal',
+  entitlements: resolveEntitlements('personal'),
+  billingUserId: `u_${id}`,
+  did: '',
+  hubUrl: 'wss://x',
+  substrateRef: 'ref',
+  region: 'us',
+  targetVersion: 'xnet-hub@0.0.1',
+  createdAt: 0,
+  lastActiveMs: 0,
+  dataTier: 'cold'
+})
+
+const okProbe: RestoreProbe = { ready: async () => true }
+const downProbe: RestoreProbe = { ready: async () => false }
+
+describe('verifyRestore', () => {
+  it('provisions a throwaway hub, asserts ready, and tears it down', async () => {
+    const prov = new MemoryProvisioner()
+    const destroyed: string[] = []
+    const origDestroy = prov.destroy.bind(prov)
+    prov.destroy = async (ref: string) => {
+      destroyed.push(ref)
+      return origDestroy(ref)
+    }
+    const res = await verifyRestore(prov, okProbe, {
+      tenantId: 't_a',
+      entitlements: resolveEntitlements('personal'),
+      targetVersion: 'xnet-hub@0.0.1'
+    })
+    expect(res).toEqual({ tenantId: 't_a', ok: true })
+    expect(destroyed).toHaveLength(1) // throwaway hub always torn down
+  })
+
+  it('reports a not-ready restored hub as a failure', async () => {
+    const res = await verifyRestore(new MemoryProvisioner(), downProbe, {
+      tenantId: 't_b',
+      entitlements: resolveEntitlements('personal'),
+      targetVersion: 'xnet-hub@0.0.1'
+    })
+    expect(res.ok).toBe(false)
+    expect(res.error).toMatch(/not ready/)
+  })
+
+  it('captures a provisioning failure instead of throwing', async () => {
+    const broken = {
+      substrate: 'broken',
+      provision: async () => {
+        throw new Error('R2 replica missing')
+      }
+    } as unknown as MemoryProvisioner
+    const res = await verifyRestore(broken, okProbe, {
+      tenantId: 't_c',
+      entitlements: resolveEntitlements('personal'),
+      targetVersion: 'xnet-hub@0.0.1'
+    })
+    expect(res).toMatchObject({ tenantId: 't_c', ok: false, error: 'R2 replica missing' })
+  })
+})
+
+describe('pickDrillSample', () => {
+  it('returns all tenants when fewer than the sample size', () => {
+    const ts = [tenant('a'), tenant('b')]
+    expect(pickDrillSample(ts, 5, 0)).toHaveLength(2)
+  })
+
+  it('rotates the sample window by day so the fleet is covered over time', () => {
+    const ts = ['a', 'b', 'c', 'd'].map(tenant)
+    const day0 = pickDrillSample(ts, 2, 0).map((t) => t.tenantId)
+    const day1 = pickDrillSample(ts, 2, 1).map((t) => t.tenantId)
+    expect(day0).toEqual(['a', 'b'])
+    expect(day1).toEqual(['c', 'd'])
+  })
+})
+
+describe('runRestoreDrills', () => {
+  it('runs the drill across a sample and includes failures', async () => {
+    const prov = new MemoryProvisioner()
+    const results = await runRestoreDrills(prov, okProbe, [tenant('a'), tenant('b')])
+    expect(results.map((r) => r.ok)).toEqual([true, true])
+  })
+})

apps/cloud/src/backup/restore-drill.ts

@@ -0,0 +1,86 @@
+/**
+ * xNet Cloud — automated restore-verification drill (exploration 0193).
+ *
+ * "We replicate to R2" is not "we can restore your hub." This drill *proves* it:
+ * provision a THROWAWAY hub that restores a tenant's DB from its R2 replica
+ * (Litestream restore-on-boot), assert it comes up ready, then always tear it
+ * down. Run nightly over a rotating sample so it costs little and catches a
+ * broken backup before a real reactivation does.
+ */
+
+import type { TenantRecord } from './../registry'
+import type { Provisioner } from '@xnetjs/cloud/provisioner'
+import type { PlanEntitlements } from '@xnetjs/entitlements'
+import { snapshotKeyFor } from '../control-plane'
+
+/** Probes whether a freshly-restored hub is up + writable (`GET /ready`). */
+export interface RestoreProbe {
+  ready(hubUrl: string): Promise<boolean>
+}
+
+export interface RestoreDrillResult {
+  tenantId: string
+  ok: boolean
+  error?: string
+}
+
+/** Verify one tenant restores from R2 into a throwaway hub, then tear it down. */
+export async function verifyRestore(
+  provisioner: Provisioner,
+  probe: RestoreProbe,
+  tenant: { tenantId: string; entitlements: PlanEntitlements; targetVersion: string }
+): Promise<RestoreDrillResult> {
+  let substrateRef: string | null = null
+  try {
+    const handle = await provisioner.provision({
+      tenantId: `drill-${tenant.tenantId}`,
+      entitlements: tenant.entitlements,
+      targetVersion: tenant.targetVersion,
+      env: {},
+      restoreFromR2: snapshotKeyFor(tenant.tenantId)
+    })
+    substrateRef = handle.substrateRef
+    const ok = await probe.ready(handle.hubUrl)
+    return { tenantId: tenant.tenantId, ok, ...(ok ? {} : { error: 'restored hub not ready' }) }
+  } catch (err) {
+    return { tenantId: tenant.tenantId, ok: false, error: (err as Error).message }
+  } finally {
+    if (substrateRef) await provisioner.destroy(substrateRef).catch(() => undefined)
+  }
+}
+
+/**
+ * Deterministically pick `sampleSize` tenants for tonight's drill, rotating by a
+ * day index so the whole fleet is covered over time without drilling all of it
+ * every night (a silent cap is logged by the caller — see exploration 0193).
+ */
+export function pickDrillSample(
+  tenants: TenantRecord[],
+  sampleSize: number,
+  dayIndex: number
+): TenantRecord[] {
+  const eligible = tenants.filter((t) => t.tenantId) // every tenant has an R2 replica path
+  if (eligible.length <= sampleSize) return eligible
+  const start = (dayIndex * sampleSize) % eligible.length
+  const rotated = [...eligible.slice(start), ...eligible.slice(0, start)]
+  return rotated.slice(0, sampleSize)
+}
+
+/** Run the drill across a sample; returns per-tenant results (failures included). */
+export async function runRestoreDrills(
+  provisioner: Provisioner,
+  probe: RestoreProbe,
+  sample: TenantRecord[]
+): Promise<RestoreDrillResult[]> {
+  const results: RestoreDrillResult[] = []
+  for (const t of sample) {
+    results.push(
+      await verifyRestore(provisioner, probe, {
+        tenantId: t.tenantId,
+        entitlements: t.entitlements,
+        targetVersion: t.targetVersion
+      })
+    )
+  }
+  return results
+}

apps/cloud/src/control-plane.ts

@@ -63,6 +63,11 @@ export function tenantIdForBilling(billingUserId: string): string {
   return `t_${billingUserId.replace(/[^a-zA-Z0-9_-]/g, '')}`
 }
 
+/** R2 object path holding a tenant's SQLite snapshot (matches the Litestream replica path). */
+export function snapshotKeyFor(tenantId: string): string {
+  return `t/${tenantId}/db`
+}
+
 export class ControlPlane {
   constructor(private readonly deps: ControlPlaneDeps) {}
 
@@ -299,9 +304,14 @@ export class ControlPlane {
     return this.deps.tenants.get(tenantId)
   }
 
+  /** Every tenant the control plane knows about (fleet observability + rollouts). */
+  listTenants(): Promise<TenantRecord[]> {
+    return this.deps.tenants.list()
+  }
+
   /** R2 object path holding a tenant's SQLite snapshot (matches the Litestream replica path). */
   private snapshotKey(tenantId: string): string {
-    return `t/${tenantId}/db`
+    return snapshotKeyFor(tenantId)
   }
 
   /** Record activity so the cold-demotion clock resets (exploration 0178). */

apps/cloud/src/dashboard.ts

@@ -10,6 +10,7 @@
 
 import type { TenantRecord } from './registry'
 import type { PlanId } from '@xnetjs/entitlements'
+import { sloForPlan } from './observability/slo'
 
 export interface DashboardView {
   billingUserId: string
@@ -69,6 +70,8 @@ function hubCard(tenant: TenantRecord): string {
         <div><dt>Region</dt><dd>${esc(tenant.region || 'auto')}</dd></div>
         <div><dt>Storage</dt><dd>${fmtBytes(e.quotaBytes)}</dd></div>
         <div><dt>Seats</dt><dd>${e.seats}</dd></div>
+        <div><dt>Uptime</dt><dd>${esc(sloForPlan(tenant.plan).label)}</dd></div>
+        <div><dt>Backups</dt><dd>Continuous → object storage</dd></div>
         <div><dt>Data identity</dt><dd>${
           tenant.did
             ? `<code>${esc(tenant.did)}</code>`

apps/cloud/src/fleet.test.ts

@@ -0,0 +1,79 @@
+import { MemoryBillingIdentityProvider } from '@xnetjs/cloud/identity'
+import { describe, expect, it } from 'vitest'
+import { FakeTenantBillingGateway } from './billing-gateway'
+import { HealthSampleStore } from './observability/health'
+import { createControlPlaneApp } from './server'
+import { buildControlPlane } from './index'
+
+const INTERNAL = 'secret123'
+
+function fleetApp() {
+  const billing = new MemoryBillingIdentityProvider('https://auth.test/authorize')
+  const { controlPlane } = buildControlPlane({ billing })
+  const health = new HealthSampleStore()
+  const app = createControlPlaneApp({
+    controlPlane,
+    billing,
+    payments: new FakeTenantBillingGateway(),
+    health,
+    internalSecret: INTERNAL,
+    sessionSecret: 'sess',
+    baseUrl: '',
+    nowMs: () => 2_000_000 // fixed clock so recorded samples fall inside the SLO window
+  })
+  return { app, controlPlane, health }
+}
+
+async function provision(
+  app: ReturnType<typeof fleetApp>['app'],
+  customerRef: string,
+  plan = 'community'
+) {
+  await app.request('/webhook', {
+    method: 'POST',
+    headers: { 'content-type': 'application/json' },
+    body: JSON.stringify({ type: 'checkout.completed', customerRef, plan })
+  })
+}
+
+describe('GET /internal/fleet/health', () => {
+  it('guards behind the internal secret', async () => {
+    const { app } = fleetApp()
+    expect((await app.request('/internal/fleet/health')).status).toBe(403)
+  })
+
+  it('reports per-tenant SLIs + a fleet aggregate', async () => {
+    const { app, controlPlane, health } = fleetApp()
+    await provision(app, 'user_a', 'community')
+    const tenant = await controlPlane.getTenantForBilling('user_a')
+
+    // Record some failing samples for the live tenant → budget should drain.
+    for (let i = 0; i < 20; i++) {
+      health.record(tenant!.tenantId, { ok: i % 2 === 0, latencyMs: 10, atMs: 1000 + i })
+    }
+
+    const res = await app.request('/internal/fleet/health', {
+      headers: { 'x-internal-secret': INTERNAL }
+    })
+    expect(res.status).toBe(200)
+    const body = (await res.json()) as {
+      fleet: { tenantCount: number; freezing: number; worstBudgetRemaining: number }
+      cold: number
+      tenants: { tenantId: string; availability: number; policy: string }[]
+    }
+    expect(body.fleet.tenantCount).toBe(1)
+    expect(body.tenants[0].availability).toBeCloseTo(0.5, 5)
+    expect(body.tenants[0].policy).toBe('freeze')
+    expect(body.fleet.freezing).toBe(1)
+  })
+
+  it('503s when observability is not configured', async () => {
+    const billing = new MemoryBillingIdentityProvider()
+    const { controlPlane } = buildControlPlane({ billing })
+    const app = createControlPlaneApp({ controlPlane, billing, internalSecret: INTERNAL })
+    const res = await app.request('/internal/fleet/health', {
+      headers: { 'x-internal-secret': INTERNAL }
+    })
+    expect(res.status).toBe(503)
+  })
+})

apps/cloud/src/index.ts

@@ -40,6 +40,53 @@ export {
   type DeviceGrantStore,
   type CodeGenerator
 } from './device-grant'
+export {
+  availability,
+  errorRate,
+  latencyPercentile,
+  errorBudgetRemaining,
+  burnRate,
+  backupHealthy,
+  windowed,
+  type HealthSample
+} from './observability/sli'
+export {
+  sloForSla,
+  sloForPlan,
+  errorBudgetMs,
+  budgetPolicy,
+  type SloTarget,
+  type BudgetPolicy
+} from './observability/slo'
+export {
+  HealthSampleStore,
+  FakeHealthProbe,
+  httpHealthProbe,
+  sampleTenantHealth,
+  tenantSli,
+  fleetSummary,
+  type HealthProbe,
+  type TenantSli,
+  type FleetSummary
+} from './observability/health'
+export {
+  rollWave,
+  runRollout,
+  type RolloutEngineDeps,
+  type RolloutPlan,
+  type RolloutReport,
+  type WaveResult,
+  type WaveOptions
+} from './rollout/engine'
+export { controlPlaneRolloutDeps } from './rollout/control-plane-deps'
+export {
+  verifyRestore,
+  runRestoreDrills,
+  pickDrillSample,
+  type RestoreProbe,
+  type RestoreDrillResult
+} from './backup/restore-drill'
+export { reconcileTenant, type ReconcileInput, type ReconcileAction } from './reconcile/reconcile'
 
 /**
  * Pick the billing identity provider from the environment. WorkOS AuthKit (free