stage1: consolidate items 2-4 and fix webhook notification schema gate

.github/workflows/comment-copilot-ci.yml

@@ -8,24 +8,17 @@ on:
 jobs:
   phase-boundary-check:
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: comment-copilot
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: Setup pnpm
         uses: pnpm/action-setup@v4
-        with:
-          version: 9
 
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
           node-version: 22
-          cache: pnpm
-          cache-dependency-path: comment-copilot/pnpm-lock.yaml
 
       - name: Install dependencies
         run: pnpm install --frozen-lockfile=false
@@ -36,24 +29,17 @@ jobs:
   validate:
     needs: [phase-boundary-check]
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: comment-copilot
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: Setup pnpm
         uses: pnpm/action-setup@v4
-        with:
-          version: 9
 
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
           node-version: 22
-          cache: pnpm
-          cache-dependency-path: comment-copilot/pnpm-lock.yaml
 
       - name: Install dependencies
         run: pnpm install --frozen-lockfile=false
@@ -70,13 +56,13 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: oauth-vitest-junit
-          path: comment-copilot/apps/web/test-results/oauth.junit.xml
+          path: apps/web/test-results/oauth.junit.xml
           if-no-files-found: warn
 
       - name: Upload Webhooks E2E Test Report
         if: always()
         uses: actions/upload-artifact@v4
         with:
           name: webhooks-e2e-vitest-junit
-          path: comment-copilot/apps/web/test-results/webhooks.e2e.junit.xml
+          path: apps/web/test-results/webhooks.e2e.junit.xml
           if-no-files-found: warn

.github/workflows/isolation-check.yml

@@ -14,20 +14,14 @@ jobs:
 
       - name: Setup pnpm
         uses: pnpm/action-setup@v4
-        with:
-          version: 9
 
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
           node-version: 22
-          cache: pnpm
-          cache-dependency-path: comment-copilot/pnpm-lock.yaml
 
       - name: Install dependencies
-        working-directory: comment-copilot
         run: pnpm install --frozen-lockfile=false
 
       - name: Run isolation guard
-        working-directory: comment-copilot
         run: pnpm check:isolation

README.md

@@ -44,6 +44,12 @@ This project is intentionally isolated:
 - Stage 3 boundary (controlled beta): `docs/dev-phase-stage-3-controlled-beta.md`
 - Stage 4 boundary (scale launch): `docs/dev-phase-stage-4-scale-launch.md`
 - Stage evidence records: `docs/ops/stage-1-evidence.md`, `docs/ops/stage-2-evidence.md`, `docs/ops/stage-3-evidence.md`, `docs/ops/stage-4-evidence.md`
+- Webhook replay runbook: `docs/ops/webhook-replay-runbook.md`
+- Provider outage runbook: `docs/ops/provider-outage-runbook.md`
+- Token and billing incident runbook: `docs/ops/token-billing-incident-runbook.md`
+- Incident triage and escalation flow: `docs/ops/incident-triage-escalation-flow.md`
+- Production deploy checklist: `docs/ops/production-deploy-checklist.md`
+- Deploy checklist dry-run evidence (2026-03-04): `docs/ops/deploy-checklist-dry-run-2026-03-04.md`
 
 ## Testing
 

apps/web/app/api/_lib/errorTracking.ts

@@ -0,0 +1,50 @@
+const ERROR_TRACKING_TIMEOUT_MS = 1500;
+
+type ErrorTrackingArgs = {
+  source: string;
+  category: string;
+  message: string;
+  metadata?: Record<string, unknown>;
+};
+
+function getWebhookUrl() {
+  const url = process.env.ERROR_TRACKING_WEBHOOK_URL;
+  if (!url) return undefined;
+  return url.trim() || undefined;
+}
+
+export async function reportErrorTrackingEvent(args: ErrorTrackingArgs) {
+  const webhookUrl = getWebhookUrl();
+  if (!webhookUrl) return;
+
+  const payload = {
+    source: args.source,
+    category: args.category,
+    message: args.message.slice(0, 1000),
+    metadata: args.metadata ?? {},
+    capturedAt: new Date().toISOString(),
+    environment: process.env.NODE_ENV ?? "unknown"
+  };
+
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), ERROR_TRACKING_TIMEOUT_MS);
+
+  try {
+    const token = process.env.ERROR_TRACKING_WEBHOOK_TOKEN;
+
+    await fetch(webhookUrl, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        ...(token ? { authorization: `Bearer ${token}` } : {})
+      },
+      body: JSON.stringify(payload),
+      signal: controller.signal
+    });
+  } catch (error) {
+    const err = error instanceof Error ? error.message : `${error}`;
+    console.error(`[error-tracking] failed to report event: ${err}`);
+  } finally {
+    clearTimeout(timeout);
+  }
+}

apps/web/app/api/_lib/webhookObservability.ts

@@ -0,0 +1,119 @@
+export type WebhookProvider = "instagram" | "tiktok" | "stripe";
+
+export interface WebhookObservabilityContext {
+  provider: WebhookProvider;
+  route: string;
+  method: "GET" | "POST";
+  startedAtMs: number;
+}
+
+const ALERT_ROUTING = {
+  primary: "app-on-call",
+  secondary: "infra-platform-owner",
+  runbook: "docs/ops/incident-triage-escalation-flow.md"
+} as const;
+
+function elapsedMs(startedAtMs: number) {
+  return Math.max(0, Date.now() - startedAtMs);
+}
+
+function emit(level: "info" | "warn", payload: Record<string, unknown>) {
+  const writer = level === "warn" ? console.warn : console.info;
+  writer(
+    JSON.stringify({
+      timestamp: new Date().toISOString(),
+      ...payload
+    })
+  );
+}
+
+function toAlertSeverity(statusCode: number) {
+  if (statusCode >= 500) {
+    return "sev2";
+  }
+  return "sev3";
+}
+
+export function createWebhookObservabilityContext(args: {
+  provider: WebhookProvider;
+  route: string;
+  method: "GET" | "POST";
+}): WebhookObservabilityContext {
+  return {
+    provider: args.provider,
+    route: args.route,
+    method: args.method,
+    startedAtMs: Date.now()
+  };
+}
+
+export function logWebhookCompleted(
+  context: WebhookObservabilityContext,
+  details: {
+    statusCode?: number;
+    accountId?: string;
+    eventType?: string;
+    workflowStarted?: boolean;
+  } = {}
+) {
+  emit("info", {
+    event: "webhook_observability.request_completed",
+    outcome: "success",
+    provider: context.provider,
+    route: context.route,
+    method: context.method,
+    statusCode: details.statusCode ?? 200,
+    durationMs: elapsedMs(context.startedAtMs),
+    accountId: details.accountId,
+    eventType: details.eventType,
+    workflowStarted: details.workflowStarted
+  });
+}
+
+export function logWebhookIgnored(
+  context: WebhookObservabilityContext,
+  details: {
+    statusCode?: number;
+    eventType?: string;
+  } = {}
+) {
+  emit("info", {
+    event: "webhook_observability.request_completed",
+    outcome: "ignored",
+    provider: context.provider,
+    route: context.route,
+    method: context.method,
+    statusCode: details.statusCode ?? 200,
+    durationMs: elapsedMs(context.startedAtMs),
+    eventType: details.eventType
+  });
+}
+
+export function logWebhookFailed(
+  context: WebhookObservabilityContext,
+  details: {
+    statusCode: number;
+    errorCode: string;
+    errorMessage: string;
+    accountId?: string;
+    eventType?: string;
+  }
+) {
+  emit("warn", {
+    event: "webhook_observability.request_completed",
+    outcome: "failure",
+    provider: context.provider,
+    route: context.route,
+    method: context.method,
+    statusCode: details.statusCode,
+    durationMs: elapsedMs(context.startedAtMs),
+    errorCode: details.errorCode,
+    errorMessage: details.errorMessage,
+    accountId: details.accountId,
+    eventType: details.eventType,
+    alertSeverity: toAlertSeverity(details.statusCode),
+    alertRoutePrimary: ALERT_ROUTING.primary,
+    alertRouteSecondary: ALERT_ROUTING.secondary,
+    alertRunbook: ALERT_ROUTING.runbook
+  });
+}

apps/web/app/api/webhooks/instagram/comments/route.ts

@@ -1,5 +1,11 @@
 import { NextRequest, NextResponse } from "next/server";
 import { getConvexServerClient } from "../../../_lib/convexServer";
+import { reportErrorTrackingEvent } from "../../../_lib/errorTracking";
+import {
+  createWebhookObservabilityContext,
+  logWebhookCompleted,
+  logWebhookFailed
+} from "../../../_lib/webhookObservability";
 import { startCommentWorkflow } from "../../../_lib/temporal";
 import { verifyInstagramWebhookSignature } from "../../../_lib/webhookSignatures";
 
@@ -33,6 +39,13 @@ export async function GET(request: NextRequest) {
 }
 
 export async function POST(request: NextRequest) {
+  const observability = createWebhookObservabilityContext({
+    provider: "instagram",
+    route: "/api/webhooks/instagram/comments",
+    method: "POST"
+  });
+  let accountId: string | undefined;
+
   try {
     const rawBody = await request.text();
     const verification = verifyInstagramWebhookSignature({
@@ -43,6 +56,11 @@ export async function POST(request: NextRequest) {
     });
 
     if (!verification.ok) {
+      logWebhookFailed(observability, {
+        statusCode: verification.status,
+        errorCode: "instagram_signature_verification_failed",
+        errorMessage: verification.error
+      });
       return NextResponse.json(
         { ok: false, error: verification.error },
         { status: verification.status }
@@ -61,6 +79,7 @@ export async function POST(request: NextRequest) {
       commenterLatestVideoId?: string;
       commenterLatestVideoTitle?: string;
     };
+    accountId = body.accountId;
     const client = getConvexServerClient();
 
     const ingestion = (await client.mutation(
@@ -80,16 +99,37 @@ export async function POST(request: NextRequest) {
       } as never
     )) as { commentId: string; created?: boolean };
 
-    if (ingestion.created ?? true) {
+    const workflowStarted = ingestion.created ?? true;
+    if (workflowStarted) {
       await startCommentWorkflow({
         accountId: body.accountId,
         commentId: ingestion.commentId
       });
     }
 
+    logWebhookCompleted(observability, {
+      accountId: body.accountId,
+      workflowStarted
+    });
     return NextResponse.json({ ok: true });
   } catch (error) {
     const message = error instanceof Error ? error.message : "Unknown error";
+    logWebhookFailed(observability, {
+      statusCode: 500,
+      errorCode: "instagram_webhook_processing_failed",
+      errorMessage: message,
+      accountId
+    });
+    await reportErrorTrackingEvent({
+      source: "webhook:instagram_comments",
+      category: "webhook_processing_failed",
+      message,
+      metadata: {
+        route: "/api/webhooks/instagram/comments",
+        accountId,
+        statusCode: 500
+      }
+    });
     return NextResponse.json({ ok: false, error: message }, { status: 500 });
   }
 }