feat(observability): surface ops-anomaly signals as metrics/alerts/panels

grafana/dashboards/gittensory.json

@@ -3094,6 +3094,94 @@
           "legendFormat": "clamped"
         }
       ]
+    },
+    {
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 213 },
+      "id": 162,
+      "title": "Reviews & Ops Anomalies (#ops-anomaly-metric)",
+      "type": "row"
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "custom": { "lineWidth": 2, "fillOpacity": 10 },
+          "unit": "short"
+        }
+      },
+      "gridPos": { "h": 8, "w": 8, "x": 0, "y": 214 },
+      "id": 163,
+      "options": {
+        "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "title": "Ops Anomaly Detections (review burst / review failure burst)",
+      "description": "runOpsAlerts' hourly scan over gittensory's own outcome data, labeled by kind (review_burst / review_failure_burst) and repo. Any nonzero value here means the scan flagged a stuck-CI finalize loop or retry storm -- see the ops_anomaly structured log for the human-readable detail line.",
+      "type": "timeseries",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+          "expr": "sum by (kind, repo) (increase(gittensory_ops_anomaly_total[1h])) or vector(0)",
+          "legendFormat": "{{kind}} {{repo}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "custom": { "lineWidth": 2, "fillOpacity": 10 },
+          "unit": "ops"
+        }
+      },
+      "gridPos": { "h": 8, "w": 8, "x": 8, "y": 214 },
+      "id": 164,
+      "options": {
+        "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "title": "Published Review Comments (rate)",
+      "description": "Published review comments/summaries per repo -- the gittensory review pipeline's actual output rate.",
+      "type": "timeseries",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+          "expr": "sum by (repo) (rate(gittensory_reviews_published_total[5m])) or vector(0)",
+          "legendFormat": "{{repo}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "custom": { "lineWidth": 2, "fillOpacity": 10 },
+          "unit": "ops"
+        }
+      },
+      "gridPos": { "h": 8, "w": 8, "x": 16, "y": 214 },
+      "id": 165,
+      "options": {
+        "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "title": "Gate Decisions by Conclusion (rate)",
+      "description": "Rendered gate verdict rate by conclusion (merge/close/hold) -- the \"are we rubber-stamping?\" signal (#reviews-dashboard).",
+      "type": "timeseries",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+          "expr": "sum by (conclusion) (rate(gittensory_gate_decisions_total[5m])) or vector(0)",
+          "legendFormat": "{{conclusion}}",
+          "refId": "A"
+        }
+      ]
     }
   ],
   "refresh": "30s",
@@ -3118,5 +3206,5 @@
   "timezone": "browser",
   "title": "Gittensory Self-Host",
   "uid": "gittensory-selfhost",
-  "version": 8
+  "version": 9
 }

prometheus/rules/alerts.yml

@@ -503,6 +503,25 @@ groups:
           description: "Provider {{ $labels.provider }} has failed repeatedly and its circuit breaker is skipping calls fast during its cooldown (sustained 5m)."
           runbook: "Check that provider's credentials/reachability (CLI auth for claude-code/codex, or the configured API key/base URL for HTTP providers) via gittensory_ai_provider_failures_total{provider=\"...\"} and recent selfhost_ai_provider_failed logs."
 
+  # ── Ops anomaly scan (review burst / review failure burst, #ops-anomaly-metric) ────
+  - name: gittensory-ops-anomalies
+    rules:
+      - alert: GittensoryOpsAnomalyDetected
+        # runOpsAlerts (src/review/ops-wire.ts) scans gittensory's own outcome data hourly and increments this
+        # counter, labeled by kind, when it catches a review burst (a stuck-CI finalize loop or sweep retry
+        # storm re-publishing the same PR far more than normal iteration ever does) or a review failure burst
+        # (repeated inconclusive AI-review calls with zero successful publish -- the #3747 incident shape).
+        # Same absolute-increase style as GittensoryDeadLetterJobsGrowing: any occurrence over the scan's own
+        # 2h detection window is worth a look, not a rate/ratio.
+        expr: increase(gittensory_ops_anomaly_total[2h]) > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "gittensory ops anomaly scan flagged a {{ $labels.kind }} on {{ $labels.repo }}"
+          description: "{{ $value | printf \"%.0f\" }} {{ $labels.kind }} detection(s) over the last 2h for {{ $labels.repo }} (sustained 5m). Check the ops_anomaly structured log for the full detail line."
+          runbook: "Tail logs for level=error event=ops_anomaly repo={{ $labels.repo }} for the human-readable anomaly text. A review_burst or review_failure_burst usually means a stuck-CI finalize loop or a sweep retry storm -- see #orb-ci-stuck-repeat / #review-burst-blind-spot."
+
   # ── Host clock sync (#3811) ───────────────────────────────────────────────
   - name: gittensory-system-health
     rules:

src/review/ops-wire.ts

@@ -31,6 +31,7 @@
 // design. This module is READ-ONLY observability: it reports drift; it never changes what blocks a live PR.
 
 import { findHottestInconclusiveReviewTargetForRepo, findHottestReviewTargetForRepo, listRepositories, sumByokAiUsageForRepoSince } from "../db/repositories";
+import { incr } from "../selfhost/metrics";
 import { isAgentConfigured } from "../settings/autonomy";
 import { resolveRepositorySettings } from "../settings/repository-settings";
 import { loadGatePrecisionReport, type GatePrecisionReport } from "../services/gate-precision";
@@ -191,6 +192,16 @@ export async function runOpsAlerts(env: Env): Promise<Record<string, string[]>>
         // Structured log = gittensory's notify path (no Discord/operator webhook exists) AND the Sentry path
         // (level:"error" + an `event` field reaches forwardStructuredLogToSentry). One line per repo.
         console.error(JSON.stringify({ level: "error", event: "ops_anomaly", repo: repoFullName, at: nowIso(), anomalies }));
+        // #ops-anomaly-metric: Prometheus counterpart to the log line above so a self-host operator can alert on
+        // /metrics instead of grepping Workers Logs. Scoped to reviewBurst/reviewFailureBurst -- the two anomalies
+        // this module exists to catch fast (#orb-ci-stuck-repeat / #review-burst-blind-spot) -- rather than every
+        // anomaly kind, so the counter stays a precise "stuck-CI/retry-storm" signal, not a catch-all.
+        if (reviewBurst && reviewBurst.count >= REVIEW_BURST_THRESHOLD) {
+          incr("gittensory_ops_anomaly_total", { repo: repoFullName, kind: "review_burst" });
+        }
+        if (reviewFailureBurst && reviewFailureBurst.count >= REVIEW_FAILURE_BURST_THRESHOLD) {
+          incr("gittensory_ops_anomaly_total", { repo: repoFullName, kind: "review_failure_burst" });
+        }
       } catch (error) {
         console.error(JSON.stringify({ level: "error", event: "ops_anomaly_repo_error", repo: repoFullName, message: errorMessage(error).slice(0, 200) }));
       }

src/selfhost/metrics.ts

@@ -133,6 +133,7 @@ const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [
   ["gittensory_github_pr_files_fetch_total", { help: "GitHub pull-request file fetch attempts.", type: "counter" }],
   ["gittensory_pr_state_cache_total", { help: "Pull-request state cache outcomes.", type: "counter" }],
   ["gittensory_ci_state_cache_total", { help: "CI-state snapshot cache outcomes.", type: "counter" }],
+  ["gittensory_ops_anomaly_total", { help: "Ops anomaly scan detections (review burst / review failure burst), by repo and kind.", type: "counter" }],
 ];
 const metricMeta = new Map<string, MetricMeta>(DEFAULT_METRIC_META);
 
@@ -151,6 +152,7 @@ export function setSelfHostedMetricsMode(isSelfHosted: boolean): void {
 const PRIVATE_REPO_LABEL_METRICS = new Set([
   "gittensory_gate_decisions_total",
   "gittensory_reviews_published_total",
+  "gittensory_ops_anomaly_total",
 ]);
 const ALWAYS_REDACT_REPO_LABEL_METRICS = new Set([
   "gittensory_agent_disposition_total",

test/unit/ops-wire.test.ts

@@ -8,6 +8,7 @@ import {
   type RepoOutcomeSnapshot,
   runOpsAlerts,
 } from "../../src/review/ops-wire";
+import { counterValue, resetMetrics, setSelfHostedMetricsMode } from "../../src/selfhost/metrics";
 import { createTestEnv } from "../helpers/d1";
 
 // Wrap env.DB.prepare so any SQL matching `pattern` throws, exercising a fail-safe catch; every other
@@ -187,7 +188,11 @@ async function seedGateFalsePositiveAnomaly(env: Env, repoFullName: string): Pro
 }
 
 describe("runOpsAlerts — cron path over gittensory's outcome data", () => {
-  afterEach(() => vi.restoreAllMocks());
+  afterEach(() => {
+    vi.restoreAllMocks();
+    resetMetrics();
+    setSelfHostedMetricsMode(false);
+  });
 
   it("emits a structured ops_anomaly log naming the repo + drift on a seeded anomaly, at error level (#orb-ci-stuck-repeat -- so it reaches Sentry)", async () => {
     const env = createTestEnv();
@@ -224,6 +229,7 @@ describe("runOpsAlerts — cron path over gittensory's outcome data", () => {
   });
 
   it("detects and reports a review burst end-to-end (a PR published far more review surfaces than normal in the window)", async () => {
+    setSelfHostedMetricsMode(true); // keep the repo label so the counter assertion can target the exact series
     const env = createTestEnv();
     await seedRegisteredRepo(env, "owner/repo");
     for (let i = 0; i < 7; i += 1) {
@@ -242,6 +248,9 @@ describe("runOpsAlerts — cron path over gittensory's outcome data", () => {
     const stats = await computeOpsStats(env);
     const row = stats.repos.find((r) => r.repoFullName === "owner/repo");
     expect(row?.anomalies.some((a) => /review burst/.test(a))).toBe(true);
+    // #ops-anomaly-metric: the Prometheus counterpart to the log line, labeled by kind (self-host mode preserves
+    // the repo label so the assertion can target the exact series without relying on cloud-worker redaction).
+    expect(counterValue("gittensory_ops_anomaly_total", { repo: "owner/repo", kind: "review_burst" })).toBe(1);
   });
 
   it("detects and reports a review FAILURE burst end-to-end -- reproduces the #3747 incident shape (repeated inconclusive calls, zero publishes) (#review-burst-blind-spot)", async () => {

test/unit/selfhost-metrics.test.ts

@@ -120,6 +120,15 @@ describe("metrics registry (#982)", () => {
     expect(await renderMetrics()).toContain('gittensory_gate_decisions_total{conclusion="hold"} 1');
   });
 
+  it("redacts the repository label from the ops anomaly counter but keeps the kind label (#ops-anomaly-metric)", async () => {
+    incr("gittensory_ops_anomaly_total", { repo: "private-owner/secret-repo", kind: "review_burst" });
+
+    const out = await renderMetrics();
+    expect(out).toContain('gittensory_ops_anomaly_total{kind="review_burst"} 1');
+    expect(out).not.toContain("private-owner/secret-repo");
+    expect(out).not.toContain('repo="');
+  });
+
   it("preserves repository labels for unrelated metrics", async () => {
     incr("debug_total", { repo: "public-owner/public-repo" });
     expect(await renderMetrics()).toContain('debug_total{repo="public-owner/public-repo"} 1');
@@ -132,10 +141,12 @@ describe("metrics registry (#982)", () => {
     setSelfHostedMetricsMode(true);
     incr("gittensory_gate_decisions_total", { repo: "owner/repo", conclusion: "success" });
     incr("gittensory_reviews_published_total", { repo: "owner/repo" });
+    incr("gittensory_ops_anomaly_total", { repo: "owner/repo", kind: "review_burst" });
 
     const out = await renderMetrics();
     expect(out).toContain('gittensory_gate_decisions_total{conclusion="success",repo="owner/repo"} 1');
     expect(out).toContain('gittensory_reviews_published_total{repo="owner/repo"} 1');
+    expect(out).toContain('gittensory_ops_anomaly_total{kind="review_burst",repo="owner/repo"} 1');
     expect(out).toContain('repo="owner/repo"');
   });