fix(observability): correct GittensoryHighJobFailureRatio alert formula

prometheus/rules/alerts.yml

@@ -39,22 +39,30 @@ groups:
   - name: gittensory-jobs
     rules:
       - alert: GittensoryHighJobFailureRatio
-        # Fraction of processed jobs that failed over the last 10m. The `> 0` guard on
-        # the denominator avoids a 0/0 = NaN (which never fires but is noise in /rules).
-        # 0.10 = 10% of jobs failing. Raise if your workload has expected transient
-        # failures; lower if any failure is a real problem.
+        # Fraction of jobs ATTEMPTED (failed + succeeded) that failed over the last 10m.
+        # gittensory_jobs_processed_total only increments on success, so the denominator
+        # must be failed+processed, not processed alone (#3892) -- dividing by processed
+        # alone computes failed:success and can read as high as 100% at a true 50% failure
+        # rate. The `> 0` guard on the ratio avoids a 0/0 = NaN (which never fires but is
+        # noise in /rules) when no jobs ran at all. 0.10 = 10% of jobs failing. Raise if
+        # your workload has expected transient failures; lower if any failure is a real
+        # problem. Mirrors the Grafana "Job Failure Rate" panel's formula.
         expr: |
           (
             sum(rate(gittensory_jobs_failed_total[10m]))
             /
-            sum(rate(gittensory_jobs_processed_total[10m])) > 0
+            (
+              sum(rate(gittensory_jobs_failed_total[10m]))
+              +
+              sum(rate(gittensory_jobs_processed_total[10m]))
+            ) > 0
           ) > 0.10
         for: 15m
         labels:
           severity: warning
         annotations:
           summary: "gittensory job failure ratio above 10%"
-          description: "{{ $value | humanizePercentage }} of jobs processed in the last 10m failed (sustained 15m). Expected: well under 10%."
+          description: "{{ $value | humanizePercentage }} of jobs attempted in the last 10m failed (sustained 15m). Expected: well under 10%."
           runbook: "Tail logs for level=error job events (e.g. selfhost_cron_error). A spike usually means a bad upstream (GitHub API / AI provider / DB) or a poison payload — check what changed."
 
       - alert: GittensoryDeadLetterJobsGrowing

test/unit/alerts-job-failure-ratio-formula.test.ts

@@ -0,0 +1,46 @@
+import { readFileSync } from "node:fs";
+import { parse as parseYaml } from "yaml";
+import { describe, expect, it } from "vitest";
+
+// Regression test (#3892): GittensoryHighJobFailureRatio's expr used to divide failed-job rate by
+// gittensory_jobs_processed_total alone. That metric only increments on SUCCESS (src/selfhost/pg-queue.ts,
+// src/selfhost/sqlite-queue.ts), so the old expr computed failed:success, not a true failure percentage --
+// at a genuine 50% failure rate it evaluated to 100%. This pins the corrected failed/(failed+processed)
+// shape (matching the Grafana "Job Failure Rate" panel's formula) so the bug can't silently return.
+
+interface AlertRule {
+  alert: string;
+  expr: string;
+}
+interface AlertGroup {
+  name: string;
+  rules: AlertRule[];
+}
+interface AlertsDoc {
+  groups: AlertGroup[];
+}
+
+const alertsDoc = parseYaml(readFileSync("prometheus/rules/alerts.yml", "utf8")) as AlertsDoc;
+
+function findAlert(name: string): AlertRule {
+  for (const group of alertsDoc.groups) {
+    const rule = group.rules.find((r) => r.alert === name);
+    if (rule) return rule;
+  }
+  throw new Error(`alert ${name} not found in prometheus/rules/alerts.yml`);
+}
+
+describe("GittensoryHighJobFailureRatio alert formula (#3892)", () => {
+  const expr = findAlert("GittensoryHighJobFailureRatio").expr;
+  const flat = expr.replace(/\s+/g, " ").trim();
+
+  it("divides failed by (failed + processed), not by processed alone", () => {
+    expect(flat).toMatch(
+      /sum\(rate\(gittensory_jobs_failed_total\[10m\]\)\) \/ \( sum\(rate\(gittensory_jobs_failed_total\[10m\]\)\) \+ sum\(rate\(gittensory_jobs_processed_total\[10m\]\)\) \) > 0/,
+    );
+  });
+
+  it("still guards the ratio comparison against a 0/0 NaN before applying the 10% threshold", () => {
+    expect(flat).toMatch(/\) > 0 \) > 0\.10/);
+  });
+});