JSONbored · loopover-orb · Jul 12, 2026 · Jul 12, 2026
@@ -344,6 +344,9 @@ REDIS_URL=redis://redis:6379                # REQUIRED for the self-host review
 # DOCKER_PROXY_MEM_LIMIT=64m                  # --profile observability
 # PROMTAIL_MEM_LIMIT=256m                     # --profile observability
 # OTEL_COLLECTOR_MEM_LIMIT=512m               # --profile observability
+# NODE_EXPORTER_MEM_LIMIT=128m                # --profile observability (#5366)
+# CADVISOR_MEM_LIMIT=256m                     # --profile observability (#5366)
+# REDIS_EXPORTER_MEM_LIMIT=64m                # --profile observability (#5366)
 
 # --- Continuous backup (optional; the Litestream sidecar in docker-compose.yml) ---
 # Blank is valid until --profile litestream is enabled.

@@ -254,6 +254,27 @@ services:
       interval: 10s
       retries: 5
 
+  # Redis has no native Prometheus endpoint (unlike Postgres/Qdrant) -- the official sidecar exporter is the
+  # standard way to get visibility into an always-on, required backend that otherwise has zero metrics today.
+  # Gated on --profile observability (not its own profile): Redis itself always runs, but nothing needs its
+  # metrics unless the observability stack is up.
+  redis-exporter:
+    image: oliver006/redis_exporter:v1.79.0
+    restart: unless-stopped
+    <<: *default-logging
+    profiles: ["observability"]
+    depends_on:
+      redis:
+        condition: service_healthy
+    environment:
+      REDIS_ADDR: "redis://redis:6379"
+    expose:
+      - "9121"
+    deploy:
+      resources:
+        limits:
+          memory: "${REDIS_EXPORTER_MEM_LIMIT:-64m}"
+
   # ── Postgres (--profile postgres | --profile pgbouncer) ───────────────────
   postgres:
     image: pgvector/pgvector:pg16
@@ -434,6 +455,58 @@ services:
     expose:
       - "9835"
 
+  # ── Host + container resource metrics (--profile observability) ────────────
+  # node-exporter: host-level CPU/memory/disk/network -- previously invisible in Grafana (docker stats-only).
+  # Read-only bind mounts of the HOST's own /proc, /sys, / -- the standard, widely-used node-exporter Docker
+  # recipe. Deliberately NOT network_mode: host or privileged: true (this repo's established posture favors
+  # isolated bridge networking over broader host access; a bind-mounted host /proc/net/dev already reflects
+  # the real host network interfaces without needing host networking).
+  node-exporter:
+    image: prom/node-exporter:v1.9.1
+    restart: unless-stopped
+    <<: *default-logging
+    profiles: ["observability"]
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - "--path.procfs=/host/proc"
+      - "--path.sysfs=/host/sys"
+      - "--path.rootfs=/rootfs"
+      - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)"
+    expose:
+      - "9100"
+    deploy:
+      resources:
+        limits:
+          memory: "${NODE_EXPORTER_MEM_LIMIT:-128m}"
+
+  # cAdvisor: per-container CPU/memory/network/disk-io -- which container is actually using the resources it's
+  # been given. Deliberately does NOT mount /var/run/docker.sock: this repo's own docker-proxy service exists
+  # specifically because even a READ-ONLY docker.sock bind mount doesn't limit what Docker API calls a
+  # container can make (the read-only flag only applies to the socket's filesystem entry, not the API surface
+  # behind it) -- see the docker-proxy service's own comment below for the same reasoning. Running cAdvisor
+  # on cgroupfs/rootfs alone (no Docker API access) gives real per-container resource metrics at the cost of
+  # friendlier labels (containers show by cgroup path/id rather than their docker-compose service name). A
+  # self-hoster who wants richer labels can add a docker.sock mount via their own docker-compose.override.yml.
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:v0.49.2
+    restart: unless-stopped
+    <<: *default-logging
+    profiles: ["observability"]
+    volumes:
+      - /:/rootfs:ro
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+      - /dev/disk/:/dev/disk:ro
+    expose:
+      - "8080"
+    deploy:
+      resources:
+        limits:
+          memory: "${CADVISOR_MEM_LIMIT:-256m}"
+
   # ── Browserless / visual review (--profile visual-review) ──────────────────
   # Headless Chromium for automated before/after PR screenshot capture (src/review/visual/**).
   # Set BOTH of these in .env (browserless always requires a token; there is no way around setting

@@ -66,7 +66,7 @@
       "gridPos": { "h": 13, "w": 12, "x": 12, "y": 0 },
       "options": {
         "mode": "markdown",
-        "content": "## 📊 Dashboards\n- **[Upstream PRs & issues (GitHub)](/d/gittensory-github)** — live, accurate census + open-PR triage (GitHub API).\n- **[Reviews & PRs (maintainer)](/d/gittensory-maintainer)** — gittensory's own review activity + reviewed-PR log.\n- **[AI usage](/d/gittensory-ai-usage)** — durable cross-provider ai_usage_events (filterable by provider/feature/model), live Prometheus counters, and Claude Code's own OTEL session telemetry, all in one place.\n- **[Gittensory (infra)](/d/gittensory)** — queue, jobs, HTTP, GitHub API cache/rate limits.\n- **[GPU metrics](/d/gittensory-gpu)** — utilization/VRAM for a self-hosted Ollama GPU box.\n\n## 📈 Metrics & logs\n- **Prometheus** — [targets](http://localhost:9090/targets) · [graph](http://localhost:9090)\n- **Alertmanager** — [alerts](http://localhost:9093)\n- **Loki** — query in [Explore](/explore) (pick the *Loki* datasource), e.g. `{compose_service=\"gittensory\"}`\n- **Sentry** — release/source-map enriched errors. Edit the dashboard link if your project URL differs.\n\n## 🩺 Quick health checks\n| What | Where |\n|---|---|\n| App serving | `GET /ready` → 200 |\n| AI wired | boot log `selfhost_ai_provider` |\n| Embeds wired | boot log `selfhost_embed_provider` |\n| Vectors wired | boot log `selfhost_vectorize` |\n| Token spend | **[AI usage](/d/gittensory-ai-usage)** dashboard |\n\n## 📚 Docs\n- [Maintainer self-hosting](https://gittensory.aethereal.dev/docs/maintainer-self-hosting) — setup, configuration, AI, REES, RAG, operations, and troubleshooting."
+        "content": "## 📊 Dashboards\n- **[Upstream PRs & issues (GitHub)](/d/gittensory-github)** — live, accurate census + open-PR triage (GitHub API).\n- **[Reviews & PRs (maintainer)](/d/gittensory-maintainer)** — gittensory's own review activity + reviewed-PR log.\n- **[AI usage](/d/gittensory-ai-usage)** — durable cross-provider ai_usage_events (filterable by provider/feature/model), live Prometheus counters, and Claude Code's own OTEL session telemetry, all in one place.\n- **[Gittensory (infra)](/d/gittensory)** — queue, jobs, HTTP, GitHub API cache/rate limits.\n- **[GPU metrics](/d/gittensory-gpu)** — utilization/VRAM for a self-hosted Ollama GPU box.\n- **[Infra health](/d/gittensory-infra-health)** — host CPU/mem/disk/network (node-exporter), per-container resource usage (cAdvisor), Redis, Qdrant, and whether the observability stack itself is up.\n\n## 📈 Metrics & logs\n- **Prometheus** — [targets](http://localhost:9090/targets) · [graph](http://localhost:9090)\n- **Alertmanager** — [alerts](http://localhost:9093)\n- **Loki** — query in [Explore](/explore) (pick the *Loki* datasource), e.g. `{compose_service=\"gittensory\"}`\n- **Sentry** — release/source-map enriched errors. Edit the dashboard link if your project URL differs.\n\n## 🩺 Quick health checks\n| What | Where |\n|---|---|\n| App serving | `GET /ready` → 200 |\n| AI wired | boot log `selfhost_ai_provider` |\n| Embeds wired | boot log `selfhost_embed_provider` |\n| Vectors wired | boot log `selfhost_vectorize` |\n| Token spend | **[AI usage](/d/gittensory-ai-usage)** dashboard |\n\n## 📚 Docs\n- [Maintainer self-hosting](https://gittensory.aethereal.dev/docs/maintainer-self-hosting) — setup, configuration, AI, REES, RAG, operations, and troubleshooting."
       }
     }
   ]

@@ -40,6 +40,16 @@ service:
   telemetry:
     logs:
       level: warn
+    # Self-monitoring (#5366): the collector's own internal metrics (spans/metrics received, dropped,
+    # queue size, memory_limiter refusals), exposed on :8888 for prometheus.yml's observability-stack job.
+    # Distinct from the `prometheus` exporter above, which re-exposes forwarded Claude Code data on :8889.
+    metrics:
+      readers:
+        - pull:
+            exporter:
+              prometheus:
+                host: 0.0.0.0
+                port: 8888
   pipelines:
     metrics:
       receivers: [otlp]

@@ -53,3 +53,48 @@ scrape_configs:
     static_configs:
       - targets: ["otel-collector:8889"]
     scrape_interval: 15s
+
+  # Host-level CPU/memory/disk/network from node-exporter (--profile observability, #5366). Bind-mounts
+  # the HOST's own /proc, /sys, / read-only — no Docker API access needed for host-level stats.
+  - job_name: node-exporter
+    static_configs:
+      - targets: ["node-exporter:9100"]
+    scrape_interval: 15s
+
+  # Per-container CPU/memory/network/disk-io from cAdvisor (--profile observability, #5366). Runs without
+  # Docker socket access (see the cadvisor service's own comment in docker-compose.yml) — containers are
+  # labeled by cgroup path/id rather than docker-compose service name.
+  - job_name: cadvisor
+    static_configs:
+      - targets: ["cadvisor:8080"]
+    scrape_interval: 15s
+
+  # Redis health (connected clients, memory, ops/sec, hit ratio) from the official sidecar exporter
+  # (--profile observability, #5366). Redis itself is required and always-on; this just adds visibility.
+  - job_name: redis
+    static_configs:
+      - targets: ["redis-exporter:9121"]
+    scrape_interval: 15s
+
+  # Qdrant's own native /metrics (--profile qdrant, #5366). Absent/down unless the Qdrant vector-store
+  # profile is active — same "other profile" gating already used by postgres/gpu above.
+  - job_name: qdrant
+    static_configs:
+      - targets: ["qdrant:6333"]
+    scrape_interval: 30s
+
+  # Self-monitoring: the observability stack watching itself (--profile observability, #5366). Every
+  # component here ships its own native Prometheus /metrics — previously scraped by nothing, so a stack
+  # component silently degrading (e.g. Loki dropping ingested lines, Alertmanager failing to notify) had
+  # zero visibility. otel-collector's own internal telemetry (distinct from :8889's re-exported Claude
+  # Code metrics above) needs service.telemetry.metrics enabled in otel/otel-collector-config.yml.
+  - job_name: observability-stack
+    static_configs:
+      - targets:
+          - "prometheus:9090"
+          - "alertmanager:9093"
+          - "loki:3100"
+          - "tempo:3200"
+          - "grafana:3000"
+          - "otel-collector:8888"
+    scrape_interval: 30s
@@ -29,6 +29,9 @@ describe("docker-compose.yml — per-service memory limits (#1828, #2495, #3893)
     "docker-proxy": "${DOCKER_PROXY_MEM_LIMIT:-64m}",
     promtail: "${PROMTAIL_MEM_LIMIT:-256m}",
     "otel-collector": "${OTEL_COLLECTOR_MEM_LIMIT:-512m}",
+    "node-exporter": "${NODE_EXPORTER_MEM_LIMIT:-128m}",
+    cadvisor: "${CADVISOR_MEM_LIMIT:-256m}",
+    "redis-exporter": "${REDIS_EXPORTER_MEM_LIMIT:-64m}",
   };
 
   it("caps the core app and every heavyweight optional service with an operator-overridable memory limit", () => {
@@ -61,6 +64,9 @@ describe("docker-compose.yml — per-service memory limits (#1828, #2495, #3893)
       "DOCKER_PROXY_MEM_LIMIT",
       "PROMTAIL_MEM_LIMIT",
       "OTEL_COLLECTOR_MEM_LIMIT",
+      "NODE_EXPORTER_MEM_LIMIT",
+      "CADVISOR_MEM_LIMIT",
+      "REDIS_EXPORTER_MEM_LIMIT",
     ]) {
       expect(env, key).toContain(key);
     }

@@ -0,0 +1,110 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+
+type DashboardTarget = { expr?: string; legendFormat?: string };
+type DashboardPanel = {
+  id?: number;
+  title?: string;
+  type?: string;
+  description?: string;
+  datasource?: { type?: string; uid?: string };
+  targets?: DashboardTarget[];
+};
+type Dashboard = {
+  uid: string;
+  title: string;
+  tags: string[];
+  panels: DashboardPanel[];
+};
+
+const dashboardPath = join(process.cwd(), "grafana/dashboards/infra-health.json");
+
+function readDashboard(): Dashboard {
+  return JSON.parse(readFileSync(dashboardPath, "utf8")) as Dashboard;
+}
+
+function allTargets(dashboard = readDashboard()): DashboardTarget[] {
+  return dashboard.panels.flatMap((panel) => panel.targets ?? []);
+}
+
+describe("Gittensory — Infra Health dashboard (#5366)", () => {
+  it("declares the expected uid/title/tags", () => {
+    const dashboard = readDashboard();
+    expect(dashboard.uid).toBe("gittensory-infra-health");
+    expect(dashboard.title).toBe("Gittensory — Infra Health (Host/Container/Redis/Qdrant)");
+    expect(dashboard.tags).toEqual(["gittensory", "infra", "observability"]);
+  });
+
+  it("every panel target uses the Prometheus datasource variable, never a hardcoded uid", () => {
+    const dashboard = readDashboard();
+    for (const panel of dashboard.panels) {
+      if (panel.type === "row") continue;
+      expect(panel.datasource?.type, panel.title).toBe("prometheus");
+      expect(panel.datasource?.uid, panel.title).toBe("${DS_PROMETHEUS}");
+      for (const target of panel.targets ?? []) {
+        expect(target.expr, panel.title).toBeTruthy();
+      }
+    }
+  });
+
+  it("watches every stack component's own /metrics, not just the app's", () => {
+    const targets = allTargets();
+    expect(targets.some((t) => t.expr === 'up{job="observability-stack"}')).toBe(true);
+    expect(targets.some((t) => t.expr === 'up{job=~"node-exporter|cadvisor|redis|qdrant"}')).toBe(true);
+  });
+
+  it("never assumes Docker socket access for cAdvisor container labeling (matches the docker-compose.yml service's own no-socket posture)", () => {
+    const dashboard = readDashboard();
+    const serialized = JSON.stringify(dashboard);
+    // No panel query relies on a docker-compose-service label -- that label only exists with Docker API
+    // access, which the cadvisor service deliberately doesn't have (see its docker-compose.yml comment).
+    expect(serialized).not.toContain("container_label_com_docker_compose_service");
+
+    const cadvisorPanels = dashboard.panels.filter(
+      (p) => p.type !== "row" && (p.title ?? "").toLowerCase().includes("container"),
+    );
+    expect(cadvisorPanels.length).toBeGreaterThan(0);
+    for (const panel of cadvisorPanels) {
+      // Every container-scoped panel must document the raw-cgroup-ID tradeoff so an operator isn't
+      // surprised by unfriendly labels (see the cadvisor service's own comment in docker-compose.yml).
+      expect(panel.description ?? "", panel.title).toMatch(/container.?id|docker ps/i);
+    }
+  });
+
+  it("shortens raw cAdvisor cgroup IDs to a 12-char docker-ps-style prefix via label_replace, never raw multi-segment cgroup paths as the legend", () => {
+    const targets = allTargets().filter((t) => t.expr?.includes("container_"));
+    expect(targets.length).toBeGreaterThan(0);
+    for (const target of targets) {
+      expect(target.expr).toContain('label_replace(');
+      expect(target.expr).toContain('"short_id"');
+      expect(target.legendFormat).toContain("{{short_id}}");
+    }
+  });
+
+  it("scopes host disk/network panels to real devices, excluding pseudo-filesystems and loopback", () => {
+    const targets = allTargets();
+    const diskTarget = targets.find((t) => t.expr?.includes("node_filesystem_avail_bytes"));
+    expect(diskTarget?.expr).toContain('fstype!~"tmpfs|overlay|squashfs"');
+
+    const netTargets = targets.filter((t) => t.expr?.includes("node_network_"));
+    expect(netTargets.length).toBeGreaterThan(0);
+    for (const target of netTargets) {
+      expect(target.expr).toContain('device!="lo"');
+    }
+  });
+
+  it("guards the Redis hit-ratio panel against a divide-by-zero on a cold cache", () => {
+    const target = allTargets().find((t) => t.expr?.includes("keyspace_hits_total") && t.expr?.includes("/"));
+    expect(target?.expr).toContain("clamp_min(");
+  });
+
+  it("uses the real Qdrant metric names confirmed on the live server (collections_total, collections_vector_total, collection_points, rest_responses_total/duration)", () => {
+    const targets = allTargets();
+    expect(targets.some((t) => t.expr === "collections_total")).toBe(true);
+    expect(targets.some((t) => t.expr === "collections_vector_total")).toBe(true);
+    expect(targets.some((t) => t.expr === "collection_points")).toBe(true);
+    expect(targets.some((t) => t.expr?.includes("rest_responses_total"))).toBe(true);
+    expect(targets.some((t) => t.expr?.includes("rest_responses_duration_seconds_bucket"))).toBe(true);
+  });
+});