From 96edf920f1a157e70e774c02cead2f85e0cdb17a Mon Sep 17 00:00:00 2001
From: JSONbored <49853598+JSONbored@users.noreply.github.com>
Date: Sun, 12 Jul 2026 08:37:17 -0700
Subject: [PATCH] feat(observability): host/container/Redis/Qdrant metrics +
 stack self-monitoring

Adds node-exporter (host CPU/mem/disk/network), cAdvisor (per-container
resource usage, no Docker socket access), and redis_exporter, wires Qdrant's
native /metrics, and scrapes every observability-stack component's own
/metrics (Prometheus/Alertmanager/Loki/Tempo/Grafana/OTEL collector) so a
degrading piece of the stack itself is no longer invisible. Ships a new
Grafana dashboard for all of it.

Closes #5366
---
 .env.example                                  |   3 +
 docker-compose.yml                            |  73 +++++
 grafana/dashboards/infra-health.json          | 297 ++++++++++++++++++
 grafana/dashboards/resource-hub.json          |   2 +-
 otel/otel-collector-config.yml                |  10 +
 prometheus/prometheus.yml                     |  45 +++
 .../selfhost-compose-resource-limits.test.ts  |   6 +
 ...ost-grafana-infra-health-dashboard.test.ts | 110 +++++++
 .../selfhost-observability-config.test.ts     |  98 ++++++
 9 files changed, 643 insertions(+), 1 deletion(-)
 create mode 100644 grafana/dashboards/infra-health.json
 create mode 100644 test/unit/selfhost-grafana-infra-health-dashboard.test.ts

diff --git a/.env.example b/.env.example
index 844b48eedd..9d66eb8fbb 100644
--- a/.env.example
+++ b/.env.example
@@ -344,6 +344,9 @@ REDIS_URL=redis://redis:6379                # REQUIRED for the self-host review
 # DOCKER_PROXY_MEM_LIMIT=64m                  # --profile observability
 # PROMTAIL_MEM_LIMIT=256m                     # --profile observability
 # OTEL_COLLECTOR_MEM_LIMIT=512m               # --profile observability
+# NODE_EXPORTER_MEM_LIMIT=128m                # --profile observability (#5366)
+# CADVISOR_MEM_LIMIT=256m                     # --profile observability (#5366)
+# REDIS_EXPORTER_MEM_LIMIT=64m                # --profile observability (#5366)
 
 # --- Continuous backup (optional; the Litestream sidecar in docker-compose.yml) ---
 # Blank is valid until --profile litestream is enabled.
diff --git a/docker-compose.yml b/docker-compose.yml
index 7e61da7418..128af1d23a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -254,6 +254,27 @@ services:
       interval: 10s
       retries: 5
 
+  # Redis has no native Prometheus endpoint (unlike Postgres/Qdrant) -- the official sidecar exporter is the
+  # standard way to get visibility into an always-on, required backend that otherwise has zero metrics today.
+  # Gated on --profile observability (not its own profile): Redis itself always runs, but nothing needs its
+  # metrics unless the observability stack is up.
+  redis-exporter:
+    image: oliver006/redis_exporter:v1.79.0
+    restart: unless-stopped
+    <<: *default-logging
+    profiles: ["observability"]
+    depends_on:
+      redis:
+        condition: service_healthy
+    environment:
+      REDIS_ADDR: "redis://redis:6379"
+    expose:
+      - "9121"
+    deploy:
+      resources:
+        limits:
+          memory: "${REDIS_EXPORTER_MEM_LIMIT:-64m}"
+
   # ── Postgres (--profile postgres | --profile pgbouncer) ───────────────────
   postgres:
     image: pgvector/pgvector:pg16
@@ -434,6 +455,58 @@ services:
     expose:
       - "9835"
 
+  # ── Host + container resource metrics (--profile observability) ────────────
+  # node-exporter: host-level CPU/memory/disk/network -- previously invisible in Grafana (docker stats-only).
+  # Read-only bind mounts of the HOST's own /proc, /sys, / -- the standard, widely-used node-exporter Docker
+  # recipe. Deliberately NOT network_mode: host or privileged: true (this repo's established posture favors
+  # isolated bridge networking over broader host access; a bind-mounted host /proc/net/dev already reflects
+  # the real host network interfaces without needing host networking).
+  node-exporter:
+    image: prom/node-exporter:v1.9.1
+    restart: unless-stopped
+    <<: *default-logging
+    profiles: ["observability"]
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - "--path.procfs=/host/proc"
+      - "--path.sysfs=/host/sys"
+      - "--path.rootfs=/rootfs"
+      - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)"
+    expose:
+      - "9100"
+    deploy:
+      resources:
+        limits:
+          memory: "${NODE_EXPORTER_MEM_LIMIT:-128m}"
+
+  # cAdvisor: per-container CPU/memory/network/disk-io -- which container is actually using the resources it's
+  # been given. Deliberately does NOT mount /var/run/docker.sock: this repo's own docker-proxy service exists
+  # specifically because even a READ-ONLY docker.sock bind mount doesn't limit what Docker API calls a
+  # container can make (the read-only flag only applies to the socket's filesystem entry, not the API surface
+  # behind it) -- see the docker-proxy service's own comment below for the same reasoning. Running cAdvisor
+  # on cgroupfs/rootfs alone (no Docker API access) gives real per-container resource metrics at the cost of
+  # friendlier labels (containers show by cgroup path/id rather than their docker-compose service name). A
+  # self-hoster who wants richer labels can add a docker.sock mount via their own docker-compose.override.yml.
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:v0.49.2
+    restart: unless-stopped
+    <<: *default-logging
+    profiles: ["observability"]
+    volumes:
+      - /:/rootfs:ro
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+      - /dev/disk/:/dev/disk:ro
+    expose:
+      - "8080"
+    deploy:
+      resources:
+        limits:
+          memory: "${CADVISOR_MEM_LIMIT:-256m}"
+
   # ── Browserless / visual review (--profile visual-review) ──────────────────
   # Headless Chromium for automated before/after PR screenshot capture (src/review/visual/**).
   # Set BOTH of these in .env (browserless always requires a token; there is no way around setting
diff --git a/grafana/dashboards/infra-health.json b/grafana/dashboards/infra-health.json
new file mode 100644
index 0000000000..9de1da964a
--- /dev/null
+++ b/grafana/dashboards/infra-health.json
@@ -0,0 +1,297 @@
+{
+  "__inputs": [],
+  "__requires": [
+    { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "10.0.0" },
+    { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }
+  ],
+  "annotations": { "list": [] },
+  "editable": false,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "description": "Host CPU/mem/disk/network (node-exporter), per-container resource usage (cAdvisor, no Docker socket access — containers show by cgroup ID, not compose service name), Redis health (redis_exporter), Qdrant (--profile qdrant), and whether the observability stack itself (Prometheus/Alertmanager/Loki/Tempo/Grafana/OTEL collector) is up (#5366).",
+  "panels": [
+    { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 1, "title": "Stack self-health", "type": "row" },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "mappings": [
+            { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
+          ],
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] },
+          "noValue": "no data"
+        }
+      },
+      "gridPos": { "h": 5, "w": 12, "x": 0, "y": 1 },
+      "id": 2,
+      "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": true }, "textMode": "name_and_value" },
+      "title": "Observability stack — up",
+      "description": "Prometheus/Alertmanager/Loki/Tempo/Grafana/OTEL-collector scraping their own native /metrics — previously watched by nothing, so a component silently degrading had zero visibility.",
+      "type": "stat",
+      "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "up{job=\"observability-stack\"}", "legendFormat": "{{instance}}", "refId": "A" }]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "mappings": [
+            { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
+          ],
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] },
+          "noValue": "no data"
+        }
+      },
+      "gridPos": { "h": 5, "w": 12, "x": 12, "y": 1 },
+      "id": 3,
+      "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": true }, "textMode": "name_and_value" },
+      "title": "Exporters — up",
+      "description": "node-exporter/cAdvisor always run with --profile observability; redis (required, always-on) always shows up; qdrant only shows up with --profile qdrant active — absent otherwise, by design.",
+      "type": "stat",
+      "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "up{job=~\"node-exporter|cadvisor|redis|qdrant\"}", "legendFormat": "{{job}}", "refId": "A" }]
+    },
+
+    { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 }, "id": 10, "title": "Host (node-exporter)", "type": "row" },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "percent", "max": 100, "min": 0 } },
+      "gridPos": { "h": 8, "w": 6, "x": 0, "y": 7 },
+      "id": 11,
+      "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "title": "Host CPU usage",
+      "type": "timeseries",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+          "expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "bytes" } },
+      "gridPos": { "h": 8, "w": 6, "x": 6, "y": 7 },
+      "id": 12,
+      "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "title": "Host memory used vs total",
+      "type": "timeseries",
+      "targets": [
+        { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes", "legendFormat": "used ({{instance}})", "refId": "A" },
+        { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "node_memory_MemTotal_bytes", "legendFormat": "total ({{instance}})", "refId": "B" }
+      ]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "percent", "max": 100, "min": 0 } },
+      "gridPos": { "h": 8, "w": 6, "x": 12, "y": 7 },
+      "id": 13,
+      "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "title": "Host disk usage by filesystem",
+      "description": "Excludes virtual/pseudo filesystems (tmpfs, overlay, squashfs) — real, mounted disks only.",
+      "type": "timeseries",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+          "expr": "100 * (1 - (node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs\"}))",
+          "legendFormat": "{{mountpoint}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "Bps" } },
+      "gridPos": { "h": 8, "w": 6, "x": 18, "y": 7 },
+      "id": 14,
+      "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "title": "Host network I/O",
+      "description": "Excludes the loopback interface.",
+      "type": "timeseries",
+      "targets": [
+        { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(node_network_receive_bytes_total{device!=\"lo\"}[5m])", "legendFormat": "rx {{device}}", "refId": "A" },
+        { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m])", "legendFormat": "tx {{device}}", "refId": "B" }
+      ]
+    },
+
+    { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 }, "id": 20, "title": "Containers (cAdvisor)", "type": "row" },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "percentunit" } },
+      "gridPos": { "h": 8, "w": 8, "x": 0, "y": 16 },
+      "id": 21,
+      "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "title": "Top containers by CPU",
+      "description": "Legend shows a 12-char container ID (docker ps's own default width) via cAdvisor's cgroup path — no docker.sock access, so no friendly service name is available (see the cadvisor service's own comment in docker-compose.yml). Cross-reference the ID with `docker ps` on the host.",
+      "type": "timeseries",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+          "expr": "topk(8, sum by (short_id) (label_replace(rate(container_cpu_usage_seconds_total{id=~\"/system.slice/docker-.+\\\\.scope\", cpu=\"total\"}[5m]), \"short_id\", \"$1\", \"id\", \"/system.slice/docker-(.{12}).*\")))",
+          "legendFormat": "{{short_id}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "bytes" } },
+      "gridPos": { "h": 8, "w": 8, "x": 8, "y": 16 },
+      "id": 22,
+      "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "title": "Top containers by memory",
+      "description": "Same container-ID caveat as the CPU panel to the left.",
+      "type": "timeseries",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+          "expr": "topk(8, label_replace(container_memory_usage_bytes{id=~\"/system.slice/docker-.+\\\\.scope\"}, \"short_id\", \"$1\", \"id\", \"/system.slice/docker-(.{12}).*\"))",
+          "legendFormat": "{{short_id}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "Bps" } },
+      "gridPos": { "h": 8, "w": 8, "x": 16, "y": 16 },
+      "id": 23,
+      "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "title": "Top containers by network I/O",
+      "description": "Same container-ID caveat as the panels to the left.",
+      "type": "timeseries",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+          "expr": "topk(8, sum by (short_id) (label_replace(rate(container_network_receive_bytes_total{id=~\"/system.slice/docker-.+\\\\.scope\"}[5m]), \"short_id\", \"$1\", \"id\", \"/system.slice/docker-(.{12}).*\")) + sum by (short_id) (label_replace(rate(container_network_transmit_bytes_total{id=~\"/system.slice/docker-.+\\\\.scope\"}[5m]), \"short_id\", \"$1\", \"id\", \"/system.slice/docker-(.{12}).*\")))",
+          "legendFormat": "{{short_id}}",
+          "refId": "A"
+        }
+      ]
+    },
+
+    { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, "id": 30, "title": "Redis (redis_exporter)", "type": "row" },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } } },
+      "gridPos": { "h": 6, "w": 4, "x": 0, "y": 25 },
+      "id": 31,
+      "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
+      "title": "Connected clients",
+      "type": "stat",
+      "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "redis_connected_clients", "legendFormat": "clients", "refId": "A" }]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "bytes" } },
+      "gridPos": { "h": 6, "w": 4, "x": 4, "y": 25 },
+      "id": 32,
+      "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
+      "title": "Memory used",
+      "type": "stat",
+      "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "redis_memory_used_bytes", "legendFormat": "used", "refId": "A" }]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "ops" } },
+      "gridPos": { "h": 6, "w": 8, "x": 8, "y": 25 },
+      "id": 33,
+      "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "title": "Commands processed / sec",
+      "type": "timeseries",
+      "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(redis_commands_processed_total[5m])", "legendFormat": "ops/sec", "refId": "A" }]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "percentunit", "max": 1, "min": 0 } },
+      "gridPos": { "h": 6, "w": 8, "x": 16, "y": 25 },
+      "id": 34,
+      "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "title": "Keyspace hit ratio",
+      "description": "hits / (hits + misses) over a 5m window. A cold start or empty cache legitimately dips this near 0 — it's not necessarily a problem on its own.",
+      "type": "timeseries",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+          "expr": "rate(redis_keyspace_hits_total[5m]) / clamp_min(rate(redis_keyspace_hits_total[5m]) + rate(redis_keyspace_misses_total[5m]), 1e-9)",
+          "legendFormat": "hit ratio",
+          "refId": "A"
+        }
+      ]
+    },
+
+    { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 31 }, "id": 40, "title": "Qdrant (--profile qdrant)", "type": "row" },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } } },
+      "gridPos": { "h": 6, "w": 4, "x": 0, "y": 32 },
+      "id": 41,
+      "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
+      "title": "Collections",
+      "type": "stat",
+      "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "collections_total", "legendFormat": "collections", "refId": "A" }]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } } },
+      "gridPos": { "h": 6, "w": 4, "x": 4, "y": 32 },
+      "id": 42,
+      "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
+      "title": "Total vectors",
+      "type": "stat",
+      "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "collections_vector_total", "legendFormat": "vectors", "refId": "A" }]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 } } },
+      "gridPos": { "h": 6, "w": 8, "x": 8, "y": 32 },
+      "id": 43,
+      "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "title": "Points per collection",
+      "type": "timeseries",
+      "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "collection_points", "legendFormat": "{{id}}", "refId": "A" }]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "reqps" } },
+      "gridPos": { "h": 6, "w": 8, "x": 16, "y": 32 },
+      "id": 44,
+      "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "title": "REST request rate by endpoint",
+      "type": "timeseries",
+      "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (endpoint, status) (rate(rest_responses_total[5m]))", "legendFormat": "{{endpoint}} ({{status}})", "refId": "A" }]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "s" } },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 38 },
+      "id": 45,
+      "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "title": "REST request latency (p50 / p95 / p99)",
+      "type": "timeseries",
+      "targets": [
+        { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(rest_responses_duration_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A" },
+        { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(rest_responses_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B" },
+        { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(rest_responses_duration_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C" }
+      ]
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 38,
+  "tags": ["gittensory", "infra", "observability"],
+  "templating": {
+    "list": [
+      { "current": {}, "hide": 0, "includeAll": false, "name": "DS_PROMETHEUS", "options": [], "query": "prometheus", "refresh": 1, "type": "datasource" }
+    ]
+  },
+  "time": { "from": "now-6h", "to": "now" },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Gittensory — Infra Health (Host/Container/Redis/Qdrant)",
+  "uid": "gittensory-infra-health",
+  "version": 1
+}
diff --git a/grafana/dashboards/resource-hub.json b/grafana/dashboards/resource-hub.json
index 1e2183c1e5..ffec2fc46c 100644
--- a/grafana/dashboards/resource-hub.json
+++ b/grafana/dashboards/resource-hub.json
@@ -66,7 +66,7 @@
       "gridPos": { "h": 13, "w": 12, "x": 12, "y": 0 },
       "options": {
         "mode": "markdown",
-        "content": "## 📊 Dashboards\n- **[Upstream PRs & issues (GitHub)](/d/gittensory-github)** — live, accurate census + open-PR triage (GitHub API).\n- **[Reviews & PRs (maintainer)](/d/gittensory-maintainer)** — gittensory's own review activity + reviewed-PR log.\n- **[AI usage](/d/gittensory-ai-usage)** — durable cross-provider ai_usage_events (filterable by provider/feature/model), live Prometheus counters, and Claude Code's own OTEL session telemetry, all in one place.\n- **[Gittensory (infra)](/d/gittensory)** — queue, jobs, HTTP, GitHub API cache/rate limits.\n- **[GPU metrics](/d/gittensory-gpu)** — utilization/VRAM for a self-hosted Ollama GPU box.\n\n## 📈 Metrics & logs\n- **Prometheus** — [targets](http://localhost:9090/targets) · [graph](http://localhost:9090)\n- **Alertmanager** — [alerts](http://localhost:9093)\n- **Loki** — query in [Explore](/explore) (pick the *Loki* datasource), e.g. `{compose_service=\"gittensory\"}`\n- **Sentry** — release/source-map enriched errors. Edit the dashboard link if your project URL differs.\n\n## 🩺 Quick health checks\n| What | Where |\n|---|---|\n| App serving | `GET /ready` → 200 |\n| AI wired | boot log `selfhost_ai_provider` |\n| Embeds wired | boot log `selfhost_embed_provider` |\n| Vectors wired | boot log `selfhost_vectorize` |\n| Token spend | **[AI usage](/d/gittensory-ai-usage)** dashboard |\n\n## 📚 Docs\n- [Maintainer self-hosting](https://gittensory.aethereal.dev/docs/maintainer-self-hosting) — setup, configuration, AI, REES, RAG, operations, and troubleshooting."
+        "content": "## 📊 Dashboards\n- **[Upstream PRs & issues (GitHub)](/d/gittensory-github)** — live, accurate census + open-PR triage (GitHub API).\n- **[Reviews & PRs (maintainer)](/d/gittensory-maintainer)** — gittensory's own review activity + reviewed-PR log.\n- **[AI usage](/d/gittensory-ai-usage)** — durable cross-provider ai_usage_events (filterable by provider/feature/model), live Prometheus counters, and Claude Code's own OTEL session telemetry, all in one place.\n- **[Gittensory (infra)](/d/gittensory)** — queue, jobs, HTTP, GitHub API cache/rate limits.\n- **[GPU metrics](/d/gittensory-gpu)** — utilization/VRAM for a self-hosted Ollama GPU box.\n- **[Infra health](/d/gittensory-infra-health)** — host CPU/mem/disk/network (node-exporter), per-container resource usage (cAdvisor), Redis, Qdrant, and whether the observability stack itself is up.\n\n## 📈 Metrics & logs\n- **Prometheus** — [targets](http://localhost:9090/targets) · [graph](http://localhost:9090)\n- **Alertmanager** — [alerts](http://localhost:9093)\n- **Loki** — query in [Explore](/explore) (pick the *Loki* datasource), e.g. `{compose_service=\"gittensory\"}`\n- **Sentry** — release/source-map enriched errors. Edit the dashboard link if your project URL differs.\n\n## 🩺 Quick health checks\n| What | Where |\n|---|---|\n| App serving | `GET /ready` → 200 |\n| AI wired | boot log `selfhost_ai_provider` |\n| Embeds wired | boot log `selfhost_embed_provider` |\n| Vectors wired | boot log `selfhost_vectorize` |\n| Token spend | **[AI usage](/d/gittensory-ai-usage)** dashboard |\n\n## 📚 Docs\n- [Maintainer self-hosting](https://gittensory.aethereal.dev/docs/maintainer-self-hosting) — setup, configuration, AI, REES, RAG, operations, and troubleshooting."
       }
     }
   ]
diff --git a/otel/otel-collector-config.yml b/otel/otel-collector-config.yml
index 2a4055ef31..80963698df 100644
--- a/otel/otel-collector-config.yml
+++ b/otel/otel-collector-config.yml
@@ -40,6 +40,16 @@ service:
   telemetry:
     logs:
       level: warn
+    # Self-monitoring (#5366): the collector's own internal metrics (spans/metrics received, dropped,
+    # queue size, memory_limiter refusals), exposed on :8888 for prometheus.yml's observability-stack job.
+    # Distinct from the `prometheus` exporter above, which re-exposes forwarded Claude Code data on :8889.
+    metrics:
+      readers:
+        - pull:
+            exporter:
+              prometheus:
+                host: 0.0.0.0
+                port: 8888
   pipelines:
     metrics:
       receivers: [otlp]
diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml
index 44b330c3f3..ed3913a350 100644
--- a/prometheus/prometheus.yml
+++ b/prometheus/prometheus.yml
@@ -53,3 +53,48 @@ scrape_configs:
     static_configs:
       - targets: ["otel-collector:8889"]
     scrape_interval: 15s
+
+  # Host-level CPU/memory/disk/network from node-exporter (--profile observability, #5366). Bind-mounts
+  # the HOST's own /proc, /sys, / read-only — no Docker API access needed for host-level stats.
+  - job_name: node-exporter
+    static_configs:
+      - targets: ["node-exporter:9100"]
+    scrape_interval: 15s
+
+  # Per-container CPU/memory/network/disk-io from cAdvisor (--profile observability, #5366). Runs without
+  # Docker socket access (see the cadvisor service's own comment in docker-compose.yml) — containers are
+  # labeled by cgroup path/id rather than docker-compose service name.
+  - job_name: cadvisor
+    static_configs:
+      - targets: ["cadvisor:8080"]
+    scrape_interval: 15s
+
+  # Redis health (connected clients, memory, ops/sec, hit ratio) from the official sidecar exporter
+  # (--profile observability, #5366). Redis itself is required and always-on; this just adds visibility.
+  - job_name: redis
+    static_configs:
+      - targets: ["redis-exporter:9121"]
+    scrape_interval: 15s
+
+  # Qdrant's own native /metrics (--profile qdrant, #5366). Absent/down unless the Qdrant vector-store
+  # profile is active — same "other profile" gating already used by postgres/gpu above.
+  - job_name: qdrant
+    static_configs:
+      - targets: ["qdrant:6333"]
+    scrape_interval: 30s
+
+  # Self-monitoring: the observability stack watching itself (--profile observability, #5366). Every
+  # component here ships its own native Prometheus /metrics — previously scraped by nothing, so a stack
+  # component silently degrading (e.g. Loki dropping ingested lines, Alertmanager failing to notify) had
+  # zero visibility. otel-collector's own internal telemetry (distinct from :8889's re-exported Claude
+  # Code metrics above) needs service.telemetry.metrics enabled in otel/otel-collector-config.yml.
+  - job_name: observability-stack
+    static_configs:
+      - targets:
+          - "prometheus:9090"
+          - "alertmanager:9093"
+          - "loki:3100"
+          - "tempo:3200"
+          - "grafana:3000"
+          - "otel-collector:8888"
+    scrape_interval: 30s
diff --git a/test/unit/selfhost-compose-resource-limits.test.ts b/test/unit/selfhost-compose-resource-limits.test.ts
index 036ce9c7f2..3e58b169d7 100644
--- a/test/unit/selfhost-compose-resource-limits.test.ts
+++ b/test/unit/selfhost-compose-resource-limits.test.ts
@@ -29,6 +29,9 @@ describe("docker-compose.yml — per-service memory limits (#1828, #2495, #3893)
     "docker-proxy": "${DOCKER_PROXY_MEM_LIMIT:-64m}",
     promtail: "${PROMTAIL_MEM_LIMIT:-256m}",
     "otel-collector": "${OTEL_COLLECTOR_MEM_LIMIT:-512m}",
+    "node-exporter": "${NODE_EXPORTER_MEM_LIMIT:-128m}",
+    cadvisor: "${CADVISOR_MEM_LIMIT:-256m}",
+    "redis-exporter": "${REDIS_EXPORTER_MEM_LIMIT:-64m}",
   };
 
   it("caps the core app and every heavyweight optional service with an operator-overridable memory limit", () => {
@@ -61,6 +64,9 @@ describe("docker-compose.yml — per-service memory limits (#1828, #2495, #3893)
       "DOCKER_PROXY_MEM_LIMIT",
       "PROMTAIL_MEM_LIMIT",
       "OTEL_COLLECTOR_MEM_LIMIT",
+      "NODE_EXPORTER_MEM_LIMIT",
+      "CADVISOR_MEM_LIMIT",
+      "REDIS_EXPORTER_MEM_LIMIT",
     ]) {
       expect(env, key).toContain(key);
     }
diff --git a/test/unit/selfhost-grafana-infra-health-dashboard.test.ts b/test/unit/selfhost-grafana-infra-health-dashboard.test.ts
new file mode 100644
index 0000000000..87a9bf86cf
--- /dev/null
+++ b/test/unit/selfhost-grafana-infra-health-dashboard.test.ts
@@ -0,0 +1,110 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+
+type DashboardTarget = { expr?: string; legendFormat?: string };
+type DashboardPanel = {
+  id?: number;
+  title?: string;
+  type?: string;
+  description?: string;
+  datasource?: { type?: string; uid?: string };
+  targets?: DashboardTarget[];
+};
+type Dashboard = {
+  uid: string;
+  title: string;
+  tags: string[];
+  panels: DashboardPanel[];
+};
+
+const dashboardPath = join(process.cwd(), "grafana/dashboards/infra-health.json");
+
+function readDashboard(): Dashboard {
+  return JSON.parse(readFileSync(dashboardPath, "utf8")) as Dashboard;
+}
+
+function allTargets(dashboard = readDashboard()): DashboardTarget[] {
+  return dashboard.panels.flatMap((panel) => panel.targets ?? []);
+}
+
+describe("Gittensory — Infra Health dashboard (#5366)", () => {
+  it("declares the expected uid/title/tags", () => {
+    const dashboard = readDashboard();
+    expect(dashboard.uid).toBe("gittensory-infra-health");
+    expect(dashboard.title).toBe("Gittensory — Infra Health (Host/Container/Redis/Qdrant)");
+    expect(dashboard.tags).toEqual(["gittensory", "infra", "observability"]);
+  });
+
+  it("every panel target uses the Prometheus datasource variable, never a hardcoded uid", () => {
+    const dashboard = readDashboard();
+    for (const panel of dashboard.panels) {
+      if (panel.type === "row") continue;
+      expect(panel.datasource?.type, panel.title).toBe("prometheus");
+      expect(panel.datasource?.uid, panel.title).toBe("${DS_PROMETHEUS}");
+      for (const target of panel.targets ?? []) {
+        expect(target.expr, panel.title).toBeTruthy();
+      }
+    }
+  });
+
+  it("watches every stack component's own /metrics, not just the app's", () => {
+    const targets = allTargets();
+    expect(targets.some((t) => t.expr === 'up{job="observability-stack"}')).toBe(true);
+    expect(targets.some((t) => t.expr === 'up{job=~"node-exporter|cadvisor|redis|qdrant"}')).toBe(true);
+  });
+
+  it("never assumes Docker socket access for cAdvisor container labeling (matches the docker-compose.yml service's own no-socket posture)", () => {
+    const dashboard = readDashboard();
+    const serialized = JSON.stringify(dashboard);
+    // No panel query relies on a docker-compose-service label -- that label only exists with Docker API
+    // access, which the cadvisor service deliberately doesn't have (see its docker-compose.yml comment).
+    expect(serialized).not.toContain("container_label_com_docker_compose_service");
+
+    const cadvisorPanels = dashboard.panels.filter(
+      (p) => p.type !== "row" && (p.title ?? "").toLowerCase().includes("container"),
+    );
+    expect(cadvisorPanels.length).toBeGreaterThan(0);
+    for (const panel of cadvisorPanels) {
+      // Every container-scoped panel must document the raw-cgroup-ID tradeoff so an operator isn't
+      // surprised by unfriendly labels (see the cadvisor service's own comment in docker-compose.yml).
+      expect(panel.description ?? "", panel.title).toMatch(/container.?id|docker ps/i);
+    }
+  });
+
+  it("shortens raw cAdvisor cgroup IDs to a 12-char docker-ps-style prefix via label_replace, never raw multi-segment cgroup paths as the legend", () => {
+    const targets = allTargets().filter((t) => t.expr?.includes("container_"));
+    expect(targets.length).toBeGreaterThan(0);
+    for (const target of targets) {
+      expect(target.expr).toContain('label_replace(');
+      expect(target.expr).toContain('"short_id"');
+      expect(target.legendFormat).toContain("{{short_id}}");
+    }
+  });
+
+  it("scopes host disk/network panels to real devices, excluding pseudo-filesystems and loopback", () => {
+    const targets = allTargets();
+    const diskTarget = targets.find((t) => t.expr?.includes("node_filesystem_avail_bytes"));
+    expect(diskTarget?.expr).toContain('fstype!~"tmpfs|overlay|squashfs"');
+
+    const netTargets = targets.filter((t) => t.expr?.includes("node_network_"));
+    expect(netTargets.length).toBeGreaterThan(0);
+    for (const target of netTargets) {
+      expect(target.expr).toContain('device!="lo"');
+    }
+  });
+
+  it("guards the Redis hit-ratio panel against a divide-by-zero on a cold cache", () => {
+    const target = allTargets().find((t) => t.expr?.includes("keyspace_hits_total") && t.expr?.includes("/"));
+    expect(target?.expr).toContain("clamp_min(");
+  });
+
+  it("uses the real Qdrant metric names confirmed on the live server (collections_total, collections_vector_total, collection_points, rest_responses_total/duration)", () => {
+    const targets = allTargets();
+    expect(targets.some((t) => t.expr === "collections_total")).toBe(true);
+    expect(targets.some((t) => t.expr === "collections_vector_total")).toBe(true);
+    expect(targets.some((t) => t.expr === "collection_points")).toBe(true);
+    expect(targets.some((t) => t.expr?.includes("rest_responses_total"))).toBe(true);
+    expect(targets.some((t) => t.expr?.includes("rest_responses_duration_seconds_bucket"))).toBe(true);
+  });
+});
diff --git a/test/unit/selfhost-observability-config.test.ts b/test/unit/selfhost-observability-config.test.ts
index 1e18e76639..0829bc5902 100644
--- a/test/unit/selfhost-observability-config.test.ts
+++ b/test/unit/selfhost-observability-config.test.ts
@@ -155,4 +155,102 @@ describe("self-host observability trace config", () => {
       ]),
     );
   });
+
+  it("wires host/container/Redis metrics and stack self-monitoring without Docker socket access (#5366)", () => {
+    const compose = record(readYaml("docker-compose.yml"));
+    const services = record(compose.services);
+    const prometheus = record(readYaml("prometheus/prometheus.yml"));
+    const scrapeConfigs = prometheus.scrape_configs as Array<Record<string, any>>;
+
+    const nodeExporter = record(services["node-exporter"]);
+    expect(nodeExporter.image).toBe("prom/node-exporter:v1.9.1");
+    expect(nodeExporter.profiles).toEqual(["observability"]);
+    expect(nodeExporter.volumes).toEqual(
+      expect.arrayContaining(["/proc:/host/proc:ro", "/sys:/host/sys:ro", "/:/rootfs:ro"]),
+    );
+    expect(nodeExporter.command).toEqual(
+      expect.arrayContaining([
+        "--path.procfs=/host/proc",
+        "--path.sysfs=/host/sys",
+        "--path.rootfs=/rootfs",
+      ]),
+    );
+    expect(nodeExporter.expose).toEqual(["9100"]);
+    // Neither host networking nor a privileged container: read-only host-directory bind mounts alone.
+    expect(nodeExporter.network_mode).toBeUndefined();
+    expect(nodeExporter.privileged).toBeUndefined();
+
+    const cadvisor = record(services.cadvisor);
+    expect(cadvisor.image).toBe("gcr.io/cadvisor/cadvisor:v0.49.2");
+    expect(cadvisor.profiles).toEqual(["observability"]);
+    expect(cadvisor.volumes).toEqual(
+      expect.arrayContaining([
+        "/:/rootfs:ro",
+        "/sys:/sys:ro",
+        "/var/lib/docker/:/var/lib/docker:ro",
+        "/dev/disk/:/dev/disk:ro",
+      ]),
+    );
+    expect(cadvisor.expose).toEqual(["8080"]);
+    // Deliberate: no Docker socket mount, matching this repo's own docker-proxy security posture (a
+    // read-only socket bind mount does not limit the Docker API surface behind it).
+    expect(JSON.stringify(cadvisor)).not.toContain("docker.sock");
+
+    const redisExporter = record(services["redis-exporter"]);
+    expect(redisExporter.image).toBe("oliver006/redis_exporter:v1.79.0");
+    expect(redisExporter.profiles).toEqual(["observability"]);
+    expect(redisExporter.depends_on?.redis).toEqual({ condition: "service_healthy" });
+    expect(redisExporter.environment?.REDIS_ADDR).toBe("redis://redis:6379");
+    expect(redisExporter.expose).toEqual(["9121"]);
+
+    expect(scrapeConfigs).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          job_name: "node-exporter",
+          static_configs: [{ targets: ["node-exporter:9100"] }],
+        }),
+        expect.objectContaining({
+          job_name: "cadvisor",
+          static_configs: [{ targets: ["cadvisor:8080"] }],
+        }),
+        expect.objectContaining({
+          job_name: "redis",
+          static_configs: [{ targets: ["redis-exporter:9121"] }],
+        }),
+        expect.objectContaining({
+          job_name: "qdrant",
+          static_configs: [{ targets: ["qdrant:6333"] }],
+        }),
+        expect.objectContaining({
+          job_name: "observability-stack",
+          static_configs: [
+            {
+              targets: [
+                "prometheus:9090",
+                "alertmanager:9093",
+                "loki:3100",
+                "tempo:3200",
+                "grafana:3000",
+                "otel-collector:8888",
+              ],
+            },
+          ],
+        }),
+      ]),
+    );
+  });
+
+  it("exposes the OTEL collector's own internal telemetry on a port distinct from the re-exported Claude Code metrics (#5366)", () => {
+    const collector = record(readYaml("otel/otel-collector-config.yml"));
+    const readers = record(collector.service).telemetry?.metrics?.readers;
+
+    expect(Array.isArray(readers)).toBe(true);
+    expect(readers[0].pull.exporter.prometheus).toEqual({
+      host: "0.0.0.0",
+      port: 8888,
+    });
+    // 8888 (internal telemetry) must never collide with 8889 (the `prometheus` exporter re-exposing
+    // forwarded Claude Code OTLP data) -- both are scraped, but they carry structurally different metrics.
+    expect(record(collector.exporters).prometheus.endpoint).toBe("0.0.0.0:8889");
+  });
 });