Harden the supply chain, fix the purge tool, add log-based alerting

.github/workflows/ci.yml

@@ -16,9 +16,11 @@ concurrency:
 
 env:
   STACK: stacks/observability
-  PROM_IMAGE: prom/prometheus:v3.1.0
-  AM_IMAGE: prom/alertmanager:v0.28.0
-  ALLOY_IMAGE: grafana/alloy:v1.6.1
+  # Image versions are NOT duplicated here. They are resolved from compose.yaml
+  # at run time by scripts/image-for.sh, because Dependabot only updates
+  # compose.yaml — hardcoded copies went stale silently and CI ended up
+  # validating v3.1.0 configs against a stack running v3.13.2.
+  GITLEAKS_IMAGE: zricethezav/gitleaks:v8.24.0
 
 jobs:
   # ---------------------------------------------------------------------------
@@ -58,9 +60,34 @@ jobs:
           cp "$STACK/.env.example" "$STACK/.env"
           echo "GRAFANA_ADMIN_PASSWORD=validation-only" >> "$STACK/.env"
 
+      # Single source of truth: whatever compose.yaml pins is what gets tested.
+      - name: Resolve pinned images from compose.yaml
+        run: |
+          {
+            echo "PROM_IMAGE=$(./scripts/image-for.sh prometheus)"
+            echo "AM_IMAGE=$(./scripts/image-for.sh alertmanager)"
+            echo "ALLOY_IMAGE=$(./scripts/image-for.sh alloy)"
+          } >> "$GITHUB_ENV"
+          ./scripts/image-for.sh prometheus
+          ./scripts/image-for.sh alertmanager
+          ./scripts/image-for.sh alloy
+
       - name: docker compose config
         run: docker compose -f "$STACK/compose.yaml" config -q
 
+      # Guard against the duplication coming back. Any image: pin outside
+      # compose.yaml is drift waiting to happen, since Dependabot cannot see it.
+      - name: Verify image versions are not duplicated outside compose.yaml
+        run: |
+          if grep -rnE '(prom|grafana)/[a-z-]+:v?[0-9]+\.[0-9]+' \
+               --include='*.sh' --include='*.yml' --include='Makefile' \
+               scripts .github Makefile 2>/dev/null \
+             | grep -vE '^[^:]+:[0-9]+:[[:space:]]*#' ; then
+            echo "::error::pinned image version outside compose.yaml — use scripts/image-for.sh"
+            exit 1
+          fi
+          echo "no duplicated image pins"
+
       # Covers every place an image is referenced, not just compose.yaml — the
       # first version of this check only looked at the stack and let :latest
       # through in the workflow itself and in the Makefile.
@@ -76,6 +103,19 @@ jobs:
           fi
           echo "all image references pinned"
 
+      # A tag is a mutable pointer; a digest is the content hash. Every service
+      # image must carry both, so a moved tag cannot change what gets deployed.
+      - name: Verify every image is pinned by digest
+        run: |
+          missing=0
+          while read -r ref; do
+            case "$ref" in
+              *@sha256:*) ;;
+              *) echo "::error::$ref is not pinned by digest — run make pin-digests"; missing=1 ;;
+            esac
+          done < <(awk '$1 == "image:" { print $2 }' "$STACK/compose.yaml")
+          exit "$missing"
+
       - name: promtool check config
         run: |
           docker run --rm --entrypoint promtool \
@@ -98,14 +138,20 @@ jobs:
 
       # `fmt --test` exits non-zero if the file is not canonically formatted, and
       # fails outright on a syntax error. It does not validate that components
-      # are configured correctly — Alloy v1.6.1 has no `validate` subcommand, so
-      # that is only caught at load time on the host.
+      # are configured correctly — Alloy has no `validate` subcommand, so that is
+      # only caught at load time on the host.
       - name: alloy fmt --test
         run: |
           docker run --rm --entrypoint alloy \
             -v "$PWD:/repo" -w /repo "$ALLOY_IMAGE" \
             fmt --test "$STACK/alloy/config.alloy"
 
+      # promtool cannot check these — it parses PromQL and rejects every LogQL
+      # stream selector. Loki itself is the only thing that understands them,
+      # so the checker boots the pinned image with the rules mounted.
+      - name: Validate Loki rules
+        run: ./scripts/check_loki_rules.sh
+
       - name: Validate Grafana dashboards
         run: python3 scripts/check_dashboards.py
 
@@ -137,19 +183,23 @@ jobs:
       - name: gitleaks — working tree
         run: |
           docker run --rm -v "$PWD:/repo" -w /repo \
-            zricethezav/gitleaks:v8.24.0 \
+            "$GITLEAKS_IMAGE" \
             detect --no-git --no-banner --redact -c .gitleaks.toml -v
 
       - name: gitleaks — full history
         run: |
           docker run --rm -v "$PWD:/repo" -w /repo \
-            zricethezav/gitleaks:v8.24.0 \
+            "$GITLEAKS_IMAGE" \
             detect --no-banner --redact -c .gitleaks.toml --log-opts="--all" -v
 
+      # gitleaks cannot be the control here. .purge-secrets.txt is gitignored so
+      # the filesystem scan skips it, and its contents are bare literals with no
+      # keyword context for a rule to match. The control is simply that it must
+      # never be a tracked file.
       - name: Assert no decrypted artefact is tracked
         run: |
           fail=0
-          for pattern in '.env' '.rendered/'; do
+          for pattern in '.env' '.rendered/' '.purge-secrets.txt'; do
             if git ls-files | grep -E "(^|/)${pattern//./\\.}" | grep -v '\.env\.example'; then
               echo "::error::tracked file matching '${pattern}' — it must be gitignored"
               fail=1

.gitignore

@@ -20,6 +20,8 @@ secrets/*
 *.crt
 age.key
 keys.txt
+# Literals fed to scripts/purge-history.sh — never commit these.
+.purge-secrets.txt
 certificates/
 
 # ---- Rendered / decrypted config (produced by scripts/render-config.sh) ----

.gitleaks.toml

@@ -48,9 +48,6 @@ paths = [
   '''secrets/.*\.sops\.yaml$''',
   # Documents required key names and deliberately carries change-me values.
   '''secrets/.*\.example\.yaml$''',
-  # Documents the leak it teaches you to remove.
-  '''docs/runbooks/purge-git-history\.md$''',
-  '''scripts/purge-history\.sh$''',
 ]
 
 regexes = [

Makefile

@@ -96,6 +96,18 @@ check-rules: ## Validate Prometheus rules and config
 	promtool check config $(STACK_DIR)/prometheus/prometheus.yaml
 	promtool check rules $(STACK_DIR)/prometheus/rules/*.rules.yaml
 
+.PHONY: check-loki-rules
+check-loki-rules: ## Validate Loki (LogQL) alerting rules
+	./scripts/check_loki_rules.sh
+
+.PHONY: pin-digests
+pin-digests: ## Re-resolve image digests in compose.yaml (--write applies)
+	./scripts/pin-digests.sh --write
+
+.PHONY: check-digests
+check-digests: ## Verify pinned digests still match the registry
+	./scripts/pin-digests.sh
+
 .PHONY: scan
 scan: ## Scan the working tree and history for secrets
 	gitleaks detect --no-banner --redact -c .gitleaks.toml
@@ -107,13 +119,19 @@ scan: ## Scan the working tree and history for secrets
 
 .PHONY: snmp-generate
 snmp-generate: ## Regenerate snmp.yaml from generator.yaml
+	@# The generator is released in lockstep with snmp-exporter but is not a
+	@# compose service, so its version is derived from the exporter's pin rather
+	@# than duplicated — see scripts/image-for.sh.
+	@# --tag-only: the exporter's digest does not belong to the generator.
+	@gen="$$(./scripts/image-for.sh --tag-only snmp-exporter | sed 's|snmp-exporter|snmp-generator|')"; \
+	printf 'using %s\n' "$$gen"; \
 	docker run --rm \
 		-v "$(PWD)/$(STACK_DIR)/snmp-exporter:/opt/" \
 		-e SNMP_COMMUNITY_PFSENSE='$${SNMP_COMMUNITY_PFSENSE}' \
 		-e SNMP_COMMUNITY_APC='$${SNMP_COMMUNITY_APC}' \
 		-e SNMP_COMMUNITY_MOKERLINK='$${SNMP_COMMUNITY_MOKERLINK}' \
 		-e SNMP_COMMUNITY_ILO='$${SNMP_COMMUNITY_ILO}' \
-		prom/snmp-generator:v0.28.0 generate \
+		"$$gen" generate \
 		-m /opt/mibs -g /opt/generator.yaml -o /opt/snmp.yaml
 	@printf '\033[0;33mCheck the diff before committing — placeholders must survive.\033[0m\n'
 

README.md

@@ -7,15 +7,15 @@
 [![CI](https://github.com/Gerrrt/HomeLab/actions/workflows/ci.yml/badge.svg)](https://github.com/Gerrrt/HomeLab/actions/workflows/ci.yml)
 [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
 [![Secrets: SOPS + age](https://img.shields.io/badge/secrets-SOPS%20%2B%20age-6f42c1.svg)](docs/adr/0005-secrets-with-sops-and-age.md)
-[![Prometheus](https://img.shields.io/badge/Prometheus-v3.1.0-E6522C.svg?logo=prometheus&logoColor=white)](stacks/observability/prometheus)
-[![Grafana](https://img.shields.io/badge/Grafana-11.5-F46800.svg?logo=grafana&logoColor=white)](stacks/observability/grafana)
-[![Loki](https://img.shields.io/badge/Loki-3.3-F5A800.svg?logo=grafana&logoColor=white)](stacks/observability/loki)
+[![Prometheus](https://img.shields.io/badge/Prometheus-E6522C.svg?logo=prometheus&logoColor=white)](stacks/observability/prometheus)
+[![Grafana](https://img.shields.io/badge/Grafana-F46800.svg?logo=grafana&logoColor=white)](stacks/observability/grafana)
+[![Loki](https://img.shields.io/badge/Loki-F5A800.svg?logo=grafana&logoColor=white)](stacks/observability/loki)
 [![pfSense](https://img.shields.io/badge/pfSense-FreeBSD%2015-212121.svg)](docs/network.md)
 
 [Architecture](docs/architecture.md) ·
 [Network](docs/network.md) ·
 [Observability](docs/observability.md) ·
-[Security](docs/security.md) ·
+[Security](SECURITY.md) ·
 [Runbooks](docs/runbooks) ·
 [Decisions](docs/adr) ·
 [Roadmap](docs/roadmap.md)
@@ -42,16 +42,20 @@ incident.
   metrics and logs from Linux hosts; `snmp_exporter` polls the four devices that
   can't run an agent (firewall, switch, UPS, iLO). One agent config, deployed
   identically everywhere. [How](docs/architecture.md#observability-data-flow)
-- **Dashboards and alerting as code.** 5 provisioned dashboards, 79 panels, 32
-  alert rules with severity routing and inhibition. No dashboard exists only in
-  a database.
+- **Dashboards and alerting as code.** 5 provisioned dashboards, 79 panels, and
+  40 alert rules — 32 metric-based in Prometheus, 8 log-based in Loki — sharing
+  one Alertmanager routing tree. No dashboard exists only in a database.
 - **Secrets encrypted in-repo with SOPS + age.** Per-device credentials,
   decrypted at deploy time into gitignored paths, with `git log` showing which
   credential rotated and when — but never to what.
   [Why](docs/adr/0005-secrets-with-sops-and-age.md)
 - **CI that actually validates the infrastructure.** `docker compose config`,
-  `promtool`, `amtool`, `alloy fmt`, dashboard-JSON and datasource checks, every
-  dashboard's PromQL parsed, plus `gitleaks` over the full history.
+  `promtool`, `amtool`, `alloy fmt`, a real Loki boot to parse the LogQL rules,
+  dashboard-JSON and datasource checks, every dashboard's PromQL parsed, plus
+  `gitleaks` over the full history.
+- **Supply chain pinned by digest.** Every image carries both a tag and a
+  `sha256:` digest, so a moved tag cannot change what deploys. CI enforces it;
+  `make pin-digests` re-resolves them from the registry.
 - **Documented decisions and runbooks.** Five ADRs covering what was chosen and
   what was rejected; four runbooks for the operations that are easy to get wrong
   at 1am.
@@ -111,7 +115,7 @@ the internet and nothing more. Full topology and data flow in
 | Alerting | Alertmanager | Severity routing, inhibition |
 | Visualisation | Grafana | 5 provisioned dashboards |
 | Secrets | SOPS + age | Encrypted in-repo |
-| CI | GitHub Actions | Lint, config validation, secret scanning |
+| CI | GitHub Actions | Lint, config validation, secret scanning, digest pinning |
 
 ## Repository layout
 
@@ -121,12 +125,13 @@ the internet and nothing more. Full topology and data flow in
 │   ├── compose.yaml
 │   ├── prometheus/           # config, file_sd targets, 32 alert rules
 │   ├── alertmanager/         # routing and inhibition
-│   ├── loki/                 # single-binary config
+│   ├── loki/                 # single-binary config + 8 LogQL rules
 │   ├── alloy/                # one agent config, used on every host
 │   ├── snmp-exporter/        # generator.yaml is the source of truth
 │   └── grafana/              # provisioning + 5 dashboards
 ├── secrets/                  # SOPS-encrypted; see secrets/README.md
-├── scripts/                  # bootstrap, render, validate, history purge
+├── scripts/                  # bootstrap, render, validate, pin-digests, purge
+├── SECURITY.md               # disclosure policy and known exposure
 ├── docs/
 │   ├── architecture.md  network.md  hardware.md
 │   ├── observability.md  security.md  roadmap.md
@@ -184,7 +189,12 @@ owner-linked device names, camera placement) are in
 
 Historical credential exposure in this repository's git history is documented
 there too, along with the runbooks to remediate it — including the parts not yet
-done.
+done. [`SECURITY.md`](SECURITY.md) carries the disclosure policy and a summary of
+what is known.
+
+Container images are pinned by **tag and digest**. A tag is a mutable pointer; a
+digest is the content hash, so a moved tag cannot change what gets deployed. CI
+enforces it, and `make pin-digests` re-resolves them.
 
 ## Roadmap
 

SECURITY.md

@@ -0,0 +1,82 @@
+# Security policy
+
+This repository documents and configures a private home network. It is not a
+product and has no users other than its owner, so "supported versions" does not
+really apply — `main` is the only branch that means anything, and it is what the
+lab runs.
+
+What *is* useful here is a clear answer to two questions: what to do if you spot
+a problem, and what is already known.
+
+## Reporting something
+
+If you find a misconfiguration, a leaked credential, or a weakness in what is
+published here, please report it privately rather than opening a public issue:
+
+- **GitHub Security Advisories** — [open a draft advisory](https://github.com/Gerrrt/HomeLab/security/advisories/new)
+  (preferred; it stays private until fixed)
+- Failing that, a GitHub issue *without* details, asking for a contact.
+
+Please do not open a public issue containing a working credential, a capture, or
+anything that would let someone else reach the network before it can be fixed.
+
+This is a personal project, so there is no SLA. Realistically: acknowledgement
+within a few days, and credential exposure treated as urgent.
+
+### Please don't
+
+The lab is a home network, not a bug bounty target. Scanning, probing or
+attempting to reach any host described in `docs/network.md` is unwelcome and not
+authorised. Everything worth reviewing is in this repository — review the
+configuration, not the running system.
+
+## Known exposure
+
+Documented rather than quietly fixed, because a known and written-down exposure
+is a very different thing from an overlooked one. Full detail in
+[`docs/security.md`](docs/security.md).
+
+| What | Status |
+| --- | --- |
+| SNMP community committed in plaintext, shared across firewall, switch, UPS and BMC | Removed from `HEAD` and replaced with per-device SOPS-encrypted values. **Still present in git history, and not yet rotated on the devices.** Treat it as public. |
+| Grafana `admin`/`admin` with anonymous Admin access enabled | Fixed — anonymous auth off, password from SOPS |
+| Passphrase-encrypted TLS private keys under `certificates/` | Removed from `HEAD`, still reachable in history. Purge tooling and a runbook are provided; not yet run. |
+
+Remediation is tracked in [`docs/roadmap.md`](docs/roadmap.md), with procedures
+in [`docs/runbooks/rotate-snmp-community.md`](docs/runbooks/rotate-snmp-community.md)
+and [`docs/runbooks/purge-git-history.md`](docs/runbooks/purge-git-history.md).
+
+[`.gitleaksignore`](.gitleaksignore) enumerates all nine historical findings
+individually, with a note on each. It exists so the full-history scan stays
+meaningful — a job that is permanently red for a known reason gets ignored, and
+then a genuinely new leak goes unnoticed alongside it. It is an acknowledgement,
+not a fix, and it gets deleted once the purge has run.
+
+## What this repository will not contain
+
+Deliberate omissions, so their absence is not mistaken for an oversight:
+
+- **Full MAC addresses.** Truncated to the vendor OUI, which keeps the useful
+  half and drops the unique identifier.
+- **Owner-linked device names**, and no room labelled as a child's.
+- **Camera-to-room mapping.** That there are cameras is fine; which one covers
+  which door is not.
+- **The WAN address, firewall rule bodies, and Wi-Fi configuration.**
+- **Any plaintext credential.** Secrets are SOPS + age encrypted; the private key
+  never enters the repository. See [`secrets/README.md`](secrets/README.md).
+
+## Controls in CI
+
+Every push and pull request runs:
+
+- **`gitleaks`** over the working tree *and* full history, with rules for SNMP
+  communities, inline Grafana passwords, PEM private keys and age secret keys.
+- An assertion that every `secrets/*.sops.yaml` is genuinely encrypted, which
+  needs no ability to decrypt.
+- An assertion that no rendered or decrypted artefact — `.env`, `.rendered/`,
+  `.purge-secrets.txt` — is ever a tracked file.
+- Verification that every container image is pinned by **tag *and* digest**, so a
+  moved tag cannot silently change what is deployed.
+
+See [`docs/security.md`](docs/security.md) for the threat model and segmentation
+rationale.

docs/observability.md

@@ -70,6 +70,35 @@ expression in every panel is syntactically valid.
 
 ## Alerting
 
+40 rules in total: 32 metric-based in `prometheus/rules/`, and 8 log-based in
+`loki/rules/`.
+
+### Log-based (Loki ruler)
+
+Some conditions only exist in logs. A metric confirms sshd is running; only the
+log shows it rejecting forty passwords in five minutes. `loki/rules/security.rules.yaml`
+covers SSH brute force, SSH accepted from outside VLAN 50/99, repeated sudo
+failures, user/group creation, kernel OOM kills, read-only remounts and disk I/O
+errors.
+
+They use the same `severity` and `category` labels as the Prometheus rules and
+are sent to the same Alertmanager, so routing and inhibition are shared.
+
+Loki's local ruler reads `<directory>/<tenant>/`, and with `auth_enabled: false`
+the tenant is literally `fake` — hence the `loki/rules:/etc/loki/rules/fake`
+mount in `compose.yaml`. Getting that path wrong produces no error, just a ruler
+that silently evaluates nothing.
+
+`promtool` cannot validate these; it parses PromQL and rejects every LogQL
+stream selector. `scripts/check_loki_rules.sh` boots the pinned Loki image with
+the rules mounted and fails on a parse error, then asserts the ruler actually
+evaluated them. Note that `loki -verify-config` is *not* sufficient on its own —
+it validates the config file and never opens the rule files. A file containing
+`count_over_time({{{BROKEN` passes `-verify-config` and is caught only by the
+boot check.
+
+### Metric-based (Prometheus)
+
 32 rules across four files in `prometheus/rules/`:
 
 | File | Covers |