feat(ci): validate every stack, not just the estate's (#263)

.github/workflows/ci.yml

@@ -14,14 +14,18 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
-env:
-  STACK: stacks/observability
-  # Image versions are NOT duplicated here. They are resolved from compose.yaml
-  # at run time by scripts/image-for.sh, because Dependabot only updates
-  # compose.yaml — hardcoded copies went stale silently and CI ended up
-  # validating v3.1.0 configs against a stack running v3.13.2. gitleaks was the
-  # last exception to that and is now a profile-gated service in compose.yaml
-  # like every other image (#65).
+# There is no `env: STACK:` here any more. It pinned every step below to
+# stacks/observability, which is why stacks/lab landed as a stack CI had never
+# seen (#263, #264). Each step loops over ./scripts/stacks.sh instead — the one
+# place that defines what a stack is, and the thing that fails when a directory
+# under stacks/ has no compose.yaml.
+#
+# Image versions are NOT duplicated here either. They are resolved from
+# compose.yaml at run time by scripts/image-for.sh, because Dependabot only
+# updates compose.yaml — hardcoded copies went stale silently and CI ended up
+# validating v3.1.0 configs against a stack running v3.13.2. gitleaks was the
+# last exception to that and is now a profile-gated service in compose.yaml
+# like every other image (#65).
 
 jobs:
   # ---------------------------------------------------------------------------
@@ -64,8 +68,14 @@ jobs:
       # needs a decryption key. The list of variables lives in the script, which
       # scripts/validate.sh also calls — inlining it here is what let CI and the
       # local run drift apart. The .env written here stays gitignored.
+      # Per stack, because the guard list is derived from each stack's own
+      # compose.yaml — seeding one from another's guards proves nothing about
+      # the file being validated.
       - name: Seed a validation-only .env
-        run: ./scripts/seed-validation-env.sh "$STACK/.env"
+        run: |
+          for stack in $(./scripts/stacks.sh); do
+            ./scripts/seed-validation-env.sh "stacks/$stack/.env" "$stack"
+          done
 
       # Single source of truth: whatever compose.yaml pins is what gets tested.
       - name: Resolve pinned images from compose.yaml
@@ -80,7 +90,11 @@ jobs:
           ./scripts/image-for.sh alloy
 
       - name: docker compose config
-        run: docker compose -f "$STACK/compose.yaml" config -q
+        run: |
+          for stack in $(./scripts/stacks.sh); do
+            echo "== $stack"
+            docker compose -f "stacks/$stack/compose.yaml" config -q
+          done
 
       # Guard against the duplication coming back. Any image: pin outside
       # compose.yaml is drift waiting to happen, since Dependabot cannot see it.
@@ -125,7 +139,9 @@ jobs:
               *@sha256:*) ;;
               *) echo "::error::$ref is not pinned by digest — run make pin-digests"; missing=1 ;;
             esac
-          done < <(awk '$1 == "image:" { print $2 }' "$STACK/compose.yaml")
+          done < <(./scripts/stacks.sh --paths | while read -r sd; do
+                     awk '$1 == "image:" { print $2 }' "$sd/compose.yaml"
+                   done)
           exit "$missing"
 
       # The three checks above are all pattern matches, and #65 walked past all
@@ -141,33 +157,57 @@ jobs:
       - name: Verify every docker image comes from compose.yaml
         run: python3 scripts/check_image_pins.py
 
-      - name: promtool check config
+      # Every stack that runs a Prometheus. A stack with rules and no unit
+      # tests fails, which is stricter than the single-stack version was: rules
+      # that cannot fire pass `check rules` (#63), so the moment to notice a
+      # stack has none is when its rules land.
+      - name: promtool check config, check rules, test rules
         run: |
-          docker run --rm --entrypoint promtool \
-            -v "$PWD:/repo" -w /repo "$PROM_IMAGE" \
-            check config "$STACK/prometheus/prometheus.yaml"
-
-      - name: promtool check rules
-        run: |
-          docker run --rm --entrypoint promtool \
-            -v "$PWD:/repo" -w /repo "$PROM_IMAGE" \
-            check rules "$STACK"/prometheus/rules/*.rules.yaml
-
-      # check rules only parses PromQL. It passed for months against a rule that
-      # could not fire for any input (#63); these are the tests that catch that.
-      - name: promtool test rules
-        run: |
-          docker run --rm --entrypoint promtool \
-            -v "$PWD:/repo" -w /repo "$PROM_IMAGE" \
-            test rules "$STACK"/prometheus/tests/*.test.yaml
+          fail=0
+          for stack in $(./scripts/stacks.sh); do
+            sd="stacks/$stack"
+            [ -f "$sd/prometheus/prometheus.yaml" ] || { echo "== $stack: no prometheus.yaml"; continue; }
+            echo "== $stack"
+            docker run --rm --entrypoint promtool \
+              -v "$PWD:/repo" -w /repo "$PROM_IMAGE" \
+              check config "$sd/prometheus/prometheus.yaml" || fail=1
+
+            rules=$(find "$sd/prometheus/rules" -name '*.rules.yaml' 2>/dev/null | sort)
+            tests=$(find "$sd/prometheus/tests" -name '*.test.yaml' 2>/dev/null | sort)
+            if [ -z "$rules" ]; then
+              echo "   no alert rules"
+              continue
+            fi
+            # shellcheck disable=SC2086
+            docker run --rm --entrypoint promtool \
+              -v "$PWD:/repo" -w /repo "$PROM_IMAGE" check rules $rules || fail=1
+            if [ -z "$tests" ]; then
+              echo "::error::$stack has alert rules and no promtool tests —"
+              echo "::error::a rule that cannot fire still passes check rules (#63)"
+              fail=1
+              continue
+            fi
+            # shellcheck disable=SC2086
+            docker run --rm --entrypoint promtool \
+              -v "$PWD:/repo" -w /repo "$PROM_IMAGE" test rules $tests || fail=1
+          done
+          exit $fail
 
       # No secret needed: the receiver URL comes from url_file, which
       # Alertmanager reads at notify time rather than at config load time.
+      # Only stacks that run one. stacks/lab has no Alertmanager by decision
+      # (ADR-0020), and the ROUTES table below describes the estate's tree
+      # specifically.
       - name: amtool check-config
         run: |
-          docker run --rm --entrypoint amtool \
-            -v "$PWD:/repo" -w /repo "$AM_IMAGE" \
-            check-config "$STACK/alertmanager/alertmanager.yaml"
+          fail=0
+          for f in $(./scripts/stacks.sh --paths | sed 's|$|/alertmanager/alertmanager.yaml|'); do
+            [ -f "$f" ] || continue
+            echo "== $f"
+            docker run --rm --entrypoint amtool \
+              -v "$PWD:/repo" -w /repo "$AM_IMAGE" check-config "$f" || fail=1
+          done
+          exit $fail
 
       # check-config proves the tree parses and that every route names a
       # receiver that exists. It does not say WHICH receiver an alert reaches,
@@ -181,13 +221,17 @@ jobs:
       - name: amtool config routes test
         run: |
           fail=0
+          # The estate's tree. Asserted against the one stack that has an
+          # Alertmanager rather than against every stack, because these eight
+          # rows are that tree's routing and not a property all stacks share.
+          cfg=stacks/observability/alertmanager/alertmanager.yaml
           while read -r expected labels; do
             [ -n "$expected" ] || continue
             # shellcheck disable=SC2086
             docker run --rm --entrypoint amtool \
               -v "$PWD:/repo" -w /repo "$AM_IMAGE" \
               config routes test \
-              --config.file="$STACK/alertmanager/alertmanager.yaml" \
+              --config.file="$cfg" \
               --verify.receivers="$expected" $labels \
               || { echo "::error::expected $expected for $labels"; fail=1; }
           done <<'ROUTES'
@@ -209,8 +253,14 @@ jobs:
       - name: alloy fmt --test
         # Every file in the directory: the agent loads the directory, and the
         # deploy script ships a subset of it, so each file must stand alone.
+        # Every *.alloy under stacks/, found rather than assumed to live under
+        # one of them. stacks/lab has no alloy/ directory — it mounts the
+        # estate's two files rather than copying them (ADR-0007) — so a `find`
+        # here covers both today and any stack that grows its own tomorrow.
         run: |
-          for f in "$STACK"/alloy/*.alloy; do
+          found=$(find stacks -path '*/alloy/*.alloy' | sort)
+          [ -n "$found" ] || { echo "::error::no *.alloy under stacks/"; exit 1; }
+          for f in $found; do
             docker run --rm --entrypoint alloy \
               -v "$PWD:/repo" -w /repo "$ALLOY_IMAGE" \
               fmt --test "$f"
@@ -233,7 +283,16 @@ jobs:
       # the static check. scripts/validate.sh may honestly skip it on a host
       # without docker; CI may not.
       - name: Verify health dependencies are satisfiable and probe the images
-        run: python3 scripts/check_compose_health.py --probe
+        run: |
+          fail=0
+          for f in $(./scripts/stacks.sh --paths | sed 's|$|/compose.yaml|'); do
+            echo "== $f"
+            python3 scripts/check_compose_health.py --probe "$f" || fail=1
+          done
+          # The half no single file can answer: a reloaded or claimed service
+          # that exists in no stack at all.
+          python3 scripts/check_compose_health.py --cross-stack || fail=1
+          exit $fail
 
       # promtool cannot check these — it parses PromQL and rejects every LogQL
       # stream selector. Loki itself is the only thing that understands them, so
@@ -245,7 +304,12 @@ jobs:
       # so a typo in one rendered an empty panel and looked like quiet traffic
       # rather than a broken query (#82).
       - name: Validate Loki rules and dashboard LogQL
-        run: ./scripts/check_loki_rules.sh
+        run: |
+          fail=0
+          for stack in $(./scripts/stacks.sh); do
+            ./scripts/check_loki_rules.sh --stack "$stack" || fail=1
+          done
+          exit $fail
 
       # The device list is spread across snmp.yaml, generator.yaml,
       # render-config.sh's REQUIRED array and the example secrets file. Drift
@@ -283,17 +347,35 @@ jobs:
         run: ./scripts/install-timers.sh --check --require-all
 
       - name: Validate Grafana dashboards
-        run: python3 scripts/check_dashboards.py
+        run: |
+          fail=0
+          for stack in $(./scripts/stacks.sh); do
+            python3 scripts/check_dashboards.py --stack "$stack" || fail=1
+          done
+          exit $fail
 
       # Dashboard queries are as easy to typo as alert rules, and a broken one
       # shows up as an empty panel rather than an error. Parse them all — the
       # LogQL half is done by the Loki step above, which is the only thing that
       # can parse it.
       - name: Parse every dashboard PromQL expression
         run: |
-          python3 scripts/check_dashboards.py --emit-promql > /tmp/dashboard-exprs.yaml
-          docker run --rm --entrypoint promtool \
-            -v /tmp:/tmp "$PROM_IMAGE" check rules /tmp/dashboard-exprs.yaml
+          fail=0
+          for stack in $(./scripts/stacks.sh); do
+            # --emit-promql fails on a stack with no dashboards, deliberately:
+            # its output is what proves the panel queries parse, so emitting an
+            # empty file would pass promtool over nothing (#68). The guard is
+            # here, where "this stack ships none" is knowable.
+            if [ -z "$(find "stacks/$stack/grafana/dashboards" -name '*.json' 2>/dev/null)" ]; then
+              echo "== $stack: no dashboards"
+              continue
+            fi
+            python3 scripts/check_dashboards.py --stack "$stack" --emit-promql \
+              > "/tmp/dashboard-exprs-$stack.yaml"
+            docker run --rm --entrypoint promtool \
+              -v /tmp:/tmp "$PROM_IMAGE" check rules "/tmp/dashboard-exprs-$stack.yaml" || fail=1
+          done
+          exit $fail
 
       # The step above reads the dashboard JSON; this one makes Grafana serve it
       # back. Grafana does not return what it was given — it sorts keys, HTML-

Makefile

@@ -179,12 +179,25 @@ check-timers: ## Verify the schedule and its staleness thresholds agree
 	./scripts/install-timers.sh --check
 
 .PHONY: pin-digests
-pin-digests: ## Re-resolve image digests in compose.yaml (--write applies)
-	./scripts/pin-digests.sh --write
+pin-digests: ## Re-resolve image digests in every stack's compose.yaml (--write applies)
+	@# Every stack, not just the estate's. pin-digests.sh takes one compose file
+	@# and rewrites it in place, which is the right shape for the work it does —
+	@# so the loop lives here rather than inside it, driven by the same
+	@# scripts/stacks.sh that validate.sh and ci.yml read. Left single-stack,
+	@# `stacks/lab`'s digests would be re-resolved by nothing and verified by
+	@# nothing, which is the #263 defect in the one place it costs a supply-chain
+	@# guarantee rather than a test.
+	@set -e; for sd in $$(./scripts/stacks.sh --paths); do \
+		printf '\033[0;34m--\033[0m %s\n' "$$sd"; \
+		COMPOSE_FILE="$$sd/compose.yaml" ./scripts/pin-digests.sh --write; \
+	done
 
 .PHONY: check-digests
 check-digests: ## Verify pinned digests still match the registry
-	./scripts/pin-digests.sh
+	@set -e; for sd in $$(./scripts/stacks.sh --paths); do \
+		printf '\033[0;34m--\033[0m %s\n' "$$sd"; \
+		COMPOSE_FILE="$$sd/compose.yaml" ./scripts/pin-digests.sh; \
+	done
 
 .PHONY: scan
 scan: ## Scan the working tree and history for secrets

docs/roadmap.md

@@ -306,11 +306,17 @@ what left this one unfireable for months.
   (`render-config.sh` derives its required keys per stack rather than demanding
   the estate's ten, `reload-config.sh` skips services a stack does not declare,
   `bootstrap.sh` refuses to give one age key both stacks) and gave `.sops.yaml`
-  the lab rule ADR-0020 asked for. It did **not** touch
-  [#263](https://github.com/Gerrrt/HomeLab/issues/263): `STACK ?=` reaches the
-  lifecycle targets and stops there, every checker is still pinned to
-  `stacks/observability`, and so the new stack is one CI has never seen —
-  validated only by hand and by the checks that already follow `STACK`.
+  the lab rule ADR-0020 asked for. [#263](https://github.com/Gerrrt/HomeLab/issues/263)
+  followed it: `scripts/stacks.sh` is now the single definition of what a stack
+  is, and `validate.sh`, `ci.yml`, `pin-digests.sh` and the Python checkers all
+  read it instead of carrying `stacks/observability`. Both stacks are checked,
+  each line says which, and a directory under `stacks/` with no compose.yaml
+  fails rather than being skipped — a stack nothing checks being the defect the
+  list exists to prevent. Two guards got stronger on the way: rules without
+  `promtool` unit tests are now a failure rather than an absence nobody
+  measured (#63), and the reload/ABSENT_BINARIES cross-checks gained a
+  cross-stack mode, because "not in this compose file" stopped meaning "in no
+  stack at all" the moment there were two.
   [#265](https://github.com/Gerrrt/HomeLab/issues/265) the domain is what
   everything else is pointed at, and blocks both
   [#266](https://github.com/Gerrrt/HomeLab/issues/266) Wazuh — the heaviest