Rebuild: working observability stack, secrets management, CI, and documentation

.editorconfig

@@ -0,0 +1,21 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_style = space
+indent_size = 2
+
+[*.md]
+trim_trailing_whitespace = false
+
+[Makefile]
+indent_style = tab
+
+[*.{sh,bash}]
+indent_size = 2
+
+[*.alloy]
+indent_style = tab

.gitattributes

@@ -0,0 +1,11 @@
+# The snmp_exporter config is machine-generated from snmp-exporter/generator.yaml.
+# Marking it as generated keeps 14k lines of OID definitions out of GitHub's
+# language stats and collapses it by default in diffs.
+stacks/observability/snmp-exporter/snmp.yaml linguist-generated=true -diff
+
+# Grafana dashboard JSON is exported by Grafana, not hand-maintained line by line.
+stacks/observability/grafana/dashboards/*.json linguist-generated=true
+
+# Normalise line endings.
+* text=auto eol=lf
+*.png binary

.github/ISSUE_TEMPLATE/bug.yml

@@ -0,0 +1,43 @@
+---
+name: Something is broken
+description: A service, dashboard, alert or metric is not behaving
+labels: ["bug"]
+body:
+  - type: dropdown
+    id: area
+    attributes:
+      label: Area
+      options:
+        - Prometheus / alerting
+        - Grafana / dashboards
+        - Loki / log collection
+        - Alloy agent
+        - SNMP / network devices
+        - Deployment (compose, secrets, scripts)
+        - Documentation
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected
+    attributes:
+      label: What should happen
+    validations:
+      required: true
+
+  - type: textarea
+    id: actual
+    attributes:
+      label: What happens instead
+      description: Include the query, panel or alert name where relevant.
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant output
+      description: >-
+        Output of `make validate`, `docker compose logs <service>`, or the
+        failing PromQL. Redact community strings and passwords.
+      render: shell

.github/ISSUE_TEMPLATE/change.yml

@@ -0,0 +1,34 @@
+---
+name: Planned change
+description: New device, new service, or a change to the network
+labels: ["enhancement"]
+body:
+  - type: textarea
+    id: goal
+    attributes:
+      label: What and why
+    validations:
+      required: true
+
+  - type: dropdown
+    id: vlan
+    attributes:
+      label: Which VLAN does this touch
+      multiple: true
+      options:
+        - "99 — Winterfell (management)"
+        - "50 — Hicks (trusted)"
+        - "40 — CasaBonita (media)"
+        - "30 — ImaginationLAN (lab)"
+        - "20 — Skids (IoT)"
+        - "10 — Degens (guest)"
+        - None / not network-facing
+
+  - type: checkboxes
+    id: security
+    attributes:
+      label: Security review
+      options:
+        - label: Does not weaken the default-deny posture between VLANs
+        - label: Any new credential goes in `secrets/*.sops.yaml`
+        - label: Any new device is added to `docs/network.md` with an OUI-truncated MAC

.github/dependabot.yml

@@ -0,0 +1,28 @@
+---
+version: 2
+
+updates:
+  # Pinned container images in compose.yaml. This is what makes pinning
+  # sustainable — without it, pinned tags simply rot.
+  - package-ecosystem: docker-compose
+    directory: /stacks/observability
+    schedule:
+      interval: weekly
+      day: sunday
+    open-pull-requests-limit: 5
+    commit-message:
+      prefix: "chore(deps)"
+    labels: ["dependencies", "observability"]
+    groups:
+      grafana-stack:
+        patterns: ["grafana/*"]
+      prometheus-stack:
+        patterns: ["prom/*"]
+
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: monthly
+    commit-message:
+      prefix: "chore(ci)"
+    labels: ["dependencies", "ci"]

.github/pull_request_template.md

@@ -0,0 +1,23 @@
+## What changed
+
+<!-- One or two sentences. What does this change do to the running lab? -->
+
+## Why
+
+<!-- The problem this solves. Link an issue or a line in docs/roadmap.md. -->
+
+## Blast radius
+
+<!-- Which hosts, VLANs or services are affected if this is wrong? -->
+
+- [ ] No change to network segmentation or firewall rules
+- [ ] No new port published to a VLAN that could not already reach the service
+- [ ] No credential added outside `secrets/*.sops.yaml`
+
+## Verification
+
+<!-- What you actually ran, and what it printed. -->
+
+- [ ] `make validate` passes
+- [ ] Deployed to the lab and confirmed working
+- [ ] Docs updated (`docs/`, service README, or `docs/roadmap.md`)

.github/workflows/ci.yml

@@ -0,0 +1,170 @@
+---
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  STACK: stacks/observability
+  PROM_IMAGE: prom/prometheus:v3.1.0
+  AM_IMAGE: prom/alertmanager:v0.28.0
+  ALLOY_IMAGE: grafana/alloy:v1.6.1
+
+jobs:
+  # ---------------------------------------------------------------------------
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: yamllint
+        run: pipx run yamllint --strict .
+
+      - name: markdownlint
+        run: npx --yes markdownlint-cli2
+
+      - name: shellcheck
+        run: shellcheck scripts/*.sh
+
+      - name: actionlint
+        uses: raven-actions/actionlint@v2
+
+      - name: EditorConfig
+        uses: editorconfig-checker/action-editorconfig-checker@v2
+
+  # ---------------------------------------------------------------------------
+  validate:
+    name: Validate configs
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      # The ${VAR:?} guards in compose.yaml exist so a missing secret fails
+      # loudly at deploy time. CI supplies throwaway values so validation never
+      # needs a decryption key.
+      - name: Seed a validation-only .env
+        run: |
+          cp "$STACK/.env.example" "$STACK/.env"
+          echo "GRAFANA_ADMIN_PASSWORD=validation-only" >> "$STACK/.env"
+
+      - name: docker compose config
+        run: docker compose -f "$STACK/compose.yaml" config -q
+
+      # Covers every place an image is referenced, not just compose.yaml — the
+      # first version of this check only looked at the stack and let :latest
+      # through in the workflow itself and in the Makefile.
+      - name: Verify no image uses a floating tag
+        run: |
+          if grep -rnE '(^|[[:space:]])[a-z0-9._/-]+:latest([[:space:]]|$)' \
+               --include='*.yaml' --include='*.yml' --include='Makefile' \
+               --include='*.sh' . \
+             | grep -v '^\./\.git/' \
+             | grep -vE '^[^:]+:[0-9]+:[[:space:]]*#' ; then
+            echo "::error::floating :latest tag found — pin an explicit version"
+            exit 1
+          fi
+          echo "all image references pinned"
+
+      - name: promtool check config
+        run: |
+          docker run --rm --entrypoint promtool \
+            -v "$PWD:/repo" -w /repo "$PROM_IMAGE" \
+            check config "$STACK/prometheus/prometheus.yaml"
+
+      - name: promtool check rules
+        run: |
+          docker run --rm --entrypoint promtool \
+            -v "$PWD:/repo" -w /repo "$PROM_IMAGE" \
+            check rules "$STACK"/prometheus/rules/*.rules.yaml
+
+      # No secret needed: the receiver URL comes from url_file, which
+      # Alertmanager reads at notify time rather than at config load time.
+      - name: amtool check-config
+        run: |
+          docker run --rm --entrypoint amtool \
+            -v "$PWD:/repo" -w /repo "$AM_IMAGE" \
+            check-config "$STACK/alertmanager/alertmanager.yaml"
+
+      # `fmt --test` exits non-zero if the file is not canonically formatted, and
+      # fails outright on a syntax error. It does not validate that components
+      # are configured correctly — Alloy v1.6.1 has no `validate` subcommand, so
+      # that is only caught at load time on the host.
+      - name: alloy fmt --test
+        run: |
+          docker run --rm --entrypoint alloy \
+            -v "$PWD:/repo" -w /repo "$ALLOY_IMAGE" \
+            fmt --test "$STACK/alloy/config.alloy"
+
+      - name: Validate Grafana dashboards
+        run: python3 scripts/check_dashboards.py
+
+      # Dashboard queries are as easy to typo as alert rules, and a broken one
+      # shows up as an empty panel rather than an error. Parse them all.
+      - name: Parse every dashboard PromQL expression
+        run: |
+          python3 scripts/check_dashboards.py --emit-promql > /tmp/dashboard-exprs.yaml
+          docker run --rm --entrypoint promtool \
+            -v /tmp:/tmp "$PROM_IMAGE" check rules /tmp/dashboard-exprs.yaml
+
+  # ---------------------------------------------------------------------------
+  secret-scan:
+    name: Secret scan
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      # Two scans, because they answer different questions.
+      #
+      # The working-tree scan asks "is there a secret in the code as it stands"
+      # and must always be clean. The history scan asks "has one ever been
+      # committed" — it honours .gitleaksignore, which enumerates the nine known
+      # historical findings with the reason each is there. Once the history
+      # purge in docs/runbooks/purge-git-history.md has run, that file goes away
+      # and this job keeps passing on its own merits.
+      - name: gitleaks — working tree
+        run: |
+          docker run --rm -v "$PWD:/repo" -w /repo \
+            zricethezav/gitleaks:v8.24.0 \
+            detect --no-git --no-banner --redact -c .gitleaks.toml -v
+
+      - name: gitleaks — full history
+        run: |
+          docker run --rm -v "$PWD:/repo" -w /repo \
+            zricethezav/gitleaks:v8.24.0 \
+            detect --no-banner --redact -c .gitleaks.toml --log-opts="--all" -v
+
+      - name: Assert no decrypted artefact is tracked
+        run: |
+          fail=0
+          for pattern in '.env' '.rendered/'; do
+            if git ls-files | grep -E "(^|/)${pattern//./\\.}" | grep -v '\.env\.example'; then
+              echo "::error::tracked file matching '${pattern}' — it must be gitignored"
+              fail=1
+            fi
+          done
+          exit "$fail"
+
+      - name: Assert SOPS files are actually encrypted
+        run: |
+          shopt -s nullglob
+          fail=0
+          for f in secrets/*.sops.yaml; do
+            if ! grep -q '^sops:' "$f"; then
+              echo "::error file=$f::not SOPS-encrypted"
+              fail=1
+            fi
+          done
+          exit "$fail"

.gitignore

@@ -1,9 +1,47 @@
-ISOs/proxmox-ve_8.4-1.iso
-software/balenaEtcher-2.1.2.Setup.exe
-certificates/Gandalf.Gondor.Lab/
-docs/hardware/Keyboard.pdf
-docs/hardware/Mar.pdf
-docs/hardware/mouse.pdf
-docs/hardware/Server1.pdf
-prometheus/prometheus.matrix.elysium/.DS_Store
-prometheus/.DS_Store
+# ---- OS cruft ----
+.DS_Store
+._*
+Thumbs.db
+desktop.ini
+
+# ---- Secrets ----
+# Only *.sops.yaml (encrypted at rest) may be committed under secrets/.
+.env
+.env.*
+!.env.example
+secrets/*
+!secrets/*.sops.yaml
+!secrets/*.example.yaml
+!secrets/README.md
+*.pem
+*.key
+*.p12
+*.pfx
+*.crt
+age.key
+keys.txt
+certificates/
+
+# ---- Rendered / decrypted config (produced by scripts/render-config.sh) ----
+**/.rendered/
+
+# ---- Runtime state produced by the stack ----
+data/
+**/data/
+*.db
+*.wal
+
+# ---- Local tooling ----
+.venv/
+node_modules/
+*.log
+*.tmp
+*.bak
+*.orig
+
+# ---- Large local artifacts never meant for git ----
+ISOs/
+software/
+*.iso
+*.exe
+docs/hardware/*.pdf