From 48eae28ccad682619071af921e8ef1342230f179 Mon Sep 17 00:00:00 2001 From: Garrett Allen Date: Sun, 2 Aug 2026 06:22:35 +0000 Subject: [PATCH 1/8] refactor(repo): restructure into a per-stack layout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous layout was //, five directories sharded by a hostname that had exactly one value and appeared in no documentation. Collapse it to stacks///, where a stack is the set of services deployed together. Also replaces a .gitignore that listed nine hardcoded paths — all of which pointed at files no longer in the tree, one of them misspelled (Mar.pdf for the Mat.pdf that was actually committed) — with pattern-based rules covering OS cruft, secrets, rendered config and runtime state. Rationale in docs/adr/0004-one-compose-stack-per-host.md. --- .editorconfig | 21 ++ .gitattributes | 11 + .gitignore | 56 +++- LICENSE | 21 ++ .../docker-compose.yaml | 19 -- docs/hardware/Inventory.md | 269 ------------------ docs/network.md | 262 +++++++++++++++++ .../docker-compose.yaml | 19 -- .../config/loki-config.yaml | 37 --- .../docker-compose.yaml | 11 - .../config/prometheus.yaml | 75 ----- .../docker-compose.yaml | 13 - .../docker-compose.yaml | 9 - .../observability/alloy}/config.alloy | 20 +- stacks/observability/loki/loki-config.yaml | 70 +++++ .../observability/prometheus/prometheus.yaml | 81 ++++++ .../observability/snmp-exporter}/snmp.yaml | 31 +- 17 files changed, 548 insertions(+), 477 deletions(-) create mode 100644 .editorconfig create mode 100644 .gitattributes create mode 100644 LICENSE delete mode 100644 alloy/prometheus.matrix.elysium/docker-compose.yaml delete mode 100644 docs/hardware/Inventory.md create mode 100644 docs/network.md delete mode 100644 grafana/prometheus.matrix.elysium/docker-compose.yaml delete mode 100644 loki/prometheus.matrix.elysium/config/loki-config.yaml delete mode 100644 loki/prometheus.matrix.elysium/docker-compose.yaml delete mode 100644 prometheus/prometheus.matrix.elysium/config/prometheus.yaml delete mode 100644 prometheus/prometheus.matrix.elysium/docker-compose.yaml delete mode 100644 snmp-exporter/prometheus.matrix.elysium/docker-compose.yaml rename {alloy/prometheus.matrix.elysium/config => stacks/observability/alloy}/config.alloy (88%) create mode 100644 stacks/observability/loki/loki-config.yaml create mode 100644 stacks/observability/prometheus/prometheus.yaml rename {snmp-exporter/prometheus.matrix.elysium/config => stacks/observability/snmp-exporter}/snmp.yaml (99%) diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..72e940a --- /dev/null +++ b/.editorconfig @@ -0,0 +1,21 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 2 + +[*.md] +trim_trailing_whitespace = false + +[Makefile] +indent_style = tab + +[*.{sh,bash}] +indent_size = 2 + +[*.alloy] +indent_style = tab diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..8f11725 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,11 @@ +# The snmp_exporter config is machine-generated from snmp-exporter/generator.yaml. +# Marking it as generated keeps 14k lines of OID definitions out of GitHub's +# language stats and collapses it by default in diffs. +stacks/observability/snmp-exporter/snmp.yaml linguist-generated=true -diff + +# Grafana dashboard JSON is exported by Grafana, not hand-maintained line by line. +stacks/observability/grafana/dashboards/*.json linguist-generated=true + +# Normalise line endings. +* text=auto eol=lf +*.png binary diff --git a/.gitignore b/.gitignore index fd2a95e..cd92cf2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,47 @@ -ISOs/proxmox-ve_8.4-1.iso -software/balenaEtcher-2.1.2.Setup.exe -certificates/Gandalf.Gondor.Lab/ -docs/hardware/Keyboard.pdf -docs/hardware/Mar.pdf -docs/hardware/mouse.pdf -docs/hardware/Server1.pdf -prometheus/prometheus.matrix.elysium/.DS_Store -prometheus/.DS_Store +# ---- OS cruft ---- +.DS_Store +._* +Thumbs.db +desktop.ini + +# ---- Secrets ---- +# Only *.sops.yaml (encrypted at rest) may be committed under secrets/. +.env +.env.* +!.env.example +secrets/* +!secrets/*.sops.yaml +!secrets/*.example.yaml +!secrets/README.md +*.pem +*.key +*.p12 +*.pfx +*.crt +age.key +keys.txt +certificates/ + +# ---- Rendered / decrypted config (produced by scripts/render-config.sh) ---- +**/.rendered/ + +# ---- Runtime state produced by the stack ---- +data/ +**/data/ +*.db +*.wal + +# ---- Local tooling ---- +.venv/ +node_modules/ +*.log +*.tmp +*.bak +*.orig + +# ---- Large local artifacts never meant for git ---- +ISOs/ +software/ +*.iso +*.exe +docs/hardware/*.pdf diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..864d41b --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Garrett Allen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/alloy/prometheus.matrix.elysium/docker-compose.yaml b/alloy/prometheus.matrix.elysium/docker-compose.yaml deleted file mode 100644 index 0af450d..0000000 --- a/alloy/prometheus.matrix.elysium/docker-compose.yaml +++ /dev/null @@ -1,19 +0,0 @@ -services: - alloy: - container_name: alloy - image: grafana/alloy:latest - restart: unless-stopped - volumes: - - /opt/alloy/config.alloy:/etc/alloy/config.alloy - - /opt/alloy/data:/alloy - - /var/run/docker.sock:/var/run/docker.sock - - /var/lib/docker/containers:/var/lib/docker/containers:ro - - /var/log:/var/log:ro - depends_on: - - loki - command: - - run - - --server.http.listen-addr=0.0.0.0:12345 - - /etc/alloy/config.alloy - ports: - - '12345:12345' \ No newline at end of file diff --git a/docs/hardware/Inventory.md b/docs/hardware/Inventory.md deleted file mode 100644 index e1931e1..0000000 --- a/docs/hardware/Inventory.md +++ /dev/null @@ -1,269 +0,0 @@ -# ***Network Inventory*** - -## **WAN** - -| $\color{limegreen}{Hostname}$ | $\color{limegreen}{IP}$ | $\color{limegreen}{MAC}$ | $\color{limegreen}{Device}$ | $\color{limegreen}{OS}$ | $\color{limegreen}{Location}$ | $\color{limegreen}{Role}$ | -| :---------------------------: | :---------------------: | :----------------------: | :-------------------------: | :---------------------: | :---------------------------: | :-----------------------: | -| morpheus | xx.xxx.xxx.xxx | 80:e8:2c:2d:77:e2 | HP/Prodesk600G4Mini | FreeBSD15.0 | Server[5] | pfSense | - -### *Recap(WAN)* - -- Cat6 cable runs from modem[^modem] to WAN interface of ProDesk[^ProDesk]. -- WiFi still operational even in bridge mode. -- WiFi is completely run by eero devices in separate VLANs. - -#### Tasks(WAN) - -- [x] Activate Bridge Mode on Gateway -- [x] Double check firewall rules - -[^modem]: [Xfinity Gateway (XB7)]() -[^ProDesk]: [HP Prodesk 600 G4 Mini Desktop Computer]() - -## **LAN** - -| $\color{limegreen}{Hostname}$ | $\color{limegreen}{IP}$ | $\color{limegreen}{MAC}$ | $\color{limegreen}{Device}$ | $\color{limegreen}{OS}$ | $\color{limegreen}{Location}$ | $\color{limegreen}{Role}$ | -| :---------------------------: | :---------------------: | :----------------------: | :-------------------------: | :---------------------: | :---------------------------: | :-----------------------: | -| morpheus | 10.7.7.1 | 02:26:26:02:0c:15 | HP/Prodesk600G4Mini | FreeBSD15.0 | Server[5] | Firewall | -| :x: | 10.7.7.2 | 1c:2a:a3:2f:10:9b | Mokerlink | :x: | Server[9] | Switch | - -### *Recap(LAN)* - -- Cat6 cable runs from NIC [^adapter] installed on ProDesk to port 1 of switch (Trunk). -- LAN is the only interface able to route [^MokerLink] ip address. -- DHCP disabled. - -> [!CAUTION] -> Re-enabling DHCP on this interface will cause problems for all other interfaces. ->> Wife == ***PISSED!*** - -#### Tasks(LAN) - -- [ ] Figure out if DNS is possible for MokerLink Web page. -- [ ] Double check firewall rules - -[^adapter]: [NIC adapter]() -[^MokerLink]: [MokerLink 26-port Managed Switch]() - -## **Winterfell (VLAN 99 - Management)** - -| $\color{limegreen}{Hostname}$ | $\color{limegreen}{IP}$ | $\color{limegreen}{MAC}$ | $\color{limegreen}{Device}$ | $\color{limegreen}{OS}$ | $\color{limegreen}{Location}$ | $\color{limegreen}{Role}$ | -| :---------------------------: | :---------------------: | :----------------------: | :-------------------------: | :---------------------: | :---------------------------: | :-----------------------: | -| morpheus | 10.0.99.1 | 02:26:26:02:0c:15 | HP/Prodesk600G4Mini | FreeBSD15.0 | Server[5] | Firewall | -| mjolnir | 10.0.99.10 | 28:29:86:80:1a:55 | APCSmartUPS | apcOS | Server[1:2] | UPS | -| prometheus | 10.0.99.20 | 00:05:1b:dd:f8:28 | Apple/MacBookPro | Ubuntu24.04.3 | Shelf | Prometheus | -| oracle | 10.0.99.30 | 58:8a:5a:31:20:6a | Dell/Inspiron153565 | Ubuntu24.04.3 | Shelf | Prometheus(2) | - -### *Recap(Winterfell)* - -- Cat6 cables for each device. -- DHCP enabled. -- VLAN 99 takes up ports 1-3 of Switch. -- Should probably have invested in a battery for the UPS...[^UPS] -- Port 3 connects to an 8-port tp-link switch[^tp-linkswitch] where prometheus and oracle are connected. -- pfSense can only be accessed on this interface from Hicks VLAN. -- prometheus is an old 2012 MacBook Pro[^MacBookPro] with Ubuntu Server installed. -- oracle is an old Dell Laptop[^Dell] with Ubuntu Server installed. - -#### Tasks(Winterfell) - -- [ ] Get a new battery for UPS. -- [ ] Finish Network Monitoring setup. - - [ ] Prometheus - - [ ] Grafana - - [ ] Loki - - [ ] SNMP-Exporter - - [ ] Alloy -- [ ] Deploy Alloy throughout network. -- [ ] Decide on oracle's exact use. - -[^UPS]: [APC Smart UPS]() -[^tp-linkswitch]: [Tp-Link 8-Port Gigabit Switch]() -[^MacBookPro]: [Apple MacBook Pro 2012]() -[^Dell]: [Dell Inspiron 15]() - -## **Hicks (VLAN 50 - Trusted Devices)** - -| $\color{limegreen}{Hostname}$ | $\color{limegreen}{IP}$ | $\color{limegreen}{MAC}$ | $\color{limegreen}{Device}$ | $\color{limegreen}{OS}$ | $\color{limegreen}{Location}$ | $\color{limegreen}{Role}$ | -| :---------------------------: | :---------------------: | :----------------------: | :-------------------------: | :---------------------: | :---------------------------: | :-----------------------: | -| morpheus | 10.0.50.1 | 02:26:26:02:0c:15 | HP/ProDesk600G4Mini | FreeBSD_15.0 | Server[5] | Firewall | -| M_Gaming | 10.0.50.10 | 04:ed:33:52:6e:94 | HP/PavillionGaming | Windows11 | LivingRoom | Laptop | -| M_Desktop | 10.0.50.20 | 04:42:1a:ec:1f:24 | ASUS/ROGStrixX570E | Windows11 | Office | Desktop | -| M_Laptop | 10.0.50.69 | e8:f6:73:a4:32:86 | Microsoft/Surface6 | Windows11 | Office | Work_PC | -| G_Laptop | 10.0.50.70 | 4c:ea:41:66:f2:70 | Microsoft/Surface6 | Windows11 | Basement | Work_PC | -| G_MacBook | 10.0.50.80 | 4c:ea:41:68:b4:c5 | Apple/MacBookPro | MacOSTahoe26 | Basement | Laptop | -| G_Desktop | 10.0.50.90 | 04:42:1a:07:41:ee | ASUS/ROGCrosshairVIII | Windows11 | Basement | Desktop | -| eero | 10.0.50.104 | 9c:57:bc:76:e3:32 | eero/Pro6E | eeroOS | DiningRoom | WiFi | -| G_iPhone | 10.0.50.105 | fe:ee:aa:9a:c6:30 | Apple/iPhone16 | iOS26.1 | :x: | Cellphone | -| M_Pixel | 10.0.50.109 | fa:cc:aa:c6:1e:9c | Google/Pixel7 | Android13 | :x: | Cellphone | -| eero | 10.0.50.110 | 9c:57:bc:77:a6:32 | eero/Pro6E | eeroOS | MediaRoom | WiFi | -| eero | 10.0.50.111 | fc:3f:a6:2f:a1:92 | eero/Pro6E | eeroOS | Playroom | WiFi | -| G_Watch | 10.0.50.112 | f6:b8:72:17:c2:f3 | Apple/Watch10 | watchOS26.0.2 | :x: | Watch | - -### *Recap(Hicks)* - -- Cat6 cables for each computer. One eero[^eero] device wired, and the other two are connected wirelessly. -- DHCP enabled. -- Desktop's[^Desktop1] [^Desktop2] were built, Surface's[^Surface] are for work, I'm the Apple guy[^MacBook] [^iPhone16] [^Watch10], and my wife uses everything else[^Pavillion] [^Pixel] -- Certain trusted devices are cabable of reaching Management devices for administration - -#### Tasks(Hicks) - -- [ ] Plan out how to interact with eero API for Home Assistant integration. -- [ ] Decide on how to go about deploying Alloy and where to set up log aggregator. - -[^Desktop1]: [Desktop PC 1]() -[^Desktop2]: [Desktop PC 2]() -[^Surface]: [Microsoft Surface Laptop 6 for Business]() -[^MacBook]: [Apple MacBook Pro M2]() -[^iPhone16]: [Apple iPhone16]() -[^Watch10]: [Apple Watch Series 10]() -[^Pavillion]: [HP Pavillion Gaming]() -[^Pixel]: [Google Pixel 7]() -[^eero]: [Eero Pro 6E]() - -## **CasaBonita (VLAN 40 - Media)** - -| $\color{limegreen}{Hostname}$ | $\color{limegreen}{IP}$ | $\color{limegreen}{MAC}$ | $\color{limegreen}{Device}$ | $\color{limegreen}{OS}$ | $\color{limegreen}{Location}$ | $\color{limegreen}{Role}$ | -| :---------------------------: | :---------------------: | :----------------------: | :-------------------------: | :---------------------: | :---------------------------: | :-----------------------: | -| morpheus | 10.0.40.1 | 02:26:26:02:0c:15 | HP/ProDesk600G4Mini | FreeBSD15.0 | Server[5] | Firewall | -| nibelheim | 10.0.40.10 | 78:c8:81:68:03:1a | Sony/PlayStation5 | :x: | Basement | GameConsole | -| hyrule | 10.0.40.20 | 00:05:1b:DE:6a:76 | Nintendo/Switch | :x: | Basement | GameConsole | -| lgwebostv | 10.0.40.100 | 58:fd:b1:75:be:35 | LG/OLEDTV | webOS | MediaRoom | TV | -| streambox | 10.0.40.101 | f0:46:3b:ba:e2:b3 | Xumo/StreamBox | entOS | MediaRoom | StreamDevice | - -### *Recap(CasaBonita)* - -- DHCP enabled. -- TV[^OLEDTV], PS5[^PS5], Nintendo[^Nintendo], and StreamBox[^StreamBox] are all wired with Cat6. -- Devices can only access internet. -- This VLAN is where I will install a NAS, just not sure what it will look like quite yet. - -#### Tasks(CasaBonita) - -- [ ] Plan out what to do for a NAS device. -- [ ] Implement NAS plan. - -[^OLEDTV]: [LD OLED TV]() -[^PS5]: [PlayStation 5]() -[^Nintendo]: [Nintendo Switch]() -[^StreamBox]: [Xumo Stream Box]() - -## **ImaginationLAN (VLAN 30 - Testing Environment)** - -| $\color{limegreen}{Hostname}$ | $\color{limegreen}{IP}$ | $\color{limegreen}{MAC}$ | $\color{limegreen}{Device}$ | $\color{limegreen}{OS}$ | $\color{limegreen}{Location}$ | $\color{limegreen}{Role}$ | -| :---------------------------: | :---------------------: | :----------------------: | :-------------------------: | :---------------------: | :---------------------------: | :-----------------------: | -| morpheus | 10.0.30.1 | 02:26:26:02:0c:15 | HP/ProDesk600G4Mini | FreeBSD15.0 | Server[5] | Firewall | -| shiva | 10.0.30.10 | 94:57:a5:51:10:ce | HPE/ProliantDL360Gen9 | Proxmox | Server[3] | Server | - -### *Recap(ImaginationLAN)* - -- All devices wired with Cat6. -- Can only be accessed from Hicks. -- Internet access allowed -- Shiva[^Shiva] is going to be where things will be hosted for the main network as soon as I run out of physical devices. -- Ifrit will be the server that hosts the playground network. - -#### Tasks(ImaginationLAN) - -- [ ] Procure second server device. -- [ ] Build playground. - - [ ] Only AFTER main network is complete - -[^Shiva]: [HPE Proliant DL360 Gen 9]() - -## **Skids (VLAN 20 - IoT/Smart Devices)** - -| $\color{limegreen}{Hostname}$ | $\color{limegreen}{IP}$ | $\color{limegreen}{MAC}$ | $\color{limegreen}{Device}$ | $\color{limegreen}{OS}$ | $\color{limegreen}{Location}$ | $\color{limegreen}{Role}$ | -| :---------------------------: | :---------------------: | :----------------------: | :-------------------------: | :---------------------: | :---------------------------: | :-----------------------: | -| morpheus | 10.0.20.1 | 02:26:26:02:0c:15 | HP/ProDesk600G4Mini | FreeBSD15.0 | Server[5] | Firewall | -| eero | 10.0.20.101 | fc:3f:a6:3b:54:52 | eero/Pro6E | eeroOS | Bedroom | WiFiMesh | -| eero | 10.0.20.102 | fc:3f:a6:3c:15:12 | eero/Pro6E | eeroOS | Office | WiFiMesh | -| eero | 10.0.20.103 | 9c:57:bc:3f:7b:12 | eero/Pro6E | eeroOS | Basement | WiFiMesh | -| bifrost | 10.0.20.104 | ec:b5:fa:af:6c:38 | Philips/HueBridge | :x: | MediaRoom | Lighting | -| office | 10.0.20.105 | 74:d4:23:55:52:3e | Amazon/Echo | FireOS | Office | Assistant | -| litter-robot4 | 10.0.20.108 | c0:49:ef:e0:90:24 | Whisker/LitterRobot4 | :x: | LaundryRoom | LitterBox | -| kitchen | 10.0.20.109 | 58:a8:e8:7b:83:39 | Amazon/Echo | FireOS | Kitchen | Assistant | -| backfloodlight | 10.0.20.112 | 10:08:2c:22:e3:4f | Ring/Floodlight | :x: | Kitchen | Camera | -| basement | 10.0.20.113 | d4:90:9c:ed:99:0e | Apple/HomePod | audioOS | Basement | Assistant | -| zeke'sroom | 10.0.20.114 | 1c:fe:2b:88:bf:3f | Amazon/Echo | FireOS | ZekeRoom | Assistant | -| whitenoise | 10.0.20.115 | 50:8b:b9:DE:1a:a8 | Tuya/WhiteNoise | :x: | ZekeRoom | WhiteNoise | -| babymonitor | 10.0.20.117 | a4:97:5c:ae:d6:7e | VTech/Camera | :x: | ZekeRoom | Camera | -| frontfloodlight | 10.0.20.118 | b4:bc:7c:1b:bb:a2 | Ring/Floodlight | :x: | Driveway | Camera | -| frontdoorbell | 10.0.20.119 | 3c:e1:a1:5d:8e:28 | Ring/Doorbell | :x: | FrontDoor | Camera | -| basestation | 10.0.20.121 | 2c:6b:7d:0d:e7:e4 | Ring/Base | :x: | DiningRoom | Hub | -| bedroom | 10.0.20.124 | 94:ea:32:89:4e:96 | Apple/HomePod | audioOS | Bedroom | Assistant | -| garage | 10.0.20.126 | 54:e0:19:3e:77:23 | Ring/Camera | :x: | Garage | Camera | -| kitchen | 10.0.20.128 | f4:34:f0:23:a7:22 | Apple/HomePod | audioOS | Kitchen | Assistant | -| kitchen | 10.0.20.130 | 54:e0:19:3c:fe:7d | Ring/Camera | :x: | Kitchen | Camera | -| mediaroom | 10.0.20.132 | f4:34:f0:22:16:2c | Apple/HomePod | audioOS | MediaRoom | Assistant | -| livingroom | 10.0.20.133 | 4c:ef:c0:f7:95:c6 | Amazon/Echo | FireOS | LivingRoom | Assistant | -| playroom | 10.0.20.144 | 68:b6:91:33:02:3a | Amazon/Echo | FireOS | Playroom | Assistant | -| playroom | 10.0.20.145 | 18:7f:88:8c:31:f9 | Ring/Camera | :x: | Playroom | Camera | -| livingroom | 10.0.20.146 | 5c:47:5e:9e:4a:49 | Ring/Camera | :x: | Livingroom | Camera | -| tablet | 10.0.20.149 | a4:97:5c:ae:d6:83 | VTech/Tablet | :x: | Bedroom | Tablet | - -### *Recap(Skids)* - -- One eero device is wired, the other two are connected wirelessly. -- DHCP enabled. -- All IoT devices are WiFi connected. -- Brands list: Philips[^Huebridge], Amazon[^echo], Whisker[^litterrobot], Ring[^floodlight] [^doorbell] [^basestation] [^camera], Apple[^homepod], Tuya[^Whitenoise], Vtech[^Monitor] - -#### Tasks(Skids) - -- [x] Move devices over to designated WiFi network. - -[^Huebridge]: [Philips Hue Bridge]() -[^echo]: [Amazon Echo]() -[^litterrobot]: [Litter-Robot 4]() -[^floodlight]: [Ring Floodlight Cam Plus]() -[^doorbell]: [Ring Doorbell Camera]() -[^basestation]: [Ring Alarm Base Station]() -[^camera]: [Ring Indoor Cam]() -[^homepod]: [Apple HomePod]() -[^Whitenoise]: [Momcozy White Noise Machine]() -[^Monitor]: [VTech Baby Monitor]() - -## **Degens (VLAN 10 - Guest Network)** - -| $\color{limegreen}{Hostname}$ | $\color{limegreen}{IP}$ | $\color{limegreen}{MAC}$ | $\color{limegreen}{Device}$ | $\color{limegreen}{OS}$ | $\color{limegreen}{Location}$ | $\color{limegreen}{Role}$ | -| :---------------------------: | :---------------------: | :----------------------: | :-------------------------: | :---------------------: | :---------------------------: | :-----------------------: | -| morpheus | 10.0.10.1 | 02:26:26:02:0c:15 | HP/ProDesk600G4Mini | FreeBSD15.0 | Server[5] | Firewall | -| eero | 10.0.10.10 | fc:3f:a6:2c:66:d2 | eero/Pro6E | eeroOS | Kitchen | WiFiMesh | -| eero | 10.0.10.101 | 9c:57:bc:6b:29:b2 | eero/Pro6E | eeroOS | LivingRoom | WiFiMesh | - -### *Recap(Degens)* - -- Guest network. -- DHCP enabled. -- Wired AND wireless access. Extra tp-link switch in Kitchen for wired devices -- Only internet, no access to any other network - -#### Tasks(Degens) - -- [x] Lock down firewall rules. - -## Other Items - -- 10 Outlet 1U PDU[^PDU] Rackmount Power Strip -- MT-VIKI 8 Port 1U[^KVM], Rackmount KVM Switch -- JINGCHENGMEI Mini Rack Mount[^ProDeskRackmount] for HP ProDesk Mini PC -- Jadol 24 Port Patch[^Panel] Panel Rackmount -- VEVOR 9U Server Rack[^Rack] -- ViewSonic N1700W LCD TV[^Cord] -- BestParts 1U Rack Sliding Rail[^Sliderail] -- 1U Server Rack Rails[^Rail] -- RJ45 Cat6 Couplers[^Couplers] -- Cat6 patch cables[^Patchcables] - -[^PDU]: [Power Distrubution Unit]() -[^KVM]: [KVM Switch]() -[^ProDeskRackmount]: [1U Rackmount for ProDesk]() -[^Panel]: [1U Rackmount Patch Panel]() -[^Rack]: [9U Open Frame Server Rack]() -[^Cord]: [Power cord for ViewSonic TV]() -[^Sliderail]: [Sliding rail compatible with Proliant]() -[^Rail]: [1U Universal Rack Mount for APC]() -[^Couplers]: [10 Pack in line couplers]() -[^Patchcables]: [Cat6 patch cables from local electronic store]() diff --git a/docs/network.md b/docs/network.md new file mode 100644 index 0000000..c03d255 --- /dev/null +++ b/docs/network.md @@ -0,0 +1,262 @@ +# Network + +Seven VLANs behind a pfSense firewall, default-deny between segments. Each +section below lists the devices on a segment, how it is wired, and what it is +allowed to reach. + +> **On the data in this file.** MAC addresses are truncated to their OUI (the +> vendor half); personal devices are listed by role rather than by owner. This +> is a public repository, and a full device fingerprint of a house is an +> inventory for someone else. The rationale is in +> [`security.md`](security.md#what-this-repository-deliberately-does-not-publish). + +| Segment | VLAN | Subnet | Purpose | Reaches | +| --- | --- | --- | --- | --- | +| WAN | — | ISP-assigned | Uplink | — | +| LAN | — | `10.7.7.0/24` | Switch management only | Nothing | +| [Winterfell](#winterfell--vlan-99--management) | 99 | `10.0.99.0/24` | Infrastructure management | Internet | +| [Hicks](#hicks--vlan-50--trusted) | 50 | `10.0.50.0/24` | Trusted workstations | Internet, 99, 30 | +| [CasaBonita](#casabonita--vlan-40--media) | 40 | `10.0.40.0/24` | TVs and consoles | Internet | +| [ImaginationLAN](#imaginationlan--vlan-30--lab) | 30 | `10.0.30.0/24` | Hypervisor / lab | Internet | +| [Skids](#skids--vlan-20--iot) | 20 | `10.0.20.0/24` | IoT and cameras | Internet | +| [Degens](#degens--vlan-10--guest) | 10 | `10.0.10.0/24` | Guest Wi-Fi | Internet | + +Hostnames are thematic rather than functional — `morpheus` is the firewall, +`mjolnir` the UPS, `shiva` the hypervisor. The Role column is the source of +truth for what a box actually does. + +--- + +## WAN + +| Hostname | IP | MAC (OUI) | Device | OS | Location | Role | +| --- | --- | --- | --- | --- | --- | --- | +| morpheus | *(ISP-assigned)* | `80:e8:2c:xx:xx:xx` | HP ProDesk 600 G4 Mini | FreeBSD 15.0 | Rack U5 | pfSense | + +### Notes + +- Cat6 from the ISP gateway[^modem] to the WAN interface of the ProDesk[^ProDesk]. +- The gateway runs in bridge mode; its own Wi-Fi radio stays operational but is + unused. All wireless is handled by eero units on tagged VLANs. + +[^modem]: [Xfinity Gateway (XB7)](https://www.xfinity.com/support/articles/broadband-gateways-userguides) +[^ProDesk]: [HP ProDesk 600 G4 Mini](https://www.microcenter.com/product/692358/) + +--- + +## LAN + +| Hostname | IP | MAC (OUI) | Device | OS | Location | Role | +| --- | --- | --- | --- | --- | --- | --- | +| morpheus | `10.7.7.1` | `02:26:26:xx:xx:xx` | HP ProDesk 600 G4 Mini | FreeBSD 15.0 | Rack U5 | Firewall | +| neo | `10.7.7.2` | `1c:2a:a3:xx:xx:xx` | MokerLink 26-port managed | — | Rack U9 | Switch | + +### Notes + +- Cat6 from the ProDesk's add-on NIC[^adapter] to port 1 of the switch (trunk). +- This interface exists solely to reach the switch's[^MokerLink] management UI, + which will not bind to a tagged interface. +- DHCP disabled. + +> [!CAUTION] +> Do not re-enable DHCP on this interface. It races the DHCP servers on every +> tagged interface and takes the whole house offline. + +[^adapter]: [USB NIC adapter](https://a.co/d/dJ4BD2N) +[^MokerLink]: [MokerLink 26-port managed switch](https://a.co/d/gaJvCKV) + +--- + +## Winterfell — VLAN 99 — Management + +Infrastructure. The only segment that can administer other segments, and the +only one Hicks is permitted to reach for management. + +| Hostname | IP | MAC (OUI) | Device | OS | Location | Role | +| --- | --- | --- | --- | --- | --- | --- | +| morpheus | `10.0.99.1` | `02:26:26:xx:xx:xx` | HP ProDesk 600 G4 Mini | FreeBSD 15.0 | Rack U5 | Firewall | +| mjolnir | `10.0.99.10` | `28:29:86:xx:xx:xx` | APC Smart-UPS[^UPS] | — | Rack U1–U2 | UPS | +| prometheus | `10.0.99.20` | `00:05:1b:xx:xx:xx` | Apple MacBook Pro (2012)[^MacBookPro] | Ubuntu 24.04.3 | Shelf | **Observability stack** | +| oracle | `10.0.99.30` | `58:8a:5a:xx:xx:xx` | Dell Inspiron 15[^Dell] | Ubuntu 24.04.3 | Shelf | Spare / undecided | + +### Notes + +- `prometheus` runs the whole monitoring stack from + [`stacks/observability`](../stacks/observability) — a 2012 MacBook Pro with + Ubuntu Server on it, which is exactly the sort of hardware a homelab should be + built from. +- Port 3 of the main switch feeds an 8-port unmanaged switch[^tp-linkswitch] + that `prometheus` and `oracle` hang off. +- pfSense's admin UI is reachable on this interface from Hicks only. +- DHCP enabled, with static reservations for everything listed. +- `oracle` has no assigned role yet — see [roadmap](roadmap.md). + +[^UPS]: [APC Smart-UPS](https://www.apc.com/us/en/product-range/61913-smart-ups/) +[^tp-linkswitch]: [TP-Link 8-port gigabit switch](https://www.tp-link.com/us/business-networking/unmanaged-switch/) +[^MacBookPro]: [Apple MacBook Pro (2012)](https://support.apple.com/en-us/111958) +[^Dell]: [Dell Inspiron 15](https://www.dell.com/support/home/en-us/product-support/product/inspiron-15-3520-laptop) + +--- + +## Hicks — VLAN 50 — Trusted + +Personal and work machines. The only segment with a path into management. + +| Hostname | IP | MAC (OUI) | Device | OS | Zone | Role | +| --- | --- | --- | --- | --- | --- | --- | +| morpheus | `10.0.50.1` | `02:26:26:xx:xx:xx` | HP ProDesk 600 G4 Mini | FreeBSD 15.0 | Rack U5 | Firewall | +| desktop-01 | `10.0.50.20` | `04:42:1a:xx:xx:xx` | ASUS ROG Strix X570-E[^Desktop1] | Windows 11 | Upper floor | Desktop | +| desktop-02 | `10.0.50.90` | `04:42:1a:xx:xx:xx` | ASUS ROG Crosshair VIII[^Desktop2] | Windows 11 | Lower floor | Desktop | +| laptop-01 | `10.0.50.10` | `04:ed:33:xx:xx:xx` | HP Pavilion Gaming[^Pavillion] | Windows 11 | Roaming | Laptop | +| laptop-02 | `10.0.50.80` | `4c:ea:41:xx:xx:xx` | Apple MacBook Pro[^MacBook] | macOS 26 | Roaming | Laptop | +| workstation-01 | `10.0.50.69` | `e8:f6:73:xx:xx:xx` | Microsoft Surface Laptop 6[^Surface] | Windows 11 | Roaming | Corporate | +| workstation-02 | `10.0.50.70` | `4c:ea:41:xx:xx:xx` | Microsoft Surface Laptop 6[^Surface] | Windows 11 | Roaming | Corporate | +| mobile-01 | `10.0.50.105` | `fe:ee:aa:xx:xx:xx` | Apple iPhone[^iPhone16] | iOS 26 | Roaming | Phone | +| mobile-02 | `10.0.50.109` | `fa:cc:aa:xx:xx:xx` | Google Pixel[^Pixel] | Android 13 | Roaming | Phone | +| wearable-01 | `10.0.50.112` | `f6:b8:72:xx:xx:xx` | Apple Watch[^Watch10] | watchOS 26 | Roaming | Watch | +| eero-trusted-1 | `10.0.50.104` | `9c:57:bc:xx:xx:xx` | eero Pro 6E[^eero] | eeroOS | Main floor | Wi-Fi | +| eero-trusted-2 | `10.0.50.110` | `9c:57:bc:xx:xx:xx` | eero Pro 6E[^eero] | eeroOS | Main floor | Wi-Fi | +| eero-trusted-3 | `10.0.50.111` | `fc:3f:a6:xx:xx:xx` | eero Pro 6E[^eero] | eeroOS | Lower floor | Wi-Fi | + +### Notes + +- Desktops are wired Cat6; one eero is wired as backhaul, the other two mesh. +- Corporate laptops live here but have no management access. They are treated as + untrusted endpoints that happen to sit on a trusted segment. +- Only specific hosts on this VLAN may reach Winterfell, and only on management + ports. + +[^Desktop1]: [Build 1](https://pcpartpicker.com/b/KXv323) +[^Desktop2]: [Build 2](https://pcpartpicker.com/list/XgZpfd) +[^Surface]: [Microsoft Surface Laptop 6 for Business](https://www.microsoft.com/en-us/surface/business/surface-laptop-6-for-business) +[^MacBook]: [Apple MacBook Pro](https://www.apple.com/macbook-pro/) +[^iPhone16]: [Apple iPhone](https://www.apple.com/iphone/) +[^Watch10]: [Apple Watch Series 10](https://www.apple.com/apple-watch-series-10/) +[^Pavillion]: [HP Pavilion Gaming](https://www.hp.com/us-en/shop/cat/laptops/gaming-laptops) +[^Pixel]: [Google Pixel](https://store.google.com/category/phones) +[^eero]: [eero Pro 6E](https://eero.com/shop/eero-pro-6e) + +--- + +## CasaBonita — VLAN 40 — Media + +Televisions and consoles. Internet only. + +| Hostname | IP | MAC (OUI) | Device | OS | Zone | Role | +| --- | --- | --- | --- | --- | --- | --- | +| morpheus | `10.0.40.1` | `02:26:26:xx:xx:xx` | HP ProDesk 600 G4 Mini | FreeBSD 15.0 | Rack U5 | Firewall | +| nibelheim | `10.0.40.10` | `78:c8:81:xx:xx:xx` | Sony PlayStation 5[^PS5] | — | Lower floor | Console | +| hyrule | `10.0.40.20` | `00:05:1b:xx:xx:xx` | Nintendo Switch[^Nintendo] | — | Lower floor | Console | +| mediatv | `10.0.40.100` | `58:fd:b1:xx:xx:xx` | LG OLED[^OLEDTV] | webOS | Media room | TV | +| streambox | `10.0.40.101` | `f0:46:3b:xx:xx:xx` | Xumo Stream Box[^StreamBox] | entOS | Media room | Streaming | + +### Notes + +- All wired with Cat6. +- Internet only, no path to any other segment. Smart TVs run unauditable + firmware with a permanent internet connection and no patch guarantee, so they + get the same trust level as a guest. +- The planned NAS lands here — see [roadmap](roadmap.md). + +[^OLEDTV]: [LG OLED TV](https://www.lg.com/us/tvs/oled) +[^PS5]: [PlayStation 5](https://www.playstation.com/en-us/ps5/) +[^Nintendo]: [Nintendo Switch](https://www.nintendo.com/us/switch/) +[^StreamBox]: [Xumo Stream Box](https://www.xfinity.com/learn/xumostreambox) + +--- + +## ImaginationLAN — VLAN 30 — Lab + +Where things get broken on purpose. + +| Hostname | IP | MAC (OUI) | Device | OS | Location | Role | +| --- | --- | --- | --- | --- | --- | --- | +| morpheus | `10.0.30.1` | `02:26:26:xx:xx:xx` | HP ProDesk 600 G4 Mini | FreeBSD 15.0 | Rack U5 | Firewall | +| shiva | `10.0.30.10` | `94:57:a5:xx:xx:xx` | HPE ProLiant DL360 Gen9[^Shiva] | Proxmox VE | Rack U3 | Hypervisor | + +### Notes + +- Reachable from Hicks only; outbound internet permitted. +- `shiva` will host services for the main network once physical hosts run out. +- A second server ("ifrit") is planned to carry the deliberately-vulnerable + playground, isolated from everything here. + +> [!NOTE] +> `10.0.30.10` is polled over SNMP as the ProLiant's iLO. If the iLO ever moves +> to a dedicated address, update +> `stacks/observability/prometheus/targets/snmp.yaml` to match — the host and +> its BMC currently share this address. + +[^Shiva]: [HPE ProLiant DL360 Gen9](https://buy.hpe.com/us/en/servers/rack-servers/proliant-dl300-servers/proliant-dl360-server/p/1010026922) + +--- + +## Skids — VLAN 20 — IoT + +Everything with a cloud dependency and no patch story. The largest segment and +the least trusted. + +| Hostname | IP | MAC (OUI) | Device | OS | Zone | Role | +| --- | --- | --- | --- | --- | --- | --- | +| morpheus | `10.0.20.1` | `02:26:26:xx:xx:xx` | HP ProDesk 600 G4 Mini | FreeBSD 15.0 | Rack U5 | Firewall | +| eero-iot-1 | `10.0.20.101` | `fc:3f:a6:xx:xx:xx` | eero Pro 6E | eeroOS | Upper floor | Wi-Fi mesh | +| eero-iot-2 | `10.0.20.102` | `fc:3f:a6:xx:xx:xx` | eero Pro 6E | eeroOS | Main floor | Wi-Fi mesh | +| eero-iot-3 | `10.0.20.103` | `9c:57:bc:xx:xx:xx` | eero Pro 6E | eeroOS | Lower floor | Wi-Fi mesh | +| bifrost | `10.0.20.104` | `ec:b5:fa:xx:xx:xx` | Philips Hue Bridge[^Huebridge] | — | Main floor | Lighting | +| speaker-01…04 | `.113`, `.124`, `.128`, `.132` | `d4:90:9c:xx:xx:xx`, `94:ea:32:xx:xx:xx`, `f4:34:f0:xx:xx:xx` | Apple HomePod[^homepod] | audioOS | Various | Assistant | +| assistant-01…05 | `.105`, `.109`, `.114`, `.133`, `.144` | `74:d4:23:xx:xx:xx`, `58:a8:e8:xx:xx:xx`, `1c:fe:2b:xx:xx:xx`, `4c:ef:c0:xx:xx:xx`, `68:b6:91:xx:xx:xx` | Amazon Echo[^echo] | FireOS | Various | Assistant | +| camera-01…07 | `.112`, `.118`, `.119`, `.126`, `.130`, `.145`, `.146` | `10:08:2c:xx:xx:xx`, `b4:bc:7c:xx:xx:xx`, `3c:e1:a1:xx:xx:xx`, `54:e0:19:xx:xx:xx`, `18:7f:88:xx:xx:xx` | Ring cameras, floodlights, doorbell[^floodlight] [^doorbell] [^camera] | — | Interior & exterior | Camera | +| alarm-hub | `10.0.20.121` | `2c:6b:7d:xx:xx:xx` | Ring Alarm Base Station[^basestation] | — | Main floor | Hub | +| monitor-01 | `10.0.20.117` | `a4:97:5c:xx:xx:xx` | VTech camera[^Monitor] | — | Upper floor | Baby monitor | +| monitor-02 | `10.0.20.149` | `a4:97:5c:xx:xx:xx` | VTech tablet[^Monitor] | — | Upper floor | Baby monitor | +| appliance-01 | `10.0.20.108` | `c0:49:ef:xx:xx:xx` | Litter-Robot 4[^litterrobot] | — | Utility | Appliance | +| appliance-02 | `10.0.20.115` | `50:8b:b9:xx:xx:xx` | Tuya white-noise machine[^Whitenoise] | — | Upper floor | Appliance | + +### Notes + +- All wireless. One eero is wired as backhaul. +- Internet only. No device here can initiate a connection to any other segment, + which is the entire reason this VLAN exists. A camera or a $20 Tuya device + with a hardcoded credential is a foothold, not a light switch. +- Device addresses and rooms are collapsed above deliberately. The exact + camera-to-room mapping is not something a public repository needs to carry. + +[^Huebridge]: [Philips Hue Bridge](https://www.philips-hue.com/en-us/p/hue-bridge/046677458478) +[^echo]: [Amazon Echo](https://www.amazon.com/dp/B07XKF5RM3) +[^litterrobot]: [Litter-Robot 4](https://www.litter-robot.com/litter-robot-4.html) +[^floodlight]: [Ring Floodlight Cam](https://ring.com/products/floodlight-cam-plus-wired) +[^doorbell]: [Ring Doorbell](https://ring.com/products/battery-doorbell) +[^basestation]: [Ring Alarm Base Station](https://ring.com/products/alarm-base-station-v2) +[^camera]: [Ring Indoor Cam](https://ring.com/products/indoor-camera) +[^homepod]: [Apple HomePod](https://www.apple.com/homepod/) +[^Whitenoise]: [White noise machine](https://a.co/d/9NG05GM) +[^Monitor]: [VTech baby monitor](https://www.vtechkids.com/monitors) + +--- + +## Degens — VLAN 10 — Guest + +| Hostname | IP | MAC (OUI) | Device | OS | Zone | Role | +| --- | --- | --- | --- | --- | --- | --- | +| morpheus | `10.0.10.1` | `02:26:26:xx:xx:xx` | HP ProDesk 600 G4 Mini | FreeBSD 15.0 | Rack U5 | Firewall | +| eero-guest-1 | `10.0.10.10` | `fc:3f:a6:xx:xx:xx` | eero Pro 6E | eeroOS | Main floor | Wi-Fi mesh | +| eero-guest-2 | `10.0.10.101` | `9c:57:bc:xx:xx:xx` | eero Pro 6E | eeroOS | Main floor | Wi-Fi mesh | + +### Notes + +- Wired and wireless. A small unmanaged switch on the main floor serves wired + guests. +- Internet only, client isolation on, no access to any other segment. + +--- + +## Rack and hardware + +See [`hardware.md`](hardware.md). + +## Diagrams + +- [Current topology](diagrams/current/matrix_elysium.png) — high-resolution + export. An inline Mermaid version is in [`architecture.md`](architecture.md). +- [Previous topology](diagrams/previous/Network_Diagram.png) — kept for + comparison. diff --git a/grafana/prometheus.matrix.elysium/docker-compose.yaml b/grafana/prometheus.matrix.elysium/docker-compose.yaml deleted file mode 100644 index f49172c..0000000 --- a/grafana/prometheus.matrix.elysium/docker-compose.yaml +++ /dev/null @@ -1,19 +0,0 @@ -services: - grafana: - image: grafana/grafana-oss:latest - container_name: grafana - environment: - - GF_SECURITY_ADMIN_USER=admin - - GF_SECURITY_ADMIN_PASSWORD=admin - - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-piechart-panel - - GF_PATHS_PROVISIONING=/etc/grafana/provisioning - - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin - - GF_AUTH_ANONYMOUS_ENABLED=true - ports: - - "3000:3000" - volumes: - - /opt/grafana/data:/grafana - depends_on: - - prometheus - - loki - restart: unless-stopped \ No newline at end of file diff --git a/loki/prometheus.matrix.elysium/config/loki-config.yaml b/loki/prometheus.matrix.elysium/config/loki-config.yaml deleted file mode 100644 index afba4a8..0000000 --- a/loki/prometheus.matrix.elysium/config/loki-config.yaml +++ /dev/null @@ -1,37 +0,0 @@ -auth_enabled: false - -server: - http_listen_port: 3100 - -common: - path_prefix: /tmp/loki - storage: - filesystem: - chunks_directory: /tmp/loki/chunks - rules_directory: /tmp/loki/rules - replication_factor: 1 - ring: - kvstore: - store: inmemory - -schema_config: - configs: - - from: 2024-01-01 - store: tsdb - object_store: filesystem - schema: v13 - index: - prefix: index_ - period: 24h - -storage_config: - tsdb_shipper: - active_index_directory: /tmp/loki/index - cache_location: /tmp/loki/index_cache - filesystem: - directory: /tmp/loki/chunks - -limits_config: - volume_enabled: true - reject_old_samples: true - reject_old_samples_max_age: 168h \ No newline at end of file diff --git a/loki/prometheus.matrix.elysium/docker-compose.yaml b/loki/prometheus.matrix.elysium/docker-compose.yaml deleted file mode 100644 index e1c4af4..0000000 --- a/loki/prometheus.matrix.elysium/docker-compose.yaml +++ /dev/null @@ -1,11 +0,0 @@ -services: - loki: - image: grafana/loki:latest - container_name: loki - command: --config.file=/etc/loki/loki-config.yml - volumes: - - /opt/loki/loki-config.yml:/etc/loki/loki-config.yml - - /opt/loki/data:/loki - ports: - - "3100:3100" - restart: unless-stopped \ No newline at end of file diff --git a/prometheus/prometheus.matrix.elysium/config/prometheus.yaml b/prometheus/prometheus.matrix.elysium/config/prometheus.yaml deleted file mode 100644 index f1610e6..0000000 --- a/prometheus/prometheus.matrix.elysium/config/prometheus.yaml +++ /dev/null @@ -1,75 +0,0 @@ -global: - scrape_interval: 15s - -scrape_configs: - - job_name: 'prometheus' - static_configs: - - targets: ['localhost:9090'] - - - job_name: 'snmp_exporter_pfSense' - scrape_interval: 60s - scrape_timeout: 45s - static_configs: - - targets: ['10.0.99.1'] - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: 10.0.99.20:9116 - metrics_path: /snmp - params: - module: ['morpheus'] - auth: ['pfsense'] - - - job_name: 'snmp_exporter_apc' - scrape_interval: 60s - scrape_timeout: 45s - static_configs: - - targets: ['10.0.99.10'] - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: 10.0.99.20:9116 - metrics_path: /snmp - params: - module: ['mjolnir'] - auth: ['apc'] - - - job_name: 'snmp_exporter_iLO' - scrape_interval: 60s - scrape_timeout: 45s - static_configs: - - targets: ['10.0.30.10'] - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: 10.0.99.20:9116 - metrics_path: /snmp - params: - module: ['shiva'] - auth: ['iLO'] - - - job_name: 'snmp_exporter_mokerlink' - scrape_interval: 60s - scrape_timeout: 45s - static_configs: - - targets: ['10.7.7.2'] - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: 10.0.99.20:9116 - metrics_path: /snmp - params: - module: ['neo'] - auth: ['mokerlink'] \ No newline at end of file diff --git a/prometheus/prometheus.matrix.elysium/docker-compose.yaml b/prometheus/prometheus.matrix.elysium/docker-compose.yaml deleted file mode 100644 index 2a6c0dc..0000000 --- a/prometheus/prometheus.matrix.elysium/docker-compose.yaml +++ /dev/null @@ -1,13 +0,0 @@ -services: - prometheus: - image: prom/prometheus:latest - command: - - --web.enable-remote-write-receiver - - --config.file=/etc/prometheus/prometheus.yml - container_name: prometheus - volumes: - - ./prometheus.yml:/etc/prometheus/prometheus.yml - - ./data:/prometheus - ports: - - "9090:9090" - restart: unless-stopped \ No newline at end of file diff --git a/snmp-exporter/prometheus.matrix.elysium/docker-compose.yaml b/snmp-exporter/prometheus.matrix.elysium/docker-compose.yaml deleted file mode 100644 index d69be15..0000000 --- a/snmp-exporter/prometheus.matrix.elysium/docker-compose.yaml +++ /dev/null @@ -1,9 +0,0 @@ -services: - snmp-exporter: - image: prom/snmp-exporter:latest - container_name: snmp-exporter - restart: unless-stopped - ports: - - "9116:9116" - volumes: - - ./snmp.yml:/etc/snmp_exporter/snmp.yml \ No newline at end of file diff --git a/alloy/prometheus.matrix.elysium/config/config.alloy b/stacks/observability/alloy/config.alloy similarity index 88% rename from alloy/prometheus.matrix.elysium/config/config.alloy rename to stacks/observability/alloy/config.alloy index f15cabd..cbed3f3 100644 --- a/alloy/prometheus.matrix.elysium/config/config.alloy +++ b/stacks/observability/alloy/config.alloy @@ -1,3 +1,17 @@ +// ============================================================================= +// Grafana Alloy — unified metrics and log collection agent. +// +// This same file runs on every monitored host. The two remote endpoints are +// read from the environment so a host outside the compose network can point at +// the monitoring host explicitly: +// +// LOKI_URL=http://10.0.99.20:3100/loki/api/v1/push +// PROMETHEUS_REMOTE_WRITE_URL=http://10.0.99.20:9090/api/v1/write +// +// Inside the observability stack the compose service names are the defaults. +// See docs/observability.md. +// ============================================================================= + // ============================================================================ // METRICS COLLECTION // ============================================================================ @@ -236,7 +250,7 @@ loki.process "log_processor" { forward_to = [loki.write.grafana_loki.receiver] stage.regex { - expression = "(?P(?i)\b(emerg|panic|corrupt|fatal|alert|crit|critical|err|eror|error|warn|warning|info|information|informational|notice|opened|closed|dbug|debug|dbg)\b)" + expression = `(?P(?i)\b(emerg|panic|corrupt|fatal|alert|crit|critical|err|eror|error|warn|warning|info|information|informational|notice|opened|closed|dbug|debug|dbg)\b)` } stage.template { @@ -262,7 +276,7 @@ loki.process "log_processor" { loki.write "grafana_loki" { endpoint { - url = "http://10.0.99.20:3100/loki/api/v1/push" + url = coalesce(sys.env("LOKI_URL"), "http://loki:3100/loki/api/v1/push") } external_labels = { host = constants.hostname, @@ -271,7 +285,7 @@ loki.write "grafana_loki" { prometheus.remote_write "metrics_service" { endpoint { - url = "http://10.0.99.20:9090/api/v1/write" + url = coalesce(sys.env("PROMETHEUS_REMOTE_WRITE_URL"), "http://prometheus:9090/api/v1/write") } external_labels = { host = constants.hostname, diff --git a/stacks/observability/loki/loki-config.yaml b/stacks/observability/loki/loki-config.yaml new file mode 100644 index 0000000..b162003 --- /dev/null +++ b/stacks/observability/loki/loki-config.yaml @@ -0,0 +1,70 @@ +--- +# Single-binary Loki with filesystem storage. This lab produces well under +# 1 GB/day of logs, so a microservices deployment would add operational cost +# for no benefit. See docs/adr/0003-observability-stack-selection.md. + +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: info + +common: + # Everything lives under /loki, which is a named volume in compose.yaml. + # This previously pointed at /tmp/loki while the volume was mounted at /loki, + # so every log line was written to the container's writable layer and lost on + # recreate. + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +storage_config: + tsdb_shipper: + active_index_directory: /loki/index + cache_location: /loki/index_cache + filesystem: + directory: /loki/chunks + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + compaction_interval: 10m + +limits_config: + volume_enabled: true + reject_old_samples: true + reject_old_samples_max_age: 168h + # 30 days, matching the Prometheus retention default in .env.example. + retention_period: 720h + max_query_series: 5000 + allow_structured_metadata: true + +ruler: + storage: + type: local + local: + directory: /loki/rules + rule_path: /loki/rules-temp + alertmanager_url: http://alertmanager:9093 + enable_api: true + +analytics: + reporting_enabled: false diff --git a/stacks/observability/prometheus/prometheus.yaml b/stacks/observability/prometheus/prometheus.yaml new file mode 100644 index 0000000..860edca --- /dev/null +++ b/stacks/observability/prometheus/prometheus.yaml @@ -0,0 +1,81 @@ +--- +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 30s + external_labels: + site: matrix.elysium + monitor: winterfell + +rule_files: + - /etc/prometheus/rules/*.rules.yaml + +alerting: + alertmanagers: + - static_configs: + - targets: ["alertmanager:9093"] + +scrape_configs: + # --------------------------------------------------------------------------- + # The stack monitoring itself. Names resolve on the compose network, so moving + # the monitoring host no longer means editing this file. + # --------------------------------------------------------------------------- + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + + - job_name: alertmanager + static_configs: + - targets: ["alertmanager:9093"] + + - job_name: loki + static_configs: + - targets: ["loki:3100"] + + - job_name: grafana + static_configs: + - targets: ["grafana:3000"] + + - job_name: snmp-exporter + static_configs: + - targets: ["snmp-exporter:9116"] + + - job_name: alloy + static_configs: + - targets: ["alloy:12345"] + + # --------------------------------------------------------------------------- + # SNMP network devices. + # + # This replaced four near-identical jobs that differed only in target, module + # and auth. Targets now live in targets/snmp.yaml and are hot-reloaded by + # file_sd without restarting Prometheus; adding a device is a three-line + # append. See docs/runbooks/add-monitored-device.md. + # + # The `module` and `auth` labels on each target are promoted to the query + # parameters snmp_exporter expects, then dropped so they do not become + # metric labels. + # --------------------------------------------------------------------------- + - job_name: snmp + scrape_interval: 60s + scrape_timeout: 45s + metrics_path: /snmp + file_sd_configs: + - files: + - /etc/prometheus/targets/snmp.yaml + refresh_interval: 5m + relabel_configs: + # The file_sd target is the *device* address, which becomes the query arg. + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - source_labels: [module] + target_label: __param_module + - source_labels: [auth] + target_label: __param_auth + - regex: (module|auth) + action: labeldrop + # ...and the address Prometheus actually connects to is the exporter. + - target_label: __address__ + replacement: snmp-exporter:9116 diff --git a/snmp-exporter/prometheus.matrix.elysium/config/snmp.yaml b/stacks/observability/snmp-exporter/snmp.yaml similarity index 99% rename from snmp-exporter/prometheus.matrix.elysium/config/snmp.yaml rename to stacks/observability/snmp-exporter/snmp.yaml index 07cfc85..7475604 100644 --- a/snmp-exporter/prometheus.matrix.elysium/config/snmp.yaml +++ b/stacks/observability/snmp-exporter/snmp.yaml @@ -1,31 +1,36 @@ -# WARNING: This file was auto-generated using snmp_exporter generator, manual changes will be lost. +# WARNING: generated by the snmp_exporter generator from generator.yaml. +# Do not edit by hand — run `make snmp-generate` instead. +# +# Community strings are ${PLACEHOLDERS}. scripts/render-config.sh decrypts +# secrets/observability.sops.yaml and renders the real file into +# .rendered/snmp.yaml, which is gitignored and is what the container mounts. auths: - apc: - community: 7H3r315N05p00N! + auth_apc: + community: ${SNMP_COMMUNITY_APC} security_level: noAuthNoPriv auth_protocol: MD5 priv_protocol: DES version: 2 - iLO: - community: 7H3r315N05p00N! + auth_ilo: + community: ${SNMP_COMMUNITY_ILO} security_level: noAuthNoPriv auth_protocol: MD5 priv_protocol: DES version: 2 - mokerlink: - community: 7H3r315N05p00N! + auth_mokerlink: + community: ${SNMP_COMMUNITY_MOKERLINK} security_level: noAuthNoPriv auth_protocol: MD5 priv_protocol: DES version: 2 - pfsense: - community: 7H3r315N05p00N! + auth_pfsense: + community: ${SNMP_COMMUNITY_PFSENSE} security_level: noAuthNoPriv auth_protocol: MD5 priv_protocol: DES version: 2 modules: - mjolnir: + apc_ups: walk: - 1.3.6.1.2.1.33.1.3.2 - 1.3.6.1.2.1.33.1.3.3.1.1 @@ -350,7 +355,7 @@ modules: type: gauge help: The minimum input line voltage allowed before the UPS system transfers to battery backup. - 1.3.6.1.2.1.33.1.9.9 - morpheus: + pfsense: walk: - 1.3.6.1.4.1.12325.1.200.1.11.2 - 1.3.6.1.4.1.12325.1.200.1.8.2 @@ -1057,7 +1062,7 @@ modules: oid: 1.3.6.1.4.1.12325.1.3.1.1 type: counter help: The speed of the fastest interface in ifTable in bps. - 1.3.6.1.4.1.12325.1.3.1.1 - neo: + mokerlink: walk: - 1.3.6.1.2.1.2.2.1.1 - 1.3.6.1.2.1.2.2.1.10 @@ -1346,7 +1351,7 @@ modules: 5: dormant 6: notPresent 7: lowerLayerDown - shiva: + ilo: walk: - 1.3.6.1.2.1.1.2 - 1.3.6.1.2.1.1.3.0 From 688414241e6bf1dc56cd1837a3064f0f87d714d8 Mon Sep 17 00:00:00 2001 From: Garrett Allen Date: Sun, 2 Aug 2026 06:22:49 +0000 Subject: [PATCH 2/8] fix(observability): replace five broken compose files with one working stack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit None of the previous compose files could start: * every mount source pointed at a path that does not exist in the repo (./prometheus.yml vs config/prometheus.yaml, /opt/loki/loki-config.yml, /opt/alloy/config.alloy) * grafana's file declared depends_on: [prometheus, loki], neither of which was defined in it, which compose rejects outright * grafana mounted its data at /grafana instead of /var/lib/grafana, so its database was written to the container layer and lost on every recreate * loki wrote to /tmp/loki while its volume was mounted at /loki, losing every log line on restart One compose.yaml now defines all six services on a shared network with healthchecks and health-gated ordering. Services resolve each other by name, which removed the hardcoded 10.0.99.20 from both prometheus.yaml and config.alloy. Other changes: * all images pinned to explicit versions; CI fails on :latest * anonymous Grafana Admin access disabled, password sourced from SOPS * snmp-exporter is no longer published to a host interface * Alloy's debug UI binds to 127.0.0.1 only * Prometheus/Alertmanager/snmp-exporter run as nobody * Alertmanager added, with severity routing and inhibit rules * commit generator.yaml, the snmp_exporter source of truth that was missing — the generated file carried a 'manual changes will be lost' warning with no way to regenerate it Fixes a live bug in config.alloy: the log_processor regex used \\b inside a double-quoted Alloy string, where it is a backspace escape rather than a word boundary. The regex never matched, so the template's {{ else }}info{{ end }} fallback labelled every log line 'info'. It is now a backtick string. --- stacks/observability/.env.example | 25 +++ stacks/observability/README.md | 62 ++++++ .../alertmanager/alertmanager.yaml | 72 ++++++ stacks/observability/compose.yaml | 208 ++++++++++++++++++ .../snmp-exporter/generator.yaml | 71 ++++++ 5 files changed, 438 insertions(+) create mode 100644 stacks/observability/.env.example create mode 100644 stacks/observability/README.md create mode 100644 stacks/observability/alertmanager/alertmanager.yaml create mode 100644 stacks/observability/compose.yaml create mode 100644 stacks/observability/snmp-exporter/generator.yaml diff --git a/stacks/observability/.env.example b/stacks/observability/.env.example new file mode 100644 index 0000000..f11573d --- /dev/null +++ b/stacks/observability/.env.example @@ -0,0 +1,25 @@ +# Non-sensitive tunables for the observability stack. +# +# Copy to .env and edit, or just rely on the defaults baked into compose.yaml. +# Secrets do NOT belong here — see secrets/README.md. +# +# scripts/render-config.sh appends the decrypted secret values to .env at +# deploy time, which is why .env itself is gitignored. + +# Address the published ports bind to. 0.0.0.0 exposes the stack to every VLAN +# the host can reach; set this to the management IP to restrict it to VLAN 99. +BIND_ADDR=0.0.0.0 + +# Used for Alertmanager's external URL in notification links. +MONITORING_HOST=10.0.99.20 + +# Published ports. +PROMETHEUS_PORT=9090 +ALERTMANAGER_PORT=9093 +LOKI_PORT=3100 +GRAFANA_PORT=3000 +ALLOY_PORT=12345 + +# Metric retention. Loki's log retention is set separately in +# loki/loki-config.yaml (retention_period) — keep the two in step. +PROMETHEUS_RETENTION=30d diff --git a/stacks/observability/README.md b/stacks/observability/README.md new file mode 100644 index 0000000..d7f8bc9 --- /dev/null +++ b/stacks/observability/README.md @@ -0,0 +1,62 @@ +# Observability stack + +Runs on `prometheus` (10.0.99.20), VLAN 99. + +```bash +make up # from the repository root +``` + +| Service | Image | Port | Purpose | +| --- | --- | --- | --- | +| `prometheus` | `prom/prometheus:v3.1.0` | 9090 | Metrics store, remote-write receiver, rule evaluation | +| `alertmanager` | `prom/alertmanager:v0.28.0` | 9093 | Alert routing, grouping, inhibition | +| `loki` | `grafana/loki:3.3.2` | 3100 | Log store | +| `grafana` | `grafana/grafana-oss:11.5.2` | 3000 | Dashboards | +| `snmp-exporter` | `prom/snmp-exporter:v0.28.0` | *internal* | SNMP polling proxy | +| `alloy` | `grafana/alloy:v1.6.1` | 12345 (localhost) | Metric and log collection | + +## Layout + +```text +compose.yaml all six services, one network, health-gated ordering +.env.example non-sensitive tunables (ports, retention, bind address) +prometheus/ + prometheus.yaml scrape config; SNMP via file_sd + targets/snmp.yaml SNMP targets — hot-reloaded, no restart needed + rules/*.rules.yaml 32 alert rules across host/network/ups/containers +alertmanager/ + alertmanager.yaml severity + category routing, inhibition +loki/loki-config.yaml single-binary, filesystem, 30-day retention +alloy/config.alloy the agent config, identical on every monitored host +snmp-exporter/ + generator.yaml source of truth — edit this + snmp.yaml generated, 14k lines, ${PLACEHOLDER} communities +grafana/ + provisioning/ datasources + dashboard provider + dashboards/*.json 5 dashboards, 79 panels +``` + +## Things worth knowing before editing + +- **`snmp-exporter/snmp.yaml` is generated.** Edit `generator.yaml` and run + `make snmp-generate`. Its community strings are `${PLACEHOLDERS}`; + `scripts/render-config.sh` renders the real file into a gitignored + `.rendered/` directory at deploy time. +- **Grafana UI edits are discarded on restart** (`allowUiUpdates: false`). Export + the JSON model and commit it — see + [`docs/observability.md`](../../docs/observability.md#dashboards). +- **Rules and routes hot-reload** with `make reload`. No restart, no TSDB head + block dropped. +- **Adding an SNMP target needs no restart** — file_sd re-reads every 5 minutes. + Adding a *module* does, because snmp-exporter reads its config once. +- **Image tags are pinned.** CI fails on `:latest`. Dependabot proposes bumps. + +## Validate before deploying + +```bash +make validate +``` + +Runs `docker compose config`, `promtool check config`, `promtool check rules`, +`amtool check-config`, `alloy fmt --verify`, the dashboard checks, yamllint, +markdownlint, shellcheck and gitleaks. Same set CI runs. diff --git a/stacks/observability/alertmanager/alertmanager.yaml b/stacks/observability/alertmanager/alertmanager.yaml new file mode 100644 index 0000000..c9cbe20 --- /dev/null +++ b/stacks/observability/alertmanager/alertmanager.yaml @@ -0,0 +1,72 @@ +--- +# Alert routing. +# +# The receiver URL is injected from the SOPS-encrypted secrets file at deploy +# time — Alertmanager expands $ENV_VAR references in *_url fields natively, so +# the webhook never touches the repository. + +global: + resolve_timeout: 5m + +templates: + - /etc/alertmanager/templates/*.tmpl + +route: + receiver: default + group_by: ["alertname", "device", "instance"] + group_wait: 30s + group_interval: 5m + # Don't re-nag every hour for something already known to be broken. + repeat_interval: 12h + + routes: + # Power events page immediately and repeat aggressively — a UPS on battery + # with no spare battery installed is a short fuse. + - receiver: urgent + matchers: + - severity = "critical" + - category = "power" + group_wait: 0s + repeat_interval: 30m + + - receiver: urgent + matchers: + - severity = "critical" + group_wait: 10s + repeat_interval: 4h + + - receiver: default + matchers: + - severity = "warning" + + # Informational alerts are recorded but never notified. + - receiver: "null" + matchers: + - severity = "info" + +inhibit_rules: + # If a host is down, don't also complain about its disks and its scrape. + - source_matchers: [severity = "critical"] + target_matchers: [severity = "warning"] + equal: ["alertname", "instance"] + + - source_matchers: [alertname = "InstanceDown"] + target_matchers: [severity =~ "warning|critical"] + equal: ["instance"] + + # A dead SNMP exporter makes every device look dead. + - source_matchers: [alertname = "SnmpExporterDown"] + target_matchers: [alertname = "SnmpTargetUnreachable"] + +receivers: + - name: "null" + + - name: default + webhook_configs: + - url: $ALERTMANAGER_WEBHOOK_URL + send_resolved: true + + - name: urgent + webhook_configs: + - url: $ALERTMANAGER_WEBHOOK_URL + send_resolved: true diff --git a/stacks/observability/compose.yaml b/stacks/observability/compose.yaml new file mode 100644 index 0000000..ea6cca4 --- /dev/null +++ b/stacks/observability/compose.yaml @@ -0,0 +1,208 @@ +--- +# Observability stack for the matrix.elysium homelab. +# +# Deployed on: prometheus (10.0.99.20, VLAN 99 / Winterfell) +# Deploy with: make up (from the repository root) +# +# Image tags are pinned deliberately — `:latest` makes a rollback impossible to +# describe and lets a breaking upstream release land silently. Dependabot opens +# a PR for each bump, and CI validates the configs against the new tag. + +name: observability + +x-logging: &default-logging + driver: json-file + options: + max-size: "10m" + max-file: "3" + +x-service-defaults: &service-defaults + restart: unless-stopped + logging: *default-logging + networks: [observability] + +services: + # --------------------------------------------------------------------------- + # Metrics store. Also acts as the remote-write sink for Alloy agents running + # on other hosts, hence --web.enable-remote-write-receiver. + # --------------------------------------------------------------------------- + prometheus: + <<: *service-defaults + image: prom/prometheus:v3.1.0 + container_name: prometheus + user: "65534:65534" + command: + - --config.file=/etc/prometheus/prometheus.yaml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-30d} + - --web.enable-remote-write-receiver + - --web.enable-lifecycle + volumes: + - ./prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro + - ./prometheus/rules:/etc/prometheus/rules:ro + - ./prometheus/targets:/etc/prometheus/targets:ro + - prometheus-data:/prometheus + ports: + - "${BIND_ADDR:-0.0.0.0}:${PROMETHEUS_PORT:-9090}:9090" + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + + # --------------------------------------------------------------------------- + # Alert routing. Receiver URLs come from SOPS-encrypted secrets, never the + # compose file — see secrets/README.md. + # --------------------------------------------------------------------------- + alertmanager: + <<: *service-defaults + image: prom/alertmanager:v0.28.0 + container_name: alertmanager + user: "65534:65534" + command: + - --config.file=/etc/alertmanager/alertmanager.yaml + - --storage.path=/alertmanager + - --web.external-url=http://${MONITORING_HOST:-10.0.99.20}:${ALERTMANAGER_PORT:-9093} + environment: + ALERTMANAGER_WEBHOOK_URL: ${ALERTMANAGER_WEBHOOK_URL:?set in secrets/observability.sops.yaml} + volumes: + - ./alertmanager/alertmanager.yaml:/etc/alertmanager/alertmanager.yaml:ro + - alertmanager-data:/alertmanager + ports: + - "${BIND_ADDR:-0.0.0.0}:${ALERTMANAGER_PORT:-9093}:9093" + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:9093/-/healthy"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 15s + + # --------------------------------------------------------------------------- + # Log store. Single-binary mode with filesystem storage: this lab pushes well + # under 1 GB/day, so the operational cost of a microservices deployment buys + # nothing. See docs/adr/0003-observability-stack-selection.md. + # --------------------------------------------------------------------------- + loki: + <<: *service-defaults + image: grafana/loki:3.3.2 + container_name: loki + user: "10001:10001" + command: -config.file=/etc/loki/loki-config.yaml + volumes: + - ./loki/loki-config.yaml:/etc/loki/loki-config.yaml:ro + - loki-data:/loki + ports: + - "${BIND_ADDR:-0.0.0.0}:${LOKI_PORT:-3100}:3100" + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:3100/ready"] + interval: 30s + timeout: 5s + retries: 5 + start_period: 45s + + # --------------------------------------------------------------------------- + # SNMP polling proxy for pfSense, the MokerLink switch, the APC UPS and iLO. + # snmp.yaml is rendered from the committed template by scripts/render-config.sh + # so the community string is never written to disk in the repo. + # --------------------------------------------------------------------------- + snmp-exporter: + <<: *service-defaults + image: prom/snmp-exporter:v0.28.0 + container_name: snmp-exporter + user: "65534:65534" + command: + - --config.file=/etc/snmp_exporter/snmp.yaml + volumes: + - ./snmp-exporter/.rendered/snmp.yaml:/etc/snmp_exporter/snmp.yaml:ro + expose: + - "9116" + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:9116/health"] + interval: 60s + timeout: 10s + retries: 3 + start_period: 20s + + # --------------------------------------------------------------------------- + # Visualisation. Datasources and dashboards are provisioned from files, so a + # wiped volume costs nothing but annotations — the dashboards are in git. + # --------------------------------------------------------------------------- + grafana: + <<: *service-defaults + image: grafana/grafana-oss:11.5.2 + container_name: grafana + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set in secrets/observability.sops.yaml} + GF_SECURITY_COOKIE_SECURE: "false" + GF_SECURITY_DISABLE_GRAVATAR: "true" + GF_USERS_ALLOW_SIGN_UP: "false" + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_INSTALL_PLUGINS: grafana-clock-panel,grafana-piechart-panel + GF_PATHS_PROVISIONING: /etc/grafana/provisioning + PROMETHEUS_URL: http://prometheus:9090 + LOKI_URL: http://loki:3100 + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana-data:/var/lib/grafana + ports: + - "${BIND_ADDR:-0.0.0.0}:${GRAFANA_PORT:-3000}:3000" + depends_on: + prometheus: + condition: service_healthy + loki: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + + # --------------------------------------------------------------------------- + # Collection agent for this host: cAdvisor + node metrics, Docker/journal/ + # syslog/auth logs. The same config.alloy runs on every other monitored host + # with LOKI_URL / PROMETHEUS_REMOTE_WRITE_URL pointed back here. + # --------------------------------------------------------------------------- + alloy: + <<: *service-defaults + image: grafana/alloy:v1.6.1 + container_name: alloy + privileged: true + environment: + LOKI_URL: http://loki:3100/loki/api/v1/push + PROMETHEUS_REMOTE_WRITE_URL: http://prometheus:9090/api/v1/write + command: + - run + - --server.http.listen-addr=0.0.0.0:12345 + - --storage.path=/var/lib/alloy/data + - /etc/alloy/config.alloy + volumes: + - ./alloy/config.alloy:/etc/alloy/config.alloy:ro + - alloy-data:/var/lib/alloy/data + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/log:/var/log:ro + - /:/rootfs:ro + ports: + - "127.0.0.1:${ALLOY_PORT:-12345}:12345" + depends_on: + loki: + condition: service_healthy + prometheus: + condition: service_healthy + +networks: + observability: + driver: bridge + +volumes: + prometheus-data: + alertmanager-data: + loki-data: + grafana-data: + alloy-data: diff --git a/stacks/observability/snmp-exporter/generator.yaml b/stacks/observability/snmp-exporter/generator.yaml new file mode 100644 index 0000000..d6195db --- /dev/null +++ b/stacks/observability/snmp-exporter/generator.yaml @@ -0,0 +1,71 @@ +--- +# Source of truth for snmp.yaml. +# +# The committed snmp.yaml is 14k lines of expanded OID definitions produced from +# this file by the snmp_exporter generator. Before this file existed, snmp.yaml +# carried a "manual changes will be lost" warning with no way to regenerate it — +# editing it was a trap. +# +# Regenerate with: make snmp-generate +# +# Community strings are ${PLACEHOLDERS} resolved at deploy time by +# scripts/render-config.sh. Each device has its own credential: a single shared +# community is a read credential for every device at once, and SNMPv2c sends it +# in cleartext. See docs/runbooks/rotate-snmp-community.md. + +auths: + auth_apc: + community: ${SNMP_COMMUNITY_APC} + security_level: noAuthNoPriv + auth_protocol: MD5 + priv_protocol: DES + version: 2 + + auth_ilo: + community: ${SNMP_COMMUNITY_ILO} + security_level: noAuthNoPriv + auth_protocol: MD5 + priv_protocol: DES + version: 2 + + auth_mokerlink: + community: ${SNMP_COMMUNITY_MOKERLINK} + security_level: noAuthNoPriv + auth_protocol: MD5 + priv_protocol: DES + version: 2 + + auth_pfsense: + community: ${SNMP_COMMUNITY_PFSENSE} + security_level: noAuthNoPriv + auth_protocol: MD5 + priv_protocol: DES + version: 2 + +modules: + # APC Smart-UPS (mjolnir, 10.0.99.10) — standard UPS-MIB. + # Battery charge, runtime remaining, input/output voltage and load, alarms. + apc_ups: + walk: + - 1.3.6.1.2.1.33.1 # UPS-MIB::upsObjects + + # pfSense (morpheus, 10.0.99.1) — BEGEMOT-PF-MIB. + # Packet filter counters, state table usage, per-interface pf statistics. + pfsense: + walk: + - 1.3.6.1.4.1.12325.1 # BEGEMOT-PF-MIB::begemotPf + + # MokerLink 26-port managed switch (neo, 10.7.7.2) — IF-MIB ifTable. + # NOTE: 32-bit counters only. Adding ifXTable (64-bit) is tracked in + # docs/roadmap.md — on gigabit links ifInOctets wraps in ~34 seconds at + # line rate, which the 60s scrape interval cannot detect. + mokerlink: + walk: + - 1.3.6.1.2.1.2.2 # IF-MIB::ifTable + + # HPE ProLiant DL360 Gen9 iLO (shiva, 10.0.30.10) — SNMPv2-MIB system group + # plus the Compaq/HP Insight enterprise tree for hardware health. + ilo: + walk: + - 1.3.6.1.2.1.1 # SNMPv2-MIB::system + - 1.3.6.1.4.1.232 # CPQ (HP Insight) enterprise subtree From 24dd8e016b886ea5405217f91a10f0a805de4343 Mon Sep 17 00:00:00 2001 From: Garrett Allen Date: Sun, 2 Aug 2026 06:23:04 +0000 Subject: [PATCH 3/8] feat(secrets): manage credentials with SOPS and age The repository committed a plaintext SNMP community string shared across the firewall, switch, UPS and BMC, plus Grafana admin/admin alongside anonymous Admin access. Credentials are now encrypted with age and committed as ciphertext. SOPS leaves keys in plaintext and encrypts values, so git log shows which credential rotated and when without revealing what it rotated to. The private key lives at ~/.config/sops/age/keys.txt and never enters the repository. Each device now has its own SNMP community. SNMPv2c transmits the community in cleartext, so a single shared string meant one captured packet granted read access to every device on the network. snmp_exporter does no environment expansion and reads its config once at startup, so render-config.sh substitutes the community strings into a gitignored .rendered/ directory at deploy time. The tracked snmp.yaml keeps its ${PLACEHOLDERS}, and the script fails loudly if any survive substitution. make secrets-init generates the keypair and creates the encrypted file; it cannot be pre-seeded here without committing a private key. Rationale in docs/adr/0005-secrets-with-sops-and-age.md. --- .sops.yaml | 21 +++++ Makefile | 132 +++++++++++++++++++++++++++++ scripts/bootstrap.sh | 83 ++++++++++++++++++ scripts/render-config.sh | 111 ++++++++++++++++++++++++ secrets/README.md | 72 ++++++++++++++++ secrets/observability.example.yaml | 34 ++++++++ 6 files changed, 453 insertions(+) create mode 100644 .sops.yaml create mode 100644 Makefile create mode 100755 scripts/bootstrap.sh create mode 100755 scripts/render-config.sh create mode 100644 secrets/README.md create mode 100644 secrets/observability.example.yaml diff --git a/.sops.yaml b/.sops.yaml new file mode 100644 index 0000000..80719bf --- /dev/null +++ b/.sops.yaml @@ -0,0 +1,21 @@ +--- +# SOPS encryption policy. +# +# Everything under secrets/ matching *.sops.yaml is encrypted with the age +# recipient below. +# +# SOPS encrypts values and leaves keys in plaintext by default, which is exactly +# what we want: `git log -p secrets/observability.sops.yaml` shows *which* +# credential rotated and when, without revealing what it rotated to. +# +# The public key here is safe to commit; it can only encrypt. The matching +# private key lives at ~/.config/sops/age/keys.txt on the deployment host and is +# never in this repository. +# +# Run `make secrets-init` on a fresh machine to generate a keypair and write its +# public half into this file. + +creation_rules: + - path_regex: secrets/.*\.sops\.ya?ml$ + age: >- + REPLACE_WITH_YOUR_AGE_PUBLIC_KEY diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..fcc726d --- /dev/null +++ b/Makefile @@ -0,0 +1,132 @@ +SHELL := /bin/bash +.DEFAULT_GOAL := help + +STACK ?= observability +STACK_DIR := stacks/$(STACK) +COMPOSE := docker compose -f $(STACK_DIR)/compose.yaml +SECRETS := secrets/$(STACK).sops.yaml + +.PHONY: help +help: ## Show this help + @printf '\033[1mHomeLab\033[0m — make [STACK=observability]\n\n' + @grep -hE '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \ + | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-16s\033[0m %s\n", $$1, $$2}' + @printf '\n' + +# --------------------------------------------------------------------------- +# Lifecycle +# --------------------------------------------------------------------------- + +.PHONY: up +up: render ## Render config and start the stack + $(COMPOSE) up -d --remove-orphans + @printf '\n\033[0;32mup\033[0m — Grafana: http://localhost:$${GRAFANA_PORT:-3000}\n' + +.PHONY: down +down: ## Stop the stack (volumes are preserved) + $(COMPOSE) down --remove-orphans + +.PHONY: restart +restart: down up ## Restart the stack + +.PHONY: pull +pull: ## Pull the pinned images + $(COMPOSE) pull + +.PHONY: ps +ps: ## Show container status + $(COMPOSE) ps + +.PHONY: logs +logs: ## Tail logs (SERVICE=grafana to narrow) + $(COMPOSE) logs -f --tail=100 $(SERVICE) + +.PHONY: reload +reload: ## Hot-reload Prometheus and Alertmanager without a restart + $(COMPOSE) exec prometheus wget -q -O- --post-data='' http://localhost:9090/-/reload + $(COMPOSE) exec alertmanager wget -q -O- --post-data='' http://localhost:9093/-/reload + @printf '\033[0;32mreloaded\033[0m\n' + +.PHONY: nuke +nuke: ## Stop the stack AND delete its volumes (destroys all metrics and logs) + @printf '\033[0;33mThis deletes every metric and log stored by the stack.\033[0m\n' + @read -p "Type 'nuke' to continue: " c; [ "$$c" = "nuke" ] || exit 1 + $(COMPOSE) down --volumes --remove-orphans + +# --------------------------------------------------------------------------- +# Secrets +# --------------------------------------------------------------------------- + +.PHONY: secrets-init +secrets-init: ## Generate an age keypair and create the encrypted secrets file + ./scripts/bootstrap.sh $(STACK) + +.PHONY: secrets-edit +secrets-edit: ## Edit the encrypted secrets in $$EDITOR + sops $(SECRETS) + +.PHONY: secrets-show +secrets-show: ## Print the decrypted secrets to stdout (careful) + sops --decrypt $(SECRETS) + +.PHONY: render +render: ## Decrypt secrets and render runtime config + ./scripts/render-config.sh $(STACK) + +# --------------------------------------------------------------------------- +# Validation +# --------------------------------------------------------------------------- + +.PHONY: validate +validate: ## Run every check CI runs + ./scripts/validate.sh + +.PHONY: lint +lint: ## Lint YAML, Markdown and shell + yamllint . + markdownlint-cli2 + shellcheck scripts/*.sh + +.PHONY: check-dashboards +check-dashboards: ## Validate dashboard JSON and datasource references + python3 scripts/check_dashboards.py + +.PHONY: check-rules +check-rules: ## Validate Prometheus rules and config + promtool check config $(STACK_DIR)/prometheus/prometheus.yaml + promtool check rules $(STACK_DIR)/prometheus/rules/*.rules.yaml + +.PHONY: scan +scan: ## Scan the working tree and history for secrets + gitleaks detect --no-banner --redact -c .gitleaks.toml + gitleaks detect --no-banner --redact -c .gitleaks.toml --log-opts="--all" + +# --------------------------------------------------------------------------- +# Maintenance +# --------------------------------------------------------------------------- + +.PHONY: snmp-generate +snmp-generate: ## Regenerate snmp.yaml from generator.yaml + docker run --rm \ + -v "$(PWD)/$(STACK_DIR)/snmp-exporter:/opt/" \ + -e SNMP_COMMUNITY_PFSENSE='$${SNMP_COMMUNITY_PFSENSE}' \ + -e SNMP_COMMUNITY_APC='$${SNMP_COMMUNITY_APC}' \ + -e SNMP_COMMUNITY_MOKERLINK='$${SNMP_COMMUNITY_MOKERLINK}' \ + -e SNMP_COMMUNITY_ILO='$${SNMP_COMMUNITY_ILO}' \ + prom/snmp-generator:latest generate \ + -m /opt/mibs -g /opt/generator.yaml -o /opt/snmp.yaml + @printf '\033[0;33mCheck the diff before committing — placeholders must survive.\033[0m\n' + +.PHONY: backup +backup: ## Back up the stack's volumes to ./backups/ + @mkdir -p backups + @for v in prometheus-data loki-data grafana-data alertmanager-data; do \ + printf 'backing up %s\n' "$$v"; \ + docker run --rm -v $(STACK)_$$v:/data -v "$(PWD)/backups:/backup" \ + alpine tar czf "/backup/$$v.tar.gz" -C /data . ; \ + done + @printf '\033[0;32mwrote backups/\033[0m\n' + +.PHONY: purge-history-dry-run +purge-history-dry-run: ## Preview the git-history secret purge (safe) + ./scripts/purge-history.sh --dry-run diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh new file mode 100755 index 0000000..9135658 --- /dev/null +++ b/scripts/bootstrap.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# +# One-time setup on a new deployment host: generate an age keypair, register its +# public half in .sops.yaml, and create the encrypted secrets file. +# +# Safe to re-run — it refuses to overwrite an existing key or secrets file. +# +# Usage: scripts/bootstrap.sh [stack] (default: observability) + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +STACK="${1:-observability}" +KEY_FILE="${SOPS_AGE_KEY_FILE:-${HOME}/.config/sops/age/keys.txt}" +SECRETS_FILE="${REPO_ROOT}/secrets/${STACK}.sops.yaml" +EXAMPLE_FILE="${REPO_ROOT}/secrets/${STACK}.example.yaml" +SOPS_CONFIG="${REPO_ROOT}/.sops.yaml" + +die() { printf '\033[0;31merror:\033[0m %s\n' "$*" >&2; exit 1; } +info() { printf '\033[0;34m--\033[0m %s\n' "$*"; } +warn() { printf '\033[0;33m!!\033[0m %s\n' "$*"; } + +for tool in age-keygen sops; do + command -v "${tool}" >/dev/null 2>&1 || die "${tool} not found. + age: https://github.com/FiloSottile/age/releases + sops: https://github.com/getsops/sops/releases" +done + +[[ -f "${EXAMPLE_FILE}" ]] || die "missing template: ${EXAMPLE_FILE}" + +# --------------------------------------------------------------------------- +# 1. age keypair +# --------------------------------------------------------------------------- +if [[ -f "${KEY_FILE}" ]]; then + info "reusing existing key at ${KEY_FILE}" +else + info "generating age keypair at ${KEY_FILE}" + mkdir -p "$(dirname "${KEY_FILE}")" + age-keygen -o "${KEY_FILE}" 2>/dev/null + chmod 600 "${KEY_FILE}" + warn "BACK THIS FILE UP OFF THIS MACHINE. Without it the encrypted secrets" + warn "in this repository are unrecoverable." +fi + +PUBLIC_KEY="$(grep -oE 'age1[a-z0-9]+' "${KEY_FILE}" | head -n1)" +[[ -n "${PUBLIC_KEY}" ]] || die "could not read a public key out of ${KEY_FILE}" +info "public key: ${PUBLIC_KEY}" + +# --------------------------------------------------------------------------- +# 2. register it in .sops.yaml +# --------------------------------------------------------------------------- +if grep -q "REPLACE_WITH_YOUR_AGE_PUBLIC_KEY" "${SOPS_CONFIG}"; then + info "writing public key into .sops.yaml" + sed -i.bak "s|REPLACE_WITH_YOUR_AGE_PUBLIC_KEY|${PUBLIC_KEY}|" "${SOPS_CONFIG}" + rm -f "${SOPS_CONFIG}.bak" +elif grep -q "${PUBLIC_KEY}" "${SOPS_CONFIG}"; then + info ".sops.yaml already lists this key" +else + warn ".sops.yaml lists a different age recipient." + warn "Add this key as an additional recipient by hand, then run:" + warn " sops updatekeys ${SECRETS_FILE}" +fi + +# --------------------------------------------------------------------------- +# 3. encrypted secrets file +# --------------------------------------------------------------------------- +if [[ -f "${SECRETS_FILE}" ]]; then + info "$(basename "${SECRETS_FILE}") already exists — leaving it alone" +else + info "creating $(basename "${SECRETS_FILE}") from the template" + sops --encrypt "${EXAMPLE_FILE}" > "${SECRETS_FILE}" + chmod 600 "${SECRETS_FILE}" + warn "It still contains the placeholder values. Edit it now:" + warn " make secrets-edit" +fi + +cat <&2; exit 1; } +info() { printf '\033[0;34m--\033[0m %s\n' "$*"; } + +[[ -d "${STACK_DIR}" ]] || die "no such stack: ${STACK_DIR}" + +command -v sops >/dev/null 2>&1 \ + || die "sops not found. Install it: https://github.com/getsops/sops/releases" + +if [[ ! -f "${SECRETS_FILE}" ]]; then + die "${SECRETS_FILE} does not exist. Run 'make secrets-init' first." +fi + +# --------------------------------------------------------------------------- +# Decrypt. Keep the plaintext in a variable, never in a file. +# --------------------------------------------------------------------------- +info "decrypting $(basename "${SECRETS_FILE}")" +PLAINTEXT="$(sops --decrypt "${SECRETS_FILE}")" \ + || die "decryption failed. Is ~/.config/sops/age/keys.txt present?" + +# Turn "key: value" YAML into shell exports. Values are single-quoted, with +# embedded single quotes escaped, so passwords containing shell metacharacters +# survive intact. +while IFS= read -r line; do + [[ "${line}" =~ ^[[:space:]]*# ]] && continue + [[ "${line}" =~ ^[[:space:]]*$ ]] && continue + [[ "${line}" =~ ^---$ ]] && continue + key="${line%%:*}" + value="${line#*: }" + [[ "${key}" == "${line}" ]] && continue + # strip surrounding quotes if the YAML had them + value="${value%\"}"; value="${value#\"}" + value="${value%\'}"; value="${value#\'}" + printf -v "${key}" '%s' "${value}" + export "${key?}" +done <<< "${PLAINTEXT}" + +REQUIRED=( + GRAFANA_ADMIN_PASSWORD + ALERTMANAGER_WEBHOOK_URL + SNMP_COMMUNITY_PFSENSE + SNMP_COMMUNITY_APC + SNMP_COMMUNITY_MOKERLINK + SNMP_COMMUNITY_ILO +) +missing=() +for var in "${REQUIRED[@]}"; do + [[ -n "${!var:-}" ]] || missing+=("${var}") +done +((${#missing[@]} == 0)) || die "missing keys in ${SECRETS_FILE}: ${missing[*]}" + +# --------------------------------------------------------------------------- +# Render snmp.yaml +# --------------------------------------------------------------------------- +SNMP_SRC="${STACK_DIR}/snmp-exporter/snmp.yaml" +SNMP_OUT_DIR="${STACK_DIR}/snmp-exporter/.rendered" +if [[ -f "${SNMP_SRC}" ]]; then + info "rendering snmp.yaml" + mkdir -p "${SNMP_OUT_DIR}" + # Restrict substitution to the SNMP_COMMUNITY_* names so that any other + # ${...} sequence inside 14k lines of OID definitions is left untouched. + # The single quotes are deliberate: envsubst takes the variable list as a + # literal string, not as already-expanded values. + # shellcheck disable=SC2016 + envsubst '${SNMP_COMMUNITY_PFSENSE} ${SNMP_COMMUNITY_APC} ${SNMP_COMMUNITY_MOKERLINK} ${SNMP_COMMUNITY_ILO}' \ + < "${SNMP_SRC}" > "${SNMP_OUT_DIR}/snmp.yaml" + chmod 600 "${SNMP_OUT_DIR}/snmp.yaml" + + # shellcheck disable=SC2016 # matching the literal text "${SNMP_COMMUNITY..." + if grep -q '\${SNMP_COMMUNITY' "${SNMP_OUT_DIR}/snmp.yaml"; then + die "unsubstituted placeholders remain in the rendered snmp.yaml" + fi +fi + +# --------------------------------------------------------------------------- +# Write .env for compose interpolation +# --------------------------------------------------------------------------- +ENV_FILE="${STACK_DIR}/.env" +info "writing $(basename "${STACK_DIR}")/.env" + +{ + echo "# Generated by scripts/render-config.sh — do not edit, do not commit." + echo "# Non-sensitive defaults come from .env.example; secrets from SOPS." + if [[ -f "${STACK_DIR}/.env.example" ]]; then + grep -vE '^\s*#|^\s*$' "${STACK_DIR}/.env.example" || true + fi + for var in GRAFANA_ADMIN_USER "${REQUIRED[@]}"; do + [[ -n "${!var:-}" ]] && printf '%s=%s\n' "${var}" "${!var}" + done +} > "${ENV_FILE}" +chmod 600 "${ENV_FILE}" + +info "done — ${STACK} is ready to start" diff --git a/secrets/README.md b/secrets/README.md new file mode 100644 index 0000000..2f041a4 --- /dev/null +++ b/secrets/README.md @@ -0,0 +1,72 @@ +# Secrets + +Secrets are encrypted with [SOPS](https://github.com/getsops/sops) using an +[age](https://github.com/FiloSottile/age) key, committed in encrypted form, and +decrypted only in memory at deploy time. + +| File | Committed? | Encrypted? | Contains | +| --- | --- | --- | --- | +| `observability.example.yaml` | yes | no | Key names and placeholder values | +| `observability.sops.yaml` | yes | **yes** | Real credentials | +| `~/.config/sops/age/keys.txt` | **never** | n/a | The private key | + +## Why encrypted-in-git rather than a gitignored `.env` + +A gitignored `.env` keeps secrets out of the repository, but it also keeps them +out of any backup, review or history. When the host dies, the secrets die with +it, and nothing records that a value ever changed. + +Committing them encrypted keeps one copy alongside the config it belongs to. +`git log -p secrets/observability.sops.yaml` shows *when* a credential rotated +without showing what it rotated to: SOPS encrypts values and leaves keys in +plaintext. + +The tradeoff is that the ciphertext is public. It is only as strong as age's +X25519 encryption and the secrecy of the private key — so the private key never +enters the repository, and losing it means re-keying every secret rather than +recovering them. + +## First-time setup + +```bash +make secrets-init +``` + +This generates an age keypair at `~/.config/sops/age/keys.txt`, writes the +public half into `.sops.yaml`, and encrypts `observability.example.yaml` into +`observability.sops.yaml` for you to fill in. + +**Back up `~/.config/sops/age/keys.txt` somewhere outside this machine.** +Without it the encrypted file is unrecoverable. + +## Editing + +```bash +make secrets-edit +``` + +Decrypts to a temporary file, opens `$EDITOR`, re-encrypts on save. The +plaintext never lands on disk unencrypted. + +## How they reach the containers + +`scripts/render-config.sh` runs before `docker compose up`. It decrypts this +file, exports the values as environment variables, and: + +- writes `stacks/observability/.env` for compose to interpolate (gitignored); +- renders `snmp-exporter/snmp.yaml`'s `${SNMP_COMMUNITY_*}` placeholders into + `snmp-exporter/.rendered/snmp.yaml` (gitignored), which is what the container + actually mounts. + +Nothing writes a secret into a tracked file. + +## Rotating + +See [`docs/runbooks/rotate-snmp-community.md`](../docs/runbooks/rotate-snmp-community.md). + +> **Note on this repository's history.** Earlier commits contained a plaintext +> SNMP community string shared across all four devices, and encrypted TLS +> private keys. Both must be treated as compromised regardless of the current +> file contents — removing a secret in a later commit does not remove it from +> git history. See +> [`docs/runbooks/purge-git-history.md`](../docs/runbooks/purge-git-history.md). diff --git a/secrets/observability.example.yaml b/secrets/observability.example.yaml new file mode 100644 index 0000000..d461528 --- /dev/null +++ b/secrets/observability.example.yaml @@ -0,0 +1,34 @@ +--- +# Template for secrets/observability.sops.yaml. +# +# Copy, fill in real values, and encrypt: +# +# make secrets-init # first time only — creates the age keypair +# make secrets-edit # opens the encrypted file in $EDITOR +# +# The encrypted result is committed; this template is not a secret and stays +# in plaintext so the required keys are discoverable without a decryption key. + +# Grafana initial admin login. Changing this after first boot has no effect — +# Grafana only reads it when it creates the admin user. Change it in the UI, or +# wipe the grafana-data volume. +GRAFANA_ADMIN_USER: admin +GRAFANA_ADMIN_PASSWORD: change-me-to-something-long + +# One SNMP community per device, not one shared across all of them. +# +# SNMPv2c sends the community in cleartext on the wire, so anyone with a port +# on the management VLAN can read it off a single packet. A distinct string per +# device limits what one captured packet grants, and lets you rotate a single +# device without an outage across the whole fleet. +# +# These grant read access to firewall rules, interface counters and hardware +# health. Treat them as credentials, not as configuration. +SNMP_COMMUNITY_PFSENSE: change-me-pfsense +SNMP_COMMUNITY_APC: change-me-apc +SNMP_COMMUNITY_MOKERLINK: change-me-mokerlink +SNMP_COMMUNITY_ILO: change-me-ilo + +# Where Alertmanager delivers notifications. Any webhook receiver works — ntfy, +# Gotify, a Home Assistant automation, or a Discord/Slack incoming webhook. +ALERTMANAGER_WEBHOOK_URL: https://ntfy.example.invalid/homelab-alerts From b190e534f74cf6c96a22f86378dc193b2d358e90 Mon Sep 17 00:00:00 2001 From: Garrett Allen Date: Sun, 2 Aug 2026 06:23:04 +0000 Subject: [PATCH 4/8] feat(observability): provision dashboards and alerting as code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grafana previously had no config at all: no datasources, no dashboards, and GF_PATHS_PROVISIONING pointing at an empty directory. Nothing about the visualisation layer was version-controlled. Adds: * provisioned Prometheus, Loki and Alertmanager datasources with fixed UIDs * 5 dashboards / 79 panels — host overview, Docker containers, network and firewall, UPS and power, logs * 32 alert rules across host, network, UPS and container concerns allowUiUpdates is false, so the JSON in git is the source of truth and UI edits are discarded on restart. Disk alerting is predictive (predict_linear over 6h, gated on being under 30% free) rather than a fixed threshold — a disk stable at 86% is not an emergency, one climbing fast at 60% is. Also collapses four near-identical SNMP scrape jobs, which differed only in target/module/auth, into one file_sd job reading targets/snmp.yaml. Targets are re-read every 5 minutes, so adding a device no longer needs a restart. --- .../grafana/dashboards/docker-containers.json | 1031 +++++++++++ .../grafana/dashboards/host-overview.json | 1367 +++++++++++++++ .../grafana/dashboards/logs-explorer.json | 828 +++++++++ .../grafana/dashboards/network-snmp.json | 1512 +++++++++++++++++ .../grafana/dashboards/ups-power.json | 1091 ++++++++++++ .../provisioning/dashboards/dashboards.yaml | 17 + .../provisioning/datasources/datasources.yaml | 44 + .../prometheus/rules/containers.rules.yaml | 94 + .../prometheus/rules/host.rules.yaml | 87 + .../prometheus/rules/network.rules.yaml | 125 ++ .../prometheus/rules/ups.rules.yaml | 79 + .../prometheus/targets/snmp.yaml | 41 + 12 files changed, 6316 insertions(+) create mode 100644 stacks/observability/grafana/dashboards/docker-containers.json create mode 100644 stacks/observability/grafana/dashboards/host-overview.json create mode 100644 stacks/observability/grafana/dashboards/logs-explorer.json create mode 100644 stacks/observability/grafana/dashboards/network-snmp.json create mode 100644 stacks/observability/grafana/dashboards/ups-power.json create mode 100644 stacks/observability/grafana/provisioning/dashboards/dashboards.yaml create mode 100644 stacks/observability/grafana/provisioning/datasources/datasources.yaml create mode 100644 stacks/observability/prometheus/rules/containers.rules.yaml create mode 100644 stacks/observability/prometheus/rules/host.rules.yaml create mode 100644 stacks/observability/prometheus/rules/network.rules.yaml create mode 100644 stacks/observability/prometheus/rules/ups.rules.yaml create mode 100644 stacks/observability/prometheus/targets/snmp.yaml diff --git a/stacks/observability/grafana/dashboards/docker-containers.json b/stacks/observability/grafana/dashboards/docker-containers.json new file mode 100644 index 0000000..3b5bc93 --- /dev/null +++ b/stacks/observability/grafana/dashboards/docker-containers.json @@ -0,0 +1,1031 @@ +{ + "uid": "homelab-docker", + "title": "Docker Containers", + "description": "cAdvisor metrics collected by the Alloy agent on each Docker host. Covers CPU, memory, network, restarts and OOM kills.", + "tags": [ + "homelab", + "docker", + "cadvisor" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "graphTooltip": 1, + "refresh": "1m", + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "templating": { + "list": [ + { + "name": "host", + "label": "Host", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "query": "label_values(container_last_seen, host)", + "refId": "host" + }, + "definition": "label_values(container_last_seen, host)", + "refresh": 1, + "includeAll": true, + "multi": true, + "regex": "", + "sort": 1, + "current": {}, + "options": [], + "hide": 0 + }, + { + "name": "container", + "label": "Container", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "query": "label_values(container_last_seen{name!=\"\"}, name)", + "refId": "container" + }, + "definition": "label_values(container_last_seen{name!=\"\"}, name)", + "refresh": 1, + "includeAll": true, + "multi": true, + "regex": "", + "sort": 1, + "current": {}, + "options": [], + "hide": 0 + } + ] + }, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "panels": [ + { + "type": "stat", + "title": "Containers running", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(count by (name) (container_last_seen{name!=\"\"}))", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Total container memory", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 4, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{name=~\"$container\"})", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Total container CPU", + "description": "Cores consumed across all selected containers.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 9, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "mappings": [], + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(container_cpu_usage_seconds_total{name=~\"$container\"}[5m]))", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Restarts (1h)", + "description": "Each container start increments this. Anything above a couple is usually a crash loop rather than a deploy.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 14, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 3 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(changes(container_start_time_seconds{name=~\"$container\"}[1h]))", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "OOM kills (24h)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 19, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(container_oom_events_total{name=~\"$container\"}[24h]))", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "row", + "title": "Resource usage", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "CPU by container", + "description": "Cores consumed. 1.0 = one full core.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 5 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 35, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(container_cpu_usage_seconds_total{name=~\"$container\"}[5m])", + "legendFormat": "{{name}}", + "range": true, + "instant": false, + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "Memory by container", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 5 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 35, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "container_memory_working_set_bytes{name=~\"$container\"}", + "legendFormat": "{{name}}", + "range": true, + "instant": false, + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "Network by container", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 14 + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(container_network_receive_bytes_total{name=~\"$container\"}[5m])", + "legendFormat": "{{name}} rx", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(container_network_transmit_bytes_total{name=~\"$container\"}[5m])", + "legendFormat": "{{name}} tx", + "range": true, + "instant": false, + "refId": "B" + } + ] + }, + { + "type": "timeseries", + "title": "Filesystem writes by container", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 14 + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(container_fs_writes_bytes_total{name=~\"$container\"}[5m])", + "legendFormat": "{{name}}", + "range": true, + "instant": false, + "refId": "A" + } + ] + }, + { + "type": "row", + "title": "Stability", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "Restarts", + "description": "Restarts in a rolling 15 minute window.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "changes(container_start_time_seconds{name=~\"$container\"}[15m])", + "legendFormat": "{{name}}", + "range": true, + "instant": false, + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "CPU throttling", + "description": "Fraction of CPU periods in which the container hit its quota. Only non-zero where a CPU limit is set.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(container_cpu_cfs_throttled_periods_total{name=~\"$container\"}[5m]) / clamp_min(rate(container_cpu_cfs_periods_total{name=~\"$container\"}[5m]), 1)", + "legendFormat": "{{name}}", + "range": true, + "instant": false, + "refId": "A" + } + ] + }, + { + "type": "table", + "title": "Container inventory", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 32 + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Uptime" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "time() - container_start_time_seconds{name=~\"$container\"}", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A", + "format": "table" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "job": true, + "id": true, + "image": true, + "site": true, + "monitor": true, + "instance": true + }, + "renameByName": { + "Value": "Uptime", + "name": "Container" + } + } + } + ] + } + ], + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "homelab" + ], + "title": "HomeLab dashboards", + "type": "dashboards", + "targetBlank": false, + "tooltip": "", + "url": "" + } + ] +} diff --git a/stacks/observability/grafana/dashboards/host-overview.json b/stacks/observability/grafana/dashboards/host-overview.json new file mode 100644 index 0000000..57dd840 --- /dev/null +++ b/stacks/observability/grafana/dashboards/host-overview.json @@ -0,0 +1,1367 @@ +{ + "uid": "homelab-host-overview", + "title": "Host Overview", + "description": "Per-host CPU, memory, storage and network for every machine running a Grafana Alloy agent. Metrics arrive by remote_write, so a host appears here as soon as its agent starts.", + "tags": [ + "homelab", + "hosts", + "node-exporter" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "graphTooltip": 1, + "refresh": "1m", + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "templating": { + "list": [ + { + "name": "instance", + "label": "Host", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "query": "label_values(node_uname_info, instance)", + "refId": "instance" + }, + "definition": "label_values(node_uname_info, instance)", + "refresh": 1, + "includeAll": true, + "multi": true, + "regex": "", + "sort": 1, + "current": {}, + "options": [], + "hide": 0 + } + ] + }, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "panels": [ + { + "type": "stat", + "title": "Hosts reporting", + "description": "node_exporter targets currently scraping successfully.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(up{job=~\".*-metrics\"} == 1)", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "gauge", + "title": "CPU busy", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 4, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\",instance=~\"$instance\"}[5m])) * 100)", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "gauge", + "title": "Memory used", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 9, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "100 * (1 - avg(node_memory_MemAvailable_bytes{instance=~\"$instance\"} / node_memory_MemTotal_bytes{instance=~\"$instance\"}))", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "gauge", + "title": "Root filesystem used", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 14, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "100 * (1 - avg(node_filesystem_avail_bytes{mountpoint=\"/\",instance=~\"$instance\"} / node_filesystem_size_bytes{mountpoint=\"/\",instance=~\"$instance\"}))", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Shortest uptime", + "description": "Lowest uptime across the selected hosts — a sudden drop means a reboot.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 19, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min(time() - node_boot_time_seconds{instance=~\"$instance\"})", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Load (15m, normalised)", + "description": "Load average divided by core count, so it is comparable across a 2012 MacBook and a DL360.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "red", + "value": 2 + } + ] + }, + "mappings": [], + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(node_load15{instance=~\"$instance\"} / count without (cpu, mode) (node_cpu_seconds_total{mode=\"idle\",instance=~\"$instance\"}))", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "row", + "title": "CPU, memory and load", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "CPU utilisation by mode", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 35, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "avg by (mode) (rate(node_cpu_seconds_total{mode!=\"idle\",instance=~\"$instance\"}[5m])) * 100", + "legendFormat": "{{mode}}", + "range": true, + "instant": false, + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "Memory", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_MemTotal_bytes{instance=~\"$instance\"} - node_memory_MemAvailable_bytes{instance=~\"$instance\"}", + "legendFormat": "{{instance}} used", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_MemAvailable_bytes{instance=~\"$instance\"}", + "legendFormat": "{{instance}} available", + "range": true, + "instant": false, + "refId": "B" + } + ] + }, + { + "type": "timeseries", + "title": "Load average", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_load1{instance=~\"$instance\"}", + "legendFormat": "{{instance}} 1m", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_load5{instance=~\"$instance\"}", + "legendFormat": "{{instance}} 5m", + "range": true, + "instant": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_load15{instance=~\"$instance\"}", + "legendFormat": "{{instance}} 15m", + "range": true, + "instant": false, + "refId": "C" + } + ] + }, + { + "type": "timeseries", + "title": "Context switches and interrupts", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_context_switches_total{instance=~\"$instance\"}[5m])", + "legendFormat": "{{instance}} ctx switches", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_intr_total{instance=~\"$instance\"}[5m])", + "legendFormat": "{{instance}} interrupts", + "range": true, + "instant": false, + "refId": "B" + } + ] + }, + { + "type": "row", + "title": "Storage", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "Filesystem used", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "100 * (1 - node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay\",instance=~\"$instance\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay\",instance=~\"$instance\"})", + "legendFormat": "{{instance}} {{mountpoint}}", + "range": true, + "instant": false, + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "Disk I/O", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_disk_read_bytes_total{instance=~\"$instance\"}[5m])", + "legendFormat": "{{instance}} {{device}} read", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_disk_written_bytes_total{instance=~\"$instance\"}[5m])", + "legendFormat": "{{instance}} {{device}} write", + "range": true, + "instant": false, + "refId": "B" + } + ] + }, + { + "type": "table", + "title": "Filesystems", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Used %" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "gauge", + "mode": "gradient" + } + }, + { + "id": "max", + "value": 100 + }, + { + "id": "min", + "value": 0 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + } + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "100 * (1 - node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay\",instance=~\"$instance\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay\",instance=~\"$instance\"})", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A", + "format": "table" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "job": true, + "fstype": true, + "site": true, + "monitor": true + }, + "renameByName": { + "Value": "Used %" + } + } + } + ] + }, + { + "type": "row", + "title": "Network", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "Network throughput", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 43 + }, + "fieldConfig": { + "defaults": { + "unit": "bps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|br-.*\",instance=~\"$instance\"}[5m]) * 8", + "legendFormat": "{{instance}} {{device}} rx", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|br-.*\",instance=~\"$instance\"}[5m]) * 8", + "legendFormat": "{{instance}} {{device}} tx", + "range": true, + "instant": false, + "refId": "B" + } + ] + }, + { + "type": "timeseries", + "title": "Network errors and drops", + "description": "Should be flat at zero. Sustained errors usually mean a bad cable or a duplex mismatch.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, + "fieldConfig": { + "defaults": { + "unit": "pps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_network_receive_errs_total{instance=~\"$instance\"}[5m])", + "legendFormat": "{{instance}} {{device}} rx errs", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_network_transmit_errs_total{instance=~\"$instance\"}[5m])", + "legendFormat": "{{instance}} {{device}} tx errs", + "range": true, + "instant": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_network_receive_drop_total{instance=~\"$instance\"}[5m])", + "legendFormat": "{{instance}} {{device}} rx drops", + "range": true, + "instant": false, + "refId": "C" + } + ] + } + ], + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "homelab" + ], + "title": "HomeLab dashboards", + "type": "dashboards", + "targetBlank": false, + "tooltip": "", + "url": "" + } + ] +} diff --git a/stacks/observability/grafana/dashboards/logs-explorer.json b/stacks/observability/grafana/dashboards/logs-explorer.json new file mode 100644 index 0000000..877415a --- /dev/null +++ b/stacks/observability/grafana/dashboards/logs-explorer.json @@ -0,0 +1,828 @@ +{ + "uid": "homelab-logs", + "title": "Logs", + "description": "Loki view across every host running an Alloy agent: Docker, systemd journal, syslog and auth.log.", + "tags": [ + "homelab", + "logs", + "loki" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "graphTooltip": 1, + "refresh": "1m", + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "templating": { + "list": [ + { + "name": "host", + "label": "Host", + "type": "query", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "query": { + "query": "label_values(host)", + "refId": "host" + }, + "definition": "label_values(host)", + "refresh": 1, + "includeAll": true, + "multi": true, + "regex": "", + "sort": 1, + "current": {}, + "options": [], + "hide": 0 + }, + { + "name": "level", + "label": "Level", + "type": "custom", + "query": "critical,error,warning,info,debug", + "includeAll": true, + "multi": true, + "current": {}, + "options": [], + "hide": 0 + }, + { + "name": "search", + "label": "Search", + "type": "textbox", + "query": "", + "current": { + "text": "", + "value": "" + }, + "options": [], + "hide": 0 + } + ] + }, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "panels": [ + { + "type": "stat", + "title": "Lines/s (all hosts)", + "description": "", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "mappings": [], + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "sum(rate({host=~\".+\"}[5m]))", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Errors (1h)", + "description": "", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 5, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "red", + "value": 50 + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "sum(count_over_time({host=~\"$host\", level=~\"error|critical\"}[1h]))", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Critical (24h)", + "description": "", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 10, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "sum(count_over_time({host=~\"$host\", level=\"critical\"}[24h]))", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Failed SSH auth (24h)", + "description": "Counted from /var/log/auth.log. On an internet-facing host a steady background rate is normal; a sudden spike is not.", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 9, + "x": 15, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 10 + }, + { + "color": "red", + "value": 100 + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "sum(count_over_time({host=~\"$host\", log_type=\"authlog\"} |~ \"(?i)failed (password|publickey)\" [24h]))", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "row", + "title": "Volume", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "Log volume by level", + "description": "Levels are normalised by the log_processor stage in config.alloy, which maps ~20 spellings (err, eror, ERROR, crit, fatal...) onto five canonical values.", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 35, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "sum by (level) (rate({host=~\"$host\", level=~\"$level\"}[5m]))", + "legendFormat": "{{level}}", + "range": true, + "instant": false, + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "Log volume by source", + "description": "", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 35, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "sum by (log_type) (rate({host=~\"$host\", level=~\"$level\"}[5m]))", + "legendFormat": "{{log_type}}", + "range": true, + "instant": false, + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "Errors by host", + "description": "", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "sum by (host) (rate({host=~\"$host\", level=~\"error|critical\"}[5m]))", + "legendFormat": "{{host}}", + "range": true, + "instant": false, + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "Docker log volume by container", + "description": "", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 35, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "sum by (service_name) (rate({host=~\"$host\", platform=\"docker\"}[5m]))", + "legendFormat": "{{service_name}}", + "range": true, + "instant": false, + "refId": "A" + } + ] + }, + { + "type": "row", + "title": "Streams", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "panels": [] + }, + { + "type": "logs", + "title": "Errors and above", + "description": "", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 22 + }, + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Descending" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "{host=~\"$host\", level=~\"error|critical\"}", + "queryType": "range", + "refId": "A" + } + ] + }, + { + "type": "logs", + "title": "Authentication log", + "description": "sshd, sudo and PAM events.", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 33 + }, + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Descending" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "{host=~\"$host\", log_type=\"authlog\"}", + "queryType": "range", + "refId": "A" + } + ] + }, + { + "type": "logs", + "title": "All selected logs", + "description": "Free-text filter via the Search variable at the top.", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 44 + }, + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Descending" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "{host=~\"$host\", level=~\"$level\"} |= \"$search\"", + "queryType": "range", + "refId": "A" + } + ] + } + ], + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "homelab" + ], + "title": "HomeLab dashboards", + "type": "dashboards", + "targetBlank": false, + "tooltip": "", + "url": "" + } + ] +} diff --git a/stacks/observability/grafana/dashboards/network-snmp.json b/stacks/observability/grafana/dashboards/network-snmp.json new file mode 100644 index 0000000..a48fb4b --- /dev/null +++ b/stacks/observability/grafana/dashboards/network-snmp.json @@ -0,0 +1,1512 @@ +{ + "uid": "homelab-network", + "title": "Network & Firewall", + "description": "SNMP view of the pfSense firewall, the managed switch and the ProLiant's iLO. Polled every 60s through snmp_exporter.", + "tags": [ + "homelab", + "network", + "snmp", + "pfsense" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "graphTooltip": 1, + "refresh": "1m", + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": {}, + "templating": { + "list": [] + }, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "panels": [ + { + "type": "stat", + "title": "Packet filter", + "description": "pf enforces inter-VLAN isolation. If this reads DOWN the network is flat.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red", + "index": 0 + }, + "1": { + "text": "RUNNING", + "color": "green", + "index": 1 + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "pfStatusRunning", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "SNMP devices reachable", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "#EAB839", + "value": 3 + }, + { + "color": "green", + "value": 4 + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(up{job=\"snmp\"} == 1)", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "gauge", + "title": "pf state table", + "description": "New connections are dropped once the state table is full.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 8, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "100 * pfStateTableCount / pfLimitsStates", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Active states", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 13, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "pfStateTableCount", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Firewall uptime", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "pfStatusRuntime", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Interfaces up", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(ifOperStatus == 1)", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "State inserts/s", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "mappings": [], + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(pfStateTableInserts[5m])", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Interfaces down", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 13, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 1 + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(ifOperStatus == 2) or vector(0)", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "pf memory drops/s", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.001 + } + ] + }, + "mappings": [], + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(pfCounterMemDrop[5m])", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "row", + "title": "Firewall (pfSense)", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "State table", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "pfStateTableCount", + "legendFormat": "states in use", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "pfLimitsStates", + "legendFormat": "limit", + "range": true, + "instant": false, + "refId": "B" + } + ] + }, + { + "type": "timeseries", + "title": "State churn", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(pfStateTableInserts[5m])", + "legendFormat": "inserts/s", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(pfStateTableRemovals[5m])", + "legendFormat": "removals/s", + "range": true, + "instant": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(pfStateTableSearches[5m])", + "legendFormat": "searches/s", + "range": true, + "instant": false, + "refId": "C" + } + ] + }, + { + "type": "timeseries", + "title": "Packet filter drops", + "description": "Non-zero memory drops mean pf is out of memory and silently discarding traffic.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "fieldConfig": { + "defaults": { + "unit": "pps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(pfCounterMemDrop[5m])", + "legendFormat": "memory drops", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(pfCounterShort[5m])", + "legendFormat": "short packets", + "range": true, + "instant": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(pfCounterBadOffset[5m])", + "legendFormat": "bad offset", + "range": true, + "instant": false, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(pfCounterFragment[5m])", + "legendFormat": "fragments", + "range": true, + "instant": false, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(pfCounterNormalize[5m])", + "legendFormat": "normalise", + "range": true, + "instant": false, + "refId": "E" + } + ] + }, + { + "type": "timeseries", + "title": "Source node tracking", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "pfSrcNodesCount", + "legendFormat": "source nodes", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(pfSrcNodesInserts[5m])", + "legendFormat": "inserts/s", + "range": true, + "instant": false, + "refId": "B" + } + ] + }, + { + "type": "row", + "title": "Switch (MokerLink, IF-MIB)", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "Interface throughput", + "description": "From 32-bit IF-MIB counters, which wrap in roughly 34 seconds at gigabit line rate. Sustained high-throughput ports will under-report until ifXTable is added — see docs/roadmap.md.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 26 + }, + "fieldConfig": { + "defaults": { + "unit": "bps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(ifInOctets[5m]) * 8", + "legendFormat": "{{ifDescr}} in", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(ifOutOctets[5m]) * 8", + "legendFormat": "{{ifDescr}} out", + "range": true, + "instant": false, + "refId": "B" + } + ] + }, + { + "type": "table", + "title": "Interface status", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 35 + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Oper status" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "1": { + "text": "up", + "color": "green", + "index": 0 + }, + "2": { + "text": "down", + "color": "red", + "index": 1 + }, + "3": { + "text": "testing", + "color": "yellow", + "index": 2 + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "ifOperStatus", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A", + "format": "table" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "job": true, + "site": true, + "monitor": true, + "ifIndex": true + }, + "renameByName": { + "Value": "Oper status", + "ifDescr": "Interface" + } + } + } + ] + }, + { + "type": "row", + "title": "Hypervisor BMC (HPE iLO)", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "Chassis power draw", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 45 + }, + "fieldConfig": { + "defaults": { + "unit": "watt", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "cpqHePowerMeterCurrReading", + "legendFormat": "{{device}} watts", + "range": true, + "instant": false, + "refId": "A" + } + ] + }, + { + "type": "table", + "title": "Hardware health", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 45 + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Condition" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "1": { + "text": "other", + "color": "text", + "index": 0 + }, + "2": { + "text": "ok", + "color": "green", + "index": 1 + }, + "3": { + "text": "degraded", + "color": "orange", + "index": 2 + }, + "4": { + "text": "failed", + "color": "red", + "index": 3 + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "cpqHeTemperatureCondition or cpqHeFltTolPowerSupplyCondition or cpqDaLogDrvCondition or cpqDaPhyDrvCondition", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A", + "format": "table" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "job": true, + "site": true, + "monitor": true, + "instance": true + }, + "renameByName": { + "Value": "Condition", + "__name__": "Component" + } + } + } + ] + } + ], + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "homelab" + ], + "title": "HomeLab dashboards", + "type": "dashboards", + "targetBlank": false, + "tooltip": "", + "url": "" + } + ] +} diff --git a/stacks/observability/grafana/dashboards/ups-power.json b/stacks/observability/grafana/dashboards/ups-power.json new file mode 100644 index 0000000..aac5af0 --- /dev/null +++ b/stacks/observability/grafana/dashboards/ups-power.json @@ -0,0 +1,1091 @@ +{ + "uid": "homelab-ups", + "title": "UPS & Power", + "description": "APC Smart-UPS (mjolnir) via UPS-MIB. Note: this UPS currently has no working battery, so runtime figures are not to be trusted until it is replaced — see docs/roadmap.md.", + "tags": [ + "homelab", + "power", + "ups", + "snmp" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "graphTooltip": 1, + "refresh": "1m", + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "templating": { + "list": [] + }, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "panels": [ + { + "type": "stat", + "title": "Power source", + "description": "Seconds spent on battery. Anything above zero means mains power is gone.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "MAINS", + "color": "green", + "index": 0 + } + } + }, + { + "type": "range", + "options": { + "from": 1, + "to": 999999, + "result": { + "text": "ON BATTERY", + "color": "red", + "index": 1 + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "upsSecondsOnBattery", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "gauge", + "title": "Battery charge", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 5, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "#EAB839", + "value": 50 + }, + { + "color": "green", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "upsEstimatedChargeRemaining", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "gauge", + "title": "Output load", + "description": "Sustained load above 80% shortens runtime non-linearly.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 10, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "upsOutputPercentLoad", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Runtime remaining", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 15, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "#EAB839", + "value": 300 + }, + { + "color": "green", + "value": 900 + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "upsEstimatedMinutesRemaining * 60", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Battery status", + "description": "UPS-MIB: 1 unknown, 2 normal, 3 low, 4 depleted.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 3 + }, + { + "color": "red", + "value": 4 + } + ] + }, + "mappings": [ + { + "type": "value", + "options": { + "1": { + "text": "unknown", + "color": "text", + "index": 0 + }, + "2": { + "text": "normal", + "color": "green", + "index": 1 + }, + "3": { + "text": "low", + "color": "orange", + "index": 2 + }, + "4": { + "text": "depleted", + "color": "red", + "index": 3 + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "upsBatteryStatus", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Output power", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 0, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "watt", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "upsOutputWatts", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Active alarms", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 15, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "upsAlarmsPresent", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Battery temp", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "celsius", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 35 + }, + { + "color": "red", + "value": 40 + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "upsBatteryTemperature", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "row", + "title": "Trends", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "Battery charge and runtime", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "upsEstimatedChargeRemaining", + "legendFormat": "charge %", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "upsEstimatedMinutesRemaining", + "legendFormat": "minutes remaining", + "range": true, + "instant": false, + "refId": "B" + } + ] + }, + { + "type": "timeseries", + "title": "Load", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "upsOutputPercentLoad", + "legendFormat": "output load %", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "upsOutputWatts", + "legendFormat": "output watts", + "range": true, + "instant": false, + "refId": "B" + } + ] + }, + { + "type": "timeseries", + "title": "Voltage", + "description": "A sagging input voltage that never quite fails is what actually kills equipment.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "fieldConfig": { + "defaults": { + "unit": "volt", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "upsInputVoltage", + "legendFormat": "input V", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "upsOutputVoltage", + "legendFormat": "output V", + "range": true, + "instant": false, + "refId": "B" + } + ] + }, + { + "type": "timeseries", + "title": "Input frequency and current", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "upsInputFrequency", + "legendFormat": "input Hz", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "upsInputCurrent", + "legendFormat": "input A", + "range": true, + "instant": false, + "refId": "B" + } + ] + }, + { + "type": "timeseries", + "title": "Time on battery", + "description": "Every non-zero excursion is a mains event worth correlating against the log volume panel on the Logs dashboard.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 25 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 12, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "upsSecondsOnBattery", + "legendFormat": "seconds on battery", + "range": true, + "instant": false, + "refId": "A" + } + ] + } + ], + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "homelab" + ], + "title": "HomeLab dashboards", + "type": "dashboards", + "targetBlank": false, + "tooltip": "", + "url": "" + } + ] +} diff --git a/stacks/observability/grafana/provisioning/dashboards/dashboards.yaml b/stacks/observability/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..54f6e73 --- /dev/null +++ b/stacks/observability/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: 1 + +providers: + - name: homelab + orgId: 1 + folder: HomeLab + type: file + # Dashboards are version-controlled in this repository, so edits made in the + # Grafana UI are intentionally discarded on restart. Export the JSON and + # commit it instead — see docs/observability.md. + disableDeletion: false + allowUiUpdates: false + updateIntervalSeconds: 30 + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/stacks/observability/grafana/provisioning/datasources/datasources.yaml b/stacks/observability/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..915d3ab --- /dev/null +++ b/stacks/observability/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,44 @@ +--- +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: ${PROMETHEUS_URL} + isDefault: true + editable: false + jsonData: + timeInterval: 15s + httpMethod: POST + manageAlerts: false + prometheusType: Prometheus + exemplarTraceIdDestinations: [] + + - name: Loki + type: loki + uid: loki + access: proxy + url: ${LOKI_URL} + editable: false + jsonData: + maxLines: 2000 + # Clicking a hostname in a log line jumps to that host's metrics rather + # than making you retype it into a new query. + derivedFields: + - name: host + matcherRegex: 'host="([^"]+)"' + datasourceUid: prometheus + url: '$${__value.raw}' + urlDisplayLabel: "View host metrics" + + - name: Alertmanager + type: alertmanager + uid: alertmanager + access: proxy + url: http://alertmanager:9093 + editable: false + jsonData: + implementation: prometheus + handleGrafanaManagedAlerts: false diff --git a/stacks/observability/prometheus/rules/containers.rules.yaml b/stacks/observability/prometheus/rules/containers.rules.yaml new file mode 100644 index 0000000..f9988d0 --- /dev/null +++ b/stacks/observability/prometheus/rules/containers.rules.yaml @@ -0,0 +1,94 @@ +--- +# Container metrics arrive via the cAdvisor exporter embedded in Alloy. +groups: + - name: containers + interval: 60s + rules: + - alert: ContainerRestartLoop + # container_start_time_seconds changes on every (re)start; more than + # three changes in 15 minutes is a crash loop rather than a deploy. + expr: changes(container_start_time_seconds{name!=""}[15m]) > 3 + for: 5m + labels: + severity: warning + category: availability + annotations: + summary: "Container {{ $labels.name }} on {{ $labels.host }} is restarting repeatedly" + description: "{{ $value }} restarts in the last 15 minutes." + + - alert: ContainerOomKilled + expr: increase(container_oom_events_total{name!=""}[15m]) > 0 + labels: + severity: warning + category: capacity + annotations: + summary: "Container {{ $labels.name }} was OOM-killed" + + - alert: ContainerHighMemory + # Only meaningful where a limit is actually set; the spec_memory_limit + # guard excludes unlimited containers, which report the host total. + expr: | + container_memory_working_set_bytes{name!=""} + / container_spec_memory_limit_bytes{name!=""} > 0.90 + and container_spec_memory_limit_bytes{name!=""} > 0 + for: 15m + labels: + severity: warning + category: capacity + annotations: + summary: "Container {{ $labels.name }} at {{ $value | humanizePercentage }} of its memory limit" + + - alert: ContainerCpuThrottled + expr: | + rate(container_cpu_cfs_throttled_periods_total{name!=""}[15m]) + / clamp_min(rate(container_cpu_cfs_periods_total{name!=""}[15m]), 1) > 0.25 + for: 30m + labels: + severity: info + category: saturation + annotations: + summary: "Container {{ $labels.name }} throttled {{ $value | humanizePercentage }} of CPU periods" + + # --- The observability stack watching itself -------------------------- + - alert: PrometheusConfigReloadFailed + expr: prometheus_config_last_reload_successful == 0 + for: 5m + labels: + severity: critical + category: correctness + annotations: + summary: "Prometheus rejected its own config reload on {{ $labels.instance }}" + + - alert: PrometheusRuleEvaluationFailures + expr: increase(prometheus_rule_evaluation_failures_total[15m]) > 0 + for: 5m + labels: + severity: warning + category: correctness + annotations: + summary: "{{ $value }} rule evaluation failures in the last 15 minutes" + + - alert: AlertmanagerNotificationsFailing + expr: increase(alertmanager_notifications_failed_total[15m]) > 0 + for: 5m + labels: + severity: critical + category: correctness + annotations: + summary: "Alertmanager cannot deliver notifications via {{ $labels.integration }}" + description: >- + Alerts are firing but not reaching anyone — this makes every other + rule in this repository silently useless. + + - alert: LokiIngestionStalled + expr: | + sum(rate(loki_distributor_lines_received_total[30m])) == 0 + for: 30m + labels: + severity: warning + category: availability + annotations: + summary: "Loki has received no log lines for 30 minutes" + description: >- + Either every Alloy agent is down or the push endpoint is + unreachable. Logs are being lost, not queued indefinitely. diff --git a/stacks/observability/prometheus/rules/host.rules.yaml b/stacks/observability/prometheus/rules/host.rules.yaml new file mode 100644 index 0000000..7776db6 --- /dev/null +++ b/stacks/observability/prometheus/rules/host.rules.yaml @@ -0,0 +1,87 @@ +--- +groups: + - name: host + interval: 30s + rules: + - alert: InstanceDown + expr: up == 0 + for: 5m + labels: + severity: critical + category: availability + annotations: + summary: "{{ $labels.job }} target {{ $labels.instance }} is down" + description: >- + Prometheus has failed to scrape {{ $labels.instance }} + ({{ $labels.job }}) for 5 minutes. + + - alert: HostDiskWillFillIn24h + # Predictive rather than a static threshold: a disk sitting at 86% and + # stable is not an emergency, one climbing fast at 60% is. + expr: | + predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"}[6h], 24 * 3600) < 0 + and on (instance, device) + node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes < 0.30 + for: 1h + labels: + severity: warning + category: capacity + annotations: + summary: "{{ $labels.instance }}:{{ $labels.mountpoint }} fills within 24h" + description: >- + {{ $labels.mountpoint }} on {{ $labels.instance }} ({{ $labels.device }}) + is below 30% free and, extrapolating the last 6 hours, reaches zero + within a day. + + - alert: HostDiskCritical + expr: | + node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} + / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"} < 0.10 + for: 15m + labels: + severity: critical + category: capacity + annotations: + summary: "{{ $labels.instance }}:{{ $labels.mountpoint }} below 10% free" + + - alert: HostMemoryPressure + expr: | + node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.10 + for: 15m + labels: + severity: warning + category: capacity + annotations: + summary: "{{ $labels.instance }} has less than 10% memory available" + + - alert: HostHighLoad + # Normalised by core count so the same rule works on the 2012 MacBook + # and the DL360 without tuning. + expr: | + node_load15 / count without (cpu, mode) (node_cpu_seconds_total{mode="idle"}) > 2 + for: 30m + labels: + severity: warning + category: saturation + annotations: + summary: "{{ $labels.instance }} load is {{ $value | humanize }}x core count" + + - alert: HostClockSkew + expr: abs(node_timex_offset_seconds) > 0.5 + for: 10m + labels: + severity: warning + category: correctness + annotations: + summary: "{{ $labels.instance }} clock is {{ $value | humanizeDuration }} off" + description: >- + Clock skew breaks log correlation across hosts and can invalidate + rate() calculations. + + - alert: HostRebooted + expr: time() - node_boot_time_seconds < 600 + labels: + severity: info + category: availability + annotations: + summary: "{{ $labels.instance }} rebooted in the last 10 minutes" diff --git a/stacks/observability/prometheus/rules/network.rules.yaml b/stacks/observability/prometheus/rules/network.rules.yaml new file mode 100644 index 0000000..81fbbc4 --- /dev/null +++ b/stacks/observability/prometheus/rules/network.rules.yaml @@ -0,0 +1,125 @@ +--- +groups: + - name: network + interval: 60s + rules: + - alert: SnmpExporterDown + expr: up{job="snmp-exporter"} == 0 + for: 5m + labels: + severity: critical + category: availability + annotations: + summary: "snmp-exporter is down — every SNMP device will appear unreachable" + + - alert: SnmpTargetUnreachable + expr: up{job="snmp"} == 0 + for: 10m + labels: + severity: warning + category: availability + annotations: + summary: "SNMP device {{ $labels.device }} ({{ $labels.instance }}) is not responding" + description: >- + Role {{ $labels.role }} on VLAN {{ $labels.vlan }}. Check the device + is up and that its SNMP community still matches the one in + secrets/observability.sops.yaml. + + - alert: SnmpScrapeSlow + expr: scrape_duration_seconds{job="snmp"} > 30 + for: 15m + labels: + severity: warning + category: performance + annotations: + summary: "SNMP scrape of {{ $labels.device }} takes {{ $value | humanizeDuration }}" + description: >- + The scrape timeout is 45s. A walk creeping toward it will start + producing gaps — narrow the module's OID list in generator.yaml. + + # --- Firewall (pfSense / BEGEMOT-PF-MIB) ------------------------------ + - alert: PfNotRunning + expr: pfStatusRunning == 0 + for: 2m + labels: + severity: critical + category: security + annotations: + summary: "Packet filter is NOT running on {{ $labels.device }}" + description: >- + pf is disabled. Inter-VLAN isolation and the default-deny posture + are not being enforced. + + - alert: PfStateTableNearLimit + expr: pfStateTableCount / pfLimitsStates > 0.80 + for: 10m + labels: + severity: warning + category: saturation + annotations: + summary: "pf state table at {{ $value | humanizePercentage }} of limit" + description: >- + New connections are dropped once the state table fills. + + - alert: PfMemoryDropsIncreasing + expr: rate(pfCounterMemDrop[15m]) > 0 + for: 15m + labels: + severity: warning + category: saturation + annotations: + summary: "pf is dropping packets for lack of memory on {{ $labels.device }}" + + # --- Switch (MokerLink / IF-MIB) -------------------------------------- + - alert: SwitchInterfaceDown + # ifOperStatus: 1=up 2=down 3=testing. Only alerts on ports that were + # up at some point in the last hour, so unused ports stay quiet. + expr: | + ifOperStatus == 2 + and on (instance, ifIndex) (max_over_time(ifOperStatus[1h]) == 1) + for: 5m + labels: + severity: warning + category: availability + annotations: + summary: "Interface {{ $labels.ifDescr }} on {{ $labels.device }} went down" + + - alert: SwitchCounterWrapSuspected + # ifInOctets/ifOutOctets are 32-bit and wrap in ~34s at gigabit line + # rate. A negative delta at a 60s scrape means the counter wrapped and + # the throughput figure for that window is wrong. + # Fix is ifXTable (64-bit) — tracked in docs/roadmap.md. + expr: rate(ifInOctets[5m]) < 0 or rate(ifOutOctets[5m]) < 0 + for: 10m + labels: + severity: info + category: correctness + annotations: + summary: "32-bit interface counter wrapped on {{ $labels.device }}:{{ $labels.ifDescr }}" + + # --- Hypervisor BMC (HPE iLO / CPQ MIBs) ------------------------------ + # CPQ condition values: 1=other 2=ok 3=degraded 4=failed + - alert: IloHardwareDegraded + expr: | + cpqHeTemperatureCondition > 2 + or cpqHeFltTolPowerSupplyCondition > 2 + or cpqDaLogDrvCondition > 2 + or cpqDaPhyDrvCondition > 2 + for: 5m + labels: + severity: critical + category: hardware + annotations: + summary: "iLO on {{ $labels.device }} reports degraded or failed hardware" + description: >- + CPQ condition code {{ $value }} (3 = degraded, 4 = failed). Check + the iLO web UI for the specific component. + + - alert: IloBatteryCondition + expr: cpqHeSysBackupBatteryCondition > 2 + for: 30m + labels: + severity: warning + category: hardware + annotations: + summary: "RAID cache / system backup battery degraded on {{ $labels.device }}" diff --git a/stacks/observability/prometheus/rules/ups.rules.yaml b/stacks/observability/prometheus/rules/ups.rules.yaml new file mode 100644 index 0000000..7ac59c5 --- /dev/null +++ b/stacks/observability/prometheus/rules/ups.rules.yaml @@ -0,0 +1,79 @@ +--- +# APC Smart-UPS (mjolnir, 10.0.99.10) via UPS-MIB. +# +# The UPS currently has no working battery — see docs/roadmap.md. Until that is +# fixed a mains loss is an immediate hard shutdown of the whole rack, which is +# why every rule here is deliberately loud and routed to the `urgent` receiver +# by category=power in alertmanager.yaml. +groups: + - name: ups + interval: 60s + rules: + - alert: UpsOnBattery + expr: upsSecondsOnBattery > 0 + for: 30s + labels: + severity: critical + category: power + annotations: + summary: "UPS {{ $labels.device }} is running on battery" + description: >- + On battery for {{ $value | humanizeDuration }}. Estimated runtime + remaining is reported by upsEstimatedMinutesRemaining. + + - alert: UpsBatteryLow + # UPS-MIB upsBatteryStatus: 1=unknown 2=normal 3=low 4=depleted + expr: upsBatteryStatus > 2 + for: 1m + labels: + severity: critical + category: power + annotations: + summary: "UPS {{ $labels.device }} battery status is {{ $value }} (>2 = low/depleted)" + + - alert: UpsRuntimeCritical + expr: upsEstimatedMinutesRemaining < 5 + for: 1m + labels: + severity: critical + category: power + annotations: + summary: "UPS {{ $labels.device }} has {{ $value }} minutes of runtime left" + + - alert: UpsChargeLow + expr: upsEstimatedChargeRemaining < 50 + for: 10m + labels: + severity: warning + category: power + annotations: + summary: "UPS {{ $labels.device }} charge at {{ $value }}%" + + - alert: UpsOverloaded + expr: upsOutputPercentLoad > 80 + for: 15m + labels: + severity: warning + category: power + annotations: + summary: "UPS {{ $labels.device }} output load at {{ $value }}%" + description: >- + Sustained load above 80% shortens battery runtime non-linearly. + + - alert: UpsAlarmPresent + expr: upsAlarmsPresent > 0 + for: 5m + labels: + severity: warning + category: power + annotations: + summary: "UPS {{ $labels.device }} reports {{ $value }} active alarm(s)" + + - alert: UpsBatteryTemperatureHigh + expr: upsBatteryTemperature > 40 + for: 15m + labels: + severity: warning + category: power + annotations: + summary: "UPS {{ $labels.device }} battery at {{ $value }}°C" diff --git a/stacks/observability/prometheus/targets/snmp.yaml b/stacks/observability/prometheus/targets/snmp.yaml new file mode 100644 index 0000000..72ce3df --- /dev/null +++ b/stacks/observability/prometheus/targets/snmp.yaml @@ -0,0 +1,41 @@ +--- +# SNMP scrape targets, consumed by the `snmp` job via file_sd_configs. +# Hot-reloaded every 5 minutes — no Prometheus restart required. +# +# `module` selects the OID set from snmp-exporter/generator.yaml. +# `auth` selects the credential block (one per device, not shared). +# +# To add a device, append a block here and add a matching module + auth to +# snmp-exporter/generator.yaml. See docs/runbooks/add-monitored-device.md. + +- targets: ["10.0.99.1"] + labels: + module: pfsense + auth: auth_pfsense + device: morpheus + role: firewall + vlan: "99" + +- targets: ["10.0.99.10"] + labels: + module: apc_ups + auth: auth_apc + device: mjolnir + role: ups + vlan: "99" + +- targets: ["10.7.7.2"] + labels: + module: mokerlink + auth: auth_mokerlink + device: neo + role: switch + vlan: lan + +- targets: ["10.0.30.10"] + labels: + module: ilo + auth: auth_ilo + device: shiva + role: hypervisor-bmc + vlan: "30" From 665e2ef68f9dd68c8a8850fd37909be83d0c8903 Mon Sep 17 00:00:00 2001 From: Garrett Allen Date: Sun, 2 Aug 2026 06:23:16 +0000 Subject: [PATCH 5/8] ci: validate configs, dashboards and secrets on every push MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The repository is almost entirely YAML and had nothing checking that any of it was valid. The defects fixed in the preceding commits — an invalid depends_on, four mount paths pointing at nothing, a regex that silently never matched — would all have been caught by a pipeline on the day they were introduced. Three jobs: lint yamllint, markdownlint, shellcheck, actionlint, editorconfig validate docker compose config, promtool check config/rules, amtool check-config, alloy fmt --verify, dashboard structure, and every dashboard PromQL expression parsed by promtool secrets gitleaks over the working tree and over full history Dashboard queries are as easy to typo as alert rules but fail as an empty panel rather than an error, so all 79 are extracted and parsed. gitleaks compiles with RE2, so the custom rules avoid lookaheads and exclude ${PLACEHOLDER} values by requiring the value's first character not to be '$'. The full-history scan honours .gitleaksignore, which enumerates all nine known historical findings with an explanation of each. That file is an acknowledgement rather than a fix: a job permanently red for a known reason gets ignored, and then a genuinely new leak goes unnoticed with it. It gets deleted once the history purge runs. Also adds Dependabot for the pinned images, which is what makes pinning sustainable rather than just stale. --- .github/ISSUE_TEMPLATE/bug.yml | 43 +++++++ .github/ISSUE_TEMPLATE/change.yml | 34 ++++++ .github/dependabot.yml | 28 +++++ .github/pull_request_template.md | 23 ++++ .github/workflows/ci.yml | 161 ++++++++++++++++++++++++++ .gitleaks.toml | 65 +++++++++++ .gitleaksignore | 33 ++++++ .markdownlint-cli2.yaml | 21 ++++ .yamllint.yaml | 29 +++++ scripts/check_dashboards.py | 164 +++++++++++++++++++++++++++ scripts/validate.sh | 180 ++++++++++++++++++++++++++++++ 11 files changed, 781 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug.yml create mode 100644 .github/ISSUE_TEMPLATE/change.yml create mode 100644 .github/dependabot.yml create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/ci.yml create mode 100644 .gitleaks.toml create mode 100644 .gitleaksignore create mode 100644 .markdownlint-cli2.yaml create mode 100644 .yamllint.yaml create mode 100755 scripts/check_dashboards.py create mode 100755 scripts/validate.sh diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml new file mode 100644 index 0000000..2daeb49 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -0,0 +1,43 @@ +--- +name: Something is broken +description: A service, dashboard, alert or metric is not behaving +labels: ["bug"] +body: + - type: dropdown + id: area + attributes: + label: Area + options: + - Prometheus / alerting + - Grafana / dashboards + - Loki / log collection + - Alloy agent + - SNMP / network devices + - Deployment (compose, secrets, scripts) + - Documentation + validations: + required: true + + - type: textarea + id: expected + attributes: + label: What should happen + validations: + required: true + + - type: textarea + id: actual + attributes: + label: What happens instead + description: Include the query, panel or alert name where relevant. + validations: + required: true + + - type: textarea + id: logs + attributes: + label: Relevant output + description: >- + Output of `make validate`, `docker compose logs `, or the + failing PromQL. Redact community strings and passwords. + render: shell diff --git a/.github/ISSUE_TEMPLATE/change.yml b/.github/ISSUE_TEMPLATE/change.yml new file mode 100644 index 0000000..579958a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/change.yml @@ -0,0 +1,34 @@ +--- +name: Planned change +description: New device, new service, or a change to the network +labels: ["enhancement"] +body: + - type: textarea + id: goal + attributes: + label: What and why + validations: + required: true + + - type: dropdown + id: vlan + attributes: + label: Which VLAN does this touch + multiple: true + options: + - "99 — Winterfell (management)" + - "50 — Hicks (trusted)" + - "40 — CasaBonita (media)" + - "30 — ImaginationLAN (lab)" + - "20 — Skids (IoT)" + - "10 — Degens (guest)" + - None / not network-facing + + - type: checkboxes + id: security + attributes: + label: Security review + options: + - label: Does not weaken the default-deny posture between VLANs + - label: Any new credential goes in `secrets/*.sops.yaml` + - label: Any new device is added to `docs/network.md` with an OUI-truncated MAC diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..174843d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,28 @@ +--- +version: 2 + +updates: + # Pinned container images in compose.yaml. This is what makes pinning + # sustainable — without it, pinned tags simply rot. + - package-ecosystem: docker-compose + directory: /stacks/observability + schedule: + interval: weekly + day: sunday + open-pull-requests-limit: 5 + commit-message: + prefix: "chore(deps)" + labels: ["dependencies", "observability"] + groups: + grafana-stack: + patterns: ["grafana/*"] + prometheus-stack: + patterns: ["prom/*"] + + - package-ecosystem: github-actions + directory: / + schedule: + interval: monthly + commit-message: + prefix: "chore(ci)" + labels: ["dependencies", "ci"] diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..471e26a --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,23 @@ +## What changed + + + +## Why + + + +## Blast radius + + + +- [ ] No change to network segmentation or firewall rules +- [ ] No new port published to a VLAN that could not already reach the service +- [ ] No credential added outside `secrets/*.sops.yaml` + +## Verification + + + +- [ ] `make validate` passes +- [ ] Deployed to the lab and confirmed working +- [ ] Docs updated (`docs/`, service README, or `docs/roadmap.md`) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a890254 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,161 @@ +--- +name: CI + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + STACK: stacks/observability + PROM_IMAGE: prom/prometheus:v3.1.0 + AM_IMAGE: prom/alertmanager:v0.28.0 + ALLOY_IMAGE: grafana/alloy:v1.6.1 + +jobs: + # --------------------------------------------------------------------------- + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: yamllint + run: pipx run yamllint --strict . + + - name: markdownlint + run: npx --yes markdownlint-cli2 + + - name: shellcheck + run: shellcheck scripts/*.sh + + - name: actionlint + uses: raven-actions/actionlint@v2 + + - name: EditorConfig + uses: editorconfig-checker/action-editorconfig-checker@v2 + + # --------------------------------------------------------------------------- + validate: + name: Validate configs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # The ${VAR:?} guards in compose.yaml exist so a missing secret fails + # loudly at deploy time. CI supplies throwaway values so validation never + # needs a decryption key. + - name: Seed a validation-only .env + run: | + cp "$STACK/.env.example" "$STACK/.env" + { + echo "GRAFANA_ADMIN_PASSWORD=validation-only" + echo "ALERTMANAGER_WEBHOOK_URL=https://example.invalid/hook" + } >> "$STACK/.env" + + - name: docker compose config + run: docker compose -f "$STACK/compose.yaml" config -q + + - name: Verify no image uses a floating tag + run: | + if grep -nE '^\s*image:.*:latest\s*$' "$STACK/compose.yaml"; then + echo "::error::floating :latest tag found — pin an explicit version" + exit 1 + fi + echo "all images pinned" + + - name: promtool check config + run: | + docker run --rm --entrypoint promtool \ + -v "$PWD:/repo" -w /repo "$PROM_IMAGE" \ + check config "$STACK/prometheus/prometheus.yaml" + + - name: promtool check rules + run: | + docker run --rm --entrypoint promtool \ + -v "$PWD:/repo" -w /repo "$PROM_IMAGE" \ + check rules "$STACK"/prometheus/rules/*.rules.yaml + + - name: amtool check-config + run: | + docker run --rm --entrypoint amtool \ + -e ALERTMANAGER_WEBHOOK_URL=https://example.invalid/hook \ + -v "$PWD:/repo" -w /repo "$AM_IMAGE" \ + check-config "$STACK/alertmanager/alertmanager.yaml" + + - name: alloy fmt --verify + run: | + docker run --rm --entrypoint alloy \ + -v "$PWD:/repo" -w /repo "$ALLOY_IMAGE" \ + fmt --verify "$STACK/alloy/config.alloy" + + - name: Validate Grafana dashboards + run: python3 scripts/check_dashboards.py + + # Dashboard queries are as easy to typo as alert rules, and a broken one + # shows up as an empty panel rather than an error. Parse them all. + - name: Parse every dashboard PromQL expression + run: | + python3 scripts/check_dashboards.py --emit-promql > /tmp/dashboard-exprs.yaml + docker run --rm --entrypoint promtool \ + -v /tmp:/tmp "$PROM_IMAGE" check rules /tmp/dashboard-exprs.yaml + + # --------------------------------------------------------------------------- + secret-scan: + name: Secret scan + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # Two scans, because they answer different questions. + # + # The working-tree scan asks "is there a secret in the code as it stands" + # and must always be clean. The history scan asks "has one ever been + # committed" — it honours .gitleaksignore, which enumerates the nine known + # historical findings with the reason each is there. Once the history + # purge in docs/runbooks/purge-git-history.md has run, that file goes away + # and this job keeps passing on its own merits. + - name: gitleaks — working tree + run: | + docker run --rm -v "$PWD:/repo" -w /repo \ + zricethezav/gitleaks:latest \ + detect --no-git --no-banner --redact -c .gitleaks.toml -v + + - name: gitleaks — full history + run: | + docker run --rm -v "$PWD:/repo" -w /repo \ + zricethezav/gitleaks:latest \ + detect --no-banner --redact -c .gitleaks.toml --log-opts="--all" -v + + - name: Assert no decrypted artefact is tracked + run: | + fail=0 + for pattern in '.env' '.rendered/'; do + if git ls-files | grep -E "(^|/)${pattern//./\\.}" | grep -v '\.env\.example'; then + echo "::error::tracked file matching '${pattern}' — it must be gitignored" + fail=1 + fi + done + exit "$fail" + + - name: Assert SOPS files are actually encrypted + run: | + shopt -s nullglob + fail=0 + for f in secrets/*.sops.yaml; do + if ! grep -q '^sops:' "$f"; then + echo "::error file=$f::not SOPS-encrypted" + fail=1 + fi + done + exit "$fail" diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000..48ff397 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,65 @@ +title = "HomeLab gitleaks configuration" + +[extend] +useDefault = true + +# --------------------------------------------------------------------------- +# Homelab-specific rules. The default ruleset is tuned for cloud provider keys +# and misses the credentials that actually matter here. +# --------------------------------------------------------------------------- + +# gitleaks compiles with RE2, which has no lookahead. Both rules below exclude +# ${PLACEHOLDER} values by requiring the first character of the value to be +# something other than '$', rather than by using a negative lookahead. + +[[rules]] +id = "snmp-community-plaintext" +description = "SNMP community string committed in plaintext" +regex = '''(?i)community\s*:\s*["']?[^\s"'#$][^\s"'#]{3,}''' +path = '''.*\.(ya?ml)$''' +tags = ["snmp", "credential"] + +[[rules]] +id = "grafana-inline-password" +description = "Grafana admin password set inline instead of from the environment" +regex = '''GF_SECURITY_ADMIN_PASSWORD\s*[:=]\s*[^\s$]+''' +tags = ["grafana", "credential"] + +[[rules]] +id = "private-key-block" +description = "PEM private key" +regex = '''-----BEGIN (?:RSA |EC |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY-----''' +tags = ["key"] + +[[rules]] +id = "age-secret-key" +description = "age private key" +regex = '''AGE-SECRET-KEY-1[0-9A-Z]{50,}''' +tags = ["age", "key"] + +# --------------------------------------------------------------------------- +# Allowlist +# --------------------------------------------------------------------------- +[allowlist] +description = "Templates, placeholders and encrypted files" + +paths = [ + # Encrypted at rest — the ciphertext is meant to be committed. + '''secrets/.*\.sops\.yaml$''', + # Documents required key names and deliberately carries change-me values. + '''secrets/.*\.example\.yaml$''', + # Documents the leak it teaches you to remove. + '''docs/runbooks/purge-git-history\.md$''', + '''scripts/purge-history\.sh$''', +] + +regexes = [ + # ${PLACEHOLDER} substitution targets are not secrets. + '''\$\{[A-Z_]+\}''', + '''change-me[a-z-]*''', + '''REPLACE_WITH_YOUR_AGE_PUBLIC_KEY''', + '''example\.invalid''', + '''validation-only''', + # age *public* keys are safe to commit; only the secret key matters. + '''age1[a-z0-9]{58}''', +] diff --git a/.gitleaksignore b/.gitleaksignore new file mode 100644 index 0000000..f0f56d1 --- /dev/null +++ b/.gitleaksignore @@ -0,0 +1,33 @@ +# Known historical exposure, acknowledged rather than hidden. +# +# These nine findings are real. They are secrets that were genuinely committed +# to this repository and are still reachable in git history. They are listed +# here so that CI's full-history scan stays meaningful — a scan that is red for +# a known reason gets ignored, and then a *new* leak goes unnoticed too. +# +# This file is not remediation. Remediation is: +# 1. Rotate the credentials -> docs/runbooks/rotate-snmp-community.md +# 2. Purge them from history -> docs/runbooks/purge-git-history.md +# Both are tracked in docs/roadmap.md and described in docs/security.md. +# +# After the history purge these fingerprints become stale and this file should +# be deleted. If gitleaks then reports nothing, the job is finished. + +# --- Shared SNMP community string, committed in plaintext (commit ee3d443) --- +# One string reused across pfSense, the APC UPS, the MokerLink switch and iLO. +ee3d443abc19029f182edbcf9beadb3e6ca8f7c5:snmp-exporter/prometheus.matrix.elysium/config/snmp.yaml:snmp-community-plaintext:4 +ee3d443abc19029f182edbcf9beadb3e6ca8f7c5:snmp-exporter/prometheus.matrix.elysium/config/snmp.yaml:snmp-community-plaintext:10 +ee3d443abc19029f182edbcf9beadb3e6ca8f7c5:snmp-exporter/prometheus.matrix.elysium/config/snmp.yaml:snmp-community-plaintext:16 +ee3d443abc19029f182edbcf9beadb3e6ca8f7c5:snmp-exporter/prometheus.matrix.elysium/config/snmp.yaml:snmp-community-plaintext:22 + +# --- Grafana admin password inline in the compose file (commit ee3d443) --- +# admin/admin, alongside GF_AUTH_ANONYMOUS_ENABLED=true with Admin role. +ee3d443abc19029f182edbcf9beadb3e6ca8f7c5:grafana/prometheus.matrix.elysium/docker-compose.yaml:grafana-inline-password:7 + +# --- TLS private keys (introduced efb2632, deleted in 647d90a) --- +# Passphrase-encrypted, which mitigates but does not resolve. Still recoverable +# with `git show 647d90a~1:certificates/Gandalf.Gondor.Lab/ca-key.pem`. +efb263225de6ee4961421596fd43e3f69b3bce8f:certificates/Gandalf.Gondor.Lab/ca-key.pem:private-key:1 +efb263225de6ee4961421596fd43e3f69b3bce8f:certificates/Gandalf.Gondor.Lab/ca-key.pem:private-key-block:1 +efb263225de6ee4961421596fd43e3f69b3bce8f:certificates/Gandalf.Gondor.Lab/cert-key.pem:private-key:1 +efb263225de6ee4961421596fd43e3f69b3bce8f:certificates/Gandalf.Gondor.Lab/cert-key.pem:private-key-block:1 diff --git a/.markdownlint-cli2.yaml b/.markdownlint-cli2.yaml new file mode 100644 index 0000000..2ff1c01 --- /dev/null +++ b/.markdownlint-cli2.yaml @@ -0,0 +1,21 @@ +--- +config: + default: true + # Tables in docs/network.md and the README badge row exceed any sane limit. + MD013: false + # Duplicate headings are meaningful in per-VLAN sections (each has "Notes"). + MD024: + siblings_only: true + # Inline HTML is used for centred badges and
blocks. + MD033: false + # The README's first heading is preceded by a centred badge block. + MD041: false + # Bare URLs appear inside code fences only. + MD034: true + +globs: + - "**/*.md" + +ignores: + - node_modules/** + - .venv/** diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 0000000..4a500f6 --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,29 @@ +--- +extends: default + +ignore: | + stacks/observability/snmp-exporter/snmp.yaml + stacks/observability/snmp-exporter/.rendered/ + node_modules/ + .venv/ + +rules: + line-length: + max: 120 + allow-non-breakable-words: true + allow-non-breakable-inline-mappings: true + comments: + min-spaces-from-content: 1 + truthy: + # GitHub Actions' `on:` key is not a boolean. + allowed-values: ["true", "false"] + check-keys: false + document-start: + present: true + indentation: + spaces: 2 + indent-sequences: consistent + braces: + max-spaces-inside: 1 + brackets: + max-spaces-inside: 1 diff --git a/scripts/check_dashboards.py b/scripts/check_dashboards.py new file mode 100755 index 0000000..782d85c --- /dev/null +++ b/scripts/check_dashboards.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Validate provisioned Grafana dashboards. + +Grafana silently accepts a dashboard that references a datasource UID which +does not exist — the panels simply render empty, which looks like "no data" +rather than "misconfigured". This checks the things Grafana will not: + + * the JSON parses; + * every datasource UID resolves to one declared in provisioning; + * every dashboard UID is unique across the folder; + * panels fit the 24-column grid and do not overlap; + * every panel has at least one target. + +Additionally, every PromQL expression is emitted to stdout in Prometheus +recording-rule form when --emit-promql is passed, so promtool can parse them. +""" +from __future__ import annotations + +import argparse +import json +import pathlib +import sys + +REPO = pathlib.Path(__file__).resolve().parent.parent +DASHBOARDS = REPO / "stacks/observability/grafana/dashboards" +DATASOURCES = REPO / "stacks/observability/grafana/provisioning/datasources/datasources.yaml" + +# UIDs Grafana provides itself. +BUILTIN_UIDS = {"-- Grafana --", "-- Mixed --", "-- Dashboard --", "grafana"} + + +def declared_datasource_uids() -> set[str]: + """Read provisioned UIDs without requiring PyYAML.""" + uids = set() + if not DATASOURCES.exists(): + return uids + for line in DATASOURCES.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if stripped.startswith("uid:"): + uids.add(stripped.split(":", 1)[1].strip()) + return uids + + +def walk_datasource_uids(node, found: set[str]) -> None: + if isinstance(node, dict): + if set(node) <= {"type", "uid"} and "uid" in node: + found.add(node["uid"]) + for value in node.values(): + walk_datasource_uids(value, found) + elif isinstance(node, list): + for value in node: + walk_datasource_uids(value, found) + + +def overlaps(a: dict, b: dict) -> bool: + return not ( + a["x"] + a["w"] <= b["x"] + or b["x"] + b["w"] <= a["x"] + or a["y"] + a["h"] <= b["y"] + or b["y"] + b["h"] <= a["y"] + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--emit-promql", action="store_true", + help="print dashboard PromQL as a rules file for promtool") + args = parser.parse_args() + + known = declared_datasource_uids() | BUILTIN_UIDS + problems: list[str] = [] + seen_dashboard_uids: dict[str, str] = {} + promql: list[str] = [] + + files = sorted(DASHBOARDS.glob("*.json")) + if not files: + print(f"no dashboards found in {DASHBOARDS}", file=sys.stderr) + return 1 + + for path in files: + name = path.name + try: + dash = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + problems.append(f"{name}: invalid JSON — {exc}") + continue + + uid = dash.get("uid") + if not uid: + problems.append(f"{name}: missing top-level 'uid'") + elif uid in seen_dashboard_uids: + problems.append(f"{name}: uid '{uid}' already used by {seen_dashboard_uids[uid]}") + else: + seen_dashboard_uids[uid] = name + + if not dash.get("title"): + problems.append(f"{name}: missing 'title'") + + found: set[str] = set() + walk_datasource_uids(dash, found) + for unknown in sorted(found - known): + problems.append( + f"{name}: datasource uid '{unknown}' is not provisioned — " + f"panels using it will render empty" + ) + + panels = dash.get("panels", []) + for i, panel in enumerate(panels): + title = panel.get("title") or f"" + grid = panel.get("gridPos") + if not grid: + problems.append(f"{name}: panel '{title}' has no gridPos") + continue + if grid["x"] + grid["w"] > 24: + problems.append( + f"{name}: panel '{title}' overflows the 24-column grid " + f"(x={grid['x']} w={grid['w']})" + ) + if panel.get("type") not in ("row", "text") and not panel.get("targets"): + problems.append(f"{name}: panel '{title}' has no targets") + + for other in panels[i + 1:]: + og = other.get("gridPos") + if og and overlaps(grid, og): + problems.append( + f"{name}: panels '{title}' and " + f"'{other.get('title')}' overlap" + ) + + for target in panel.get("targets", []): + expr = target.get("expr") + ds_uid = (target.get("datasource") or {}).get("uid") + if expr and ds_uid != "loki": + promql.append(expr) + + if args.emit_promql: + print("groups:") + print(" - name: dashboard-expressions") + print(" rules:") + for i, expr in enumerate(promql): + # Block scalar keeps multi-line expressions and quoting intact. + print(f" - record: dashboard:expr{i}") + print(" expr: |") + for line in expr.splitlines(): + print(f" {line}") + return 0 + + for problem in problems: + print(f" {problem}", file=sys.stderr) + + if problems: + print(f"\n{len(problems)} problem(s) in {len(files)} dashboard(s)", file=sys.stderr) + return 1 + + print( + f"{len(files)} dashboards OK " + f"({sum(len(json.loads(p.read_text())['panels']) for p in files)} panels, " + f"{len(promql)} PromQL expressions)" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/validate.sh b/scripts/validate.sh new file mode 100755 index 0000000..ce90e45 --- /dev/null +++ b/scripts/validate.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +# +# Everything CI runs, runnable locally. Uses containers for the Prometheus and +# Alertmanager tooling so the versions match what actually runs in production. +# +# Usage: scripts/validate.sh + +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${REPO_ROOT}" || exit 1 + +STACK="stacks/observability" +PROM_IMAGE="prom/prometheus:v3.1.0" +AM_IMAGE="prom/alertmanager:v0.28.0" +ALLOY_IMAGE="grafana/alloy:v1.6.1" + +FAILED=0 +pass() { printf '\033[0;32m PASS\033[0m %s\n' "$*"; } +fail() { printf '\033[0;31m FAIL\033[0m %s\n' "$*"; FAILED=1; } +skip() { printf '\033[0;33m SKIP\033[0m %s\n' "$*"; } +head_() { printf '\n\033[1m%s\033[0m\n' "$*"; } + +have() { command -v "$1" >/dev/null 2>&1; } +have_docker() { have docker && docker info >/dev/null 2>&1; } + +# --------------------------------------------------------------------------- +head_ "Compose" +# --------------------------------------------------------------------------- +if have docker; then + # A .env is required for the ${VAR:?} guards; use the example values plus + # throwaway secrets so validation does not depend on a decryption key. + TMP_ENV="$(mktemp)" + trap 'rm -f "${TMP_ENV}"' EXIT + cat "${STACK}/.env.example" > "${TMP_ENV}" + cat >> "${TMP_ENV}" <<'EOF' +GRAFANA_ADMIN_PASSWORD=validation-only +ALERTMANAGER_WEBHOOK_URL=https://example.invalid/hook +EOF + if docker compose --env-file "${TMP_ENV}" -f "${STACK}/compose.yaml" config -q 2>/dev/null; then + pass "docker compose config" + else + docker compose --env-file "${TMP_ENV}" -f "${STACK}/compose.yaml" config -q + fail "docker compose config" + fi +else + skip "docker not installed" +fi + +# --------------------------------------------------------------------------- +head_ "Prometheus" +# --------------------------------------------------------------------------- +if have promtool; then + PROMTOOL=(promtool) +elif have_docker; then + PROMTOOL=(docker run --rm --entrypoint promtool -v "${REPO_ROOT}:/repo" -w /repo "${PROM_IMAGE}") +else + PROMTOOL=() +fi + +if ((${#PROMTOOL[@]})); then + if "${PROMTOOL[@]}" check config "${STACK}/prometheus/prometheus.yaml" 2>&1 \ + | grep -qE '^\s*SUCCESS'; then + pass "promtool check config" + else + "${PROMTOOL[@]}" check config "${STACK}/prometheus/prometheus.yaml" + fail "promtool check config" + fi + + if "${PROMTOOL[@]}" check rules "${STACK}"/prometheus/rules/*.rules.yaml >/dev/null 2>&1; then + pass "promtool check rules" + else + "${PROMTOOL[@]}" check rules "${STACK}"/prometheus/rules/*.rules.yaml + fail "promtool check rules" + fi +else + skip "no promtool and no docker daemon" +fi + +# --------------------------------------------------------------------------- +head_ "Alertmanager" +# --------------------------------------------------------------------------- +if have amtool; then + AMTOOL=(amtool) +elif have_docker; then + AMTOOL=(docker run --rm --entrypoint amtool -e ALERTMANAGER_WEBHOOK_URL=https://example.invalid/hook \ + -v "${REPO_ROOT}:/repo" -w /repo "${AM_IMAGE}") +else + AMTOOL=() +fi + +if ((${#AMTOOL[@]})); then + export ALERTMANAGER_WEBHOOK_URL="${ALERTMANAGER_WEBHOOK_URL:-https://example.invalid/hook}" + if "${AMTOOL[@]}" check-config "${STACK}/alertmanager/alertmanager.yaml" >/dev/null 2>&1; then + pass "amtool check-config" + else + "${AMTOOL[@]}" check-config "${STACK}/alertmanager/alertmanager.yaml" + fail "amtool check-config" + fi +else + skip "no amtool and no docker daemon" +fi + +# --------------------------------------------------------------------------- +head_ "Alloy" +# --------------------------------------------------------------------------- +if have_docker; then + if docker run --rm -v "${REPO_ROOT}:/repo" -w /repo --entrypoint alloy "${ALLOY_IMAGE}" \ + fmt --verify "${STACK}/alloy/config.alloy" >/dev/null 2>&1; then + pass "alloy fmt --verify" + else + docker run --rm -v "${REPO_ROOT}:/repo" -w /repo --entrypoint alloy "${ALLOY_IMAGE}" \ + fmt --verify "${STACK}/alloy/config.alloy" + fail "alloy fmt --verify" + fi +else + skip "alloy fmt (needs a docker daemon)" +fi + +# --------------------------------------------------------------------------- +head_ "Grafana dashboards" +# --------------------------------------------------------------------------- +if have python3; then + if python3 scripts/check_dashboards.py; then + pass "dashboard JSON and datasource references" + else + fail "dashboard JSON and datasource references" + fi +else + skip "python3 not installed" +fi + +# --------------------------------------------------------------------------- +head_ "YAML / Markdown / shell" +# --------------------------------------------------------------------------- +if have yamllint; then + if yamllint . >/dev/null 2>&1; then pass "yamllint"; else yamllint .; fail "yamllint"; fi +else + skip "yamllint not installed (pip install yamllint)" +fi + +if have markdownlint-cli2; then + if markdownlint-cli2 >/dev/null 2>&1; then pass "markdownlint"; else markdownlint-cli2; fail "markdownlint"; fi +else + skip "markdownlint-cli2 not installed (npm i -g markdownlint-cli2)" +fi + +if have shellcheck; then + if shellcheck scripts/*.sh; then pass "shellcheck"; else fail "shellcheck"; fi +else + skip "shellcheck not installed" +fi + +# --------------------------------------------------------------------------- +head_ "Secrets" +# --------------------------------------------------------------------------- +if have gitleaks; then + if gitleaks detect --no-banner --redact -c .gitleaks.toml >/dev/null 2>&1; then + pass "gitleaks (working tree)" + else + gitleaks detect --no-banner --redact -c .gitleaks.toml + fail "gitleaks (working tree)" + fi +else + skip "gitleaks not installed" +fi + +# Cheap belt-and-braces check that no rendered/decrypted artefact is staged. +if git ls-files --error-unmatch "${STACK}/.env" >/dev/null 2>&1 \ + || git ls-files "${STACK}/snmp-exporter/.rendered" | grep -q .; then + fail "a rendered or decrypted file is tracked by git" +else + pass "no rendered or decrypted files tracked" +fi + +printf '\n' +if ((FAILED)); then + printf '\033[0;31mvalidation failed\033[0m\n'; exit 1 +fi +printf '\033[0;32mall checks passed\033[0m\n' From 50bf192631c90e45f4569846c8920007cabddd0f Mon Sep 17 00:00:00 2001 From: Garrett Allen Date: Sun, 2 Aug 2026 06:23:33 +0000 Subject: [PATCH 6/8] docs: rewrite as project documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README was a personal journal entry. It described the lab entirely in the future tense, listed an objective (red team, Kubernetes, CTF, reverse engineering) none of which exists in the repository, and contained no architecture, quick start, or link to anything else. Neither network diagram was referenced by any file. Replaces it with a project README, and splits the inventory — which was the best work in the repository — into documentation that can be navigated: architecture.md Mermaid topology and data flow that render on GitHub network.md the per-VLAN inventory, restructured and sanitised hardware.md rack layout and compute, split out of the inventory observability.md what is collected, retention, cardinality, operations security.md threat model, segmentation rationale, secrets handling roadmap.md the task checkboxes that were buried in the inventory adr/ 5 decision records runbooks/ deploy, add a device, rotate credentials, purge history Sanitisation of network.md: MAC addresses truncated to their OUI, personal devices listed by role rather than by owner, and camera-to-room mapping removed. The file previously published the full MAC, OS version and room of every device in the house, including cameras and a baby monitor, in a public repository. The vendor half of the MAC keeps everything that was actually useful. Also drops the $\color{limegreen}{...}$ table headers, which render only on github.com and make the header text ungreppable. Mermaid diagrams are now the maintained ones. The existing PNG is 12.5 MB at 9871x4466, which GitHub will not render usefully, and its .drawio source was deleted in an earlier commit — so it is kept and linked but is no longer the primary reference. Adds scripts/purge-history.sh with a --dry-run mode that rewrites a scratch mirror and reports, leaving the real repository untouched. --- README.md | 232 +++++++++++++++--- .../adr/0001-record-architecture-decisions.md | 37 +++ docs/adr/0002-vlan-segmentation-strategy.md | 57 +++++ .../adr/0003-observability-stack-selection.md | 60 +++++ docs/adr/0004-one-compose-stack-per-host.md | 60 +++++ docs/adr/0005-secrets-with-sops-and-age.md | 66 +++++ docs/architecture.md | 166 +++++++++++++ docs/hardware.md | 57 +++++ docs/images/README.md | 43 ++++ docs/observability.md | 139 +++++++++++ docs/roadmap.md | 72 ++++++ docs/runbooks/add-monitored-device.md | 135 ++++++++++ docs/runbooks/deploy-stack.md | 105 ++++++++ docs/runbooks/purge-git-history.md | 115 +++++++++ docs/runbooks/rotate-snmp-community.md | 124 ++++++++++ docs/security.md | 125 ++++++++++ scripts/purge-history.sh | 131 ++++++++++ 17 files changed, 1687 insertions(+), 37 deletions(-) create mode 100644 docs/adr/0001-record-architecture-decisions.md create mode 100644 docs/adr/0002-vlan-segmentation-strategy.md create mode 100644 docs/adr/0003-observability-stack-selection.md create mode 100644 docs/adr/0004-one-compose-stack-per-host.md create mode 100644 docs/adr/0005-secrets-with-sops-and-age.md create mode 100644 docs/architecture.md create mode 100644 docs/hardware.md create mode 100644 docs/images/README.md create mode 100644 docs/observability.md create mode 100644 docs/roadmap.md create mode 100644 docs/runbooks/add-monitored-device.md create mode 100644 docs/runbooks/deploy-stack.md create mode 100644 docs/runbooks/purge-git-history.md create mode 100644 docs/runbooks/rotate-snmp-community.md create mode 100644 docs/security.md create mode 100755 scripts/purge-history.sh diff --git a/README.md b/README.md index 3c97ba0..8a852c0 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,197 @@ -# HomeLab - -## Introduction - -Alright, well... Here we go I guess? - -I've been wanting to build my own HomeLab for years, and now that I have plenty of time on my hands due to an indefinitely delayed West-Coast move, and an empty house except for 2 anxious puppies, why not? - -## Objective - -I want this lab to be an all-encompassing Cybersecurity lab for sharpening my skills and continuous learning. I want to include Red Team, Blue Team, Penetration Testing, Web Apps, Docker, Kubernetes, Development, Analysis, Reverse Engineering, Threat Hunting, Intrusion Detection, CTF's, Mobile, VulnHub, annnnnnnnnnnnd I'm sure I'll add more when/if I remember. I have no idea how it's going to look at this time, nor do I know yet what sorts of extra hardware I'll need to purchase to supplement what I already have. - -As a bonus, maybe I'll start to graduate from script-kiddie and improve my automation game with some coding. I forgot how many times I've attempted to START trying to learn how to code, but it always ends with me naturally gravitating back to one of my other hobbies out of frustration and wanting to be comfortable again. Now, with the power of ~*Adderall*~ immense focus, perhaps now's the time to harness this energy towards something resume-building, educational, time-consuming, wallet-shrinking, and spouse-angering since she will likely wonder just what in the double h-e-double hockey stick am I doing all alone in my office all day. Love ya, but I gotta hack. It's what I gotta do! - -_*UPDATE*_: As with most projects this one has evolved over time to include not just a playground, but my entire home network set up. I've only made my wife upset a certain number of times that remain <10 but unfortunately still >1. I'd say that average is pretty good. - -## The Human Element +
-Will I have enough resources? Heck I don't know, I'm making this up as I go. I will be scouring Amazon, Save My Server, Newegg, PCPartPicker I guess, eBay duh, and a few other places to search for deals that don't sound too jank sketch like the one refurbished absolute monster of a server I found for only $75. - -***...tempting, but no...*** - -Yes, I considered it. Times are hard. - -## Possibilities - -So far, Amazon appears to be the most promising with some of the best deals I've seen. They even have plenty of seemingly honest reviews that slightly boost my confidence it would be a good choice. - -Servers I'm considering: - -- Dell PowerEdge -- Enterprise Proliant -- ...and that's about it for now. - -Of course whichever machine I get is going to be beefed up enough in order to host all the machines I want, and it will be virtualized with Proxmox. No license purchases here, no Sir. - -## Engage +# HomeLab -After that the real fun starts. I absolutely can't wait to stupidly mess up the entire environment and then frantically try to fix it, but inevitably make things worse through spastic troubleshooting and desperation. It's all in the name of bettering myself and learning, right? Even if it degrades my health? ***Hell yea***. +**A segmented home network and its observability stack, managed as code.** + +[![CI](https://github.com/Gerrrt/HomeLab/actions/workflows/ci.yml/badge.svg)](https://github.com/Gerrrt/HomeLab/actions/workflows/ci.yml) +[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) +[![Secrets: SOPS + age](https://img.shields.io/badge/secrets-SOPS%20%2B%20age-6f42c1.svg)](docs/adr/0005-secrets-with-sops-and-age.md) +[![Prometheus](https://img.shields.io/badge/Prometheus-v3.1.0-E6522C.svg?logo=prometheus&logoColor=white)](stacks/observability/prometheus) +[![Grafana](https://img.shields.io/badge/Grafana-11.5-F46800.svg?logo=grafana&logoColor=white)](stacks/observability/grafana) +[![Loki](https://img.shields.io/badge/Loki-3.3-F5A800.svg?logo=grafana&logoColor=white)](stacks/observability/loki) +[![pfSense](https://img.shields.io/badge/pfSense-FreeBSD%2015-212121.svg)](docs/network.md) + +[Architecture](docs/architecture.md) · +[Network](docs/network.md) · +[Observability](docs/observability.md) · +[Security](docs/security.md) · +[Runbooks](docs/runbooks) · +[Decisions](docs/adr) · +[Roadmap](docs/roadmap.md) + +
+ +--- + +Seven VLANs behind a pfSense firewall, default-deny between every segment, with +a Prometheus/Loki/Grafana stack watching all of it. Every config in this +repository is the config that runs, validated on every push. + +It started as a place to practise security work and turned into the network the +house actually depends on, which changed the requirements considerably — a +broken experiment is a learning opportunity, a broken DHCP server is a domestic +incident. + +## Highlights + +- **Network segmented by trust, not by function.** Seven VLANs; IoT, media and + guest segments are terminal — egress only, no path to anything else. Exactly + two inter-VLAN rules exist. [Why](docs/adr/0002-vlan-segmentation-strategy.md) +- **Full observability pipeline for a mixed estate.** Grafana Alloy agents push + metrics and logs from Linux hosts; `snmp_exporter` polls the four devices that + can't run an agent (firewall, switch, UPS, iLO). One agent config, deployed + identically everywhere. [How](docs/architecture.md#observability-data-flow) +- **Dashboards and alerting as code.** 5 provisioned dashboards, 79 panels, 32 + alert rules with severity routing and inhibition. No dashboard exists only in + a database. +- **Secrets encrypted in-repo with SOPS + age.** Per-device credentials, + decrypted at deploy time into gitignored paths, with `git log` showing which + credential rotated and when — but never to what. + [Why](docs/adr/0005-secrets-with-sops-and-age.md) +- **CI that actually validates the infrastructure.** `docker compose config`, + `promtool`, `amtool`, `alloy fmt`, dashboard-JSON and datasource checks, every + dashboard's PromQL parsed, plus `gitleaks` over the full history. +- **Documented decisions and runbooks.** Five ADRs covering what was chosen and + what was rejected; four runbooks for the operations that are easy to get wrong + at 1am. + +## Architecture + +```mermaid +graph TB + INET([Internet]) --- FW{{"morpheus · pfSense
HP ProDesk 600 G4"}} + FW --- SW[neo · 26-port managed switch] + + subgraph V99["VLAN 99 · Management"] + MON["prometheus
observability stack"] + UPS["mjolnir · UPS"] + end + subgraph V50["VLAN 50 · Trusted"] + WS["workstations"] + end + subgraph V30["VLAN 30 · Lab"] + HV["shiva · Proxmox"] + end + subgraph Terminal["VLANs 40 / 20 / 10 · egress only"] + UNTRUSTED["media · IoT · guest"] + end + + SW --- V99 + SW --- V50 + SW --- V30 + SW --- Terminal + WS -.->|management| V99 + WS -.->|lab| V30 + + classDef mgmt fill:#1f6f4a,stroke:#2ea043,color:#fff + classDef trusted fill:#1f4e79,stroke:#388bfd,color:#fff + classDef untrusted fill:#6e2c2c,stroke:#f85149,color:#fff + classDef infra fill:#4a3f7a,stroke:#a371f7,color:#fff + class MON,UPS mgmt + class WS trusted + class UNTRUSTED untrusted + class HV,FW,SW infra +``` + +Dotted lines are the only two paths between segments. Everything else reaches +the internet and nothing more. Full topology and data flow in +[`docs/architecture.md`](docs/architecture.md). + +## Stack + +| Layer | Tool | Role | +| --- | --- | --- | +| Firewall / routing | pfSense on FreeBSD 15 | VLANs, DHCP, default-deny | +| Virtualisation | Proxmox VE | Lab hypervisor | +| Metrics | Prometheus | 30-day retention, remote-write receiver | +| Logs | Loki | Single-binary, filesystem storage | +| Collection | Grafana Alloy | node + cAdvisor metrics, Docker/journal/syslog/auth logs | +| Network polling | snmp_exporter | pfSense, switch, UPS, iLO | +| Alerting | Alertmanager | Severity routing, inhibition | +| Visualisation | Grafana | 5 provisioned dashboards | +| Secrets | SOPS + age | Encrypted in-repo | +| CI | GitHub Actions | Lint, config validation, secret scanning | + +## Repository layout + +```text +. +├── stacks/observability/ # the deployed stack — one compose file, six services +│ ├── compose.yaml +│ ├── prometheus/ # config, file_sd targets, 32 alert rules +│ ├── alertmanager/ # routing and inhibition +│ ├── loki/ # single-binary config +│ ├── alloy/ # one agent config, used on every host +│ ├── snmp-exporter/ # generator.yaml is the source of truth +│ └── grafana/ # provisioning + 5 dashboards +├── secrets/ # SOPS-encrypted; see secrets/README.md +├── scripts/ # bootstrap, render, validate, history purge +├── docs/ +│ ├── architecture.md network.md hardware.md +│ ├── observability.md security.md roadmap.md +│ ├── adr/ # 5 architecture decision records +│ └── runbooks/ # deploy, add device, rotate creds, purge history +└── Makefile # make help +``` + +## Quick start + +Requires Docker with the compose plugin, plus [`sops`](https://github.com/getsops/sops) +and [`age`](https://github.com/FiloSottile/age). + +```bash +git clone https://github.com/Gerrrt/HomeLab.git && cd HomeLab + +make secrets-init # generate an age keypair, create the encrypted secrets file +make secrets-edit # fill in real values +make validate # everything CI runs +make up # render config and start the stack +``` + +Grafana on `:3000`, Prometheus on `:9090`. Full procedure, verification steps and +troubleshooting in [`docs/runbooks/deploy-stack.md`](docs/runbooks/deploy-stack.md). + +```console +$ make help + up Render config and start the stack + down Stop the stack (volumes are preserved) + reload Hot-reload Prometheus and Alertmanager without a restart + secrets-init Generate an age keypair and create the encrypted secrets file + secrets-edit Edit the encrypted secrets in $EDITOR + validate Run every check CI runs + backup Back up the stack's volumes to ./backups/ + ... +``` + + + +## What runs it + +The entire observability stack runs on a 2012 MacBook Pro with Ubuntu Server on +it. Four SNMP devices at a 60-second interval, Alloy agents, and 30 days of +metrics, on hardware that was otherwise going to landfill. Hardware details in +[`docs/hardware.md`](docs/hardware.md). + +## Security posture + +Segmentation rationale, threat model, secrets handling, and an explicit account +of what this repository deliberately does not publish (full MAC addresses, +owner-linked device names, camera placement) are in +[`docs/security.md`](docs/security.md). + +Historical credential exposure in this repository's git history is documented +there too, along with the runbooks to remediate it — including the parts not yet +done. + +## Roadmap + +Open work is tracked in [`docs/roadmap.md`](docs/roadmap.md). The current top +items: rotate the SNMP communities, purge the old ones from git history, replace +the UPS battery, and get 64-bit interface counters off the switch. + +## License + +[MIT](LICENSE) diff --git a/docs/adr/0001-record-architecture-decisions.md b/docs/adr/0001-record-architecture-decisions.md new file mode 100644 index 0000000..66778bc --- /dev/null +++ b/docs/adr/0001-record-architecture-decisions.md @@ -0,0 +1,37 @@ +# ADR-0001: Record architecture decisions + +**Status:** Accepted · 2025-11 + +## Context + +This lab has been rebuilt several times. Each rebuild re-litigated decisions +that had already been made and forgotten — why the IoT VLAN is terminal, why +Loki instead of an ELK stack, why one compose file instead of five. The +reasoning lived in memory, so it evaporated. + +The configs record *what* is deployed. Nothing recorded *why*, or what was +rejected and for what reason. Six months later that distinction is the whole +difference between maintaining a system and re-deriving it. + +## Decision + +Record every architecturally significant decision as a short Markdown file in +`docs/adr/`, numbered sequentially, in the format popularised by Michael Nygard: +context, decision, consequences. + +A decision is "architecturally significant" if reversing it would mean changing +more than one file, or if a reasonable person would ask "why is it done that +way?" + +ADRs are immutable once accepted. A decision that changes gets a new ADR that +supersedes the old one, and the old one is marked Superseded rather than edited. +The history of what was believed and when is the point. + +## Consequences + +- Every non-obvious choice has a written justification, so a reader — including + a future me — can evaluate it rather than guess at it. +- Reversing a decision requires articulating why, which raises the bar slightly + and usefully. +- There is a small ongoing cost: a new ADR per significant change. Skipping it + is easy and self-punishing. diff --git a/docs/adr/0002-vlan-segmentation-strategy.md b/docs/adr/0002-vlan-segmentation-strategy.md new file mode 100644 index 0000000..b41b13b --- /dev/null +++ b/docs/adr/0002-vlan-segmentation-strategy.md @@ -0,0 +1,57 @@ +# ADR-0002: Segment the network by trust, not by function + +**Status:** Accepted · 2025-06 + +## Context + +A flat home network puts a $20 Wi-Fi plug on the same broadcast domain as a +work laptop. The plug runs unauditable firmware, often has no update mechanism +at all, and is frequently the softest target on the network. Once it is +compromised, everything else is one hop away. + +The obvious alternative — segment by function (a "media" network, an "office" +network) — sorts devices by what they do rather than by how much damage they can +do. A smart TV and a workstation both belong to "the household", but they +warrant completely different trust. + +There were three plausible options: + +1. **Flat network, host firewalls.** Cheapest. Relies on every endpoint + defending itself, which a Ring doorbell cannot. +2. **Segment by function.** Intuitive to explain, but produces segments with + mixed trust levels, which means the rules between them end up permissive. +3. **Segment by trust level.** More VLANs, more rules to reason about, but the + rules are simple because each segment has one trust level. + +## Decision + +Seven VLANs, assigned by how much a compromise of that segment would cost, with +default deny between all of them. + +| VLAN | Trust | Rationale | +| --- | --- | --- | +| 99 Winterfell | Highest | Infrastructure. Compromise here is total. | +| 50 Hicks | High | Workstations. The only segment with a management path. | +| 30 ImaginationLAN | Contained | Deliberately broken things live here. | +| 40 CasaBonita | Low | Vendor firmware, permanent internet connection. | +| 20 Skids | Lowest | IoT. Assume every device is already compromised. | +| 10 Degens | Untrusted | Guests. | + +Exactly two inter-VLAN rules exist: trusted → management (administration), and +trusted → lab (usability). Everything else is egress-only. + +## Consequences + +- A compromised IoT device reaches the internet and nothing else. This is the + main thing the design buys, and it holds even for devices that will never be + patched. +- Adding a device requires deciding its trust level first, which is a useful + forcing function. +- Cross-VLAN conveniences break by default. Chromecast and AirPlay discovery use + mDNS, which does not cross VLANs without an explicit reflector — an ongoing + annoyance that is the honest cost of this design. +- Trusted devices are a single point of failure for management access. A + compromised workstation reaches Winterfell. Mitigating that properly needs a + bastion, which is not worth it at this scale. +- The rule set is small enough to hold in your head, which matters more than any + individual rule. diff --git a/docs/adr/0003-observability-stack-selection.md b/docs/adr/0003-observability-stack-selection.md new file mode 100644 index 0000000..cd61263 --- /dev/null +++ b/docs/adr/0003-observability-stack-selection.md @@ -0,0 +1,60 @@ +# ADR-0003: Prometheus, Loki and Alloy over the alternatives + +**Status:** Accepted · 2025-11 + +## Context + +The lab needs metrics, logs and alerting on a monitoring host with modest +resources (a 2012 MacBook Pro), covering both Linux hosts and network appliances +that can only speak SNMP. + +Options considered: + +**Metrics.** Prometheus, InfluxDB, or Zabbix. Zabbix handles SNMP natively and +would have been fewer moving parts, but its data model and alerting are far less +expressive than PromQL, and its config lives in a database rather than in files — +which rules out managing it in git. InfluxDB's push model suits IoT telemetry +better than infrastructure monitoring, and Flux is a smaller ecosystem. + +**Logs.** Loki or an ELK/OpenSearch stack. Elasticsearch full-text indexes +everything, which is powerful and expensive; its baseline JVM heap alone exceeds +what this host can spare. Loki indexes only labels and leaves log bodies +compressed, so its footprint is roughly proportional to what you actually query. + +**Collection.** Promtail + node_exporter + cAdvisor as three separate agents, or +Grafana Alloy as one. Three agents means three configs, three deployment units +and three sets of version skew per host. + +**Deployment mode.** Loki microservices or single binary. Microservices scale +horizontally; nothing here needs that. + +## Decision + +- **Prometheus** for metrics, with `--web.enable-remote-write-receiver` so agents + push rather than being scraped. +- **Loki** in single-binary mode with filesystem storage. +- **Grafana Alloy** as the single collection agent, replacing Promtail, + node_exporter and cAdvisor. +- **snmp_exporter** as a polling proxy for devices that cannot run an agent. +- **Grafana** for visualisation, provisioned entirely from files. + +Remote-write rather than scrape is the load-bearing choice: a new host appears in +Prometheus the moment its agent starts, with no target list to maintain and no +firewall rule allowing the monitoring VLAN to reach into the monitored one. + +## Consequences + +- One agent binary and one config file per host. `config.alloy` is identical + everywhere; only two environment variables differ. +- PromQL and LogQL are close enough that a metric query translates to a log query + with little friction, and Grafana correlates the two on one dashboard. +- Loki's label-only indexing means a badly chosen label (a request ID, an IP) + explodes cardinality in a way Elasticsearch would have absorbed. This is + documented in `observability.md` and is a real ongoing constraint. +- Single-binary Loki cannot scale out. At this volume — well under 1 GB/day — + that is not a limitation, and converting later is a config change, not a + rewrite. +- SNMP devices are polled at 60s rather than pushed, so their resolution is + coarser than host metrics. Acceptable for interface counters and UPS state. +- Prometheus retention is bounded by local disk. 30 days fits; anything longer + needs remote storage (Thanos, Mimir), which would be a new ADR. diff --git a/docs/adr/0004-one-compose-stack-per-host.md b/docs/adr/0004-one-compose-stack-per-host.md new file mode 100644 index 0000000..bb4c3a4 --- /dev/null +++ b/docs/adr/0004-one-compose-stack-per-host.md @@ -0,0 +1,60 @@ +# ADR-0004: One compose stack per host, not one directory per service + +**Status:** Accepted · 2025-11 + +## Context + +The repository previously used `//docker-compose.yaml` — five +directories, each with its own compose file, all for services running on the +same host: + +```text +prometheus/prometheus.matrix.elysium/docker-compose.yaml +grafana/prometheus.matrix.elysium/docker-compose.yaml +loki/prometheus.matrix.elysium/docker-compose.yaml +alloy/prometheus.matrix.elysium/docker-compose.yaml +snmp-exporter/prometheus.matrix.elysium/docker-compose.yaml +``` + +This looked like it was designed for many hosts, but the hostname level had +exactly one value, and that value appeared nowhere in the documentation. Worse, +the split actively broke things: + +- `grafana`'s compose declared `depends_on: [prometheus, loki]`, but those + services were defined in *other* compose files. Compose rejects this outright — + the file could not start. +- Services could not resolve each other by name, so every cross-reference was a + hardcoded IP (`10.0.99.20` appeared five times across two files). +- Five separate `docker compose up` invocations, in an order nothing recorded. +- No shared network, no ordering, no health gating. + +## Decision + +One directory per **stack**, where a stack is the set of services deployed +together as a unit: + +```text +stacks/observability/ +├── compose.yaml # all six services +├── prometheus/ loki/ grafana/ alertmanager/ alloy/ snmp-exporter/ +``` + +Host-to-stack mapping lives in `docs/architecture.md`, not in the directory +tree. A second host means a second directory under `stacks/`. + +## Consequences + +- Services resolve each other by name on a shared compose network, so the + hardcoded monitoring-host IP disappeared from both `prometheus.yaml` and + `config.alloy`. +- `depends_on` with `condition: service_healthy` now works, so Grafana does not + start before its datasources are ready. +- One `make up` instead of five ordered invocations. +- `docker compose config` validates the whole stack at once, which is what makes + the CI job meaningful. +- The tradeoff: restarting one service means `docker compose up -d ` + rather than acting on an isolated directory. In practice these six services are + always deployed and upgraded together, which is the definition of a stack. +- Directory names no longer encode which host they run on. That information moved + to documentation, where it can be kept accurate — the FQDN in the old paths had + already drifted out of every other document. diff --git a/docs/adr/0005-secrets-with-sops-and-age.md b/docs/adr/0005-secrets-with-sops-and-age.md new file mode 100644 index 0000000..1d4d55c --- /dev/null +++ b/docs/adr/0005-secrets-with-sops-and-age.md @@ -0,0 +1,66 @@ +# ADR-0005: Encrypt secrets in-repo with SOPS and age + +**Status:** Accepted · 2025-11 + +## Context + +The repository committed a plaintext SNMP community string shared across the +firewall, the switch, the UPS and the server's BMC, plus Grafana `admin`/`admin` +alongside anonymous Admin access. Both are now public forever, because git +history keeps everything. + +Any replacement had to satisfy four constraints: + +- No plaintext credential in a tracked file, ever. +- Recoverable if the monitoring host dies — a secret that exists only on the box + it protects disappears with the box. +- Auditable: it should be possible to see *that* a credential rotated. +- Deployable by one person with `make up`, without standing up a secrets server. + +Options considered: + +1. **Gitignored `.env`.** Trivial, and what most homelabs do. But the secrets + exist in exactly one place, are in no backup, and nothing records that they + ever changed. It also fails silently — a missing `.env` produces a subtly + misconfigured stack rather than an error. +2. **HashiCorp Vault.** The correct answer at organisational scale, and absurd + here: a highly-available service, an unseal ceremony, and a hard dependency + whose own failure takes down the thing meant to monitor failures. +3. **SOPS + age.** Encrypted files committed to the repo, decrypted at deploy + time with a local key. +4. **git-crypt.** Similar, but encrypts whole files, so a diff shows only that + the blob changed — no visibility into which key rotated. + +## Decision + +SOPS with an age key. + +SOPS encrypts values and leaves keys in plaintext, so +`git log -p secrets/observability.sops.yaml` shows *which* credential changed +and when, without revealing what it changed to. + +The age private key lives at `~/.config/sops/age/keys.txt` on the deployment +host and never enters the repository. `scripts/render-config.sh` decrypts in +memory at deploy time and writes only to gitignored paths. + +Each device gets its own SNMP community rather than one shared string. + +## Consequences + +- The ciphertext is public. Security rests entirely on age's X25519 encryption + and on the private key staying private — an explicit and understood tradeoff, + and the reason the key is never committed. +- Losing `keys.txt` means every secret must be re-created, not recovered. The + bootstrap script warns about this loudly; backing that file up off-host is a + hard requirement. +- One extra tool on the deployment host (`sops`, plus `age` for bootstrap). +- `snmp_exporter` does no environment expansion and reads its config once at + startup, so the community strings must be substituted into a real file. That + file is written to a gitignored `.rendered/` directory — the tracked + `snmp.yaml` keeps its `${PLACEHOLDERS}`, and CI asserts nothing rendered is + ever tracked. +- Per-device communities mean four values to rotate instead of one. That is the + point: one captured SNMPv2c packet no longer yields read access to every + device on the network. +- CI can verify encryption without any ability to decrypt, by asserting each + `secrets/*.sops.yaml` carries a `sops:` metadata block. diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..0e19a46 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,166 @@ +# Architecture + +## Network topology + +Every segment terminates on pfSense. There is no route between segments unless a +rule creates one, and only two such rules exist. + +```mermaid +graph TB + INET([Internet]) + GW[ISP Gateway
bridge mode] + FW{{"morpheus
pfSense · FreeBSD 15
HP ProDesk 600 G4"}} + SW[neo · MokerLink 26-port
802.1Q trunk] + + INET --- GW --- FW --- SW + + subgraph V99["VLAN 99 · Winterfell · Management"] + MON["prometheus · 10.0.99.20
observability stack"] + UPS["mjolnir · APC Smart-UPS"] + SPARE["oracle · spare"] + end + + subgraph V50["VLAN 50 · Hicks · Trusted"] + WS["workstations · laptops · phones"] + end + + subgraph V40["VLAN 40 · CasaBonita · Media"] + TV["TV · consoles · streaming"] + end + + subgraph V30["VLAN 30 · ImaginationLAN · Lab"] + HV["shiva · ProLiant DL360 Gen9
Proxmox VE"] + end + + subgraph V20["VLAN 20 · Skids · IoT"] + IOT["cameras · assistants · sensors"] + end + + subgraph V10["VLAN 10 · Degens · Guest"] + GUEST["guest devices"] + end + + SW --- V99 + SW --- V50 + SW --- V40 + SW --- V30 + SW --- V20 + SW --- V10 + + WS -.->|"management access
(the only inbound path)"| V99 + WS -.->|lab access| V30 + + classDef mgmt fill:#1f6f4a,stroke:#2ea043,color:#fff + classDef trusted fill:#1f4e79,stroke:#388bfd,color:#fff + classDef untrusted fill:#6e2c2c,stroke:#f85149,color:#fff + classDef infra fill:#4a3f7a,stroke:#a371f7,color:#fff + class MON,UPS,SPARE mgmt + class WS trusted + class IOT,GUEST,TV untrusted + class HV,FW,SW infra +``` + +Everything reaches the internet. Nothing reaches anything else, with two +exceptions drawn as dotted lines above: trusted workstations may administer +management, and may reach the lab. IoT, media and guest are terminal — traffic +goes out, nothing comes back in. + +The full device inventory is in [`network.md`](network.md); the reasoning behind +the split is in [`security.md`](security.md). + +## Observability data flow + +```mermaid +graph LR + subgraph Sources["Monitored estate"] + direction TB + PF["morpheus
pfSense"] + UPSD["mjolnir
APC UPS"] + SWD["neo
switch"] + ILO["shiva
iLO"] + HOSTS["Linux hosts
+ Docker"] + end + + subgraph Stack["prometheus · 10.0.99.20 · one compose stack"] + direction TB + SNMP["snmp-exporter
:9116"] + ALLOY["Alloy
cAdvisor · node · logs"] + PROM[("Prometheus
:9090 · 30d")] + LOKI[("Loki
:3100 · 30d")] + AM["Alertmanager
:9093"] + GRAF["Grafana
:3000"] + end + + OUT([Webhook
notification]) + + PF -->|SNMP v2c| SNMP + UPSD -->|SNMP v2c| SNMP + SWD -->|SNMP v2c| SNMP + ILO -->|SNMP v2c| SNMP + HOSTS -->|metrics + logs| ALLOY + + SNMP -->|scrape /snmp| PROM + ALLOY -->|remote_write| PROM + ALLOY -->|push| LOKI + PROM -->|alerts| AM + AM --> OUT + PROM --> GRAF + LOKI --> GRAF + + classDef store fill:#4a3f7a,stroke:#a371f7,color:#fff + classDef agent fill:#1f4e79,stroke:#388bfd,color:#fff + class PROM,LOKI store + class SNMP,ALLOY agent +``` + +Two collection paths, because the estate has two kinds of device: + +- **Things that run an agent.** Linux hosts get a Grafana Alloy agent, which + gathers node metrics, cAdvisor container metrics, the systemd journal, syslog + and `auth.log`, then pushes to Loki and remote-writes to Prometheus. The same + `config.alloy` runs everywhere; only two environment variables differ. +- **Things that cannot.** The firewall, switch, UPS and BMC are polled over SNMP + through `snmp-exporter`, which Prometheus scrapes as a proxy. + +Remote-write rather than scrape for agents means a new host appears in +Prometheus as soon as its agent starts — no target list to edit, no firewall +hole from the monitoring VLAN into the monitored one. + +## Host and stack mapping + +| Host | VLAN | Stack | Contents | +| --- | --- | --- | --- | +| `prometheus` (10.0.99.20) | 99 | [`stacks/observability`](../stacks/observability) | Prometheus, Alertmanager, Loki, Grafana, snmp-exporter, Alloy | +| `shiva` (10.0.30.10) | 30 | *(none yet)* | Proxmox VE — see [roadmap](roadmap.md) | +| `oracle` (10.0.99.30) | 99 | *(none yet)* | Undecided | + +One directory per stack, not one per service. A stack is the unit that gets +deployed together; a second host means a second directory under `stacks/`, not a +re-shard of everything. Reasoning in +[ADR-0004](adr/0004-one-compose-stack-per-host.md). + +## Ports + +| Service | Port | Bound to | Notes | +| --- | --- | --- | --- | +| Grafana | 3000 | `${BIND_ADDR}` | The only UI meant to be opened by a human | +| Prometheus | 9090 | `${BIND_ADDR}` | Also the remote-write receiver for agents | +| Loki | 3100 | `${BIND_ADDR}` | Push endpoint for agents | +| Alertmanager | 9093 | `${BIND_ADDR}` | | +| Alloy | 12345 | `127.0.0.1` | Debug UI, deliberately not exposed | +| snmp-exporter | 9116 | *compose network only* | Never published to a host interface | + +`BIND_ADDR` defaults to `0.0.0.0` and is set in `.env`. Setting it to the host's +VLAN 99 address confines the whole stack to the management segment; the +published ports exist because agents on other hosts need to reach Prometheus and +Loki. + +## Reference diagrams + +The Mermaid diagrams above are the maintained ones — they render on GitHub, diff +as text, and cannot drift out of sync with the repo without a visible change. + +- [Current topology export](diagrams/current/matrix_elysium.png) — detailed + physical drawing, 9871×4466. Its editable `.drawio` source was lost in an + earlier commit, which is a large part of why the diagrams above are Mermaid. +- [Previous topology](diagrams/previous/Network_Diagram.png) diff --git a/docs/hardware.md b/docs/hardware.md new file mode 100644 index 0000000..2dc2e55 --- /dev/null +++ b/docs/hardware.md @@ -0,0 +1,57 @@ +# Hardware + +A 9U open-frame rack, a firewall built from a refurbished mini PC, a +decommissioned enterprise server, and two laptops that were headed for a +landfill. + +## Rack + +| U | Device | Role | +| --- | --- | --- | +| U1–U2 | APC Smart-UPS[^UPS] | Power | +| U3 | HPE ProLiant DL360 Gen9[^Shiva] | Proxmox hypervisor (`shiva`) | +| U5 | HP ProDesk 600 G4 Mini[^ProDesk] | pfSense firewall (`morpheus`) | +| U6 | MT-VIKI 8-port KVM[^KVM] | Console access | +| U7 | Jadol 24-port patch panel[^Panel] | Cabling | +| U8 | 10-outlet PDU[^PDU] | Power distribution | +| U9 | MokerLink 26-port managed switch[^MokerLink] | Core switching (`neo`) | + +Off-rack: two Ubuntu Server laptops on a shelf (`prometheus`, `oracle`), an +8-port unmanaged TP-Link switch feeding them, and eero Pro 6E units distributed +through the house. + +## Compute + +| Host | Hardware | CPU | RAM | Storage | OS | +| --- | --- | --- | --- | --- | --- | +| `morpheus` | HP ProDesk 600 G4 Mini | i5-8500T | 32 GB | 1 TB SSD | FreeBSD 15.0 (pfSense) | +| `shiva` | HPE ProLiant DL360 Gen9 | 2× Xeon E5 v3/v4 | — | — | Proxmox VE | +| `prometheus` | Apple MacBook Pro (2012) | i5/i7 | — | SSD | Ubuntu Server 24.04.3 | +| `oracle` | Dell Inspiron 15 | i5-1235U | 32 GB | 2 TB SSD | Ubuntu Server 24.04.3 | + +The observability stack runs on a thirteen-year-old MacBook. It handles four +SNMP devices at a 60-second interval, two Alloy agents, and 30 days of metric +retention without complaint — which is a useful thing to know before spending +money on a monitoring host. + +## Accessories + +- 1U rackmount tray for the ProDesk Mini[^ProDeskRackmount] +- Sliding rails for the ProLiant[^Sliderail] +- 1U universal rack mount for the APC[^Rail] +- ViewSonic N1700W LCD, used as a rack console via the KVM +- RJ45 Cat6 in-line couplers[^Couplers] +- Cat6 patch cables[^Patchcables] + +[^UPS]: [APC Smart-UPS](https://www.apc.com/us/en/product-range/61913-smart-ups/) +[^Shiva]: [HPE ProLiant DL360 Gen9](https://buy.hpe.com/us/en/servers/rack-servers/proliant-dl300-servers/proliant-dl360-server/p/1010026922) +[^ProDesk]: [HP ProDesk 600 G4 Mini](https://www.microcenter.com/product/692358/) +[^KVM]: [MT-VIKI 8-port rackmount KVM](https://a.co/d/2yQl4KH) +[^Panel]: [Jadol 24-port patch panel](https://a.co/d/izggRoK) +[^PDU]: [10-outlet 1U PDU](https://a.co/d/ibEygxZ) +[^MokerLink]: [MokerLink 26-port managed switch](https://a.co/d/gaJvCKV) +[^ProDeskRackmount]: [1U rackmount for ProDesk Mini](https://a.co/d/4d7klOL) +[^Sliderail]: [Sliding rails for ProLiant](https://a.co/d/5d4A4FO) +[^Rail]: [1U universal rack mount](https://a.co/d/6R0vjHz) +[^Couplers]: [Cat6 in-line couplers](https://a.co/d/gP3b948) +[^Patchcables]: [Cat6 patch cables](https://vetco.net/collections/cables-cat6-patch-cables) diff --git a/docs/images/README.md b/docs/images/README.md new file mode 100644 index 0000000..2106e90 --- /dev/null +++ b/docs/images/README.md @@ -0,0 +1,43 @@ +# Screenshots + +Dashboard screenshots go here and are referenced from the root `README.md`. + +They are deliberately absent rather than faked — a mocked-up dashboard image in +a monitoring repository is worse than none, because it cannot be checked against +the JSON that produced it. + +## Capturing them + +Once the stack has a few days of real data: + +```bash +make up +# open http://:3000, log in, HomeLab folder +``` + +For each dashboard, set the time range to something with visible activity +(24h works well), then use Grafana's **Share → Export → Save as image**, or take +a full-page browser screenshot. + +Save as: + +| File | Dashboard | +| --- | --- | +| `host-overview.png` | Host Overview | +| `docker-containers.png` | Docker Containers | +| `network-snmp.png` | Network & Firewall | +| `ups-power.png` | UPS & Power | +| `logs-explorer.png` | Logs | + +Then uncomment the screenshot block in the root `README.md`. + +## Before publishing + +These are going into a public repository. Check each image for: + +- Full MAC addresses in table panels +- The WAN IP address in any interface panel +- Hostnames or usernames in log lines +- Anything in a Grafana annotation or query bar you did not mean to publish + +Crop or blur rather than re-shooting — it is easier to be thorough. diff --git a/docs/observability.md b/docs/observability.md new file mode 100644 index 0000000..edd9c74 --- /dev/null +++ b/docs/observability.md @@ -0,0 +1,139 @@ +# Observability + +What is collected, where it goes, and how to change it. + +## What is collected + +| Source | Via | Interval | Examples | +| --- | --- | --- | --- | +| Linux hosts | Alloy → `node_exporter` | 60s | CPU, memory, filesystem, network, load, clock offset | +| Docker containers | Alloy → cAdvisor | 60s | Per-container CPU, memory, network, restarts, OOM | +| Container logs | Alloy → Docker socket | stream | stdout/stderr per container | +| systemd journal | Alloy | stream | unit, boot ID, transport, priority | +| `/var/log/auth.log` | Alloy | 60s poll | sshd, sudo, PAM | +| syslog, `/var/log/*.log` | Alloy | 60s poll | Everything else | +| pfSense | snmp-exporter | 60s | pf state table, counters, interface stats | +| MokerLink switch | snmp-exporter | 60s | Interface status and octet counters | +| APC UPS | snmp-exporter | 60s | Charge, runtime, load, voltage, alarms | +| ProLiant iLO | snmp-exporter | 60s | Temperature, PSU, drive and battery health | +| The stack itself | Prometheus | 15s | Every component scrapes itself | + +Retention is 30 days for both metrics (`PROMETHEUS_RETENTION` in `.env`) and +logs (`retention_period` in `loki/loki-config.yaml`). Change both together or +dashboards will show metrics with no matching logs at the far end of the range. + +## Log level normalisation + +Logs arrive spelling severity about twenty different ways — `ERROR`, `err`, +`eror`, `crit`, `fatal`, `panic`, `dbug`. The `log_processor` stage in +`config.alloy` maps all of them onto five canonical values before they reach +Loki: + +```text +emerg, panic, corrupt, fatal, alert, crit, critical → critical +err, eror, error → error +warn, warning → warning +info, information, informational, notice → info +dbug, debug, dbg → debug +``` + +This is what makes `{level="error"}` a useful query across a FreeBSD firewall, a +Ubuntu host and a Go container at the same time. + +> **This was silently broken until recently.** The extracting regex used `\b` +> inside a double-quoted Alloy string, where `\b` is a backspace escape rather +> than a word boundary. The regex never matched, so the template's +> `{{ else }}info{{ end }}` fallback labelled *every* line `info`. It is now a +> backtick string. If you edit that stage, verify afterwards that +> `sum by (level) (count_over_time({host=~".+"}[1h]))` returns more than one +> series. + +## Dashboards + +Five dashboards are provisioned from `grafana/dashboards/` into a **HomeLab** +folder: + +| Dashboard | UID | Covers | +| --- | --- | --- | +| Host Overview | `homelab-host-overview` | CPU, memory, storage, network per host | +| Docker Containers | `homelab-docker` | Per-container resources, restarts, OOM kills | +| Network & Firewall | `homelab-network` | pf state table, switch interfaces, iLO health | +| UPS & Power | `homelab-ups` | Battery, runtime, load, input voltage | +| Logs | `homelab-logs` | Volume by level and source, error and auth streams | + +`allowUiUpdates` is `false`, so edits made in the Grafana UI are discarded on +restart. That is deliberate — the JSON in git is the source of truth. To change +a dashboard: edit it in the UI, **Dashboard settings → JSON Model**, copy, and +commit it over the file. CI checks the result parses, that every datasource UID +resolves, that panels fit the grid and do not overlap, and that every PromQL +expression in every panel is syntactically valid. + +## Alerting + +32 rules across four files in `prometheus/rules/`: + +| File | Covers | +| --- | --- | +| `host.rules.yaml` | Instance down, predictive disk fill, memory, load, clock skew, reboots | +| `network.rules.yaml` | SNMP reachability, pf not running, state table, switch links, iLO hardware | +| `ups.rules.yaml` | On battery, low battery, runtime, load, temperature | +| `containers.rules.yaml` | Restart loops, OOM kills, throttling, and the stack watching itself | + +Routing is by `severity` and `category` (see `alertmanager/alertmanager.yaml`). +`critical` + `category=power` pages immediately and repeats every 30 minutes; +other criticals repeat every 4 hours; warnings every 12; `info` is recorded but +never notified. + +Inhibit rules stop cascades: a down host suppresses its own disk warnings, and a +dead `snmp-exporter` suppresses the "every device is unreachable" storm that +would otherwise follow. + +Disk alerting is predictive rather than a fixed threshold — `predict_linear` over +a 6-hour window, firing when the extrapolation reaches zero within a day *and* +free space is already under 30%. A disk sitting at 86% and stable is not an +emergency; one climbing fast at 60% is. + +## Adding a monitored device + +See [`runbooks/add-monitored-device.md`](runbooks/add-monitored-device.md). In +short: + +- **A Linux host:** run Alloy with `LOKI_URL` and + `PROMETHEUS_REMOTE_WRITE_URL` pointed at `10.0.99.20`. Nothing on the + monitoring host changes. +- **An SNMP device:** append a target to + `prometheus/targets/snmp.yaml` and a module plus auth to + `snmp-exporter/generator.yaml`. file_sd picks the target up within five + minutes without a restart. + +## Cardinality + +The stack is small, but two things will bite if ignored: + +- **The `ilo` SNMP module exposes ~1,355 metrics.** The HP Insight tree is + enormous. It is scraped once a minute from one device, which is fine — but do + not add a second module that broad without trimming the OID list in + `generator.yaml`. +- **Loki labels must stay low-cardinality.** `host`, `level`, `log_type`, + `service_name` and `unit` are bounded. Never promote a request ID, IP address + or timestamp to a label; use `|=` line filters instead. + +`module` and `auth` are deliberately dropped by `labeldrop` in `prometheus.yaml` +after being converted to query parameters, so they never become metric labels. + +## Operating + +```bash +make up # render secrets, start everything +make ps # container status +make logs SERVICE=grafana # tail one service +make reload # hot-reload Prometheus + Alertmanager config +make validate # everything CI runs +make backup # tar the data volumes into ./backups/ +make down # stop, keep data +make nuke # stop, destroy data (prompts) +``` + +Prometheus and Alertmanager are started with lifecycle endpoints enabled, so +rule and route changes apply via `make reload` without dropping the TSDB head +block. diff --git a/docs/roadmap.md b/docs/roadmap.md new file mode 100644 index 0000000..ed89ec4 --- /dev/null +++ b/docs/roadmap.md @@ -0,0 +1,72 @@ +# Roadmap + +Open work, extracted from the per-VLAN task lists that used to live inside the +inventory. Ordered roughly by how much it matters. + +## Security + +- [ ] **Rotate the SNMP communities on all four devices.** The previous shared + string was committed in plaintext and must be considered public. + → [runbook](runbooks/rotate-snmp-community.md) +- [ ] **Purge `certificates/` and the old community string from git history**, + then regenerate the CA and leaf certificates. + → [runbook](runbooks/purge-git-history.md) +- [ ] Move to SNMPv3 authPriv where the hardware supports it. pfSense, the APC + and iLO all do; the MokerLink switch does not, which is the blocker for + doing it uniformly. +- [ ] Put Grafana behind TLS rather than plain HTTP on the management VLAN. +- [ ] Decide whether the lab VLAN needs egress filtering before the + deliberately-vulnerable playground exists. + +## Monitoring + +- [ ] **Add `ifXTable` (64-bit counters) to the `mokerlink` SNMP module.** The + current `ifTable` counters are 32-bit and wrap in roughly 34 seconds at + gigabit line rate, so sustained high-throughput ports under-report. The + `SwitchCounterWrapSuspected` alert detects this but does not fix it. +- [ ] Deploy Alloy to the remaining hosts — currently only the monitoring host + and one other report in. `shiva` and `oracle` are next. +- [ ] Add blackbox-exporter for uptime and TLS-expiry checks on internal + services. +- [ ] Confirm whether `10.0.30.10` is genuinely both the Proxmox host and its + iLO, or whether one of the two records is stale. The SNMP target and the + inventory currently agree on the address but describe different things. +- [ ] Capture dashboard screenshots for the README once the stack has a few days + of real data. → [`images/README.md`](images/README.md) +- [ ] Loki alerting rules — the ruler is configured and pointed at Alertmanager + but no log-based rules exist yet. Repeated SSH auth failure is the obvious + first one. + +## Infrastructure + +- [ ] **Replace the UPS battery.** `mjolnir` currently has none, so a mains loss + is an immediate hard shutdown of the rack. Every rule in + `ups.rules.yaml` is currently reporting on a UPS that cannot actually hold + the load. +- [ ] Decide what `oracle` (10.0.99.30) is for. It is a 32 GB / 2 TB machine + sitting idle on the management VLAN. +- [ ] Plan and build the NAS on VLAN 40. +- [ ] Procure a second server ("ifrit") for the isolated playground network. +- [ ] Build the playground — **only** after the main network is finished. +- [ ] Work out DNS for the MokerLink management UI so it is not reached by IP. + +## Automation + +- [ ] Home Assistant integration with the eero API, so device joins and leaves + show up as events rather than being discovered by accident. +- [ ] Move stack deployment from `make up` over SSH to something pull-based, so + the monitoring host converges on the repo rather than being pushed to. +- [ ] Automate the Grafana dashboard export step — the current loop (edit in UI, + copy JSON, commit) is manual and therefore skipped under pressure. + +## Done + +- [x] Bridge mode on the ISP gateway +- [x] Lock down guest VLAN firewall rules +- [x] Move IoT devices onto their own SSID and VLAN +- [x] Stand up Prometheus, Grafana, Loki, snmp-exporter and Alloy +- [x] Consolidate five broken compose files into one working stack +- [x] Provision Grafana datasources and dashboards from files +- [x] Add alerting (32 rules) and Alertmanager routing +- [x] Move secrets to SOPS + age +- [x] Add CI: lint, config validation, secret scanning diff --git a/docs/runbooks/add-monitored-device.md b/docs/runbooks/add-monitored-device.md new file mode 100644 index 0000000..a4d4bf6 --- /dev/null +++ b/docs/runbooks/add-monitored-device.md @@ -0,0 +1,135 @@ +# Runbook: Add a monitored device + +Two paths, depending on whether the device can run an agent. + +--- + +## A Linux host + +Nothing on the monitoring host changes. Alloy pushes; Prometheus does not need +to be told the host exists. + +On the new host: + +```bash +sudo mkdir -p /opt/alloy +sudo cp /path/to/HomeLab/stacks/observability/alloy/config.alloy /opt/alloy/ + +sudo docker run -d \ + --name alloy --restart unless-stopped --privileged \ + -e LOKI_URL=http://10.0.99.20:3100/loki/api/v1/push \ + -e PROMETHEUS_REMOTE_WRITE_URL=http://10.0.99.20:9090/api/v1/write \ + -v /opt/alloy/config.alloy:/etc/alloy/config.alloy:ro \ + -v /var/run/docker.sock:/var/run/docker.sock:ro \ + -v /var/lib/docker/containers:/var/lib/docker/containers:ro \ + -v /var/log:/var/log:ro \ + -v /:/rootfs:ro \ + -p 127.0.0.1:12345:12345 \ + grafana/alloy:v1.6.1 \ + run --server.http.listen-addr=0.0.0.0:12345 \ + --storage.path=/var/lib/alloy/data \ + /etc/alloy/config.alloy +``` + +The two `*_URL` variables are the only difference from the monitoring host's own +agent — inside the compose stack they default to service names. + +**Verify** (from the monitoring host, within a minute or two): + +```promql +up{instance=~".*.*"} +``` + +```logql +{host=""} +``` + +The host appears on the Host Overview and Logs dashboards automatically — both +template their host variable from live label values. + +**Firewall:** the new host must be able to reach `10.0.99.20` on 9090 and 3100. +If it is not on VLAN 99 or 50, that is a rule you have to add, and one worth +thinking about before you do. + +--- + +## An SNMP device + +Three edits, no restart. + +### 1. Define how to poll it — `snmp-exporter/generator.yaml` + +```yaml +auths: + auth_newdevice: + community: ${SNMP_COMMUNITY_NEWDEVICE} + security_level: noAuthNoPriv + version: 2 + +modules: + newdevice: + walk: + - 1.3.6.1.2.1.2.2 # IF-MIB::ifTable +``` + +Keep the OID list tight. The `ilo` module walks the whole HP Insight tree and +produces ~1,355 metrics from one device; that is fine once and a cardinality +problem if repeated. + +### 2. Add the credential + +```bash +make secrets-edit # add SNMP_COMMUNITY_NEWDEVICE +``` + +Give it its own community. Reusing one across devices means a single captured +SNMPv2c packet — which is cleartext — grants read access to all of them. + +Then add the variable to the `envsubst` list and the `REQUIRED` array in +`scripts/render-config.sh`. + +### 3. Add the target — `prometheus/targets/snmp.yaml` + +```yaml +- targets: ["10.0.99.40"] + labels: + module: newdevice + auth: auth_newdevice + device: + role: + vlan: "99" +``` + +`module` and `auth` become query parameters and are then dropped, so they never +land as metric labels. `device`, `role` and `vlan` do persist and are what alert +annotations use. + +### 4. Apply + +```bash +make render && make up # re-render snmp.yaml with the new community +``` + +Prometheus re-reads the targets file every 5 minutes on its own — no restart +needed for the target itself. The `make up` is for snmp-exporter picking up the +new module. + +### Verify + +```bash +# Does the device answer at all? +snmpwalk -v2c -c '' 10.0.99.40 1.3.6.1.2.1.1.1.0 + +# Does the exporter understand it? +curl -s 'http://localhost:9116/snmp?target=10.0.99.40&module=newdevice&auth=auth_newdevice' | head +``` + +Then **Prometheus → Status → Targets**, job `snmp`, and confirm the new instance +is `UP`. + +### 5. Document it + +Add a row to [`docs/network.md`](../network.md) with the MAC truncated to its +OUI, and consider whether it needs an alert rule in +`prometheus/rules/network.rules.yaml`. A device nobody alerts on is a device +nobody notices failing. diff --git a/docs/runbooks/deploy-stack.md b/docs/runbooks/deploy-stack.md new file mode 100644 index 0000000..797a681 --- /dev/null +++ b/docs/runbooks/deploy-stack.md @@ -0,0 +1,105 @@ +# Runbook: Deploy the observability stack + +**Target:** `prometheus` (10.0.99.20), VLAN 99 +**Time:** ~10 minutes on a clean host + +## Prerequisites + +```bash +docker --version # 24+ with the compose plugin +sops --version # https://github.com/getsops/sops/releases +age --version # https://github.com/FiloSottile/age/releases +``` + +## First deployment + +```bash +git clone HomeLab && cd HomeLab + +make secrets-init # generates an age keypair, creates the encrypted file +make secrets-edit # replace every change-me value +make validate # confirm the configs are sound before starting anything +make up +``` + +`make up` renders the decrypted config and starts all six services. Give it a +minute — Grafana waits on Prometheus and Loki reporting healthy. + +> **Back up `~/.config/sops/age/keys.txt` off this machine now.** Without it the +> encrypted secrets in the repository cannot be decrypted by anything, including +> you. + +## Verify + +```bash +make ps # all six services healthy +curl -s localhost:9090/-/healthy # Prometheus +curl -s localhost:3100/ready # Loki +curl -s localhost:9093/-/healthy # Alertmanager +curl -s localhost:3000/api/health # Grafana +``` + +Then in the UI: + +1. **Prometheus → Status → Targets.** Every job `UP`. The four `snmp` targets + take up to 45 seconds on their first scrape. +2. **Prometheus → Status → Rules.** 32 rules loaded, none in error. +3. **Grafana → Dashboards → HomeLab.** Five dashboards, populated. +4. **Grafana → Explore → Loki**, run `{host=~".+"}`. Logs should be arriving. +5. Confirm level normalisation is working — this has been silently broken + before: + + ```logql + sum by (level) (count_over_time({host=~".+"}[1h])) + ``` + + More than one series means the regex is matching. Only `info` means it is not + (see `docs/observability.md`). + +## Updating + +```bash +git pull +make validate +make up # recreates only what changed +``` + +Config-only changes to Prometheus rules or Alertmanager routing do not need a +restart: + +```bash +make reload +``` + +## Rolling back + +Images are pinned, so rolling back is a git operation: + +```bash +git revert +make up +``` + +Data volumes survive `make down` and `make up`. Only `make nuke` destroys them, +and it prompts. + +## Troubleshooting + +| Symptom | Cause | Fix | +| --- | --- | --- | +| `GRAFANA_ADMIN_PASSWORD: unset` | `.env` not rendered | `make render` | +| `unsubstituted placeholders remain` | A `SNMP_COMMUNITY_*` key is missing from the secrets file | `make secrets-edit` | +| SNMP targets `DOWN` | Community mismatch, or the device is not reachable from VLAN 99 | `snmpwalk -v2c -c '' 1.3.6.1.2.1.1.1.0` | +| Grafana panels empty, no error | Datasource UID mismatch | `make check-dashboards` | +| Loki `ready` returns 503 for a while | Normal on first start | Wait ~45s | +| Every log line labelled `info` | The level regex is not matching | See `docs/observability.md` | + +## Backups + +```bash +make backup # tars each data volume into ./backups/ +``` + +Prometheus and Loki data is reproducible-ish (it re-accumulates), but Grafana's +volume holds annotations and users. The dashboards themselves are in git, so a +lost Grafana volume is an inconvenience rather than a loss. diff --git a/docs/runbooks/purge-git-history.md b/docs/runbooks/purge-git-history.md new file mode 100644 index 0000000..753248c --- /dev/null +++ b/docs/runbooks/purge-git-history.md @@ -0,0 +1,115 @@ +# Runbook: Purge secrets from git history + +**This rewrites every commit SHA in the repository and requires a force-push to +`main`.** Read the whole page before starting. + +## What is still in history + +| What | Introduced | Removed from HEAD | Still reachable at | +| --- | --- | --- | --- | +| Shared SNMP community | `ee3d443` | yes | every commit in between | +| Grafana `admin`/`admin` inline | `ee3d443` | yes | every commit in between | +| TLS private keys under `certificates/` | `efb2632` | `647d90a` | `647d90a~1` | + +All nine individual findings are enumerated in +[`.gitleaksignore`](../../.gitleaksignore) with their fingerprints. + +Verify for yourself before and after: + +```bash +git show 647d90a~1:certificates/Gandalf.Gondor.Lab/ca-key.pem | head -1 +git log --all -S '7H3r315N05p00N' --oneline +``` + +Deleting a file in a later commit does not remove it from history. `git show` +hands it straight back, GitHub's API serves it from unreachable objects for a +while after a force-push, and forks keep it indefinitely. + +## Before you start + +1. **Rotate the credentials first.** See + [`rotate-snmp-community.md`](rotate-snmp-community.md) and regenerate the CA + and leaf certificates. Anything ever pushed to a public repository is + compromised regardless of what you do to the history — the purge stops it + being *trivially* discoverable, not from having been seen. +2. Merge or close every open pull request. A rewrite orphans them. +3. Make sure no other clone has unpushed work. +4. Install the tool: + + ```bash + pipx install git-filter-repo + ``` + +## Dry run + +```bash +make purge-history-dry-run +``` + +This clones a scratch mirror, rewrites *that*, and reports whether the secrets +are gone. Your repository is untouched. The scratch copy is left in place for +inspection — check it before continuing: + +```bash +cd /tmp//repo +git log --oneline | head +git log --all -S '7H3r315N05p00N' --oneline # must be empty +``` + +## Execute + +```bash +./scripts/purge-history.sh --execute +``` + +It requires a clean working tree, prompts for confirmation, and writes a full +backup bundle to `../HomeLab-backup-.bundle` before touching anything. + +Restore from that bundle if something goes wrong: + +```bash +git clone HomeLab-backup-.bundle HomeLab-restored +``` + +## Push + +`git-filter-repo` removes the remote deliberately, so a rewrite cannot be pushed +by reflex. Re-add it and force-push: + +```bash +git remote add origin git@github.com:Gerrrt/HomeLab.git +git push --force --all origin +git push --force --tags origin +``` + +## Afterwards + +- **Every existing clone must be re-cloned.** A stale clone that pushes will + reintroduce the removed objects. +- GitHub keeps unreachable objects for a while. To have them purged sooner, ask + GitHub Support to run garbage collection on the repository. +- **Forks keep their own copy of everything.** If the repository has been forked, + the secrets are still public through the fork and no amount of rewriting your + copy changes that. This is the single strongest argument for rotating first + and treating the purge as tidying rather than as remediation. +- **Delete `.gitleaksignore`.** Its fingerprints reference commits that no longer + exist, and leaving it in place means a future finding could be masked by a + stale entry. CI should pass on full history with no ignore file at all — that + is how you know the purge worked. + +## Verify + +```bash +git log --all --oneline -- certificates/ # empty +git grep -I '7H3r315N05p00N' $(git rev-list --all) # no matches +gitleaks detect --no-banner --redact -c .gitleaks.toml --log-opts="--all" +``` + +## If you would rather not rewrite history + +Defensible, and the cost is that you must be explicit about it. The credentials +are rotated, so what remains in history is a dead string and a set of +passphrase-encrypted keys. Document that decision in +[`security.md`](../security.md) rather than leaving a reader to discover it — a +known, accepted, written-down exposure reads very differently from an +overlooked one. diff --git a/docs/runbooks/rotate-snmp-community.md b/docs/runbooks/rotate-snmp-community.md new file mode 100644 index 0000000..6d3b73e --- /dev/null +++ b/docs/runbooks/rotate-snmp-community.md @@ -0,0 +1,124 @@ +# Runbook: Rotate the SNMP communities + +**Why this is not optional.** A single SNMP community string was committed to +this public repository in plaintext and shared across pfSense, the MokerLink +switch, the APC UPS and the ProLiant's iLO. It is still in git history. Anyone +who cloned the repository at any point has it. + +These are read-only communities, which sounds mild. On a firewall, read-only +means the complete state table, every interface and the full pf configuration +surface. Treat them as credentials. + +**Order matters:** change the device first, then the repository. Doing it the +other way round means the exporter starts failing before the device is ready. + +--- + +## 1. Generate four distinct communities + +```bash +for d in pfsense apc mokerlink ilo; do + printf '%-10s %s\n' "$d" "$(openssl rand -base64 24 | tr -d '/+=' | head -c 24)" +done +``` + +One per device. The whole reason the old arrangement was dangerous is that a +single string unlocked everything. + +Note that SNMPv2c sends these in cleartext on every poll. Distinct communities +limit the blast radius of a captured packet; they do not make the protocol +secure. Moving to SNMPv3 authPriv is tracked in [`roadmap.md`](../roadmap.md) — +the MokerLink switch not supporting it is the blocker. + +## 2. Change each device + +### pfSense (`morpheus`, 10.0.99.1) + +**Services → SNMP.** Replace the read community string. Confirm the daemon binds +only to the VLAN 99 interface — not WAN, not all interfaces. + +### APC Smart-UPS (`mjolnir`, 10.0.99.10) + +Network Management Card web UI, **Configuration → Network → SNMPv1 → Access +Control**. Replace the community for the `prometheus` host entry. Set access to +**Read** and restrict the NMS address to `10.0.99.20` if the card supports it. + +### MokerLink switch (`neo`, 10.7.7.2) + +Web UI at `http://10.7.7.2`, **SNMP → Community**. Replace the read-only +community. Delete any default `public`/`private` entries while you are in there. + +### HPE iLO (`shiva`, 10.0.30.10) + +**Administration → Management → SNMP Settings.** Replace the read community. + +## 3. Verify each device before touching the repo + +```bash +snmpwalk -v2c -c '' 10.0.99.1 1.3.6.1.2.1.1.1.0 # pfSense +snmpwalk -v2c -c '' 10.0.99.10 1.3.6.1.2.1.1.1.0 # APC +snmpwalk -v2c -c '' 10.7.7.2 1.3.6.1.2.1.1.1.0 # switch +snmpwalk -v2c -c '' 10.0.30.10 1.3.6.1.2.1.1.1.0 # iLO +``` + +Each must return a sysDescr string. Also confirm the **old** community now +fails — some devices append rather than replace: + +```bash +snmpwalk -v2c -c '' 10.0.99.1 1.3.6.1.2.1.1.1.0 # must time out +``` + +## 4. Update the repository + +```bash +make secrets-edit +``` + +Replace all four values: + +```yaml +SNMP_COMMUNITY_PFSENSE: +SNMP_COMMUNITY_APC: +SNMP_COMMUNITY_MOKERLINK: +SNMP_COMMUNITY_ILO: +``` + +Then re-render and restart: + +```bash +make render +make up +``` + +`make render` fails loudly if any placeholder is left unsubstituted, so a typo in +a key name is caught before the container starts. + +## 5. Confirm monitoring recovered + +**Prometheus → Status → Targets**, job `snmp`. All four instances `UP` within +60 seconds. Or: + +```promql +up{job="snmp"} +``` + +The `SnmpTargetUnreachable` alert fires after 10 minutes, so a mistake here +announces itself. + +## 6. Commit + +```bash +git add secrets/observability.sops.yaml +git commit -m "chore(secrets): rotate SNMP communities" +``` + +The diff shows *which* keys changed and nothing about their values — SOPS +encrypts values and leaves keys in plaintext. + +--- + +## Also required + +Rotating the live credential does not remove the old one from git history. +Follow [`purge-git-history.md`](purge-git-history.md) as well — one without the +other leaves the job half done. diff --git a/docs/security.md b/docs/security.md new file mode 100644 index 0000000..405eec0 --- /dev/null +++ b/docs/security.md @@ -0,0 +1,125 @@ +# Security + +The lab is a security project, so the interesting question is not "is it +secure" but "what is it defending against, and what is it knowingly not." + +## Threat model + +What this network is actually built to survive: + +| Threat | Control | +| --- | --- | +| A compromised IoT device pivoting to a workstation | VLAN 20 is terminal — no route to any other segment | +| A guest on the Wi-Fi enumerating the LAN | VLAN 10 is terminal, client isolation on | +| A smart TV's firmware phoning somewhere unexpected | VLAN 40 is terminal, egress only | +| A corporate laptop carrying something in from outside | Sits on VLAN 50 but has no management access | +| A lab VM escaping into the house | VLAN 30 reachable only *from* trusted, never *to* it | +| Losing visibility of a failure | 32 alert rules, 30 days of metrics and logs | +| Mains power loss | UPS on the management VLAN, monitored, alerts on `category=power` | + +What it explicitly does **not** defend against: a determined attacker with +physical access to the rack, a supply-chain compromise in an upstream container +image, or a vulnerability in pfSense itself. There is no IDS/IPS, no egress +filtering by domain, and no MFA on the internal services. + +## Segmentation + +Default deny between every segment. Two exceptions: + +1. Specific hosts on **Hicks (50)** may reach **Winterfell (99)** on management + ports. Without this there is no way to administer anything. +2. **Hicks (50)** may reach **ImaginationLAN (30)** so the lab is usable. + +Everything else — IoT, media, guest — gets internet and nothing more. + +The IoT segment is the one that justifies the whole exercise. It holds cameras, +a doorbell, an alarm hub, smart speakers, a baby monitor and a $20 Tuya +white-noise machine. Every one of those is a network-connected computer running +firmware nobody outside its vendor has audited, several with no update +mechanism at all. Treating them as untrusted is not paranoia; it is the only +assumption consistent with what they are. + +## Secrets + +- Credentials are encrypted with [SOPS](https://github.com/getsops/sops) + age + and committed in encrypted form. See [`secrets/README.md`](../secrets/README.md). +- The private key lives at `~/.config/sops/age/keys.txt` on the deployment host + and is never in the repository. +- `scripts/render-config.sh` decrypts at deploy time into gitignored files. + Nothing writes a plaintext secret into a tracked path. +- CI runs `gitleaks` with rules specifically for SNMP communities, inline + Grafana passwords, PEM private keys and age secret keys, and separately + asserts that every `secrets/*.sops.yaml` is genuinely encrypted. + +### Known historical exposure + +This repository previously committed real credentials. Removing them from `HEAD` +does not remove them from history, and anything ever pushed to a public +repository must be treated as compromised: + +| What | Where | Status | +| --- | --- | --- | +| SNMP community shared across all four devices | `snmp.yaml`, from commit `ee3d443` | Replaced with per-device placeholders. **Rotate on the devices** — see [runbook](runbooks/rotate-snmp-community.md) | +| Grafana `admin` / `admin` with anonymous Admin access | compose file | Fixed: password from SOPS, anonymous auth disabled | +| Passphrase-encrypted TLS private keys | `certificates/`, added in `efb2632`, deleted in `647d90a` but reachable at `647d90a~1` | Still in history. **Purge and regenerate** — see [runbook](runbooks/purge-git-history.md) | + +CI scans both the working tree and the full history. The working-tree scan must +be clean unconditionally. The history scan honours +[`.gitleaksignore`](../.gitleaksignore), which lists all nine historical +findings individually, each annotated with what it is and why it is still there. + +That file is an acknowledgement, not a fix. It exists because a CI job that is +permanently red for a known reason gets ignored — and then a genuinely new leak +goes unnoticed alongside it. Once the history purge runs, the fingerprints go +stale and the file gets deleted. + +### Why SNMPv2c is still a weak point + +The devices are polled with SNMPv2c, which transmits the community string in +cleartext. Anyone with a port on the management VLAN can read it off a single +packet. Two mitigations are in place and one is not: + +- **Done:** each device now has a distinct community, so one captured packet + does not grant read access to the whole fleet. +- **Done:** SNMP is only reachable on the management VLAN, which nothing but + specific trusted hosts can enter. +- **Not done:** SNMPv3 with authPriv. The MokerLink switch does not support it. + Tracked in [roadmap](roadmap.md). + +These communities are read-only, but "read-only" on a firewall means the +complete state table and interface topology. They are credentials. + +## Hardening applied to the stack + +- Anonymous Grafana access disabled; sign-up disabled; admin password from SOPS. +- Prometheus, Alertmanager and snmp-exporter run as `nobody` (65534); Loki as + its own unprivileged UID. +- `snmp-exporter` is never published to a host interface — it is reachable only + on the compose network. +- The Alloy debug UI binds to `127.0.0.1` only. +- The Docker socket is mounted read-only into Alloy. +- All images are pinned to explicit versions, so an upstream compromise cannot + arrive silently via `:latest`. Dependabot proposes the bumps; CI validates + them. +- Grafana telemetry and update checks disabled. + +Alloy still runs `privileged: true`, which it needs for host-level metric +collection. That is a real tradeoff and is noted rather than hidden. + +## What this repository deliberately does not publish + +Being able to describe a network precisely is useful; publishing a complete +fingerprint of a house is not. Withheld on purpose: + +- **Full MAC addresses.** Truncated to the OUI, which keeps the useful + information (vendor, and therefore what the device is) and drops the unique + identifier. Full MACs enable device tracking and, on some networks, MAC-based + access control bypass. +- **Owner-linked device names.** Personal devices are listed by role + (`laptop-01`) rather than by person, and a child's bedroom is not labelled. +- **Camera-to-room mapping.** Knowing there are seven cameras is fine. Knowing + which one covers which door is a physical-security detail. +- **The WAN address**, firewall rule bodies, and Wi-Fi configuration. + +The public IP was already redacted in the original inventory — the rest of this +is the same instinct applied consistently. diff --git a/scripts/purge-history.sh b/scripts/purge-history.sh new file mode 100755 index 0000000..2a87cc1 --- /dev/null +++ b/scripts/purge-history.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# +# Remove committed secrets from the *entire* git history. +# +# Deleting a secret in a later commit does not remove it from history — every +# earlier commit still contains it, and `git show :` will hand it +# straight back. This rewrites history so those blobs stop existing. +# +# What it removes: +# * certificates/ — TLS private keys deleted in commit 647d90a but +# still reachable at 647d90a~1 +# * the shared SNMP community string, wherever it appears +# +# This rewrites every commit SHA. Read docs/runbooks/purge-git-history.md before +# running it, and rotate the credentials regardless — assume anything that was +# ever pushed to a public repository is compromised. +# +# Usage: +# scripts/purge-history.sh --dry-run # rewrite a scratch mirror, report +# scripts/purge-history.sh --execute # rewrite ./ for real + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +MODE="${1:---dry-run}" + +die() { printf '\033[0;31merror:\033[0m %s\n' "$*" >&2; exit 1; } +info() { printf '\033[0;34m--\033[0m %s\n' "$*"; } +warn() { printf '\033[0;33m!!\033[0m %s\n' "$*"; } + +# git resolves `git filter-repo` to a git-filter-repo executable on PATH, so +# this single check covers both installation styles. +if ! git filter-repo --help >/dev/null 2>&1; then + die "git-filter-repo not found. Install it: + pipx install git-filter-repo (or: pip install git-filter-repo) + https://github.com/newren/git-filter-repo" +fi + +# The literal to scrub. Kept here rather than in a tracked replacements file so +# the string itself is not re-committed by the very script meant to remove it. +LEAKED_COMMUNITY='7H3r315N05p00N!' + +REPLACEMENTS="$(mktemp)" +PATHS_FILE="$(mktemp)" +trap 'rm -f "${REPLACEMENTS}" "${PATHS_FILE}"' EXIT + +printf '%s==>REDACTED-ROTATED-CREDENTIAL\n' "${LEAKED_COMMUNITY}" > "${REPLACEMENTS}" +cat > "${PATHS_FILE}" <<'EOF' +certificates/ +EOF + +run_filter() { + local target="$1" + git -C "${target}" filter-repo --force \ + --invert-paths --paths-from-file "${PATHS_FILE}" \ + --replace-text "${REPLACEMENTS}" +} + +report() { + local target="$1" + printf '\n\033[1mVerification\033[0m\n' + if git -C "${target}" log --all --oneline -- certificates/ 2>/dev/null | grep -q .; then + printf '\033[0;31m FAIL\033[0m certificates/ still referenced in history\n' + else + printf '\033[0;32m PASS\033[0m no commit touches certificates/\n' + fi + + if git -C "${target}" grep -qI "${LEAKED_COMMUNITY}" "$(git -C "${target}" rev-list --all)" -- 2>/dev/null; then + printf '\033[0;31m FAIL\033[0m leaked community string still present\n' + else + printf '\033[0;32m PASS\033[0m leaked community string absent from all commits\n' + fi + + printf ' commits: %s\n' "$(git -C "${target}" rev-list --all --count)" +} + +case "${MODE}" in + --dry-run) + SCRATCH="$(mktemp -d)" + info "cloning a scratch mirror to ${SCRATCH}/repo" + git clone --no-local --quiet "${REPO_ROOT}" "${SCRATCH}/repo" + git -C "${SCRATCH}/repo" fetch --quiet origin '+refs/heads/*:refs/heads/*' 2>/dev/null || true + info "rewriting the scratch copy (the real repository is untouched)" + run_filter "${SCRATCH}/repo" >/dev/null + report "${SCRATCH}/repo" + printf '\nScratch copy left at %s for inspection.\n' "${SCRATCH}/repo" + printf 'Re-run with --execute to rewrite this repository for real.\n' + ;; + + --execute) + warn "This rewrites every commit SHA in ${REPO_ROOT}." + warn "Anyone with an existing clone will have to re-clone." + read -r -p "Type 'rewrite' to continue: " confirm + [[ "${confirm}" == "rewrite" ]] || die "aborted" + + [[ -z "$(git -C "${REPO_ROOT}" status --porcelain)" ]] \ + || die "working tree is not clean — commit or stash first" + + BACKUP="${REPO_ROOT}/../HomeLab-backup-$(git -C "${REPO_ROOT}" rev-parse --short HEAD).bundle" + info "writing a full backup bundle to ${BACKUP}" + git -C "${REPO_ROOT}" bundle create "${BACKUP}" --all + + info "rewriting history" + run_filter "${REPO_ROOT}" + report "${REPO_ROOT}" + + cat < + 3. Force-push every branch and tag: + git push --force --all origin + git push --force --tags origin + 4. Tell anyone with a clone to re-clone. Old clones can reintroduce the + removed blobs on their next push. + 5. Rotate the credentials — see docs/runbooks/rotate-snmp-community.md. + Assume everything that was ever public is compromised. + +Backup bundle: ${BACKUP} +Restore with: git clone ${BACKUP} HomeLab-restored +EOF + ;; + + *) + die "usage: $(basename "$0") [--dry-run|--execute]" + ;; +esac From f1e201d52833192a60812f0d3b03cefa8ee244eb Mon Sep 17 00:00:00 2001 From: Garrett Allen Date: Sun, 2 Aug 2026 06:30:14 +0000 Subject: [PATCH 7/8] fix(alertmanager): read the webhook URL from url_file, and address review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI caught the first problem: amtool rejected the config with 'unsupported scheme "" for URL'. Alertmanager does not expand environment variables in its configuration, so `url: $ALERTMANAGER_WEBHOOK_URL` was parsed as a literal string. The supported mechanism is url_file, read at notify time. scripts/render-config.sh now writes alertmanager/.rendered/webhook_url and compose mounts it read-only. Because url_file resolves at notify time rather than at load time, amtool validates the committed config with no secret present at all, so CI no longer needs a throwaway webhook URL. Also replaces envsubst with bash parameter expansion in render-config.sh. envsubst lives in gettext-base and is not guaranteed on a minimal server install, so a fresh host would have failed with 'command not found' after decrypting secrets. Testing that substitution against a community string containing / & \ and $ surfaced a second bug: bash 5.2 enables patsub_replacement by default, which makes an unescaped '&' in the replacement expand to the matched text — so a community containing '&' rendered as the literal placeholder it was meant to replace. Disabling the option makes the replacement literal; verified that such a value now survives intact. Narrows the generated .env to the two values compose actually interpolates, so each secret now lives in exactly one rendered file rather than being copied into .env as well. Drops the empty Alertmanager templates: glob. Review feedback, all of it correct: * gitleaks ran as :latest in both scan steps, and snmp-generate used prom/snmp-generator:latest. Both pinned. A scanner that changes underneath you gives results you cannot reproduce; a generator that changes gives diffs you cannot explain. * The 'no floating tags' check only grepped compose.yaml, which is exactly why those two slipped through — it could not see the file it was defined in. It now covers every YAML, Makefile and shell script in the repo. * scripts/validate.sh ran only the working-tree gitleaks scan while CI ran that plus full history, so `make validate` could pass while CI failed. It now runs both, which is what "the same set CI runs" was meant to mean. * secrets/README.md described observability.sops.yaml as committed when it is absent from the branch. It has to be: encrypting it needs an age keypair, and generating one here would mean committing a private key. The table now says so, and notes that `make up` fails loudly until `make secrets-init` has been run. --- .github/workflows/ci.yml | 23 ++++---- Makefile | 2 +- scripts/render-config.sh | 54 ++++++++++++++++--- scripts/validate.sh | 23 +++++--- secrets/README.md | 21 ++++++-- .../alertmanager/alertmanager.yaml | 19 ++++--- stacks/observability/compose.yaml | 6 ++- 7 files changed, 107 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a890254..b3d77e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,21 +56,25 @@ jobs: - name: Seed a validation-only .env run: | cp "$STACK/.env.example" "$STACK/.env" - { - echo "GRAFANA_ADMIN_PASSWORD=validation-only" - echo "ALERTMANAGER_WEBHOOK_URL=https://example.invalid/hook" - } >> "$STACK/.env" + echo "GRAFANA_ADMIN_PASSWORD=validation-only" >> "$STACK/.env" - name: docker compose config run: docker compose -f "$STACK/compose.yaml" config -q + # Covers every place an image is referenced, not just compose.yaml — the + # first version of this check only looked at the stack and let :latest + # through in the workflow itself and in the Makefile. - name: Verify no image uses a floating tag run: | - if grep -nE '^\s*image:.*:latest\s*$' "$STACK/compose.yaml"; then + if grep -rnE '(^|[[:space:]])[a-z0-9._/-]+:latest([[:space:]]|$)' \ + --include='*.yaml' --include='*.yml' --include='Makefile' \ + --include='*.sh' . \ + | grep -v '^\./\.git/' \ + | grep -vE '^[^:]+:[0-9]+:[[:space:]]*#' ; then echo "::error::floating :latest tag found — pin an explicit version" exit 1 fi - echo "all images pinned" + echo "all image references pinned" - name: promtool check config run: | @@ -84,10 +88,11 @@ jobs: -v "$PWD:/repo" -w /repo "$PROM_IMAGE" \ check rules "$STACK"/prometheus/rules/*.rules.yaml + # No secret needed: the receiver URL comes from url_file, which + # Alertmanager reads at notify time rather than at config load time. - name: amtool check-config run: | docker run --rm --entrypoint amtool \ - -e ALERTMANAGER_WEBHOOK_URL=https://example.invalid/hook \ -v "$PWD:/repo" -w /repo "$AM_IMAGE" \ check-config "$STACK/alertmanager/alertmanager.yaml" @@ -128,13 +133,13 @@ jobs: - name: gitleaks — working tree run: | docker run --rm -v "$PWD:/repo" -w /repo \ - zricethezav/gitleaks:latest \ + zricethezav/gitleaks:v8.24.0 \ detect --no-git --no-banner --redact -c .gitleaks.toml -v - name: gitleaks — full history run: | docker run --rm -v "$PWD:/repo" -w /repo \ - zricethezav/gitleaks:latest \ + zricethezav/gitleaks:v8.24.0 \ detect --no-banner --redact -c .gitleaks.toml --log-opts="--all" -v - name: Assert no decrypted artefact is tracked diff --git a/Makefile b/Makefile index fcc726d..8a26606 100644 --- a/Makefile +++ b/Makefile @@ -113,7 +113,7 @@ snmp-generate: ## Regenerate snmp.yaml from generator.yaml -e SNMP_COMMUNITY_APC='$${SNMP_COMMUNITY_APC}' \ -e SNMP_COMMUNITY_MOKERLINK='$${SNMP_COMMUNITY_MOKERLINK}' \ -e SNMP_COMMUNITY_ILO='$${SNMP_COMMUNITY_ILO}' \ - prom/snmp-generator:latest generate \ + prom/snmp-generator:v0.28.0 generate \ -m /opt/mibs -g /opt/generator.yaml -o /opt/snmp.yaml @printf '\033[0;33mCheck the diff before committing — placeholders must survive.\033[0m\n' diff --git a/scripts/render-config.sh b/scripts/render-config.sh index 3de769d..538791c 100755 --- a/scripts/render-config.sh +++ b/scripts/render-config.sh @@ -75,13 +75,30 @@ SNMP_OUT_DIR="${STACK_DIR}/snmp-exporter/.rendered" if [[ -f "${SNMP_SRC}" ]]; then info "rendering snmp.yaml" mkdir -p "${SNMP_OUT_DIR}" - # Restrict substitution to the SNMP_COMMUNITY_* names so that any other - # ${...} sequence inside 14k lines of OID definitions is left untouched. - # The single quotes are deliberate: envsubst takes the variable list as a - # literal string, not as already-expanded values. - # shellcheck disable=SC2016 - envsubst '${SNMP_COMMUNITY_PFSENSE} ${SNMP_COMMUNITY_APC} ${SNMP_COMMUNITY_MOKERLINK} ${SNMP_COMMUNITY_ILO}' \ - < "${SNMP_SRC}" > "${SNMP_OUT_DIR}/snmp.yaml" + chmod 700 "${SNMP_OUT_DIR}" + + # Substitution is done with bash parameter expansion rather than envsubst or + # sed. envsubst lives in gettext-base, which is not guaranteed on a minimal + # server install, and sed would mangle any community string containing / & or + # a backslash. This also touches only the four SNMP_COMMUNITY_* names, so no + # other ${...} sequence in 14k lines of OID definitions can be affected. + # + # bash 5.2 enables patsub_replacement by default, which makes an unescaped '&' + # in the replacement expand to the matched text — so a community string + # containing '&' would silently render as the placeholder it was meant to + # replace. Disabling it makes the replacement literal. The redirect keeps this + # a no-op on bash < 5.2, where the option does not exist. + shopt -u patsub_replacement 2>/dev/null || true + + snmp_content="$(cat "${SNMP_SRC}")" + for var in SNMP_COMMUNITY_PFSENSE SNMP_COMMUNITY_APC \ + SNMP_COMMUNITY_MOKERLINK SNMP_COMMUNITY_ILO; do + snmp_content="${snmp_content//\$\{${var}\}/${!var}}" + done + + umask 077 + printf '%s\n' "${snmp_content}" > "${SNMP_OUT_DIR}/snmp.yaml" + unset snmp_content chmod 600 "${SNMP_OUT_DIR}/snmp.yaml" # shellcheck disable=SC2016 # matching the literal text "${SNMP_COMMUNITY..." @@ -90,9 +107,30 @@ if [[ -f "${SNMP_SRC}" ]]; then fi fi +# --------------------------------------------------------------------------- +# Render the Alertmanager webhook URL +# +# Alertmanager does not expand environment variables in its config. `url_file` +# is the supported mechanism, so the URL is written to a file that compose +# mounts read-only. +# --------------------------------------------------------------------------- +AM_OUT_DIR="${STACK_DIR}/alertmanager/.rendered" +if [[ -f "${STACK_DIR}/alertmanager/alertmanager.yaml" ]]; then + info "rendering alertmanager webhook_url" + mkdir -p "${AM_OUT_DIR}" + printf '%s' "${ALERTMANAGER_WEBHOOK_URL}" > "${AM_OUT_DIR}/webhook_url" + chmod 600 "${AM_OUT_DIR}/webhook_url" +fi + # --------------------------------------------------------------------------- # Write .env for compose interpolation +# +# Only the values compose actually interpolates are written here. The SNMP +# communities go into the rendered snmp.yaml and the webhook URL into +# webhook_url; copying them into .env as well would spread the same secret +# across three files for no benefit. # --------------------------------------------------------------------------- +COMPOSE_VARS=(GRAFANA_ADMIN_USER GRAFANA_ADMIN_PASSWORD) ENV_FILE="${STACK_DIR}/.env" info "writing $(basename "${STACK_DIR}")/.env" @@ -102,7 +140,7 @@ info "writing $(basename "${STACK_DIR}")/.env" if [[ -f "${STACK_DIR}/.env.example" ]]; then grep -vE '^\s*#|^\s*$' "${STACK_DIR}/.env.example" || true fi - for var in GRAFANA_ADMIN_USER "${REQUIRED[@]}"; do + for var in "${COMPOSE_VARS[@]}"; do [[ -n "${!var:-}" ]] && printf '%s=%s\n' "${var}" "${!var}" done } > "${ENV_FILE}" diff --git a/scripts/validate.sh b/scripts/validate.sh index ce90e45..cab4082 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -33,10 +33,7 @@ if have docker; then TMP_ENV="$(mktemp)" trap 'rm -f "${TMP_ENV}"' EXIT cat "${STACK}/.env.example" > "${TMP_ENV}" - cat >> "${TMP_ENV}" <<'EOF' -GRAFANA_ADMIN_PASSWORD=validation-only -ALERTMANAGER_WEBHOOK_URL=https://example.invalid/hook -EOF + echo "GRAFANA_ADMIN_PASSWORD=validation-only" >> "${TMP_ENV}" if docker compose --env-file "${TMP_ENV}" -f "${STACK}/compose.yaml" config -q 2>/dev/null; then pass "docker compose config" else @@ -83,14 +80,15 @@ head_ "Alertmanager" if have amtool; then AMTOOL=(amtool) elif have_docker; then - AMTOOL=(docker run --rm --entrypoint amtool -e ALERTMANAGER_WEBHOOK_URL=https://example.invalid/hook \ + AMTOOL=(docker run --rm --entrypoint amtool \ -v "${REPO_ROOT}:/repo" -w /repo "${AM_IMAGE}") else AMTOOL=() fi +# The receiver URL comes from url_file, which Alertmanager reads at notify time +# rather than at load time — so this validates without any secret present. if ((${#AMTOOL[@]})); then - export ALERTMANAGER_WEBHOOK_URL="${ALERTMANAGER_WEBHOOK_URL:-https://example.invalid/hook}" if "${AMTOOL[@]}" check-config "${STACK}/alertmanager/alertmanager.yaml" >/dev/null 2>&1; then pass "amtool check-config" else @@ -154,13 +152,22 @@ fi # --------------------------------------------------------------------------- head_ "Secrets" # --------------------------------------------------------------------------- +# Both scans, matching CI exactly. Running only the working-tree scan locally +# would let `make validate` pass while CI fails on history, or vice versa. if have gitleaks; then - if gitleaks detect --no-banner --redact -c .gitleaks.toml >/dev/null 2>&1; then + if gitleaks detect --no-git --no-banner --redact -c .gitleaks.toml >/dev/null 2>&1; then pass "gitleaks (working tree)" else - gitleaks detect --no-banner --redact -c .gitleaks.toml + gitleaks detect --no-git --no-banner --redact -c .gitleaks.toml fail "gitleaks (working tree)" fi + + if gitleaks detect --no-banner --redact -c .gitleaks.toml --log-opts="--all" >/dev/null 2>&1; then + pass "gitleaks (full history)" + else + gitleaks detect --no-banner --redact -c .gitleaks.toml --log-opts="--all" + fail "gitleaks (full history)" + fi else skip "gitleaks not installed" fi diff --git a/secrets/README.md b/secrets/README.md index 2f041a4..4d15512 100644 --- a/secrets/README.md +++ b/secrets/README.md @@ -4,12 +4,20 @@ Secrets are encrypted with [SOPS](https://github.com/getsops/sops) using an [age](https://github.com/FiloSottile/age) key, committed in encrypted form, and decrypted only in memory at deploy time. -| File | Committed? | Encrypted? | Contains | +| File | In this repo? | Encrypted? | Contains | | --- | --- | --- | --- | | `observability.example.yaml` | yes | no | Key names and placeholder values | -| `observability.sops.yaml` | yes | **yes** | Real credentials | +| `observability.sops.yaml` | **not yet** — created by `make secrets-init`, then committed | **yes** | Real credentials | | `~/.config/sops/age/keys.txt` | **never** | n/a | The private key | +> `observability.sops.yaml` is deliberately absent from the repository right +> now. Encrypting it requires an age keypair, and generating that keypair here +> would mean either committing a private key or encrypting to a key nobody +> holds. `make secrets-init` creates both on the machine that will run the +> stack; the encrypted result is then committed as normal. Until that has been +> run, `make up` fails with a clear error rather than starting a stack with +> default credentials. + ## Why encrypted-in-git rather than a gitignored `.env` A gitignored `.env` keeps secrets out of the repository, but it also keeps them @@ -53,11 +61,14 @@ plaintext never lands on disk unencrypted. `scripts/render-config.sh` runs before `docker compose up`. It decrypts this file, exports the values as environment variables, and: -- writes `stacks/observability/.env` for compose to interpolate (gitignored); +- writes `stacks/observability/.env` with only the values compose interpolates + (the Grafana credentials); - renders `snmp-exporter/snmp.yaml`'s `${SNMP_COMMUNITY_*}` placeholders into - `snmp-exporter/.rendered/snmp.yaml` (gitignored), which is what the container - actually mounts. + `snmp-exporter/.rendered/snmp.yaml`, which is what the container mounts; +- writes `alertmanager/.rendered/webhook_url`, because Alertmanager does not + expand environment variables and reads receiver URLs via `url_file`. +All three are gitignored, and each secret is written to exactly one of them. Nothing writes a secret into a tracked file. ## Rotating diff --git a/stacks/observability/alertmanager/alertmanager.yaml b/stacks/observability/alertmanager/alertmanager.yaml index c9cbe20..b710213 100644 --- a/stacks/observability/alertmanager/alertmanager.yaml +++ b/stacks/observability/alertmanager/alertmanager.yaml @@ -1,16 +1,19 @@ --- # Alert routing. # -# The receiver URL is injected from the SOPS-encrypted secrets file at deploy -# time — Alertmanager expands $ENV_VAR references in *_url fields natively, so -# the webhook never touches the repository. +# The receiver URL is a secret (a webhook URL is a bearer credential — anyone +# holding it can post to it), so it is not in this file. +# +# Alertmanager does NOT expand environment variables in its config. The +# supported mechanism is `url_file`, which reads the URL from disk at notify +# time. scripts/render-config.sh decrypts secrets/observability.sops.yaml and +# writes .rendered/webhook_url, which compose mounts read-only at +# /etc/alertmanager/secrets/. Both the rendered directory and its contents are +# gitignored. global: resolve_timeout: 5m -templates: - - /etc/alertmanager/templates/*.tmpl - route: receiver: default group_by: ["alertname", "device", "instance"] @@ -63,10 +66,10 @@ receivers: - name: default webhook_configs: - - url: $ALERTMANAGER_WEBHOOK_URL + - url_file: /etc/alertmanager/secrets/webhook_url send_resolved: true - name: urgent webhook_configs: - - url: $ALERTMANAGER_WEBHOOK_URL + - url_file: /etc/alertmanager/secrets/webhook_url send_resolved: true diff --git a/stacks/observability/compose.yaml b/stacks/observability/compose.yaml index ea6cca4..9945efc 100644 --- a/stacks/observability/compose.yaml +++ b/stacks/observability/compose.yaml @@ -64,10 +64,12 @@ services: - --config.file=/etc/alertmanager/alertmanager.yaml - --storage.path=/alertmanager - --web.external-url=http://${MONITORING_HOST:-10.0.99.20}:${ALERTMANAGER_PORT:-9093} - environment: - ALERTMANAGER_WEBHOOK_URL: ${ALERTMANAGER_WEBHOOK_URL:?set in secrets/observability.sops.yaml} volumes: - ./alertmanager/alertmanager.yaml:/etc/alertmanager/alertmanager.yaml:ro + # Contains webhook_url, written by scripts/render-config.sh. Alertmanager + # does not expand environment variables, so the receiver URL is read from + # a file via `url_file`. + - ./alertmanager/.rendered:/etc/alertmanager/secrets:ro - alertmanager-data:/alertmanager ports: - "${BIND_ADDR:-0.0.0.0}:${ALERTMANAGER_PORT:-9093}:9093" From 6c3bc7bb9e048a02a17357937cabb844aed8b32d Mon Sep 17 00:00:00 2001 From: Garrett Allen Date: Sun, 2 Aug 2026 06:35:18 +0000 Subject: [PATCH 8/8] ci: use 'alloy fmt --test', the flag that actually exists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI failed with 'unknown flag: --verify'. Alloy v1.6.1's fmt command takes --write/-w and --test/-t; there is no --verify, and no 'alloy validate' subcommand in this version. Verified against the real v1.6.1 binary rather than by reading the docs a second time: config.alloy is already canonically formatted, 'fmt --test' exits 0, and loading the log_processor block in a real Alloy instance shows the component starting cleanly — which is what actually compiles the regex. That also confirmed the earlier backtick fix does what it was meant to. With the double-quoted string the pattern matched 0 of 4 representative log lines, because \b was a backspace byte; with the backtick string it matches 3 of 4, the fourth having no level token at all. scripts/validate.sh now prefers a local alloy binary over docker, matching how it already handles promtool and amtool, so the whole suite runs without a daemon. Step labels updated to name the flag they actually pass, and both note that fmt checks syntax and formatting but not component configuration. --- .github/workflows/ci.yml | 8 ++++++-- scripts/validate.sh | 24 ++++++++++++++++-------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3d77e2..d06385e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,11 +96,15 @@ jobs: -v "$PWD:/repo" -w /repo "$AM_IMAGE" \ check-config "$STACK/alertmanager/alertmanager.yaml" - - name: alloy fmt --verify + # `fmt --test` exits non-zero if the file is not canonically formatted, and + # fails outright on a syntax error. It does not validate that components + # are configured correctly — Alloy v1.6.1 has no `validate` subcommand, so + # that is only caught at load time on the host. + - name: alloy fmt --test run: | docker run --rm --entrypoint alloy \ -v "$PWD:/repo" -w /repo "$ALLOY_IMAGE" \ - fmt --verify "$STACK/alloy/config.alloy" + fmt --test "$STACK/alloy/config.alloy" - name: Validate Grafana dashboards run: python3 scripts/check_dashboards.py diff --git a/scripts/validate.sh b/scripts/validate.sh index cab4082..aa4336f 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -102,17 +102,25 @@ fi # --------------------------------------------------------------------------- head_ "Alloy" # --------------------------------------------------------------------------- -if have_docker; then - if docker run --rm -v "${REPO_ROOT}:/repo" -w /repo --entrypoint alloy "${ALLOY_IMAGE}" \ - fmt --verify "${STACK}/alloy/config.alloy" >/dev/null 2>&1; then - pass "alloy fmt --verify" +if have alloy; then + ALLOY=(alloy) +elif have_docker; then + ALLOY=(docker run --rm -v "${REPO_ROOT}:/repo" -w /repo --entrypoint alloy "${ALLOY_IMAGE}") +else + ALLOY=() +fi + +# `fmt --test` fails on a syntax error and on non-canonical formatting. It does +# not check component configuration — v1.6.1 has no `validate` subcommand. +if ((${#ALLOY[@]})); then + if "${ALLOY[@]}" fmt --test "${STACK}/alloy/config.alloy" >/dev/null 2>&1; then + pass "alloy fmt --test" else - docker run --rm -v "${REPO_ROOT}:/repo" -w /repo --entrypoint alloy "${ALLOY_IMAGE}" \ - fmt --verify "${STACK}/alloy/config.alloy" - fail "alloy fmt --verify" + "${ALLOY[@]}" fmt --test "${STACK}/alloy/config.alloy" + fail "alloy fmt --test" fi else - skip "alloy fmt (needs a docker daemon)" + skip "no alloy binary and no docker daemon" fi # ---------------------------------------------------------------------------