From af3ce44956b1da9420742d9442415f1f8b050949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey?= Date: Fri, 20 Mar 2026 10:13:16 -0700 Subject: [PATCH 1/8] chore(glyph): remove unused cockpit service Co-Authored-By: Claude Opus 4.6 --- hosts/glyph/services/default.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/hosts/glyph/services/default.nix b/hosts/glyph/services/default.nix index 59bc0e8b..7bfc1bcb 100644 --- a/hosts/glyph/services/default.nix +++ b/hosts/glyph/services/default.nix @@ -21,7 +21,6 @@ users.users.mu.extraGroups = ["media"]; users.users.${config.services.transmission.user}.extraGroups = ["media"]; - services.cockpit.enable = true; services.openssh.enable = true; services.roon-server = { enable = true; From 6f686667f294b10fc5b706f74eaa1288d49394eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey?= Date: Fri, 20 Mar 2026 10:13:46 -0700 Subject: [PATCH 2/8] feat(glyph): add Loki log aggregation service Configures Loki on glyph with filesystem storage, TSDB index, and 30-day retention. Listens on port 3100 for log ingestion from Alloy agents on all hosts. Co-Authored-By: Claude Opus 4.6 --- hosts/glyph/services/default.nix | 1 + hosts/glyph/services/loki.nix | 44 ++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 hosts/glyph/services/loki.nix diff --git a/hosts/glyph/services/default.nix b/hosts/glyph/services/default.nix index 7bfc1bcb..a081eb35 100644 --- a/hosts/glyph/services/default.nix +++ b/hosts/glyph/services/default.nix @@ -10,6 +10,7 @@ ./dns.nix ./filebrowser.nix ./jellyfin.nix + ./loki.nix ./nfs.nix ./open-terminal.nix ./open-webui.nix diff --git a/hosts/glyph/services/loki.nix b/hosts/glyph/services/loki.nix new file mode 100644 index 00000000..ba054019 --- /dev/null +++ b/hosts/glyph/services/loki.nix @@ -0,0 +1,44 @@ +{_}: { + services.loki = { + enable = true; + configuration = { + server.http_listen_port = 3100; + auth_enabled = false; + + common = { + path_prefix = "/var/lib/loki"; + replication_factor = 1; + ring.kvstore.store = "inmemory"; + }; + + schema_config.configs = [ + { + from = "2024-01-01"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index = { + prefix = "index_"; + period = "24h"; + }; + } + ]; + + storage_config.filesystem.directory = "/var/lib/loki/chunks"; + + limits_config = { + retention_period = "30d"; + ingestion_burst_size_mb = 16; + ingestion_rate_mb = 8; + }; + + compactor = { + working_directory = "/var/lib/loki/compactor"; + compaction_interval = "10m"; + retention_enabled = true; + retention_delete_delay = "2h"; + delete_request_store = "filesystem"; + }; + }; + }; +} From 01716e3020b4a138cf1cc782d65a0218b02c394c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey?= Date: Fri, 20 Mar 2026 10:14:00 -0700 Subject: [PATCH 3/8] feat(glyph): add Alloy to ship local journals to Loki Configures Grafana Alloy to read systemd journal entries and forward them to the local Loki instance with host=glyph label. Co-Authored-By: Claude Opus 4.6 --- hosts/glyph/services/alloy.nix | 18 ++++++++++++++++++ hosts/glyph/services/default.nix | 1 + 2 files changed, 19 insertions(+) create mode 100644 hosts/glyph/services/alloy.nix diff --git a/hosts/glyph/services/alloy.nix b/hosts/glyph/services/alloy.nix new file mode 100644 index 00000000..9688d1aa --- /dev/null +++ b/hosts/glyph/services/alloy.nix @@ -0,0 +1,18 @@ +{_}: { + services.alloy.enable = true; + + environment.etc."alloy/config.alloy".text = '' + loki.source.journal "systemd" { + forward_to = [loki.write.local.receiver] + } + + loki.write "local" { + endpoint { + url = "http://localhost:3100/loki/api/v1/push" + } + external_labels = { + host = "glyph", + } + } + ''; +} diff --git a/hosts/glyph/services/default.nix b/hosts/glyph/services/default.nix index a081eb35..30148950 100644 --- a/hosts/glyph/services/default.nix +++ b/hosts/glyph/services/default.nix @@ -4,6 +4,7 @@ ... }: { imports = [ + ./alloy.nix ./attic.nix ./db.nix ./avahi.nix From 68c14c018e1a5a1665002e4af7cd1775699e20c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey?= Date: Fri, 20 Mar 2026 10:14:17 -0700 Subject: [PATCH 4/8] feat(spore): add Alloy to ship journals to Loki Configures Grafana Alloy on spore to forward systemd journal entries to Loki on glyph over Tailscale. Co-Authored-By: Claude Opus 4.6 --- hosts/spore/services/alloy.nix | 18 ++++++++++++++++++ hosts/spore/services/default.nix | 1 + 2 files changed, 19 insertions(+) create mode 100644 hosts/spore/services/alloy.nix diff --git a/hosts/spore/services/alloy.nix b/hosts/spore/services/alloy.nix new file mode 100644 index 00000000..bfd5f878 --- /dev/null +++ b/hosts/spore/services/alloy.nix @@ -0,0 +1,18 @@ +{_}: { + services.alloy.enable = true; + + environment.etc."alloy/config.alloy".text = '' + loki.source.journal "systemd" { + forward_to = [loki.write.remote.receiver] + } + + loki.write "remote" { + endpoint { + url = "http://glyph.rove-duck.ts.net:3100/loki/api/v1/push" + } + external_labels = { + host = "spore", + } + } + ''; +} diff --git a/hosts/spore/services/default.nix b/hosts/spore/services/default.nix index 114b2c36..e21d5a00 100644 --- a/hosts/spore/services/default.nix +++ b/hosts/spore/services/default.nix @@ -4,6 +4,7 @@ ... }: { imports = [ + ./alloy.nix ./grafana.nix ./homepage-dashboard.nix ./mastodon.nix From 7d94cd51e6d840de426cf8b57f3d879642781982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey?= Date: Fri, 20 Mar 2026 10:14:30 -0700 Subject: [PATCH 5/8] feat(zeta): add Alloy to ship journals to Loki Configures Grafana Alloy on zeta to forward systemd journal entries to Loki on glyph over Tailscale. Co-Authored-By: Claude Opus 4.6 --- hosts/zeta/services/alloy.nix | 18 ++++++++++++++++++ hosts/zeta/services/default.nix | 1 + 2 files changed, 19 insertions(+) create mode 100644 hosts/zeta/services/alloy.nix diff --git a/hosts/zeta/services/alloy.nix b/hosts/zeta/services/alloy.nix new file mode 100644 index 00000000..1cde0262 --- /dev/null +++ b/hosts/zeta/services/alloy.nix @@ -0,0 +1,18 @@ +{_}: { + services.alloy.enable = true; + + environment.etc."alloy/config.alloy".text = '' + loki.source.journal "systemd" { + forward_to = [loki.write.remote.receiver] + } + + loki.write "remote" { + endpoint { + url = "http://glyph.rove-duck.ts.net:3100/loki/api/v1/push" + } + external_labels = { + host = "zeta", + } + } + ''; +} diff --git a/hosts/zeta/services/default.nix b/hosts/zeta/services/default.nix index 9fb999b7..f9a7cac5 100644 --- a/hosts/zeta/services/default.nix +++ b/hosts/zeta/services/default.nix @@ -4,6 +4,7 @@ ... }: { imports = [ + ./alloy.nix ./home-assistant.nix ./homebridge.nix ./znc.nix From 827bd4662449f156e02b4fdb26fa93cdfefe585e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey?= Date: Fri, 20 Mar 2026 10:14:41 -0700 Subject: [PATCH 6/8] feat(spore): provision Loki datasource in Grafana Adds Loki on glyph as a datasource so logs from all hosts are queryable via the Grafana Explore view. Co-Authored-By: Claude Opus 4.6 --- hosts/spore/services/grafana.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hosts/spore/services/grafana.nix b/hosts/spore/services/grafana.nix index 2c9b06d4..b385036c 100644 --- a/hosts/spore/services/grafana.nix +++ b/hosts/spore/services/grafana.nix @@ -73,6 +73,12 @@ isDefault = true; editable = false; } + { + name = "Loki"; + type = "loki"; + url = "http://glyph.rove-duck.ts.net:3100"; + editable = false; + } ]; }; }; From 33af89c3a0e3f072e6ff8b2b4a215de85550a4ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey?= Date: Fri, 20 Mar 2026 11:44:41 -0700 Subject: [PATCH 7/8] feat(spore): add Log Explorer dashboard to Grafana Provisioned dashboard with host selector, text search, log volume histogram, error/warning counters, and a full log stream panel. Co-Authored-By: Claude Opus 4.6 --- hosts/spore/services/dashboards/logs.json | 224 ++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 hosts/spore/services/dashboards/logs.json diff --git a/hosts/spore/services/dashboards/logs.json b/hosts/spore/services/dashboards/logs.json new file mode 100644 index 00000000..554e5e0f --- /dev/null +++ b/hosts/spore/services/dashboards/logs.json @@ -0,0 +1,224 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "datasource": { "type": "loki", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "text" } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 8, "x": 0, "y": 0 }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["count"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "title": "Total Log Lines", + "type": "stat", + "targets": [ + { + "expr": "sum(count_over_time({host=~\"$host\"} |~ \"$search\" [$__auto]))", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "loki", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "red" } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 8, "x": 8, "y": 0 }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["count"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "title": "Error Lines", + "type": "stat", + "targets": [ + { + "expr": "sum(count_over_time({host=~\"$host\"} |~ \"$search\" |~ \"(?i)error|fatal|panic\" [$__auto]))", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "loki", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "yellow" } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 8, "x": 16, "y": 0 }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["count"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "title": "Warning Lines", + "type": "stat", + "targets": [ + { + "expr": "sum(count_over_time({host=~\"$host\"} |~ \"$search\" |~ \"(?i)warn\" [$__auto]))", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "loki", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 50, + "gradientMode": "none", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "normal" }, + "thresholdsStyle": { "mode": "off" } + } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 24, "x": 0, "y": 4 }, + "id": 4, + "options": { + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "title": "Log Volume", + "type": "timeseries", + "targets": [ + { + "expr": "sum by (host) (count_over_time({host=~\"$host\"} |~ \"$search\" [$__auto]))", + "legendFormat": "{{host}}", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "loki", "uid": "${datasource}" }, + "gridPos": { "h": 18, "w": 24, "x": 0, "y": 10 }, + "id": 5, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "title": "Logs", + "type": "logs", + "targets": [ + { + "expr": "{host=~\"$host\"} |~ \"$search\"", + "refId": "A" + } + ] + } + ], + "schemaVersion": 39, + "tags": ["logs", "loki"], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": true, + "label": "Host", + "multi": true, + "name": "host", + "options": [], + "query": "label_values({}, host)", + "refresh": 2, + "regex": "", + "type": "query" + }, + { + "current": { "selected": false, "text": "", "value": "" }, + "hide": 0, + "label": "Search", + "name": "search", + "options": [ + { "selected": true, "text": "", "value": "" } + ], + "query": "", + "type": "textbox" + }, + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { "from": "now-1h", "to": "now" }, + "title": "Log Explorer", + "uid": "log-explorer" +} From b0c1b7ab537204911a574cfcf6fa08f81533063b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey?= Date: Fri, 20 Mar 2026 11:49:49 -0700 Subject: [PATCH 8/8] fix: use catch-all arg pattern in new modules `{_}` is a destructuring pattern that rejects extra specialArgs like `inputs`. Use `_:` which accepts any argument set without destructuring, satisfying both statix and the module system. Co-Authored-By: Claude Opus 4.6 --- hosts/glyph/services/alloy.nix | 2 +- hosts/glyph/services/loki.nix | 2 +- hosts/spore/services/alloy.nix | 2 +- hosts/zeta/services/alloy.nix | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hosts/glyph/services/alloy.nix b/hosts/glyph/services/alloy.nix index 9688d1aa..9f77a165 100644 --- a/hosts/glyph/services/alloy.nix +++ b/hosts/glyph/services/alloy.nix @@ -1,4 +1,4 @@ -{_}: { +_: { services.alloy.enable = true; environment.etc."alloy/config.alloy".text = '' diff --git a/hosts/glyph/services/loki.nix b/hosts/glyph/services/loki.nix index ba054019..0ed2d8fd 100644 --- a/hosts/glyph/services/loki.nix +++ b/hosts/glyph/services/loki.nix @@ -1,4 +1,4 @@ -{_}: { +_: { services.loki = { enable = true; configuration = { diff --git a/hosts/spore/services/alloy.nix b/hosts/spore/services/alloy.nix index bfd5f878..bd288806 100644 --- a/hosts/spore/services/alloy.nix +++ b/hosts/spore/services/alloy.nix @@ -1,4 +1,4 @@ -{_}: { +_: { services.alloy.enable = true; environment.etc."alloy/config.alloy".text = '' diff --git a/hosts/zeta/services/alloy.nix b/hosts/zeta/services/alloy.nix index 1cde0262..02f7479e 100644 --- a/hosts/zeta/services/alloy.nix +++ b/hosts/zeta/services/alloy.nix @@ -1,4 +1,4 @@ -{_}: { +_: { services.alloy.enable = true; environment.etc."alloy/config.alloy".text = ''