feat(observability): ordered structured log-level filtering (Loki detected_level + pino map)

k8s/observability/61-grafana-logs-dashboard.yaml

@@ -79,27 +79,28 @@ data:
             "label": "Min level",
             "type": "custom",
             "hide": 0,
-            "query": "All : . , Warning+ : (?i)(\"level\":[456]0|\"level\":\"(warn|warning|error|fatal|critical)\"|\\[(warn|warning|error|fatal|crit|critical)\\]) , Error+ : (?i)(\"level\":[56]0|\"level\":\"(error|fatal|critical)\"|\\[(error|fatal|crit|critical)\\])",
+            "description": "Ordered level filter using real level fields, not message substrings. server-2 (pino) gets a normalized `level` via structured metadata from Alloy; the other apps use Loki's query-time `detected_level`. The OR covers both so a single dropdown works across all apps.",
+            "query": "All : , Warning+ :  | level=~\"warn|error|fatal|critical\" or detected_level=~\"warn|error|fatal|critical\" , Error+ :  | level=~\"error|fatal|critical\" or detected_level=~\"error|fatal|critical\"",
             "current": {
               "selected": true,
               "text": "All",
-              "value": "."
+              "value": ""
             },
             "options": [
               {
                 "selected": true,
                 "text": "All",
-                "value": "."
+                "value": ""
               },
               {
                 "selected": false,
                 "text": "Warning+",
-                "value": "(?i)(\"level\":[456]0|\"level\":\"(warn|warning|error|fatal|critical)\"|\\[(warn|warning|error|fatal|crit|critical)\\])"
+                "value": " | level=~\"warn|error|fatal|critical\" or detected_level=~\"warn|error|fatal|critical\""
               },
               {
                 "selected": false,
                 "text": "Error+",
-                "value": "(?i)(\"level\":[56]0|\"level\":\"(error|fatal|critical)\"|\\[(error|fatal|crit|critical)\\])"
+                "value": " | level=~\"error|fatal|critical\" or detected_level=~\"error|fatal|critical\""
               }
             ],
             "includeAll": false,
@@ -161,7 +162,7 @@ data:
             {
               "refId": "A",
               "datasource": "Loki",
-              "expr": "sum by (app) (count_over_time({namespace=~\"$namespace\", app=~\"$app\"} |~ \"(?i)$search\" |~ `$level` [$__interval]))",
+              "expr": "sum by (app) (count_over_time({namespace=~\"$namespace\", app=~\"$app\"} |~ \"(?i)$search\" $level [$__interval]))",
               "legendFormat": "{{app}}",
               "queryType": "range"
             }
@@ -204,7 +205,7 @@ data:
                 {
                   "refId": "A",
                   "datasource": "Loki",
-                  "expr": "{namespace=~\"$namespace\", app=~\"$app\"} |~ \"(?i)$search\" |~ `$level`",
+                  "expr": "{namespace=~\"$namespace\", app=~\"$app\"} |~ \"(?i)$search\" $level",
                   "queryType": "range",
                   "maxLines": 500
                 }

k8s/observability/80-alloy.yaml

@@ -89,8 +89,37 @@ data:
       forward_to = [loki.process.drop_extra_labels.receiver]
     }
 
-    // Drop any remaining labels not in the 5-key allowlist (LOG-02, T-15-05)
+    // Normalize server-2's pino NUMERIC level into a real ordered level, exposed
+    // as STRUCTURED METADATA (not a label — keep the 5-key label allowlist below to
+    // avoid stream cardinality blow-up, LOG-02). Loki's query-time detected_level
+    // already gives a real level to the text-format apps (rabbitmq/postgres/glitchtip)
+    // but maps pino's numeric "level":30/40/50 to `unknown`, hiding server-2 under a
+    // level filter. This stage fixes that. See plans logs-level-filtering-todo brief.
     loki.process "drop_extra_labels" {
+      // Scope to server-2 only; other apps keep their query-time detected_level.
+      stage.match {
+        selector = "{app=\"server-2\"}"
+
+        // Pino logs JSON; pull the numeric level out of the line into `level`.
+        stage.json {
+          expressions = { "level" = "level" }
+        }
+
+        // Map pino numeric severity -> normalized text level, overwriting `level`.
+        // Pino: trace=10 debug=20 info=30 warn=40 error=50 fatal=60. `.Value` is the
+        // current `level` value; the result is written back to the `level` key.
+        stage.template {
+          source   = "level"
+          template = "{{ if eq .Value \"60\" }}fatal{{ else if eq .Value \"50\" }}error{{ else if eq .Value \"40\" }}warn{{ else if eq .Value \"30\" }}info{{ else if eq .Value \"20\" }}debug{{ else if eq .Value \"10\" }}trace{{ else }}unknown{{ end }}"
+        }
+
+        // Expose as structured metadata named `level` (queryable, not a stream label).
+        stage.structured_metadata {
+          values = { "level" = "" }
+        }
+      }
+
+      // Drop any remaining labels not in the 5-key allowlist (LOG-02, T-15-05)
       stage.label_keep {
         values = ["namespace", "pod", "container", "app", "job"]
       }