From 516f13af73fb44ca3a715fd705b2e06384220d07 Mon Sep 17 00:00:00 2001
From: Bojan <bojan.colic131@gmail.com>
Date: Fri, 31 Jul 2026 16:32:25 +0200
Subject: [PATCH 1/5] feat(observability): add worker pressure flame graph

---
 docs/use-dkg/backpressure-observability.md    |  23 ++
 packages/cli/src/daemon/backpressure-log.ts   |  23 ++
 packages/cli/src/daemon/lifecycle.ts          |   6 +-
 packages/cli/test/backpressure-log.test.ts    |  29 +++
 packages/cli/vitest.unit.config.ts            |   1 +
 .../grafana-dashboard-dkg-node-logs.json      | 209 ++++++++++++++++++
 tools/observability/lib/dashboards.mjs        |  86 ++++++-
 tools/observability/lib/queries.mjs           |  26 +++
 tools/observability/verify-profile-render.mjs |  43 ++++
 9 files changed, 444 insertions(+), 2 deletions(-)
 create mode 100644 packages/cli/src/daemon/backpressure-log.ts
 create mode 100644 packages/cli/test/backpressure-log.test.ts

diff --git a/docs/use-dkg/backpressure-observability.md b/docs/use-dkg/backpressure-observability.md
index 6ea8aec4a6..2e7eb15a8d 100644
--- a/docs/use-dkg/backpressure-observability.md
+++ b/docs/use-dkg/backpressure-observability.md
@@ -141,6 +141,29 @@ Per-item enqueue/start logs are deliberately avoided. Transition and periodic
 summary logging make sustained pressure visible without creating a log storm
 that competes with the overloaded scheduler.
 
+The daemon routes these records through its structured log sink. That keeps a
+full local copy and forwards the same redacted record to enabled syslog and
+OTLP exporters, so the records reach the Loki-backed Grafana dashboards.
+
+## Grafana worker-pressure flame graph
+
+The **DKG Node — Logs** dashboard includes a **Scheduler pressure** row. Its
+flame graph expands the bounded `activeOperations` and `queuedOperations`
+arrays, groups them by scheduler, lane, and operation source, and shows the
+largest sampled `oldestAgeMs` in the selected time range.
+
+The two top-level branches have distinct meanings:
+
+- **active / admitted**: work occupying a worker slot;
+- **queued / waiting**: work still waiting for admission.
+
+Block width is elapsed pressure age in milliseconds. It answers which source
+was present and how old its oldest observed work became. It is not CPU time,
+request share, invocation count, or an exact completed-job duration. The
+monitor intentionally emits transition, periodic summary, and recovery records
+instead of a per-item event stream, so Grafana must not claim more precision
+than the log contract provides.
+
 ## Metrics
 
 The common OpenTelemetry instruments use bounded `scheduler` and `lane`
diff --git a/packages/cli/src/daemon/backpressure-log.ts b/packages/cli/src/daemon/backpressure-log.ts
new file mode 100644
index 0000000000..a454aadb47
--- /dev/null
+++ b/packages/cli/src/daemon/backpressure-log.ts
@@ -0,0 +1,23 @@
+import type { OperationContext } from '@origintrail-official/dkg-core';
+
+export interface BackpressureLogWriter {
+  info(context: OperationContext, message: string): void;
+  warn(context: OperationContext, message: string): void;
+}
+
+/**
+ * Route monitor output through the structured daemon logger.
+ *
+ * The structured logger is the single path that persists records to the
+ * dashboard database and forwards redacted copies to syslog/OTLP. Keeping this
+ * adapter separate makes the remote-observability wiring testable without
+ * starting a daemon.
+ */
+export function createBackpressureLogEmitter(
+  logger: BackpressureLogWriter,
+  context: OperationContext,
+): (level: 'info' | 'warn', message: string) => void {
+  return (level, message) => {
+    logger[level](context, message);
+  };
+}
diff --git a/packages/cli/src/daemon/lifecycle.ts b/packages/cli/src/daemon/lifecycle.ts
index b0fae05696..6a54f04c53 100644
--- a/packages/cli/src/daemon/lifecycle.ts
+++ b/packages/cli/src/daemon/lifecycle.ts
@@ -150,6 +150,7 @@ import {
   resolveMetricsCollectorConfig,
 } from '../metrics-collector-config.js';
 import { createDaemonLogSink } from './log-sink.js';
+import { createBackpressureLogEmitter } from './backpressure-log.js';
 import { startRpcUsageTelemetry } from './rpc-usage-log.js';
 import { startDashboardLogVolumePruner } from './dashboard-log-volume-pruner.js';
 import { SqliteSnapshotPageIndexStore } from './snapshot-page-index-store.js';
@@ -1172,7 +1173,10 @@ export async function runDaemonInner(
     appendFile(logFile, line + "\n").catch(() => {});
   }
   const backpressureMonitor = new BackpressureMonitor({
-    emit: (level, message) => log(`[${level}] ${message}`),
+    emit: createBackpressureLogEmitter(
+      new Logger('backpressure'),
+      createOperationContext('system'),
+    ),
   });
 
   configureApiQueryPriority(process.env.DKG_API_QUERY_PRIORITY, {
diff --git a/packages/cli/test/backpressure-log.test.ts b/packages/cli/test/backpressure-log.test.ts
new file mode 100644
index 0000000000..dc85cbf554
--- /dev/null
+++ b/packages/cli/test/backpressure-log.test.ts
@@ -0,0 +1,29 @@
+import { describe, expect, it, vi } from 'vitest';
+import type { OperationContext } from '@origintrail-official/dkg-core';
+import { createBackpressureLogEmitter } from '../src/daemon/backpressure-log.js';
+
+describe('backpressure log bridge', () => {
+  it('routes warning and recovery records through the structured logger', () => {
+    const context: OperationContext = {
+      operationId: 'backpressure-monitor',
+      operationName: 'system',
+    };
+    const logger = {
+      info: vi.fn(),
+      warn: vi.fn(),
+    };
+    const emit = createBackpressureLogEmitter(logger, context);
+
+    emit('warn', '[backpressure] {"event":"transition"}');
+    emit('info', '[backpressure] {"event":"recovered"}');
+
+    expect(logger.warn).toHaveBeenCalledWith(
+      context,
+      '[backpressure] {"event":"transition"}',
+    );
+    expect(logger.info).toHaveBeenCalledWith(
+      context,
+      '[backpressure] {"event":"recovered"}',
+    );
+  });
+});
diff --git a/packages/cli/vitest.unit.config.ts b/packages/cli/vitest.unit.config.ts
index c1dee3b3fe..ad87481df4 100644
--- a/packages/cli/vitest.unit.config.ts
+++ b/packages/cli/vitest.unit.config.ts
@@ -31,6 +31,7 @@ export default defineConfig({
           'test/config.test.ts',
           'test/status-route-rpc.test.ts',
           'test/backpressure-route.test.ts',
+          'test/backpressure-log.test.ts',
       'test/status-route-store-quads.test.ts',
       'test/query-route-lifecycle.test.ts',
           'test/status-command-store.test.ts',
diff --git a/tools/observability/grafana-dashboard-dkg-node-logs.json b/tools/observability/grafana-dashboard-dkg-node-logs.json
index 0bdccacf73..683bb98ced 100644
--- a/tools/observability/grafana-dashboard-dkg-node-logs.json
+++ b/tools/observability/grafana-dashboard-dkg-node-logs.json
@@ -204,6 +204,215 @@
           "expr": "sum(sum_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `rpc_usage` | logfmt | method != `` | unwrap count [$__auto]))"
         }
       ]
+    },
+    {
+      "title": "Scheduler pressure",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 34
+      },
+      "type": "row",
+      "collapsed": false
+    },
+    {
+      "title": "",
+      "gridPos": {
+        "h": 3,
+        "w": 24,
+        "x": 0,
+        "y": 35
+      },
+      "type": "text",
+      "options": {
+        "mode": "markdown",
+        "content": "**Worker queue diagnostics (PR #2003).** The flame graph reads structured `[backpressure]` records from Loki. A leaf answers **which scheduler/lane/operation produced the work** and shows the **largest sampled elapsed age** observed in the selected time range. Active and queued work are separate branches. These sparse transition/summary samples describe pressure; they are not CPU profiles, invocation counts, or exact end-to-end job durations."
+      },
+      "transparent": true
+    },
+    {
+      "title": "Worker queue pressure flame graph — $node",
+      "gridPos": {
+        "h": 14,
+        "w": 24,
+        "x": 0,
+        "y": 38
+      },
+      "datasource": {
+        "type": "loki",
+        "uid": "${loki}"
+      },
+      "type": "flamegraph",
+      "description": "Each leaf is a PR #2003 worker source. Width is its peak sampled oldest elapsed age in the selected range (milliseconds), split between admitted/active work and queued work. This is pressure age, not CPU utilization, request share, or exact completed-job runtime.",
+      "options": {
+        "showFlameGraphOnly": false
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms"
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${loki}"
+          },
+          "refId": "A",
+          "expr": "sum((max by (scheduler, lane, phase, operation) ((max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[0].operation\", age=\"activeOperations[0].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`0` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[1].operation\", age=\"activeOperations[1].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`1` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[2].operation\", age=\"activeOperations[2].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`2` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[3].operation\", age=\"activeOperations[3].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`3` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[4].operation\", age=\"activeOperations[4].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`4` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[5].operation\", age=\"activeOperations[5].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`5` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[6].operation\", age=\"activeOperations[6].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`6` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[7].operation\", age=\"activeOperations[7].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`7` | unwrap age | __error__ = `` [$__range])))) or (max by (scheduler, lane, phase, operation) ((max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[0].operation\", age=\"queuedOperations[0].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`0` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[1].operation\", age=\"queuedOperations[1].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`1` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[2].operation\", age=\"queuedOperations[2].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`2` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[3].operation\", age=\"queuedOperations[3].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`3` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[4].operation\", age=\"queuedOperations[4].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`4` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[5].operation\", age=\"queuedOperations[5].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`5` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[6].operation\", age=\"queuedOperations[6].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`6` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[7].operation\", age=\"queuedOperations[7].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`7` | unwrap age | __error__ = `` [$__range])))))",
+          "legendFormat": "0|0|all sampled worker pressure",
+          "instant": true,
+          "queryType": "instant"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${loki}"
+          },
+          "refId": "B",
+          "expr": "sum(max by (scheduler, lane, phase, operation) ((max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[0].operation\", age=\"activeOperations[0].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`0` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[1].operation\", age=\"activeOperations[1].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`1` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[2].operation\", age=\"activeOperations[2].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`2` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[3].operation\", age=\"activeOperations[3].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`3` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[4].operation\", age=\"activeOperations[4].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`4` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[5].operation\", age=\"activeOperations[5].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`5` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[6].operation\", age=\"activeOperations[6].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`6` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[7].operation\", age=\"activeOperations[7].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`7` | unwrap age | __error__ = `` [$__range]))))",
+          "legendFormat": "1|0|active / admitted",
+          "instant": true,
+          "queryType": "instant"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${loki}"
+          },
+          "refId": "C",
+          "expr": "max by (scheduler, lane, phase, operation) ((max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[0].operation\", age=\"activeOperations[0].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`0` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[1].operation\", age=\"activeOperations[1].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`1` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[2].operation\", age=\"activeOperations[2].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`2` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[3].operation\", age=\"activeOperations[3].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`3` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[4].operation\", age=\"activeOperations[4].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`4` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[5].operation\", age=\"activeOperations[5].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`5` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[6].operation\", age=\"activeOperations[6].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`6` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[7].operation\", age=\"activeOperations[7].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`7` | unwrap age | __error__ = `` [$__range])))",
+          "legendFormat": "2|1|{{scheduler}}/{{lane}}/{{operation}}",
+          "instant": true,
+          "queryType": "instant"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${loki}"
+          },
+          "refId": "D",
+          "expr": "sum(max by (scheduler, lane, phase, operation) ((max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[0].operation\", age=\"queuedOperations[0].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`0` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[1].operation\", age=\"queuedOperations[1].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`1` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[2].operation\", age=\"queuedOperations[2].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`2` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[3].operation\", age=\"queuedOperations[3].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`3` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[4].operation\", age=\"queuedOperations[4].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`4` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[5].operation\", age=\"queuedOperations[5].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`5` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[6].operation\", age=\"queuedOperations[6].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`6` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[7].operation\", age=\"queuedOperations[7].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`7` | unwrap age | __error__ = `` [$__range]))))",
+          "legendFormat": "1|0|queued / waiting",
+          "instant": true,
+          "queryType": "instant"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${loki}"
+          },
+          "refId": "E",
+          "expr": "max by (scheduler, lane, phase, operation) ((max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[0].operation\", age=\"queuedOperations[0].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`0` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[1].operation\", age=\"queuedOperations[1].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`1` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[2].operation\", age=\"queuedOperations[2].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`2` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[3].operation\", age=\"queuedOperations[3].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`3` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[4].operation\", age=\"queuedOperations[4].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`4` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[5].operation\", age=\"queuedOperations[5].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`5` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[6].operation\", age=\"queuedOperations[6].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`6` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[7].operation\", age=\"queuedOperations[7].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`7` | unwrap age | __error__ = `` [$__range])))",
+          "legendFormat": "2|1|{{scheduler}}/{{lane}}/{{operation}}",
+          "instant": true,
+          "queryType": "instant"
+        }
+      ],
+      "transformations": [
+        {
+          "id": "seriesToRows",
+          "options": {}
+        },
+        {
+          "id": "extractFields",
+          "options": {
+            "source": "Metric",
+            "format": "regexp",
+            "regexp": "(?<level>\\d+)\\|(?<leaf>[01])\\|(?<label>.*)",
+            "replace": false
+          }
+        },
+        {
+          "id": "convertFieldType",
+          "options": {
+            "conversions": [
+              {
+                "targetField": "level",
+                "destinationType": "number"
+              },
+              {
+                "targetField": "leaf",
+                "destinationType": "number"
+              }
+            ]
+          }
+        },
+        {
+          "id": "calculateField",
+          "options": {
+            "mode": "binary",
+            "binary": {
+              "left": {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Value"
+                }
+              },
+              "operator": "*",
+              "right": {
+                "matcher": {
+                  "id": "byName",
+                  "options": "leaf"
+                }
+              }
+            },
+            "alias": "self",
+            "replaceFields": false
+          }
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Metric": true,
+              "leaf": true
+            },
+            "indexByName": {
+              "level": 0,
+              "label": 1,
+              "Value": 2,
+              "self": 3
+            },
+            "renameByName": {
+              "Value": "value"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "title": "Backpressure transitions and summaries — $node",
+      "gridPos": {
+        "h": 10,
+        "w": 24,
+        "x": 0,
+        "y": 52
+      },
+      "datasource": {
+        "type": "loki",
+        "uid": "${loki}"
+      },
+      "type": "logs",
+      "options": {
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": true,
+        "enableLogDetails": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${loki}"
+          },
+          "refId": "A",
+          "expr": "{service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]`"
+        }
+      ]
     }
   ]
 }
diff --git a/tools/observability/lib/dashboards.mjs b/tools/observability/lib/dashboards.mjs
index 1fdab45a49..b60f2133ec 100644
--- a/tools/observability/lib/dashboards.mjs
+++ b/tools/observability/lib/dashboards.mjs
@@ -4,7 +4,13 @@
 // helpers (row layout, panel constructors, node-identity discovery model)
 // live at module scope. Rendering, CLI handling and --check live in
 // ../generate-observability.mjs.
-import { dkgLogStream, severityIs, severityMatches, RPC_USAGE_PIPELINE } from './queries.mjs';
+import {
+  backpressurePeakAgeByOperation,
+  dkgLogStream,
+  severityIs,
+  severityMatches,
+  RPC_USAGE_PIPELINE,
+} from './queries.mjs';
 
 const LOKI = { type: 'loki', uid: '${loki}' };
 const VM = { type: 'prometheus', uid: '${vm}' };
@@ -68,6 +74,80 @@ const LOGSP = (w, h, title, expr) => ({ w, h, def: { datasource: LOKI, type: 'lo
   targets: [{ datasource: LOKI, refId: 'A', expr }] } });
 const TEXT = (content) => ({ w: 24, h: 3, def: { type: 'text', title: '', options: { mode: 'markdown', content }, transparent: true } });
 const ROW = (title) => ({ w: 24, h: 1, def: { type: 'row', title, collapsed: false } });
+const INSTANT_LOKI_TARGET = (refId, expr, legendFormat) => ({
+  datasource: LOKI,
+  refId,
+  expr,
+  legendFormat,
+  instant: true,
+  queryType: 'instant',
+});
+
+const BACKPRESSURE_FLAME = (stream) => {
+  const active = backpressurePeakAgeByOperation(stream, 'active');
+  const queued = backpressurePeakAgeByOperation(stream, 'queued');
+  return {
+    w: 24,
+    h: 14,
+    def: {
+      datasource: LOKI,
+      type: 'flamegraph',
+      title: 'Worker queue pressure flame graph — $node',
+      description: 'Each leaf is a PR #2003 worker source. Width is its peak sampled oldest elapsed age in the selected range (milliseconds), split between admitted/active work and queued work. This is pressure age, not CPU utilization, request share, or exact completed-job runtime.',
+      options: { showFlameGraphOnly: false },
+      fieldConfig: { defaults: { unit: 'ms' }, overrides: [] },
+      targets: [
+        INSTANT_LOKI_TARGET('A', `sum((${active}) or (${queued}))`, '0|0|all sampled worker pressure'),
+        INSTANT_LOKI_TARGET('B', `sum(${active})`, '1|0|active / admitted'),
+        INSTANT_LOKI_TARGET('C', active, '2|1|{{scheduler}}/{{lane}}/{{operation}}'),
+        INSTANT_LOKI_TARGET('D', `sum(${queued})`, '1|0|queued / waiting'),
+        INSTANT_LOKI_TARGET('E', queued, '2|1|{{scheduler}}/{{lane}}/{{operation}}'),
+      ],
+      transformations: [
+        { id: 'seriesToRows', options: {} },
+        {
+          id: 'extractFields',
+          options: {
+            source: 'Metric',
+            format: 'regexp',
+            regexp: '(?<level>\\d+)\\|(?<leaf>[01])\\|(?<label>.*)',
+            replace: false,
+          },
+        },
+        {
+          id: 'convertFieldType',
+          options: {
+            conversions: [
+              { targetField: 'level', destinationType: 'number' },
+              { targetField: 'leaf', destinationType: 'number' },
+            ],
+          },
+        },
+        {
+          id: 'calculateField',
+          options: {
+            mode: 'binary',
+            binary: {
+              left: { matcher: { id: 'byName', options: 'Value' } },
+              operator: '*',
+              right: { matcher: { id: 'byName', options: 'leaf' } },
+            },
+            alias: 'self',
+            replaceFields: false,
+          },
+        },
+        {
+          id: 'organize',
+          options: {
+            excludeByName: { Time: true, Metric: true, leaf: true },
+            indexByName: { level: 0, label: 1, Value: 2, self: 3 },
+            renameByName: { Value: 'value' },
+          },
+        },
+      ],
+    },
+  };
+};
 
 // Loki query building blocks come from the shared contract (lib/queries.mjs)
 // — the same module alerts.mjs composes from, so the stream selector,
@@ -131,6 +211,10 @@ const buildNodeLogsDashboard = (NODE_IDENTITY) => ({
         options: { reduceOptions: { calcs: ['sum'], fields: '', values: false } },
         targets: [{ datasource: LOKI, refId: 'A', expr: `sum(sum_over_time(${NB}${RPCPIPE}[$__auto]))` }] } },
     ],
+    [ ROW('Scheduler pressure') ],
+    [ TEXT('**Worker queue diagnostics (PR #2003).** The flame graph reads structured `[backpressure]` records from Loki. A leaf answers **which scheduler/lane/operation produced the work** and shows the **largest sampled elapsed age** observed in the selected time range. Active and queued work are separate branches. These sparse transition/summary samples describe pressure; they are not CPU profiles, invocation counts, or exact end-to-end job durations.') ],
+    [ BACKPRESSURE_FLAME(NB) ],
+    [ LOGSP(24, 10, 'Backpressure transitions and summaries — $node', `${NB} |= \`[backpressure]\``) ],
   ]),
 });
 
diff --git a/tools/observability/lib/queries.mjs b/tools/observability/lib/queries.mjs
index f3fa52d7b7..4661941563 100644
--- a/tools/observability/lib/queries.mjs
+++ b/tools/observability/lib/queries.mjs
@@ -24,6 +24,32 @@ export const severityMatches = (regex) => ` | severity_text=~\`${regex}\``;
  *  callers append the range selector directly: `...${RPC_USAGE_PIPELINE}[1h]`. */
 export const RPC_USAGE_PIPELINE = ' |= `rpc_usage` | logfmt | method != `` | unwrap count ';
 
+/**
+ * Build a Loki expression for the peak sampled age of every bounded
+ * backpressure operation in a phase. PR #2003 keeps at most eight operation
+ * summaries per lane, so the query expands those fixed indexes and collapses
+ * them back by source. `slot` prevents LogQL's `or` from dropping an operation
+ * that moved between array positions during the selected time range.
+ *
+ * The returned value is an elapsed age in milliseconds from the diagnostic
+ * sample. It is not CPU time, request count, or a completed-job duration.
+ */
+export const backpressurePeakAgeByOperation = (stream, phase) => {
+  if (phase !== 'active' && phase !== 'queued') {
+    throw new Error(`backpressure phase must be active or queued, got ${phase}`);
+  }
+  const field = phase === 'active' ? 'activeOperations' : 'queuedOperations';
+  const entries = Array.from({ length: 8 }, (_, index) => `max_over_time(${stream}`
+    + ' |= `[backpressure]`'
+    + ' | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})`'
+    + ' | line_format `{{.payload}}`'
+    + ` | json scheduler, lane, operation="${field}[${index}].operation", age="${field}[${index}].oldestAgeMs"`
+    + ' | operation != ``'
+    + ` | label_format phase=\`${phase}\`, slot=\`${index}\``
+    + ' | unwrap age | __error__ = `` [$__range])');
+  return `max by (scheduler, lane, phase, operation) (${entries.map((entry) => `(${entry})`).join(' or ')})`;
+};
+
 /** Loki-side node identity: the label pair every per-node log aggregation
  *  groups by (Tempo's equivalent is resource.service.instance.id). */
 export const LOKI_NODE_GROUP = 'service_instance_id, deployment_environment';
diff --git a/tools/observability/verify-profile-render.mjs b/tools/observability/verify-profile-render.mjs
index 6645ca7a35..5fe2c5dcb4 100644
--- a/tools/observability/verify-profile-render.mjs
+++ b/tools/observability/verify-profile-render.mjs
@@ -145,6 +145,49 @@ const dkgSelectorViolations = (rawExpr, rawNodeFilter) => {
   }
 }
 
+// ── logs dashboard: PR #2003 worker-pressure surface ──────────────────────
+{
+  const dash = load('grafana-dashboard-dkg-node-logs.json');
+  const flame = (dash.panels ?? []).find((panel) => panel.type === 'flamegraph');
+  if (!flame) {
+    fail('logs dashboard', 'worker-pressure flamegraph panel is missing');
+  } else {
+    if (!flame.title?.includes('Worker queue pressure')) {
+      fail('logs dashboard flamegraph', `unexpected title: ${JSON.stringify(flame.title)}`);
+    }
+    if ((flame.targets ?? []).length !== 5) {
+      fail('logs dashboard flamegraph', `expected 5 ordered profile targets, got ${(flame.targets ?? []).length}`);
+    }
+    const refs = (flame.targets ?? []).map((target) => target.refId).join('');
+    if (refs !== 'ABCDE') {
+      fail('logs dashboard flamegraph', `target order is ${JSON.stringify(refs)}, want ABCDE`);
+    }
+    const exprs = (flame.targets ?? []).map((target) => target.expr ?? '').join('\n');
+    if (!exprs.includes('service_instance_id="$node"')) {
+      fail('logs dashboard flamegraph', 'queries are not scoped to the selected node');
+    }
+    if (!exprs.includes('|= `[backpressure]`')) {
+      fail('logs dashboard flamegraph', 'queries do not select PR #2003 backpressure records');
+    }
+    for (const phase of ['activeOperations', 'queuedOperations']) {
+      for (let index = 0; index < 8; index++) {
+        if (!exprs.includes(`${phase}[${index}].operation`)) {
+          fail('logs dashboard flamegraph', `missing bounded ${phase}[${index}] extraction`);
+        }
+      }
+    }
+    const transforms = (flame.transformations ?? []).map((transform) => transform.id);
+    const wanted = ['seriesToRows', 'extractFields', 'convertFieldType', 'calculateField', 'organize'];
+    if (JSON.stringify(transforms) !== JSON.stringify(wanted)) {
+      fail('logs dashboard flamegraph', `transform chain is ${JSON.stringify(transforms)}, want ${JSON.stringify(wanted)}`);
+    }
+  }
+  const raw = (dash.panels ?? []).find((panel) => panel.title?.startsWith('Backpressure transitions'));
+  if (!raw?.targets?.some((target) => target.expr?.includes('|= `[backpressure]`'))) {
+    fail('logs dashboard', 'raw backpressure evidence panel is missing');
+  }
+}
+
 // ── alert payload: datasource UID integrity + profiled rules + routing ─────
 {
   const alertsFile = 'alert-rules.provisioning.json';

From 94cd86efa7395198b8bad033dfa2c724e3751d97 Mon Sep 17 00:00:00 2001
From: Bojan <bojan.colic131@gmail.com>
Date: Mon, 3 Aug 2026 11:03:58 +0200
Subject: [PATCH 2/5] feat(observability): add worker pressure heatmaps

---
 docs/use-dkg/backpressure-observability.md    |  18 +-
 .../grafana-dashboard-dkg-node-logs.json      | 160 +++++++++++++++++-
 tools/observability/lib/dashboards.mjs        |  53 +++++-
 tools/observability/lib/queries.mjs           |  23 +++
 tools/observability/verify-profile-render.mjs |  36 ++++
 5 files changed, 286 insertions(+), 4 deletions(-)

diff --git a/docs/use-dkg/backpressure-observability.md b/docs/use-dkg/backpressure-observability.md
index 2e7eb15a8d..ddad696979 100644
--- a/docs/use-dkg/backpressure-observability.md
+++ b/docs/use-dkg/backpressure-observability.md
@@ -145,7 +145,7 @@ The daemon routes these records through its structured log sink. That keeps a
 full local copy and forwards the same redacted record to enabled syslog and
 OTLP exporters, so the records reach the Loki-backed Grafana dashboards.
 
-## Grafana worker-pressure flame graph
+## Grafana worker-pressure flame graph and heatmaps
 
 The **DKG Node — Logs** dashboard includes a **Scheduler pressure** row. Its
 flame graph expands the bounded `activeOperations` and `queuedOperations`
@@ -164,6 +164,22 @@ monitor intentionally emits transition, periodic summary, and recovery records
 instead of a per-item event stream, so Grafana must not claim more precision
 than the log contract provides.
 
+Two heatmaps below the flame graph keep the active and queued meanings
+separate. Each heatmap groups samples by scheduler/lane and shows the
+distribution over time of the peak sampled age in each Grafana resolution
+bucket:
+
+- **Active / admitted pressure age** shows how old the oldest work occupying a
+  worker slot became;
+- **Queued / waiting pressure age** shows how old the oldest work waiting for
+  admission became.
+
+The heatmaps retain the maximum observed age in each time bucket because the
+source is a sparse transition/summary stream. A darker cell means more samples
+fell into that time/age bucket; it does not mean more CPU was consumed. Empty
+periods mean Loki received no matching diagnostic sample, not necessarily that
+the scheduler was idle.
+
 ## Metrics
 
 The common OpenTelemetry instruments use bounded `scheduler` and `lane`
diff --git a/tools/observability/grafana-dashboard-dkg-node-logs.json b/tools/observability/grafana-dashboard-dkg-node-logs.json
index 683bb98ced..f78fae7bf1 100644
--- a/tools/observability/grafana-dashboard-dkg-node-logs.json
+++ b/tools/observability/grafana-dashboard-dkg-node-logs.json
@@ -227,7 +227,7 @@
       "type": "text",
       "options": {
         "mode": "markdown",
-        "content": "**Worker queue diagnostics (PR #2003).** The flame graph reads structured `[backpressure]` records from Loki. A leaf answers **which scheduler/lane/operation produced the work** and shows the **largest sampled elapsed age** observed in the selected time range. Active and queued work are separate branches. These sparse transition/summary samples describe pressure; they are not CPU profiles, invocation counts, or exact end-to-end job durations."
+        "content": "**Worker queue diagnostics (PR #2003).** The flame graph answers **which scheduler/lane/operation produced the pressure**; the two heatmaps show **when active and queued pressure accumulated and how old it became**. All three views read the same bounded structured `[backpressure]` records from Loki. These sparse transition/summary samples describe elapsed pressure age; they are not CPU profiles, invocation counts, or exact end-to-end job durations."
       },
       "transparent": true
     },
@@ -384,13 +384,169 @@
         }
       ]
     },
+    {
+      "title": "Active / admitted pressure age heatmap — $node",
+      "gridPos": {
+        "h": 12,
+        "w": 12,
+        "x": 0,
+        "y": 52
+      },
+      "datasource": {
+        "type": "loki",
+        "uid": "${loki}"
+      },
+      "type": "heatmap",
+      "description": "Distribution over time of the peak sampled age of the oldest work occupying a worker slot, grouped by scheduler/lane. Values come from sparse PR #2003 transition and summary records; they are sampled pressure age in milliseconds, not CPU time or exact job duration.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms"
+        },
+        "overrides": []
+      },
+      "options": {
+        "calculate": true,
+        "calculation": {
+          "xBuckets": {
+            "mode": "count",
+            "value": "60"
+          },
+          "yBuckets": {
+            "mode": "count",
+            "value": "20",
+            "scale": {
+              "type": "linear"
+            }
+          }
+        },
+        "cellGap": 1,
+        "cellValues": {},
+        "color": {
+          "mode": "scheme",
+          "scheme": "Reds",
+          "steps": 64,
+          "reverse": false,
+          "scale": "exponential",
+          "exponent": 0.5
+        },
+        "filterValues": {
+          "le": 0
+        },
+        "legend": {
+          "show": true
+        },
+        "rowsFrame": {
+          "layout": "auto"
+        },
+        "tooltip": {
+          "mode": "single",
+          "showColorScale": true,
+          "yHistogram": false
+        },
+        "yAxis": {
+          "axisPlacement": "left",
+          "reverse": false,
+          "unit": "ms"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${loki}"
+          },
+          "refId": "A",
+          "expr": "max by (scheduler, lane) (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, age=\"oldestActiveAgeMs\" | unwrap age | __error__ = `` [$__auto]))",
+          "legendFormat": "{{scheduler}}/{{lane}}",
+          "queryType": "range"
+        }
+      ]
+    },
+    {
+      "title": "Queued / waiting pressure age heatmap — $node",
+      "gridPos": {
+        "h": 12,
+        "w": 12,
+        "x": 12,
+        "y": 52
+      },
+      "datasource": {
+        "type": "loki",
+        "uid": "${loki}"
+      },
+      "type": "heatmap",
+      "description": "Distribution over time of the peak sampled age of the oldest work waiting for admission, grouped by scheduler/lane. Values come from sparse PR #2003 transition and summary records; they are sampled pressure age in milliseconds, not CPU time or exact job duration.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms"
+        },
+        "overrides": []
+      },
+      "options": {
+        "calculate": true,
+        "calculation": {
+          "xBuckets": {
+            "mode": "count",
+            "value": "60"
+          },
+          "yBuckets": {
+            "mode": "count",
+            "value": "20",
+            "scale": {
+              "type": "linear"
+            }
+          }
+        },
+        "cellGap": 1,
+        "cellValues": {},
+        "color": {
+          "mode": "scheme",
+          "scheme": "Oranges",
+          "steps": 64,
+          "reverse": false,
+          "scale": "exponential",
+          "exponent": 0.5
+        },
+        "filterValues": {
+          "le": 0
+        },
+        "legend": {
+          "show": true
+        },
+        "rowsFrame": {
+          "layout": "auto"
+        },
+        "tooltip": {
+          "mode": "single",
+          "showColorScale": true,
+          "yHistogram": false
+        },
+        "yAxis": {
+          "axisPlacement": "left",
+          "reverse": false,
+          "unit": "ms"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${loki}"
+          },
+          "refId": "A",
+          "expr": "max by (scheduler, lane) (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, age=\"oldestQueuedAgeMs\" | unwrap age | __error__ = `` [$__auto]))",
+          "legendFormat": "{{scheduler}}/{{lane}}",
+          "queryType": "range"
+        }
+      ]
+    },
     {
       "title": "Backpressure transitions and summaries — $node",
       "gridPos": {
         "h": 10,
         "w": 24,
         "x": 0,
-        "y": 52
+        "y": 64
       },
       "datasource": {
         "type": "loki",
diff --git a/tools/observability/lib/dashboards.mjs b/tools/observability/lib/dashboards.mjs
index b60f2133ec..0c39930086 100644
--- a/tools/observability/lib/dashboards.mjs
+++ b/tools/observability/lib/dashboards.mjs
@@ -5,6 +5,7 @@
 // live at module scope. Rendering, CLI handling and --check live in
 // ../generate-observability.mjs.
 import {
+  backpressurePeakAgeOverTime,
   backpressurePeakAgeByOperation,
   dkgLogStream,
   severityIs,
@@ -149,6 +150,55 @@ const BACKPRESSURE_FLAME = (stream) => {
   };
 };
 
+const BACKPRESSURE_HEATMAP = (stream, phase) => {
+  if (phase !== 'active' && phase !== 'queued') {
+    throw new Error(`backpressure heatmap phase must be active or queued, got ${phase}`);
+  }
+  const active = phase === 'active';
+  const label = active ? 'Active / admitted' : 'Queued / waiting';
+  const meaning = active ? 'oldest work occupying a worker slot' : 'oldest work waiting for admission';
+  return {
+    w: 12,
+    h: 12,
+    def: {
+      datasource: LOKI,
+      type: 'heatmap',
+      title: `${label} pressure age heatmap — $node`,
+      description: `Distribution over time of the peak sampled age of the ${meaning}, grouped by scheduler/lane. Values come from sparse PR #2003 transition and summary records; they are sampled pressure age in milliseconds, not CPU time or exact job duration.`,
+      fieldConfig: { defaults: { unit: 'ms' }, overrides: [] },
+      options: {
+        calculate: true,
+        calculation: {
+          xBuckets: { mode: 'count', value: '60' },
+          yBuckets: { mode: 'count', value: '20', scale: { type: 'linear' } },
+        },
+        cellGap: 1,
+        cellValues: {},
+        color: {
+          mode: 'scheme',
+          scheme: active ? 'Reds' : 'Oranges',
+          steps: 64,
+          reverse: false,
+          scale: 'exponential',
+          exponent: 0.5,
+        },
+        filterValues: { le: 0 },
+        legend: { show: true },
+        rowsFrame: { layout: 'auto' },
+        tooltip: { mode: 'single', showColorScale: true, yHistogram: false },
+        yAxis: { axisPlacement: 'left', reverse: false, unit: 'ms' },
+      },
+      targets: [{
+        datasource: LOKI,
+        refId: 'A',
+        expr: backpressurePeakAgeOverTime(stream, phase),
+        legendFormat: '{{scheduler}}/{{lane}}',
+        queryType: 'range',
+      }],
+    },
+  };
+};
+
 // Loki query building blocks come from the shared contract (lib/queries.mjs)
 // — the same module alerts.mjs composes from, so the stream selector,
 // severity filter shape and rpc_usage pipeline cannot drift between the
@@ -212,8 +262,9 @@ const buildNodeLogsDashboard = (NODE_IDENTITY) => ({
         targets: [{ datasource: LOKI, refId: 'A', expr: `sum(sum_over_time(${NB}${RPCPIPE}[$__auto]))` }] } },
     ],
     [ ROW('Scheduler pressure') ],
-    [ TEXT('**Worker queue diagnostics (PR #2003).** The flame graph reads structured `[backpressure]` records from Loki. A leaf answers **which scheduler/lane/operation produced the work** and shows the **largest sampled elapsed age** observed in the selected time range. Active and queued work are separate branches. These sparse transition/summary samples describe pressure; they are not CPU profiles, invocation counts, or exact end-to-end job durations.') ],
+    [ TEXT('**Worker queue diagnostics (PR #2003).** The flame graph answers **which scheduler/lane/operation produced the pressure**; the two heatmaps show **when active and queued pressure accumulated and how old it became**. All three views read the same bounded structured `[backpressure]` records from Loki. These sparse transition/summary samples describe elapsed pressure age; they are not CPU profiles, invocation counts, or exact end-to-end job durations.') ],
     [ BACKPRESSURE_FLAME(NB) ],
+    [ BACKPRESSURE_HEATMAP(NB, 'active'), BACKPRESSURE_HEATMAP(NB, 'queued') ],
     [ LOGSP(24, 10, 'Backpressure transitions and summaries — $node', `${NB} |= \`[backpressure]\``) ],
   ]),
 });
diff --git a/tools/observability/lib/queries.mjs b/tools/observability/lib/queries.mjs
index 4661941563..38c47ecf52 100644
--- a/tools/observability/lib/queries.mjs
+++ b/tools/observability/lib/queries.mjs
@@ -50,6 +50,29 @@ export const backpressurePeakAgeByOperation = (stream, phase) => {
   return `max by (scheduler, lane, phase, operation) (${entries.map((entry) => `(${entry})`).join(' or ')})`;
 };
 
+/**
+ * Build a Loki range expression for the peak sampled scheduler/lane age in
+ * each Grafana resolution bucket. The heatmap uses these time-series samples
+ * to calculate its distribution client-side. `max_over_time` is deliberate:
+ * backpressure records are sparse transition/summary/recovery samples, so the
+ * most severe observed age in a bucket is the honest value to retain.
+ *
+ * The returned value is sampled elapsed age in milliseconds. It is not CPU
+ * time, request share, or an exact completed-job duration.
+ */
+export const backpressurePeakAgeOverTime = (stream, phase) => {
+  if (phase !== 'active' && phase !== 'queued') {
+    throw new Error(`backpressure phase must be active or queued, got ${phase}`);
+  }
+  const field = phase === 'active' ? 'oldestActiveAgeMs' : 'oldestQueuedAgeMs';
+  return `max by (scheduler, lane) (max_over_time(${stream}`
+    + ' |= `[backpressure]`'
+    + ' | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})`'
+    + ' | line_format `{{.payload}}`'
+    + ` | json scheduler, lane, age="${field}"`
+    + ' | unwrap age | __error__ = `` [$__auto]))';
+};
+
 /** Loki-side node identity: the label pair every per-node log aggregation
  *  groups by (Tempo's equivalent is resource.service.instance.id). */
 export const LOKI_NODE_GROUP = 'service_instance_id, deployment_environment';
diff --git a/tools/observability/verify-profile-render.mjs b/tools/observability/verify-profile-render.mjs
index 5fe2c5dcb4..4bb1601255 100644
--- a/tools/observability/verify-profile-render.mjs
+++ b/tools/observability/verify-profile-render.mjs
@@ -186,6 +186,42 @@ const dkgSelectorViolations = (rawExpr, rawNodeFilter) => {
   if (!raw?.targets?.some((target) => target.expr?.includes('|= `[backpressure]`'))) {
     fail('logs dashboard', 'raw backpressure evidence panel is missing');
   }
+
+  const heatmaps = (dash.panels ?? []).filter((panel) => panel.type === 'heatmap');
+  if (heatmaps.length !== 2) {
+    fail('logs dashboard', `expected 2 worker-pressure heatmaps, got ${heatmaps.length}`);
+  }
+  for (const phase of ['Active / admitted', 'Queued / waiting']) {
+    const heatmap = heatmaps.find((panel) => panel.title?.startsWith(phase));
+    const field = phase.startsWith('Active') ? 'oldestActiveAgeMs' : 'oldestQueuedAgeMs';
+    const where = `logs dashboard ${phase.toLowerCase()} heatmap`;
+    if (!heatmap) {
+      fail(where, 'panel is missing');
+      continue;
+    }
+    if (heatmap.options?.calculate !== true) {
+      fail(where, 'must calculate heatmap buckets from the Loki time series');
+    }
+    if (heatmap.fieldConfig?.defaults?.unit !== 'ms' || heatmap.options?.yAxis?.unit !== 'ms') {
+      fail(where, 'sampled pressure age must be rendered in milliseconds');
+    }
+    if ((heatmap.targets ?? []).length !== 1) {
+      fail(where, `expected one range target, got ${(heatmap.targets ?? []).length}`);
+      continue;
+    }
+    const [target] = heatmap.targets;
+    const expr = target.expr ?? '';
+    if (target.queryType !== 'range') fail(where, `queryType is ${JSON.stringify(target.queryType)}, want range`);
+    if (!expr.includes('service_instance_id="$node"')) fail(where, 'query is not scoped to the selected node');
+    if (!expr.includes('|= `[backpressure]`')) fail(where, 'query does not select PR #2003 backpressure records');
+    if (!expr.includes(`age="${field}"`)) fail(where, `query does not extract ${field}`);
+    if (!expr.includes('max_over_time(') || !expr.includes('[$__auto]')) {
+      fail(where, 'query must retain the peak sparse sample in each Grafana resolution bucket');
+    }
+    if (target.legendFormat !== '{{scheduler}}/{{lane}}') {
+      fail(where, `legend is ${JSON.stringify(target.legendFormat)}, want scheduler/lane`);
+    }
+  }
 }
 
 // ── alert payload: datasource UID integrity + profiled rules + routing ─────

From de3cd818452a4e321ae622c7ae239811decb5ce3 Mon Sep 17 00:00:00 2001
From: Bojan <bojan.colic131@gmail.com>
Date: Mon, 3 Aug 2026 11:27:32 +0200
Subject: [PATCH 3/5] feat(observability): visualize W1 sync cost

---
 .github/workflows/observability-artifacts.yml |    4 +-
 docs/use-dkg/backpressure-observability.md    |   42 +
 tools/observability/RUNBOOK.md                |    1 +
 .../observability/generate-observability.mjs  |    5 +-
 .../grafana-dashboard-dkg-sync-cost.json      | 1671 +++++++++++++++++
 tools/observability/lib/dashboards.mjs        |  247 +++
 tools/observability/lib/w1.mjs                |  117 ++
 tools/observability/verify-profile-render.mjs |  138 ++
 .../observability/w1/w1-dashboard-rules.yaml  |   70 +
 9 files changed, 2291 insertions(+), 4 deletions(-)
 create mode 100644 tools/observability/grafana-dashboard-dkg-sync-cost.json
 create mode 100644 tools/observability/w1/w1-dashboard-rules.yaml

diff --git a/.github/workflows/observability-artifacts.yml b/.github/workflows/observability-artifacts.yml
index c65e35de24..480094738c 100644
--- a/.github/workflows/observability-artifacts.yml
+++ b/.github/workflows/observability-artifacts.yml
@@ -106,12 +106,12 @@ jobs:
       # contract. `--entrypoint promtool` is REQUIRED: the image entrypoint is
       # ["/bin/prometheus"], so the un-overridden form fails with
       # `prometheus: error: unexpected promtool`.
-      - name: Parse the W1 rule fixture (pinned promtool)
+      - name: Parse the W1 decision and Grafana-dashboard rule fixtures (pinned promtool)
         run: |
           set -euo pipefail
           docker run --rm -v "${PWD}/tools/observability:/w" --entrypoint promtool \
             prom/prometheus@sha256:6559acbd5d770b15bb3c954629ce190ac3cbbdb2b7f1c30f0385c4e05104e218 \
-            check rules /w/w1/w1-rules.yaml
+            check rules /w/w1/w1-rules.yaml /w/w1/w1-dashboard-rules.yaml
 
       # `check rules` proves the expressions PARSE. It is blind to what they
       # RETURN, and the defect that motivated these tests parsed perfectly:
diff --git a/docs/use-dkg/backpressure-observability.md b/docs/use-dkg/backpressure-observability.md
index d6b1c3e99b..6e761d79e6 100644
--- a/docs/use-dkg/backpressure-observability.md
+++ b/docs/use-dkg/backpressure-observability.md
@@ -217,6 +217,48 @@ fell into that time/age bucket; it does not mean more CPU was consumed. Empty
 periods mean Loki received no matching diagnostic sample, not necessarily that
 the scheduler was idle.
 
+## Grafana source-attributed sync cost
+
+The **DKG Nodes — Sync Cost** dashboard is the interactive Grafana companion
+to the W1 measurement contract introduced by PR #2033. It reads OpenTelemetry
+metrics from VictoriaMetrics and keeps the existing Loki pressure dashboard
+intact: logs answer what was present in a bounded pressure snapshot, while the
+metrics dashboard measures completed work, transferred payload, admission
+pressure, and catch-up outcomes over time.
+
+The dashboard covers every W1 instrument:
+
+| Instrument | Dashboard view |
+| --- | --- |
+| I1 physical sync attempts | Rate by source, outcome, transport, plane, and phase |
+| I2/I3 request and response payload bytes | Throughput by source and response outcome |
+| I4 logical operation duration | Source/lane/outcome flame graph, active-worker equivalents, and duration heatmap |
+| I5 rejected operations | Rate by source, lane, and bounded reason |
+| I6 single-flight joins | Rate by scope, owner source, and joining source |
+| I7 catch-up requests | Rate by route result and shared-memory request flag |
+| I8 catch-up jobs | Rate by terminal status and admission path |
+| I9 catch-up job duration | Walk-job p95 and duration heatmap |
+
+The flame-graph width is accumulated **active wall-clock occupancy** for
+completed logical sync operations in the selected Grafana range. It is not CPU
+time. Operations rejected before they start never receive a zero-duration I4
+sample; they appear in I5 instead.
+
+The two Prometheus heatmaps read real histogram buckets rather than estimating
+a distribution from log samples:
+
+- logical sync operation duration uses I4 and follows the selected
+  source/lane/outcome filters;
+- walk catch-up duration uses I9 and deliberately excludes synthetic
+  already-ready jobs, which perform no work.
+
+The dashboard also renders the W1 evidence gates and source-family shares over
+the selected Grafana range. Those interactive panels are for investigation;
+the generated `tools/observability/w1/w1-queries.md` packet remains the fixed
+1-hour/2-hour decision contract. Any failed evidence gate makes the window
+**inconclusive** rather than healthy, and the byte counters describe encoded
+application payload rather than network-wire bandwidth.
+
 ## Metrics
 
 The common OpenTelemetry instruments use bounded `scheduler` and `lane`
diff --git a/tools/observability/RUNBOOK.md b/tools/observability/RUNBOOK.md
index cb2b92eed3..9c10c23c91 100644
--- a/tools/observability/RUNBOOK.md
+++ b/tools/observability/RUNBOOK.md
@@ -15,6 +15,7 @@ import via `POST /api/dashboards/db {dashboard, folderUid, overwrite:true}`):
 | `dkg-fleet-logs` | `grafana-dashboard-dkg-fleet-logs.json` | logs (live) |
 | `dkg-node-logs` | `grafana-dashboard-dkg-node-logs.json` | logs (live) |
 | `dkg-node-metrics` | `grafana-dashboard-dkg-node-metrics.json` | node metrics endpoint + collector→VictoriaMetrics route (collector self-monitoring row is live already; the two raw-RPC panels additionally need nodes on a post-PR-#1409 build, which ships `dkg.chain.rpc.requests.total`) |
+| `dkg-sync-cost` | `grafana-dashboard-dkg-sync-cost.json` | node metrics endpoint + collector→VictoriaMetrics route + a node build containing PR #2033 W1 instruments I1–I9 |
 | `dkg-node-traces` | `grafana-dashboard-dkg-node-traces.json` | node traces endpoint + collector→Tempo route |
 
 Datasources are template variables (`loki` / `vm` / `tempo`) — the dashboards
diff --git a/tools/observability/generate-observability.mjs b/tools/observability/generate-observability.mjs
index d5fb80e59c..c7dd274354 100644
--- a/tools/observability/generate-observability.mjs
+++ b/tools/observability/generate-observability.mjs
@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 // Source of truth for the DKG observability Grafana artifacts.
 //
-// Emits the four dashboard JSONs, alert-rules.provisioning.json,
+// Emits the five dashboard JSONs, alert-rules.provisioning.json,
 // example-alerts.md and the W1 sync-measurement artifacts (w1/) in this
 // directory. Edit the lib/ sources, regenerate, commit both — never hand-edit
 // a rendered artifact.
@@ -53,7 +53,7 @@ assertPromLabel(PROM_NODE_LABEL, usage);
 const nodeProfile = promNodeProfile(PROM_NODE_LABEL);
 
 
-const { fleet, nodeLogs, metrics, traces } = buildDashboards({ nodeProfile });
+const { fleet, nodeLogs, metrics, syncCost, traces } = buildDashboards({ nodeProfile });
 const { alerts, specs, routes } = buildAlerts({ nodeProfile, VM_UID, LOKI_UID });
 const docs = buildDocs({ specs, routes });
 // W1 sync-measurement decision queries (lib/w1.mjs) render into a NESTED key
@@ -70,6 +70,7 @@ const rendered = new Map([
   ['grafana-dashboard-dkg-fleet-logs.json', JSON.stringify(fleet, null, 2) + '\n'],
   ['grafana-dashboard-dkg-node-logs.json', JSON.stringify(nodeLogs, null, 2) + '\n'],
   ['grafana-dashboard-dkg-node-metrics.json', JSON.stringify(metrics, null, 2) + '\n'],
+  ['grafana-dashboard-dkg-sync-cost.json', JSON.stringify(syncCost, null, 2) + '\n'],
   ['grafana-dashboard-dkg-node-traces.json', JSON.stringify(traces, null, 2) + '\n'],
   ['alert-rules.provisioning.json', JSON.stringify(alerts, null, 2) + '\n'],
   ...Object.entries(docs),
diff --git a/tools/observability/grafana-dashboard-dkg-sync-cost.json b/tools/observability/grafana-dashboard-dkg-sync-cost.json
new file mode 100644
index 0000000000..129d69e007
--- /dev/null
+++ b/tools/observability/grafana-dashboard-dkg-sync-cost.json
@@ -0,0 +1,1671 @@
+{
+  "uid": "dkg-sync-cost",
+  "title": "DKG Nodes — Sync Cost",
+  "timezone": "browser",
+  "refresh": "1m",
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "tags": [
+    "dkg",
+    "metrics",
+    "sync"
+  ],
+  "links": [
+    {
+      "type": "dashboards",
+      "tags": [
+        "dkg"
+      ],
+      "asDropdown": true,
+      "title": "DKG dashboards",
+      "includeVars": false,
+      "keepTime": true
+    }
+  ],
+  "description": "PR #2033 W1 source-attributed sync telemetry (I1–I9). Node label profile: instance. The dashboard is an interactive operational view; the generated w1/w1-queries.md remains the fixed 1 h / 2 h decision contract.",
+  "templating": {
+    "list": [
+      {
+        "name": "vm",
+        "type": "datasource",
+        "query": "prometheus",
+        "label": "Metrics DS",
+        "hide": 2,
+        "current": {}
+      },
+      {
+        "name": "node",
+        "label": "Node",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${vm}"
+        },
+        "query": "label_values({__name__=~\"dkg_.+\"}, instance)",
+        "refresh": 2,
+        "includeAll": true,
+        "multi": true,
+        "allValue": ".+",
+        "allowCustomValue": true,
+        "current": {
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        }
+      },
+      {
+        "name": "sync_source",
+        "label": "Sync source",
+        "type": "custom",
+        "query": "catchup-foreground,on-connect,reconcile,catchup-background,vm-recovery,swm-recovery,control-plane,unspecified",
+        "includeAll": true,
+        "multi": true,
+        "allValue": ".+",
+        "current": {
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        }
+      },
+      {
+        "name": "sync_lane",
+        "label": "Operation lane",
+        "type": "custom",
+        "query": "durable,changelog,shared_memory,swm_recovery",
+        "includeAll": true,
+        "multi": true,
+        "allValue": ".+",
+        "current": {
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        }
+      },
+      {
+        "name": "sync_outcome",
+        "label": "Operation outcome",
+        "type": "custom",
+        "query": "resolved,error,cancelled",
+        "includeAll": true,
+        "multi": true,
+        "allValue": ".+",
+        "current": {
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        }
+      }
+    ]
+  },
+  "panels": [
+    {
+      "title": "",
+      "gridPos": {
+        "h": 3,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "type": "text",
+      "options": {
+        "mode": "markdown",
+        "content": "**Requires the extended sync diagnostics from PR #2033 and node metric export.** The flame graph and duration heatmaps use real OpenTelemetry counters/histograms, while the existing **DKG Node — Logs** pressure views continue to use Loki transition/summary records. Filter the interactive operation views with the source/lane/outcome controls above. The evidence and materiality panels intentionally retain W1's fixed durable filters and source-family rules; a failed gate means **inconclusive**, not healthy."
+      },
+      "transparent": true
+    },
+    {
+      "title": "Source-attributed sync cost (I1–I4)",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 3
+      },
+      "type": "row",
+      "collapsed": false
+    },
+    {
+      "title": "Source-attributed sync active occupancy — $node",
+      "gridPos": {
+        "h": 14,
+        "w": 24,
+        "x": 0,
+        "y": 4
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "flamegraph",
+      "description": "Hierarchy is source → requester lane → terminal outcome. Width is completed logical-sync active wall-clock occupancy accumulated over the selected Grafana range. It is admitted occupancy, not CPU time, and excludes rejected-before-start operations (shown separately).",
+      "options": {
+        "showFlameGraphOnly": false
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms"
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "sum(increase({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\"${sync_lane:regex}\", source=~\"${sync_source:regex}\", outcome=~\"${sync_outcome:regex}\", instance=~\"${node:regex}\"}[$__range]))",
+          "legendFormat": "0|0|all selected sync occupancy",
+          "instant": true,
+          "range": false,
+          "queryType": "instant",
+          "editorMode": "code"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "B",
+          "expr": "sum by (source) (increase({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\"${sync_lane:regex}\", source=~\"${sync_source:regex}\", outcome=~\"${sync_outcome:regex}\", instance=~\"${node:regex}\"}[$__range]))",
+          "legendFormat": "1|0|{{source}}",
+          "instant": true,
+          "range": false,
+          "queryType": "instant",
+          "editorMode": "code"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "C",
+          "expr": "sum by (source, lane) (increase({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\"${sync_lane:regex}\", source=~\"${sync_source:regex}\", outcome=~\"${sync_outcome:regex}\", instance=~\"${node:regex}\"}[$__range]))",
+          "legendFormat": "2|0|{{source}}/{{lane}}",
+          "instant": true,
+          "range": false,
+          "queryType": "instant",
+          "editorMode": "code"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "D",
+          "expr": "sum by (source, lane, outcome) (increase({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\"${sync_lane:regex}\", source=~\"${sync_source:regex}\", outcome=~\"${sync_outcome:regex}\", instance=~\"${node:regex}\"}[$__range]))",
+          "legendFormat": "3|1|{{source}}/{{lane}}/{{outcome}}",
+          "instant": true,
+          "range": false,
+          "queryType": "instant",
+          "editorMode": "code"
+        }
+      ],
+      "transformations": [
+        {
+          "id": "seriesToRows",
+          "options": {}
+        },
+        {
+          "id": "extractFields",
+          "options": {
+            "source": "Metric",
+            "format": "regexp",
+            "regexp": "(?<level>\\d+)\\|(?<leaf>[01])\\|(?<label>.*)",
+            "replace": false
+          }
+        },
+        {
+          "id": "convertFieldType",
+          "options": {
+            "conversions": [
+              {
+                "targetField": "level",
+                "destinationType": "number"
+              },
+              {
+                "targetField": "leaf",
+                "destinationType": "number"
+              }
+            ]
+          }
+        },
+        {
+          "id": "calculateField",
+          "options": {
+            "mode": "binary",
+            "binary": {
+              "left": {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Value"
+                }
+              },
+              "operator": "*",
+              "right": {
+                "matcher": {
+                  "id": "byName",
+                  "options": "leaf"
+                }
+              }
+            },
+            "alias": "self",
+            "replaceFields": false
+          }
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Metric": true,
+              "leaf": true
+            },
+            "indexByName": {
+              "level": 0,
+              "label": 1,
+              "Value": 2,
+              "self": 3
+            },
+            "renameByName": {
+              "Value": "value"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "title": "Physical sync attempts by source/outcome (I1)",
+      "gridPos": {
+        "h": 9,
+        "w": 8,
+        "x": 0,
+        "y": 18
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "timeseries",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "sum by (source, outcome, transport, plane, phase) (rate(dkg_sync_attempt_total{source=~\"${sync_source:regex}\", instance=~\"${node:regex}\"}[$__rate_interval]))",
+          "legendFormat": "{{source}} · {{outcome}} · {{transport}}/{{plane}}/{{phase}}"
+        }
+      ]
+    },
+    {
+      "title": "Encoded payload throughput by source (I2 + I3)",
+      "gridPos": {
+        "h": 9,
+        "w": 8,
+        "x": 8,
+        "y": 18
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "timeseries",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "Bps"
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "sum by (source) (rate({__name__=~\"dkg_sync_attempt_request_bytes(_total)?\", source=~\"${sync_source:regex}\", instance=~\"${node:regex}\"}[$__rate_interval]))",
+          "legendFormat": "request · {{source}}"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "B",
+          "expr": "sum by (source, outcome) (rate({__name__=~\"dkg_sync_attempt_response_bytes(_total)?\", source=~\"${sync_source:regex}\", instance=~\"${node:regex}\"}[$__rate_interval]))",
+          "legendFormat": "response · {{source}} · {{outcome}}"
+        }
+      ]
+    },
+    {
+      "title": "Average active worker equivalents (I4 sum)",
+      "gridPos": {
+        "h": 9,
+        "w": 8,
+        "x": 16,
+        "y": 18
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "timeseries",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "sum by (source, lane, outcome) (rate({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\"${sync_lane:regex}\", source=~\"${sync_source:regex}\", outcome=~\"${sync_outcome:regex}\", instance=~\"${node:regex}\"}[$__rate_interval])) / 1000",
+          "legendFormat": "{{source}} · {{lane}} · {{outcome}}"
+        }
+      ]
+    },
+    {
+      "title": "Logical sync operation duration heatmap (I4)",
+      "gridPos": {
+        "h": 12,
+        "w": 12,
+        "x": 0,
+        "y": 27
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "heatmap",
+      "description": "Completed logical-sync duration distribution for the selected source/lane/outcome filters. Rejected-before-start work is excluded by contract and appears in I5.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms"
+        },
+        "overrides": []
+      },
+      "options": {
+        "calculate": false,
+        "cellGap": 1,
+        "cellValues": {},
+        "color": {
+          "mode": "scheme",
+          "scheme": "Reds",
+          "steps": 64,
+          "reverse": false,
+          "scale": "exponential",
+          "exponent": 0.5
+        },
+        "filterValues": {
+          "le": 0
+        },
+        "legend": {
+          "show": true
+        },
+        "rowsFrame": {
+          "layout": "auto"
+        },
+        "tooltip": {
+          "mode": "single",
+          "showColorScale": true,
+          "yHistogram": true
+        },
+        "yAxis": {
+          "axisPlacement": "left",
+          "reverse": false,
+          "unit": "ms"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "sum by (le) (rate({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_bucket\", lane=~\"${sync_lane:regex}\", source=~\"${sync_source:regex}\", outcome=~\"${sync_outcome:regex}\", instance=~\"${node:regex}\"}[$__rate_interval]))",
+          "format": "heatmap",
+          "legendFormat": "{{le}}",
+          "queryType": "range",
+          "editorMode": "code"
+        }
+      ]
+    },
+    {
+      "title": "Sync scheduler queue-wait heatmap (corroborating P1)",
+      "gridPos": {
+        "h": 12,
+        "w": 12,
+        "x": 12,
+        "y": 27
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "heatmap",
+      "description": "Queue-wait distribution for the sync-global scheduler and selected operation lanes. This corroborates W1 cost; it is not one of I1–I9.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms"
+        },
+        "overrides": []
+      },
+      "options": {
+        "calculate": false,
+        "cellGap": 1,
+        "cellValues": {},
+        "color": {
+          "mode": "scheme",
+          "scheme": "Oranges",
+          "steps": 64,
+          "reverse": false,
+          "scale": "exponential",
+          "exponent": 0.5
+        },
+        "filterValues": {
+          "le": 0
+        },
+        "legend": {
+          "show": true
+        },
+        "rowsFrame": {
+          "layout": "auto"
+        },
+        "tooltip": {
+          "mode": "single",
+          "showColorScale": true,
+          "yHistogram": true
+        },
+        "yAxis": {
+          "axisPlacement": "left",
+          "reverse": false,
+          "unit": "ms"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "sum by (le) (rate({__name__=~\"dkg_sync_scheduler_queue_wait_ms(_milliseconds)?_bucket\", scheduler=\"sync-global\", lane=~\"${sync_lane:regex}\", instance=~\"${node:regex}\"}[$__rate_interval]))",
+          "format": "heatmap",
+          "legendFormat": "{{le}}",
+          "queryType": "range",
+          "editorMode": "code"
+        }
+      ]
+    },
+    {
+      "title": "Oldest queued sync item age by lane (corroborating P2)",
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 39
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "timeseries",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms"
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "max by (lane) ({__name__=~\"dkg_backpressure_oldest_queued_age_ms(_milliseconds)?\", scheduler=\"sync-global\", lane=~\"${sync_lane:regex}\", instance=~\"${node:regex}\"})",
+          "legendFormat": "{{lane}}"
+        }
+      ]
+    },
+    {
+      "title": "Admission and coalescing (I5–I6)",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 46
+      },
+      "type": "row",
+      "collapsed": false
+    },
+    {
+      "title": "Operations rejected before starting (I5)",
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 47
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "timeseries",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "sum by (lane, source, reason) (rate(dkg_sync_operation_rejected_total{lane=~\"${sync_lane:regex}\", source=~\"${sync_source:regex}\", instance=~\"${node:regex}\"}[$__rate_interval]))",
+          "legendFormat": "{{source}} · {{lane}} · {{reason}}"
+        }
+      ]
+    },
+    {
+      "title": "Single-flight joins (I6)",
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 47
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "timeseries",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "sum by (scope, owner_source, joiner_source) (rate(dkg_sync_singleflight_joins_total{instance=~\"${node:regex}\"}[$__rate_interval]))",
+          "legendFormat": "{{scope}} · {{owner_source}} ← {{joiner_source}}"
+        }
+      ]
+    },
+    {
+      "title": "Catch-up request and job accounting (I7–I9)",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 56
+      },
+      "type": "row",
+      "collapsed": false
+    },
+    {
+      "title": "Catch-up requests by route result (I7)",
+      "gridPos": {
+        "h": 9,
+        "w": 8,
+        "x": 0,
+        "y": 57
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "timeseries",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "sum by (result, include_shared_memory) (rate(dkg_context_graph_catchup_requests_total{instance=~\"${node:regex}\"}[$__rate_interval]))",
+          "legendFormat": "{{result}} · swm={{include_shared_memory}}"
+        }
+      ]
+    },
+    {
+      "title": "Catch-up jobs by terminal state (I8)",
+      "gridPos": {
+        "h": 9,
+        "w": 8,
+        "x": 8,
+        "y": 57
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "timeseries",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "sum by (status, admission) (rate(dkg_context_graph_catchup_jobs_total{instance=~\"${node:regex}\"}[$__rate_interval]))",
+          "legendFormat": "{{status}} · {{admission}}"
+        }
+      ]
+    },
+    {
+      "title": "Walk catch-up duration p95 (I9)",
+      "gridPos": {
+        "h": 9,
+        "w": 8,
+        "x": 16,
+        "y": 57
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "timeseries",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms"
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "histogram_quantile(0.95, sum by (le) (rate({__name__=~\"dkg_context_graph_catchup_job_duration_ms(_milliseconds)?_bucket\", admission=\"walk\", instance=~\"${node:regex}\"}[$__rate_interval])))",
+          "legendFormat": "walk p95"
+        }
+      ]
+    },
+    {
+      "title": "Walk catch-up job duration heatmap (I9)",
+      "gridPos": {
+        "h": 12,
+        "w": 24,
+        "x": 0,
+        "y": 66
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "heatmap",
+      "description": "Wall-clock duration distribution for real walk jobs. Synthetic already-ready jobs deliberately produce no zero-duration sample.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms"
+        },
+        "overrides": []
+      },
+      "options": {
+        "calculate": false,
+        "cellGap": 1,
+        "cellValues": {},
+        "color": {
+          "mode": "scheme",
+          "scheme": "Blues",
+          "steps": 64,
+          "reverse": false,
+          "scale": "exponential",
+          "exponent": 0.5
+        },
+        "filterValues": {
+          "le": 0
+        },
+        "legend": {
+          "show": true
+        },
+        "rowsFrame": {
+          "layout": "auto"
+        },
+        "tooltip": {
+          "mode": "single",
+          "showColorScale": true,
+          "yHistogram": true
+        },
+        "yAxis": {
+          "axisPlacement": "left",
+          "reverse": false,
+          "unit": "ms"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "sum by (le) (rate({__name__=~\"dkg_context_graph_catchup_job_duration_ms(_milliseconds)?_bucket\", admission=\"walk\", instance=~\"${node:regex}\"}[$__rate_interval]))",
+          "format": "heatmap",
+          "legendFormat": "{{le}}",
+          "queryType": "range",
+          "editorMode": "code"
+        }
+      ]
+    },
+    {
+      "title": "W1 evidence gates — selected Grafana range",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 78
+      },
+      "type": "row",
+      "collapsed": false
+    },
+    {
+      "title": "",
+      "gridPos": {
+        "h": 3,
+        "w": 24,
+        "x": 0,
+        "y": 79
+      },
+      "type": "text",
+      "options": {
+        "mode": "markdown",
+        "content": "These six panels are the same gate expressions as the generated W1 decision packet, evaluated over the selected Grafana range. Threshold colors encode the fixed contract where a numeric threshold exists. **All gates must pass** before interpreting the source-family shares; low export coverage must be compared with selected-range length ÷ scrape interval."
+      },
+      "transparent": true
+    },
+    {
+      "title": "Completed durable operations",
+      "gridPos": {
+        "h": 6,
+        "w": 4,
+        "x": 0,
+        "y": 82
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "stat",
+      "description": "I4 count; must be at least 200.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 200
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "sum(increase({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_count\", lane=~\"durable|changelog\", instance=~\"${node:regex}\"}[$__range]))",
+          "legendFormat": "",
+          "instant": true,
+          "range": false,
+          "queryType": "instant",
+          "editorMode": "code"
+        }
+      ]
+    },
+    {
+      "title": "Attributed durable payload",
+      "gridPos": {
+        "h": 6,
+        "w": 4,
+        "x": 4,
+        "y": 82
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "stat",
+      "description": "I2 + I3; must be at least 50 MB.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 50000000
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "(sum(increase({__name__=~\"dkg_sync_attempt_request_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", instance=~\"${node:regex}\"}[$__range])) or vector(0)) + (sum(increase({__name__=~\"dkg_sync_attempt_response_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", instance=~\"${node:regex}\"}[$__range])) or vector(0))",
+          "legendFormat": "",
+          "instant": true,
+          "range": false,
+          "queryType": "instant",
+          "editorMode": "code"
+        }
+      ]
+    },
+    {
+      "title": "Cross-family joins",
+      "gridPos": {
+        "h": 6,
+        "w": 4,
+        "x": 8,
+        "y": 82
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "stat",
+      "description": "I6; must remain zero.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "(sum(increase(dkg_sync_singleflight_joins_total{owner_source=~\"catchup-foreground\", joiner_source!~\"catchup-foreground\", instance=~\"${node:regex}\"}[$__range])) or vector(0)) + (sum(increase(dkg_sync_singleflight_joins_total{owner_source=~\"on-connect|reconcile\", joiner_source!~\"on-connect|reconcile\", instance=~\"${node:regex}\"}[$__range])) or vector(0)) + (sum(increase(dkg_sync_singleflight_joins_total{owner_source=~\"catchup-background|vm-recovery|swm-recovery|control-plane\", joiner_source!~\"catchup-background|vm-recovery|swm-recovery|control-plane\", instance=~\"${node:regex}\"}[$__range])) or vector(0)) + (sum(increase(dkg_sync_singleflight_joins_total{owner_source!~\"catchup-foreground|on-connect|reconcile|catchup-background|vm-recovery|swm-recovery|control-plane\", joiner_source=~\"catchup-foreground|on-connect|reconcile|catchup-background|vm-recovery|swm-recovery|control-plane\", instance=~\"${node:regex}\"}[$__range])) or vector(0))",
+          "legendFormat": "",
+          "instant": true,
+          "range": false,
+          "queryType": "instant",
+          "editorMode": "code"
+        }
+      ]
+    },
+    {
+      "title": "Unclassified source samples",
+      "gridPos": {
+        "h": 6,
+        "w": 4,
+        "x": 12,
+        "y": 82
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "stat",
+      "description": "I1/I4/I5; must remain zero.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "(sum(increase(dkg_sync_attempt_total{plane=\"durable\", phase=~\"data|meta|delta\", source!~\"catchup-foreground|on-connect|reconcile|catchup-background|vm-recovery|swm-recovery|control-plane\", instance=~\"${node:regex}\"}[$__range])) or vector(0)) + (sum(increase({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_count\", lane=~\"durable|changelog\", source!~\"catchup-foreground|on-connect|reconcile|catchup-background|vm-recovery|swm-recovery|control-plane\", instance=~\"${node:regex}\"}[$__range])) or vector(0)) + (sum(increase(dkg_sync_operation_rejected_total{lane=~\"durable|changelog\", source!~\"catchup-foreground|on-connect|reconcile|catchup-background|vm-recovery|swm-recovery|control-plane\", instance=~\"${node:regex}\"}[$__range])) or vector(0))",
+          "legendFormat": "",
+          "instant": true,
+          "range": false,
+          "queryType": "instant",
+          "editorMode": "code"
+        }
+      ]
+    },
+    {
+      "title": "Counter resets",
+      "gridPos": {
+        "h": 6,
+        "w": 4,
+        "x": 16,
+        "y": 82
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "stat",
+      "description": "Must remain zero inside the selected range.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "max(resets(dkg_sync_attempt_total{plane=\"durable\", phase=~\"data|meta|delta\", instance=~\"${node:regex}\"}[$__range]))",
+          "legendFormat": "",
+          "instant": true,
+          "range": false,
+          "queryType": "instant",
+          "editorMode": "code"
+        }
+      ]
+    },
+    {
+      "title": "Minimum export samples",
+      "gridPos": {
+        "h": 6,
+        "w": 4,
+        "x": 20,
+        "y": 82
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "stat",
+      "description": "Compare with selected-range length divided by scrape interval.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "none",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "min(count_over_time(dkg_sync_attempt_total{plane=\"durable\", phase=~\"data|meta|delta\", instance=~\"${node:regex}\"}[$__range]))",
+          "legendFormat": "",
+          "instant": true,
+          "range": false,
+          "queryType": "instant",
+          "editorMode": "code"
+        }
+      ]
+    },
+    {
+      "title": "W1 materiality and source-family decision",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 88
+      },
+      "type": "row",
+      "collapsed": false
+    },
+    {
+      "title": "All-source durable active ms/hour",
+      "gridPos": {
+        "h": 6,
+        "w": 6,
+        "x": 0,
+        "y": 89
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "stat",
+      "description": "I4 strain denominator; occupancy, not CPU.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms"
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "none",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "3600 * sum(rate({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\"durable|changelog\", instance=~\"${node:regex}\"}[$__range]))",
+          "legendFormat": "",
+          "instant": true,
+          "range": false,
+          "queryType": "instant",
+          "editorMode": "code"
+        }
+      ]
+    },
+    {
+      "title": "All-source durable bytes/hour",
+      "gridPos": {
+        "h": 6,
+        "w": 6,
+        "x": 6,
+        "y": 89
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "stat",
+      "description": "I2 + I3 volume denominator.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "none",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "3600 * ((sum(rate({__name__=~\"dkg_sync_attempt_request_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", instance=~\"${node:regex}\"}[$__range])) or vector(0)) + (sum(rate({__name__=~\"dkg_sync_attempt_response_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", instance=~\"${node:regex}\"}[$__range])) or vector(0)))",
+          "legendFormat": "",
+          "instant": true,
+          "range": false,
+          "queryType": "instant",
+          "editorMode": "code"
+        }
+      ]
+    },
+    {
+      "title": "Excluded-family bytes/hour",
+      "gridPos": {
+        "h": 6,
+        "w": 6,
+        "x": 12,
+        "y": 89
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "stat",
+      "description": "Background, VM/SWM recovery, and control-plane traffic.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "none",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "3600 * ((sum(rate({__name__=~\"dkg_sync_attempt_request_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", source=~\"catchup-background|vm-recovery|swm-recovery|control-plane\", instance=~\"${node:regex}\"}[$__range])) or vector(0)) + (sum(rate({__name__=~\"dkg_sync_attempt_response_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", source=~\"catchup-background|vm-recovery|swm-recovery|control-plane\", instance=~\"${node:regex}\"}[$__range])) or vector(0)))",
+          "legendFormat": "",
+          "instant": true,
+          "range": false,
+          "queryType": "instant",
+          "editorMode": "code"
+        }
+      ]
+    },
+    {
+      "title": "Excluded-family active ms/hour",
+      "gridPos": {
+        "h": 6,
+        "w": 6,
+        "x": 18,
+        "y": 89
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "stat",
+      "description": "Reported separately; never eligible for the shadow decision.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms"
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "none",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "3600 * sum(rate({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\"durable|changelog\", source=~\"catchup-background|vm-recovery|swm-recovery|control-plane\", instance=~\"${node:regex}\"}[$__range]))",
+          "legendFormat": "",
+          "instant": true,
+          "range": false,
+          "queryType": "instant",
+          "editorMode": "code"
+        }
+      ]
+    },
+    {
+      "title": "Eligible active share",
+      "gridPos": {
+        "h": 6,
+        "w": 4,
+        "x": 0,
+        "y": 95
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "stat",
+      "description": "Foreground + recurring share of all-source active occupancy; must reach 30%.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 0.3
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "sum(rate({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\"durable|changelog\", source=~\"catchup-foreground|on-connect|reconcile\", instance=~\"${node:regex}\"}[$__range])) / sum(rate({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\"durable|changelog\", instance=~\"${node:regex}\"}[$__range]))",
+          "legendFormat": "",
+          "instant": true,
+          "range": false,
+          "queryType": "instant",
+          "editorMode": "code"
+        }
+      ]
+    },
+    {
+      "title": "Foreground share — bytes",
+      "gridPos": {
+        "h": 6,
+        "w": 5,
+        "x": 4,
+        "y": 95
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "stat",
+      "description": "A candidate family must reach 60% on both axes.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "none",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "((sum(rate({__name__=~\"dkg_sync_attempt_request_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", source=~\"catchup-foreground\", instance=~\"${node:regex}\"}[$__range])) or vector(0)) + (sum(rate({__name__=~\"dkg_sync_attempt_response_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", source=~\"catchup-foreground\", instance=~\"${node:regex}\"}[$__range])) or vector(0))) / ((sum(rate({__name__=~\"dkg_sync_attempt_request_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", source=~\"catchup-foreground|on-connect|reconcile\", instance=~\"${node:regex}\"}[$__range])) or vector(0)) + (sum(rate({__name__=~\"dkg_sync_attempt_response_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", source=~\"catchup-foreground|on-connect|reconcile\", instance=~\"${node:regex}\"}[$__range])) or vector(0)))",
+          "legendFormat": "",
+          "instant": true,
+          "range": false,
+          "queryType": "instant",
+          "editorMode": "code"
+        }
+      ]
+    },
+    {
+      "title": "Foreground share — active ms",
+      "gridPos": {
+        "h": 6,
+        "w": 5,
+        "x": 9,
+        "y": 95
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "stat",
+      "description": "A candidate family must reach 60% on both axes.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "none",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "sum(rate({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\"durable|changelog\", source=~\"catchup-foreground\", instance=~\"${node:regex}\"}[$__range])) / sum(rate({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\"durable|changelog\", source=~\"catchup-foreground|on-connect|reconcile\", instance=~\"${node:regex}\"}[$__range]))",
+          "legendFormat": "",
+          "instant": true,
+          "range": false,
+          "queryType": "instant",
+          "editorMode": "code"
+        }
+      ]
+    },
+    {
+      "title": "Recurring share — bytes",
+      "gridPos": {
+        "h": 6,
+        "w": 5,
+        "x": 14,
+        "y": 95
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "stat",
+      "description": "A candidate family must reach 60% on both axes.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "none",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "((sum(rate({__name__=~\"dkg_sync_attempt_request_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", source=~\"on-connect|reconcile\", instance=~\"${node:regex}\"}[$__range])) or vector(0)) + (sum(rate({__name__=~\"dkg_sync_attempt_response_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", source=~\"on-connect|reconcile\", instance=~\"${node:regex}\"}[$__range])) or vector(0))) / ((sum(rate({__name__=~\"dkg_sync_attempt_request_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", source=~\"catchup-foreground|on-connect|reconcile\", instance=~\"${node:regex}\"}[$__range])) or vector(0)) + (sum(rate({__name__=~\"dkg_sync_attempt_response_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", source=~\"catchup-foreground|on-connect|reconcile\", instance=~\"${node:regex}\"}[$__range])) or vector(0)))",
+          "legendFormat": "",
+          "instant": true,
+          "range": false,
+          "queryType": "instant",
+          "editorMode": "code"
+        }
+      ]
+    },
+    {
+      "title": "Recurring share — active ms",
+      "gridPos": {
+        "h": 6,
+        "w": 5,
+        "x": 19,
+        "y": 95
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${vm}"
+      },
+      "type": "stat",
+      "description": "A candidate family must reach 60% on both axes.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "none",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${vm}"
+          },
+          "refId": "A",
+          "expr": "sum(rate({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\"durable|changelog\", source=~\"on-connect|reconcile\", instance=~\"${node:regex}\"}[$__range])) / sum(rate({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\"durable|changelog\", source=~\"catchup-foreground|on-connect|reconcile\", instance=~\"${node:regex}\"}[$__range]))",
+          "legendFormat": "",
+          "instant": true,
+          "range": false,
+          "queryType": "instant",
+          "editorMode": "code"
+        }
+      ]
+    }
+  ]
+}
diff --git a/tools/observability/lib/dashboards.mjs b/tools/observability/lib/dashboards.mjs
index 0c39930086..cbadb59950 100644
--- a/tools/observability/lib/dashboards.mjs
+++ b/tools/observability/lib/dashboards.mjs
@@ -12,6 +12,7 @@ import {
   severityMatches,
   RPC_USAGE_PIPELINE,
 } from './queries.mjs';
+import { buildW1GrafanaModel } from './w1.mjs';
 
 const LOKI = { type: 'loki', uid: '${loki}' };
 const VM = { type: 'prometheus', uid: '${vm}' };
@@ -84,6 +85,17 @@ const INSTANT_LOKI_TARGET = (refId, expr, legendFormat) => ({
   queryType: 'instant',
 });
 
+const INSTANT_PROM_TARGET = (refId, expr, legendFormat) => ({
+  datasource: VM,
+  refId,
+  expr,
+  legendFormat,
+  instant: true,
+  range: false,
+  queryType: 'instant',
+  editorMode: 'code',
+});
+
 const BACKPRESSURE_FLAME = (stream) => {
   const active = backpressurePeakAgeByOperation(stream, 'active');
   const queued = backpressurePeakAgeByOperation(stream, 'queued');
@@ -199,6 +211,134 @@ const BACKPRESSURE_HEATMAP = (stream, phase) => {
   };
 };
 
+const W1_FLAMEGRAPH = (targets) => ({
+  w: 24,
+  h: 14,
+  def: {
+    datasource: VM,
+    type: 'flamegraph',
+    title: 'Source-attributed sync active occupancy — $node',
+    description: 'Hierarchy is source → requester lane → terminal outcome. Width is completed logical-sync active wall-clock occupancy accumulated over the selected Grafana range. It is admitted occupancy, not CPU time, and excludes rejected-before-start operations (shown separately).',
+    options: { showFlameGraphOnly: false },
+    fieldConfig: { defaults: { unit: 'ms' }, overrides: [] },
+    targets: [
+      INSTANT_PROM_TARGET('A', targets.flame_root, '0|0|all selected sync occupancy'),
+      INSTANT_PROM_TARGET('B', targets.flame_source, '1|0|{{source}}'),
+      INSTANT_PROM_TARGET('C', targets.flame_source_lane, '2|0|{{source}}/{{lane}}'),
+      INSTANT_PROM_TARGET('D', targets.flame_source_lane_outcome, '3|1|{{source}}/{{lane}}/{{outcome}}'),
+    ],
+    transformations: [
+      { id: 'seriesToRows', options: {} },
+      {
+        id: 'extractFields',
+        options: {
+          source: 'Metric',
+          format: 'regexp',
+          regexp: '(?<level>\\d+)\\|(?<leaf>[01])\\|(?<label>.*)',
+          replace: false,
+        },
+      },
+      {
+        id: 'convertFieldType',
+        options: {
+          conversions: [
+            { targetField: 'level', destinationType: 'number' },
+            { targetField: 'leaf', destinationType: 'number' },
+          ],
+        },
+      },
+      {
+        id: 'calculateField',
+        options: {
+          mode: 'binary',
+          binary: {
+            left: { matcher: { id: 'byName', options: 'Value' } },
+            operator: '*',
+            right: { matcher: { id: 'byName', options: 'leaf' } },
+          },
+          alias: 'self',
+          replaceFields: false,
+        },
+      },
+      {
+        id: 'organize',
+        options: {
+          excludeByName: { Time: true, Metric: true, leaf: true },
+          indexByName: { level: 0, label: 1, Value: 2, self: 3 },
+          renameByName: { Value: 'value' },
+        },
+      },
+    ],
+  },
+});
+
+const PROM_HISTOGRAM_HEATMAP = (w, title, description, expr, scheme = 'Oranges') => ({
+  w,
+  h: 12,
+  def: {
+    datasource: VM,
+    type: 'heatmap',
+    title,
+    description,
+    fieldConfig: { defaults: { unit: 'ms' }, overrides: [] },
+    options: {
+      calculate: false,
+      cellGap: 1,
+      cellValues: {},
+      color: { mode: 'scheme', scheme, steps: 64, reverse: false, scale: 'exponential', exponent: 0.5 },
+      filterValues: { le: 0 },
+      legend: { show: true },
+      rowsFrame: { layout: 'auto' },
+      tooltip: { mode: 'single', showColorScale: true, yHistogram: true },
+      yAxis: { axisPlacement: 'left', reverse: false, unit: 'ms' },
+    },
+    targets: [{
+      datasource: VM,
+      refId: 'A',
+      expr,
+      format: 'heatmap',
+      legendFormat: '{{le}}',
+      queryType: 'range',
+      editorMode: 'code',
+    }],
+  },
+});
+
+const STAT = (w, title, description, expr, unit = 'short', thresholds) => ({
+  w,
+  h: 6,
+  def: {
+    datasource: VM,
+    type: 'stat',
+    title,
+    description,
+    fieldConfig: {
+      defaults: {
+        unit,
+        ...(thresholds ? { color: { mode: 'thresholds' }, thresholds } : {}),
+      },
+      overrides: [],
+    },
+    options: {
+      colorMode: thresholds ? 'value' : 'none',
+      graphMode: 'area',
+      justifyMode: 'auto',
+      reduceOptions: { calcs: ['lastNotNull'], fields: '', values: false },
+      textMode: 'auto',
+    },
+    targets: [INSTANT_PROM_TARGET('A', expr, '')],
+  },
+});
+
+const MIN_GATE = (minimum) => ({ mode: 'absolute', steps: [
+  { color: 'red', value: null },
+  { color: 'green', value: minimum },
+] });
+const ZERO_GATE = { mode: 'absolute', steps: [
+  { color: 'green', value: null },
+  { color: 'red', value: 1 },
+] };
+
 // Loki query building blocks come from the shared contract (lib/queries.mjs)
 // — the same module alerts.mjs composes from, so the stream selector,
 // severity filter shape and rpc_usage pipeline cannot drift between the
@@ -341,6 +481,112 @@ const buildMetricsDashboard = (nodeProfile, NODE_IDENTITY) => {
   };
 };
 
+const buildSyncCostDashboard = (nodeProfile, NODE_IDENTITY) => {
+  const model = buildW1GrafanaModel({ nodeProfile });
+  const Q = model.targets;
+  const multiCustom = (name, label, values) => ({
+    name,
+    label,
+    type: 'custom',
+    query: values.join(','),
+    includeAll: true,
+    multi: true,
+    allValue: '.+',
+    current: { text: ['All'], value: ['$__all'] },
+  });
+  return {
+    uid: 'dkg-sync-cost',
+    title: 'DKG Nodes — Sync Cost',
+    timezone: 'browser',
+    refresh: '1m',
+    time: { from: 'now-6h', to: 'now' },
+    tags: ['dkg', 'metrics', 'sync'],
+    links: dashLinks,
+    description: `PR #2033 W1 source-attributed sync telemetry (I1–I9). Node label profile: ${nodeProfile.label}. The dashboard is an interactive operational view; the generated w1/w1-queries.md remains the fixed 1 h / 2 h decision contract.`,
+    templating: { list: [
+      { name: 'vm', type: 'datasource', query: 'prometheus', label: 'Metrics DS', hide: 2, current: {} },
+      nodeVarMulti(NODE_IDENTITY.metrics),
+      multiCustom('sync_source', 'Sync source', model.variables.sources),
+      multiCustom('sync_lane', 'Operation lane', model.variables.lanes),
+      multiCustom('sync_outcome', 'Operation outcome', model.variables.outcomes),
+    ] },
+    panels: layout([
+      [ TEXT('**Requires the extended sync diagnostics from PR #2033 and node metric export.** The flame graph and duration heatmaps use real OpenTelemetry counters/histograms, while the existing **DKG Node — Logs** pressure views continue to use Loki transition/summary records. Filter the interactive operation views with the source/lane/outcome controls above. The evidence and materiality panels intentionally retain W1\'s fixed durable filters and source-family rules; a failed gate means **inconclusive**, not healthy.') ],
+      [ ROW('Source-attributed sync cost (I1–I4)') ],
+      [ W1_FLAMEGRAPH(Q) ],
+      [
+        TS(8, 9, VM, 'Physical sync attempts by source/outcome (I1)', [
+          { expr: Q.attempts_rate, legend: '{{source}} · {{outcome}} · {{transport}}/{{plane}}/{{phase}}' },
+        ], 'ops'),
+        TS(8, 9, VM, 'Encoded payload throughput by source (I2 + I3)', [
+          { expr: Q.request_bytes_rate, legend: 'request · {{source}}' },
+          { expr: Q.response_bytes_rate, legend: 'response · {{source}} · {{outcome}}' },
+        ], 'Bps'),
+        TS(8, 9, VM, 'Average active worker equivalents (I4 sum)', [
+          { expr: Q.active_worker_equivalents, legend: '{{source}} · {{lane}} · {{outcome}}' },
+        ], 'short'),
+      ],
+      [
+        PROM_HISTOGRAM_HEATMAP(12, 'Logical sync operation duration heatmap (I4)', 'Completed logical-sync duration distribution for the selected source/lane/outcome filters. Rejected-before-start work is excluded by contract and appears in I5.', Q.sync_operation_duration_buckets, 'Reds'),
+        PROM_HISTOGRAM_HEATMAP(12, 'Sync scheduler queue-wait heatmap (corroborating P1)', 'Queue-wait distribution for the sync-global scheduler and selected operation lanes. This corroborates W1 cost; it is not one of I1–I9.', Q.queue_wait_buckets, 'Oranges'),
+      ],
+      [ TS(24, 7, VM, 'Oldest queued sync item age by lane (corroborating P2)', [
+        { expr: Q.oldest_queued_age, legend: '{{lane}}' },
+      ], 'ms') ],
+
+      [ ROW('Admission and coalescing (I5–I6)') ],
+      [
+        TS(12, 9, VM, 'Operations rejected before starting (I5)', [
+          { expr: Q.rejected_rate, legend: '{{source}} · {{lane}} · {{reason}}' },
+        ], 'ops'),
+        TS(12, 9, VM, 'Single-flight joins (I6)', [
+          { expr: Q.singleflight_joins_rate, legend: '{{scope}} · {{owner_source}} ← {{joiner_source}}' },
+        ], 'ops'),
+      ],
+
+      [ ROW('Catch-up request and job accounting (I7–I9)') ],
+      [
+        TS(8, 9, VM, 'Catch-up requests by route result (I7)', [
+          { expr: Q.catchup_requests_rate, legend: '{{result}} · swm={{include_shared_memory}}' },
+        ], 'ops'),
+        TS(8, 9, VM, 'Catch-up jobs by terminal state (I8)', [
+          { expr: Q.catchup_jobs_rate, legend: '{{status}} · {{admission}}' },
+        ], 'ops'),
+        TS(8, 9, VM, 'Walk catch-up duration p95 (I9)', [
+          { expr: Q.catchup_duration_p95, legend: 'walk p95' },
+        ], 'ms'),
+      ],
+      [ PROM_HISTOGRAM_HEATMAP(24, 'Walk catch-up job duration heatmap (I9)', 'Wall-clock duration distribution for real walk jobs. Synthetic already-ready jobs deliberately produce no zero-duration sample.', Q.catchup_duration_buckets, 'Blues') ],
+
+      [ ROW('W1 evidence gates — selected Grafana range') ],
+      [ TEXT('These six panels are the same gate expressions as the generated W1 decision packet, evaluated over the selected Grafana range. Threshold colors encode the fixed contract where a numeric threshold exists. **All gates must pass** before interpreting the source-family shares; low export coverage must be compared with selected-range length ÷ scrape interval.') ],
+      [
+        STAT(4, 'Completed durable operations', 'I4 count; must be at least 200.', Q.gate_completed_operations, 'short', MIN_GATE(200)),
+        STAT(4, 'Attributed durable payload', 'I2 + I3; must be at least 50 MB.', Q.gate_payload_bytes, 'bytes', MIN_GATE(50_000_000)),
+        STAT(4, 'Cross-family joins', 'I6; must remain zero.', Q.gate_cross_family_joins, 'short', ZERO_GATE),
+        STAT(4, 'Unclassified source samples', 'I1/I4/I5; must remain zero.', Q.gate_invalidating_sources, 'short', ZERO_GATE),
+        STAT(4, 'Counter resets', 'Must remain zero inside the selected range.', Q.gate_counter_resets, 'short', ZERO_GATE),
+        STAT(4, 'Minimum export samples', 'Compare with selected-range length divided by scrape interval.', Q.gate_export_samples),
+      ],
+
+      [ ROW('W1 materiality and source-family decision') ],
+      [
+        STAT(6, 'All-source durable active ms/hour', 'I4 strain denominator; occupancy, not CPU.', Q.all_source_active_ms_per_hour, 'ms'),
+        STAT(6, 'All-source durable bytes/hour', 'I2 + I3 volume denominator.', Q.all_source_bytes_per_hour, 'bytes'),
+        STAT(6, 'Excluded-family bytes/hour', 'Background, VM/SWM recovery, and control-plane traffic.', Q.excluded_bytes_per_hour, 'bytes'),
+        STAT(6, 'Excluded-family active ms/hour', 'Reported separately; never eligible for the shadow decision.', Q.excluded_active_ms_per_hour, 'ms'),
+      ],
+      [
+        STAT(4, 'Eligible active share', 'Foreground + recurring share of all-source active occupancy; must reach 30%.', Q.eligible_active_share, 'percentunit', MIN_GATE(0.30)),
+        STAT(5, 'Foreground share — bytes', 'A candidate family must reach 60% on both axes.', Q.foreground_bytes_share, 'percentunit'),
+        STAT(5, 'Foreground share — active ms', 'A candidate family must reach 60% on both axes.', Q.foreground_active_share, 'percentunit'),
+        STAT(5, 'Recurring share — bytes', 'A candidate family must reach 60% on both axes.', Q.recurring_bytes_share, 'percentunit'),
+        STAT(5, 'Recurring share — active ms', 'A candidate family must reach 60% on both axes.', Q.recurring_active_share, 'percentunit'),
+      ],
+    ]),
+  };
+};
+
 const TQ = (w, h, title, q, limit) => ({ w, h, def: { datasource: TEMPO, type: 'table', title,
   targets: [{ datasource: TEMPO, refId: 'A', queryType: 'traceql', query: q, limit: limit ?? 20, tableType: 'traces', filters: [] }] } });
 const NODEQ = 'resource.service.name="dkg-node" && resource.service.instance.id=~"${node:regex}"';
@@ -369,6 +615,7 @@ export const buildDashboards = ({ nodeProfile }) => {
     fleet: buildFleetLogsDashboard(),
     nodeLogs: buildNodeLogsDashboard(NODE_IDENTITY),
     metrics: buildMetricsDashboard(nodeProfile, NODE_IDENTITY),
+    syncCost: buildSyncCostDashboard(nodeProfile, NODE_IDENTITY),
     traces: buildTracesDashboard(NODE_IDENTITY),
   };
 };
diff --git a/tools/observability/lib/w1.mjs b/tools/observability/lib/w1.mjs
index dce53b22b9..2305e22973 100644
--- a/tools/observability/lib/w1.mjs
+++ b/tools/observability/lib/w1.mjs
@@ -360,6 +360,94 @@ const buildWindowQueries = ({ nodeProfile, range }) => {
   ];
 };
 
+// ── Grafana dashboard model ──────────────────────────────────────────────
+// The W1 report above answers a fixed measurement decision over 1 h / 2 h
+// windows. The Grafana dashboard is the interactive companion: it exposes all
+// nine instruments, retains the same spelling/profile contract, and lets an
+// operator filter the source-attributed operation views without forking the
+// metric vocabulary or hand-writing a second set of selectors.
+const W1_OPERATION_LANES = ['durable', 'changelog', 'shared_memory', 'swm_recovery'];
+const W1_OPERATION_OUTCOMES = ['resolved', 'error', 'cancelled'];
+
+export const buildW1GrafanaModel = ({ nodeProfile }) => {
+  const SEL = nodeProfile.selector;
+  const RATE = '[$__rate_interval]';
+  const RANGE = '[$__range]';
+  const sourceFilter = 'source=~"${sync_source:regex}"';
+  const laneFilter = 'lane=~"${sync_lane:regex}"';
+  const outcomeFilter = 'outcome=~"${sync_outcome:regex}"';
+
+  const i1 = selector(I.I1, { matchers: [sourceFilter, SEL] });
+  const i2 = selector(I.I2, { matchers: [sourceFilter, SEL] });
+  const i3 = selector(I.I3, { matchers: [sourceFilter, SEL] });
+  const i4sum = selector(I.I4, { series: '_sum', matchers: [laneFilter, sourceFilter, outcomeFilter, SEL] });
+  const i4bucket = selector(I.I4, { series: '_bucket', matchers: [laneFilter, sourceFilter, outcomeFilter, SEL] });
+  const i5 = selector(I.I5, { matchers: [laneFilter, sourceFilter, SEL] });
+  const i6 = selector(I.I6, { matchers: [SEL] });
+  const i7 = selector(I.I7, { matchers: [SEL] });
+  const i8 = selector(I.I8, { matchers: [SEL] });
+  const i9bucket = selector(I.I9, { series: '_bucket', matchers: ['admission="walk"', SEL] });
+  const p1bucket = selector(I.P1, { series: '_bucket', matchers: ['scheduler="sync-global"', laneFilter, SEL] });
+  const p2 = selector(I.P2, { matchers: ['scheduler="sync-global"', laneFilter, SEL] });
+
+  const activeInRange = `increase(${i4sum}${RANGE})`;
+  const windowSections = buildWindowQueries({ nodeProfile, range: '$__range' });
+  const decision = Object.fromEntries(windowSections
+    .flatMap((section) => section.queries)
+    .map((query) => [query.id, query.expr]));
+  const requiredDecision = (id) => {
+    const expr = decision[id];
+    if (!expr) throw new Error(`w1 Grafana model: missing decision query ${id}`);
+    return expr;
+  };
+
+  const targets = {
+    flame_root: `sum(${activeInRange})`,
+    flame_source: `sum by (source) (${activeInRange})`,
+    flame_source_lane: `sum by (source, lane) (${activeInRange})`,
+    flame_source_lane_outcome: `sum by (source, lane, outcome) (${activeInRange})`,
+    attempts_rate: `sum by (source, outcome, transport, plane, phase) (rate(${i1}${RATE}))`,
+    request_bytes_rate: `sum by (source) (rate(${i2}${RATE}))`,
+    response_bytes_rate: `sum by (source, outcome) (rate(${i3}${RATE}))`,
+    active_worker_equivalents: `sum by (source, lane, outcome) (rate(${i4sum}${RATE})) / 1000`,
+    sync_operation_duration_buckets: `sum by (le) (rate(${i4bucket}${RATE}))`,
+    rejected_rate: `sum by (lane, source, reason) (rate(${i5}${RATE}))`,
+    singleflight_joins_rate: `sum by (scope, owner_source, joiner_source) (rate(${i6}${RATE}))`,
+    queue_wait_buckets: `sum by (le) (rate(${p1bucket}${RATE}))`,
+    oldest_queued_age: `max by (lane) (${p2})`,
+    catchup_requests_rate: `sum by (result, include_shared_memory) (rate(${i7}${RATE}))`,
+    catchup_jobs_rate: `sum by (status, admission) (rate(${i8}${RATE}))`,
+    catchup_duration_buckets: `sum by (le) (rate(${i9bucket}${RATE}))`,
+    catchup_duration_p95: `histogram_quantile(0.95, sum by (le) (rate(${i9bucket}${RATE})))`,
+
+    gate_completed_operations: requiredDecision('completed_durable_operations'),
+    gate_payload_bytes: requiredDecision('attributed_durable_payload_bytes'),
+    gate_cross_family_joins: requiredDecision('cross_family_singleflight_joins'),
+    gate_invalidating_sources: requiredDecision('invalidating_source_samples'),
+    gate_counter_resets: requiredDecision('counter_resets'),
+    gate_export_samples: requiredDecision('export_coverage_min_samples'),
+
+    all_source_active_ms_per_hour: requiredDecision('all_source_durable_active_ms_per_hour'),
+    all_source_bytes_per_hour: requiredDecision('all_source_durable_bytes_per_hour'),
+    eligible_active_share: requiredDecision('eligible_active_share_of_all_source'),
+    foreground_bytes_share: requiredDecision('foreground_share_of_eligible_durable_bytes'),
+    foreground_active_share: requiredDecision('foreground_share_of_eligible_durable_active_ms'),
+    recurring_bytes_share: requiredDecision('recurring_share_of_eligible_durable_bytes'),
+    recurring_active_share: requiredDecision('recurring_share_of_eligible_durable_active_ms'),
+    excluded_bytes_per_hour: requiredDecision('excluded_durable_bytes_per_hour'),
+    excluded_active_ms_per_hour: requiredDecision('excluded_durable_active_ms_per_hour'),
+  };
+
+  return {
+    variables: {
+      sources: W1_SOURCE_FAMILIES.flatMap((family) => family.sources),
+      lanes: W1_OPERATION_LANES,
+      outcomes: W1_OPERATION_OUTCOMES,
+    },
+    targets,
+  };
+};
+
 // ── rendering ─────────────────────────────────────────────────────────────
 const GENERATED_MD = `<!-- GENERATED FILE — do not edit. Source: tools/observability/lib/w1.mjs
      Regenerate with: node tools/observability/generate-observability.mjs
@@ -518,11 +606,40 @@ const renderRules = ({ windows }) => [
   '',
 ].join('\n');
 
+// Promtool cannot parse Grafana interpolation tokens. This fixture carries the
+// exact dashboard expressions after replacing only the five runtime macros
+// with representative safe values. verify-profile-render.mjs independently
+// performs the same substitution and compares the complete expression set,
+// so this parser target cannot drift away from the dashboard it certifies.
+const dashboardExprForPromtool = (expr) => expr
+  .replaceAll('${node:regex}', '.*')
+  .replaceAll('${sync_source:regex}', '.*')
+  .replaceAll('${sync_lane:regex}', '.*')
+  .replaceAll('${sync_outcome:regex}', '.*')
+  .replaceAll('$__rate_interval', '5m')
+  .replaceAll('$__range', '2h');
+
+const renderDashboardRules = ({ targets }) => [
+  '# GENERATED FILE — do not edit. Source: tools/observability/lib/w1.mjs',
+  '# Exact parser fixture for grafana-dashboard-dkg-sync-cost.json.',
+  '# Grafana variables use representative values solely for promtool parsing.',
+  'groups:',
+  '  - name: w1-grafana-dashboard',
+  '    rules:',
+  ...Object.entries(targets).flatMap(([id, expr]) => [
+    `      - record: w1_dashboard_${id.replaceAll(/[^A-Za-z0-9_]/g, '_')}`,
+    `        expr: ${yamlQuote(dashboardExprForPromtool(expr))}`,
+  ]),
+  '',
+].join('\n');
+
 /** The rendered W1 artifact set, keyed by path relative to the output dir. */
 export const buildW1Artifacts = ({ nodeProfile }) => {
   const windows = W1_WINDOWS.map((w) => ({ ...w, sections: buildWindowQueries({ nodeProfile, range: w.range }) }));
+  const dashboard = buildW1GrafanaModel({ nodeProfile });
   return {
     'w1/w1-queries.md': renderReport({ nodeProfile, windows }),
     'w1/w1-rules.yaml': renderRules({ windows }),
+    'w1/w1-dashboard-rules.yaml': renderDashboardRules(dashboard),
   };
 };
diff --git a/tools/observability/verify-profile-render.mjs b/tools/observability/verify-profile-render.mjs
index 4bb1601255..4b81b52da2 100644
--- a/tools/observability/verify-profile-render.mjs
+++ b/tools/observability/verify-profile-render.mjs
@@ -145,6 +145,144 @@ const dkgSelectorViolations = (rawExpr, rawNodeFilter) => {
   }
 }
 
+// ── W1 sync-cost dashboard: complete I1–I9 surface + parser fixture ───────
+{
+  const dash = load('grafana-dashboard-dkg-sync-cost.json');
+  const nodeVar = (dash.templating?.list ?? []).find((v) => v.name === 'node');
+  const wantVarQuery = `label_values({__name__=~"dkg_.+"}, ${LABEL})`;
+  if (!nodeVar) fail('sync-cost dashboard', 'no $node template variable');
+  else if (nodeVar.query !== wantVarQuery) fail('sync-cost dashboard $node variable', `query is ${JSON.stringify(nodeVar.query)}, want ${JSON.stringify(wantVarQuery)}`);
+
+  const expectedVariables = {
+    sync_source: 'catchup-foreground,on-connect,reconcile,catchup-background,vm-recovery,swm-recovery,control-plane,unspecified',
+    sync_lane: 'durable,changelog,shared_memory,swm_recovery',
+    sync_outcome: 'resolved,error,cancelled',
+  };
+  for (const [name, query] of Object.entries(expectedVariables)) {
+    const variable = (dash.templating?.list ?? []).find((v) => v.name === name);
+    if (!variable) fail('sync-cost dashboard', `missing $${name} variable`);
+    else {
+      if (variable.query !== query) fail(`sync-cost dashboard $${name}`, `query is ${JSON.stringify(variable.query)}, want ${JSON.stringify(query)}`);
+      if (!variable.includeAll || !variable.multi || variable.allValue !== '.+') {
+        fail(`sync-cost dashboard $${name}`, 'must be multi-select with an explicit .+ All value');
+      }
+    }
+  }
+
+  const allPanels = (dash.panels ?? []).flatMap((panel) => [panel, ...(panel.panels ?? [])]);
+  const allTargets = allPanels.flatMap((panel) => (panel.targets ?? []).map((target) => ({ panel, target })));
+  const nodeFilter = `${LABEL}=~"\${node:regex}"`;
+  const dashboardExprs = [];
+  for (const { panel, target } of allTargets) {
+    const expr = target.expr ?? '';
+    if (!/dkg_/.test(expr)) continue;
+    dashboardExprs.push(expr);
+    const where = `sync-cost panel "${panel.title}" target ${target.refId}`;
+    for (const sel of dkgSelectorViolations(expr, nodeFilter)) {
+      fail(where, `dkg_* selector lacks the profiled node filter ${nodeFilter}: ${sel}`);
+    }
+    if (profiled) {
+      if (usesUnprofiledLabel(expr)) fail(where, `unprofiled '${DEFAULT_LABEL}' label survived in expr: ${expr}`);
+      if (target.legendFormat && legendRe(DEFAULT_LABEL).test(target.legendFormat)) fail(where, `unprofiled legend: ${target.legendFormat}`);
+    }
+  }
+  if (profiled) {
+    for (const variable of dash.templating?.list ?? []) {
+      const query = typeof variable.query === 'string' ? variable.query : JSON.stringify(variable.query ?? '');
+      if (usesUnprofiledLabel(query)) fail(`sync-cost dashboard variable $${variable.name}`, `unprofiled '${DEFAULT_LABEL}' in query: ${query}`);
+    }
+  }
+
+  const expressionText = dashboardExprs.join('\n');
+  const expectedInstruments = [
+    'dkg_sync_attempt_total',
+    'dkg_sync_attempt_request_bytes',
+    'dkg_sync_attempt_response_bytes',
+    'dkg_sync_operation_duration_ms',
+    'dkg_sync_operation_rejected_total',
+    'dkg_sync_singleflight_joins_total',
+    'dkg_context_graph_catchup_requests_total',
+    'dkg_context_graph_catchup_jobs_total',
+    'dkg_context_graph_catchup_job_duration_ms',
+  ];
+  for (const instrument of expectedInstruments) {
+    if (!expressionText.includes(instrument)) fail('sync-cost dashboard', `W1 instrument ${instrument} is not read by any panel`);
+  }
+
+  const flame = allPanels.find((panel) => panel.type === 'flamegraph');
+  if (!flame) fail('sync-cost dashboard', 'source-attributed flamegraph is missing');
+  else {
+    const refs = (flame.targets ?? []).map((target) => target.refId).join('');
+    if (refs !== 'ABCD') fail('sync-cost dashboard flamegraph', `target order is ${JSON.stringify(refs)}, want ABCD`);
+    const transforms = (flame.transformations ?? []).map((transform) => transform.id);
+    const wanted = ['seriesToRows', 'extractFields', 'convertFieldType', 'calculateField', 'organize'];
+    if (JSON.stringify(transforms) !== JSON.stringify(wanted)) {
+      fail('sync-cost dashboard flamegraph', `transform chain is ${JSON.stringify(transforms)}, want ${JSON.stringify(wanted)}`);
+    }
+    const exprs = (flame.targets ?? []).map((target) => target.expr ?? '').join('\n');
+    for (const label of ['source', 'lane', 'outcome']) {
+      if (!exprs.includes(label)) fail('sync-cost dashboard flamegraph', `hierarchy does not include ${label}`);
+    }
+    if (!exprs.includes('$__range')) fail('sync-cost dashboard flamegraph', 'width is not evaluated over the selected Grafana range');
+  }
+
+  const heatmaps = allPanels.filter((panel) => panel.type === 'heatmap');
+  const expectedHeatmaps = [
+    ['Logical sync operation duration heatmap', 'dkg_sync_operation_duration_ms'],
+    ['Sync scheduler queue-wait heatmap', 'dkg_sync_scheduler_queue_wait_ms'],
+    ['Walk catch-up job duration heatmap', 'dkg_context_graph_catchup_job_duration_ms'],
+  ];
+  if (heatmaps.length !== expectedHeatmaps.length) {
+    fail('sync-cost dashboard', `expected ${expectedHeatmaps.length} histogram heatmaps, got ${heatmaps.length}`);
+  }
+  for (const [title, metric] of expectedHeatmaps) {
+    const panel = heatmaps.find((candidate) => candidate.title?.startsWith(title));
+    const where = `sync-cost dashboard ${title}`;
+    if (!panel) { fail(where, 'panel is missing'); continue; }
+    if (panel.options?.calculate !== false) fail(where, 'must consume server-side Prometheus histogram buckets');
+    if (panel.fieldConfig?.defaults?.unit !== 'ms' || panel.options?.yAxis?.unit !== 'ms') fail(where, 'must render duration in milliseconds');
+    const [target, ...extra] = panel.targets ?? [];
+    if (!target || extra.length) { fail(where, `expected exactly one target, got ${(panel.targets ?? []).length}`); continue; }
+    if (target.format !== 'heatmap' || target.queryType !== 'range') fail(where, 'target must be a Prometheus range heatmap');
+    if (!target.expr?.includes(metric) || !target.expr?.includes('_bucket') || !target.expr?.includes('sum by (le)')) {
+      fail(where, `target is not a bucket aggregation for ${metric}`);
+    }
+  }
+
+  for (const title of [
+    'Completed durable operations',
+    'Attributed durable payload',
+    'Cross-family joins',
+    'Unclassified source samples',
+    'Counter resets',
+    'Minimum export samples',
+  ]) {
+    if (!allPanels.some((panel) => panel.type === 'stat' && panel.title === title)) {
+      fail('sync-cost dashboard evidence gates', `stat panel "${title}" is missing`);
+    }
+  }
+
+  // The generated promtool fixture must be expression-identical to the
+  // dashboard after substituting only Grafana runtime variables. That makes
+  // the CI parser check certify every panel query, rather than a nearby copy.
+  const substituteGrafana = (expr) => expr
+    .replaceAll('${node:regex}', '.*')
+    .replaceAll('${sync_source:regex}', '.*')
+    .replaceAll('${sync_lane:regex}', '.*')
+    .replaceAll('${sync_outcome:regex}', '.*')
+    .replaceAll('$__rate_interval', '5m')
+    .replaceAll('$__range', '2h');
+  const fixtureText = fs.readFileSync(path.join(DIR, 'w1/w1-dashboard-rules.yaml'), 'utf8').replace(/\r\n/g, '\n');
+  const fixtureExprs = [...fixtureText.matchAll(/^ {8}expr: "((?:[^"\\]|\\.)*)"$/gm)]
+    .map((match) => match[1].replace(/\\(["\\])/g, '$1'));
+  const sorted = (values) => [...values].sort();
+  const rendered = sorted(dashboardExprs.map(substituteGrafana));
+  const fixture = sorted(fixtureExprs);
+  if (rendered.length !== fixture.length || rendered.some((expr, index) => expr !== fixture[index])) {
+    fail('sync-cost dashboard ↔ w1-dashboard-rules.yaml', `expression sets differ (${rendered.length} dashboard targets vs ${fixture.length} fixture rules)`);
+  }
+}
+
 // ── logs dashboard: PR #2003 worker-pressure surface ──────────────────────
 {
   const dash = load('grafana-dashboard-dkg-node-logs.json');
diff --git a/tools/observability/w1/w1-dashboard-rules.yaml b/tools/observability/w1/w1-dashboard-rules.yaml
new file mode 100644
index 0000000000..9879fe5571
--- /dev/null
+++ b/tools/observability/w1/w1-dashboard-rules.yaml
@@ -0,0 +1,70 @@
+# GENERATED FILE — do not edit. Source: tools/observability/lib/w1.mjs
+# Exact parser fixture for grafana-dashboard-dkg-sync-cost.json.
+# Grafana variables use representative values solely for promtool parsing.
+groups:
+  - name: w1-grafana-dashboard
+    rules:
+      - record: w1_dashboard_flame_root
+        expr: "sum(increase({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\".*\", source=~\".*\", outcome=~\".*\", instance=~\".*\"}[2h]))"
+      - record: w1_dashboard_flame_source
+        expr: "sum by (source) (increase({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\".*\", source=~\".*\", outcome=~\".*\", instance=~\".*\"}[2h]))"
+      - record: w1_dashboard_flame_source_lane
+        expr: "sum by (source, lane) (increase({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\".*\", source=~\".*\", outcome=~\".*\", instance=~\".*\"}[2h]))"
+      - record: w1_dashboard_flame_source_lane_outcome
+        expr: "sum by (source, lane, outcome) (increase({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\".*\", source=~\".*\", outcome=~\".*\", instance=~\".*\"}[2h]))"
+      - record: w1_dashboard_attempts_rate
+        expr: "sum by (source, outcome, transport, plane, phase) (rate(dkg_sync_attempt_total{source=~\".*\", instance=~\".*\"}[5m]))"
+      - record: w1_dashboard_request_bytes_rate
+        expr: "sum by (source) (rate({__name__=~\"dkg_sync_attempt_request_bytes(_total)?\", source=~\".*\", instance=~\".*\"}[5m]))"
+      - record: w1_dashboard_response_bytes_rate
+        expr: "sum by (source, outcome) (rate({__name__=~\"dkg_sync_attempt_response_bytes(_total)?\", source=~\".*\", instance=~\".*\"}[5m]))"
+      - record: w1_dashboard_active_worker_equivalents
+        expr: "sum by (source, lane, outcome) (rate({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\".*\", source=~\".*\", outcome=~\".*\", instance=~\".*\"}[5m])) / 1000"
+      - record: w1_dashboard_sync_operation_duration_buckets
+        expr: "sum by (le) (rate({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_bucket\", lane=~\".*\", source=~\".*\", outcome=~\".*\", instance=~\".*\"}[5m]))"
+      - record: w1_dashboard_rejected_rate
+        expr: "sum by (lane, source, reason) (rate(dkg_sync_operation_rejected_total{lane=~\".*\", source=~\".*\", instance=~\".*\"}[5m]))"
+      - record: w1_dashboard_singleflight_joins_rate
+        expr: "sum by (scope, owner_source, joiner_source) (rate(dkg_sync_singleflight_joins_total{instance=~\".*\"}[5m]))"
+      - record: w1_dashboard_queue_wait_buckets
+        expr: "sum by (le) (rate({__name__=~\"dkg_sync_scheduler_queue_wait_ms(_milliseconds)?_bucket\", scheduler=\"sync-global\", lane=~\".*\", instance=~\".*\"}[5m]))"
+      - record: w1_dashboard_oldest_queued_age
+        expr: "max by (lane) ({__name__=~\"dkg_backpressure_oldest_queued_age_ms(_milliseconds)?\", scheduler=\"sync-global\", lane=~\".*\", instance=~\".*\"})"
+      - record: w1_dashboard_catchup_requests_rate
+        expr: "sum by (result, include_shared_memory) (rate(dkg_context_graph_catchup_requests_total{instance=~\".*\"}[5m]))"
+      - record: w1_dashboard_catchup_jobs_rate
+        expr: "sum by (status, admission) (rate(dkg_context_graph_catchup_jobs_total{instance=~\".*\"}[5m]))"
+      - record: w1_dashboard_catchup_duration_buckets
+        expr: "sum by (le) (rate({__name__=~\"dkg_context_graph_catchup_job_duration_ms(_milliseconds)?_bucket\", admission=\"walk\", instance=~\".*\"}[5m]))"
+      - record: w1_dashboard_catchup_duration_p95
+        expr: "histogram_quantile(0.95, sum by (le) (rate({__name__=~\"dkg_context_graph_catchup_job_duration_ms(_milliseconds)?_bucket\", admission=\"walk\", instance=~\".*\"}[5m])))"
+      - record: w1_dashboard_gate_completed_operations
+        expr: "sum(increase({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_count\", lane=~\"durable|changelog\", instance=~\".*\"}[2h]))"
+      - record: w1_dashboard_gate_payload_bytes
+        expr: "(sum(increase({__name__=~\"dkg_sync_attempt_request_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", instance=~\".*\"}[2h])) or vector(0)) + (sum(increase({__name__=~\"dkg_sync_attempt_response_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", instance=~\".*\"}[2h])) or vector(0))"
+      - record: w1_dashboard_gate_cross_family_joins
+        expr: "(sum(increase(dkg_sync_singleflight_joins_total{owner_source=~\"catchup-foreground\", joiner_source!~\"catchup-foreground\", instance=~\".*\"}[2h])) or vector(0)) + (sum(increase(dkg_sync_singleflight_joins_total{owner_source=~\"on-connect|reconcile\", joiner_source!~\"on-connect|reconcile\", instance=~\".*\"}[2h])) or vector(0)) + (sum(increase(dkg_sync_singleflight_joins_total{owner_source=~\"catchup-background|vm-recovery|swm-recovery|control-plane\", joiner_source!~\"catchup-background|vm-recovery|swm-recovery|control-plane\", instance=~\".*\"}[2h])) or vector(0)) + (sum(increase(dkg_sync_singleflight_joins_total{owner_source!~\"catchup-foreground|on-connect|reconcile|catchup-background|vm-recovery|swm-recovery|control-plane\", joiner_source=~\"catchup-foreground|on-connect|reconcile|catchup-background|vm-recovery|swm-recovery|control-plane\", instance=~\".*\"}[2h])) or vector(0))"
+      - record: w1_dashboard_gate_invalidating_sources
+        expr: "(sum(increase(dkg_sync_attempt_total{plane=\"durable\", phase=~\"data|meta|delta\", source!~\"catchup-foreground|on-connect|reconcile|catchup-background|vm-recovery|swm-recovery|control-plane\", instance=~\".*\"}[2h])) or vector(0)) + (sum(increase({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_count\", lane=~\"durable|changelog\", source!~\"catchup-foreground|on-connect|reconcile|catchup-background|vm-recovery|swm-recovery|control-plane\", instance=~\".*\"}[2h])) or vector(0)) + (sum(increase(dkg_sync_operation_rejected_total{lane=~\"durable|changelog\", source!~\"catchup-foreground|on-connect|reconcile|catchup-background|vm-recovery|swm-recovery|control-plane\", instance=~\".*\"}[2h])) or vector(0))"
+      - record: w1_dashboard_gate_counter_resets
+        expr: "max(resets(dkg_sync_attempt_total{plane=\"durable\", phase=~\"data|meta|delta\", instance=~\".*\"}[2h]))"
+      - record: w1_dashboard_gate_export_samples
+        expr: "min(count_over_time(dkg_sync_attempt_total{plane=\"durable\", phase=~\"data|meta|delta\", instance=~\".*\"}[2h]))"
+      - record: w1_dashboard_all_source_active_ms_per_hour
+        expr: "3600 * sum(rate({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\"durable|changelog\", instance=~\".*\"}[2h]))"
+      - record: w1_dashboard_all_source_bytes_per_hour
+        expr: "3600 * ((sum(rate({__name__=~\"dkg_sync_attempt_request_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", instance=~\".*\"}[2h])) or vector(0)) + (sum(rate({__name__=~\"dkg_sync_attempt_response_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", instance=~\".*\"}[2h])) or vector(0)))"
+      - record: w1_dashboard_eligible_active_share
+        expr: "sum(rate({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\"durable|changelog\", source=~\"catchup-foreground|on-connect|reconcile\", instance=~\".*\"}[2h])) / sum(rate({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\"durable|changelog\", instance=~\".*\"}[2h]))"
+      - record: w1_dashboard_foreground_bytes_share
+        expr: "((sum(rate({__name__=~\"dkg_sync_attempt_request_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", source=~\"catchup-foreground\", instance=~\".*\"}[2h])) or vector(0)) + (sum(rate({__name__=~\"dkg_sync_attempt_response_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", source=~\"catchup-foreground\", instance=~\".*\"}[2h])) or vector(0))) / ((sum(rate({__name__=~\"dkg_sync_attempt_request_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", source=~\"catchup-foreground|on-connect|reconcile\", instance=~\".*\"}[2h])) or vector(0)) + (sum(rate({__name__=~\"dkg_sync_attempt_response_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", source=~\"catchup-foreground|on-connect|reconcile\", instance=~\".*\"}[2h])) or vector(0)))"
+      - record: w1_dashboard_foreground_active_share
+        expr: "sum(rate({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\"durable|changelog\", source=~\"catchup-foreground\", instance=~\".*\"}[2h])) / sum(rate({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\"durable|changelog\", source=~\"catchup-foreground|on-connect|reconcile\", instance=~\".*\"}[2h]))"
+      - record: w1_dashboard_recurring_bytes_share
+        expr: "((sum(rate({__name__=~\"dkg_sync_attempt_request_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", source=~\"on-connect|reconcile\", instance=~\".*\"}[2h])) or vector(0)) + (sum(rate({__name__=~\"dkg_sync_attempt_response_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", source=~\"on-connect|reconcile\", instance=~\".*\"}[2h])) or vector(0))) / ((sum(rate({__name__=~\"dkg_sync_attempt_request_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", source=~\"catchup-foreground|on-connect|reconcile\", instance=~\".*\"}[2h])) or vector(0)) + (sum(rate({__name__=~\"dkg_sync_attempt_response_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", source=~\"catchup-foreground|on-connect|reconcile\", instance=~\".*\"}[2h])) or vector(0)))"
+      - record: w1_dashboard_recurring_active_share
+        expr: "sum(rate({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\"durable|changelog\", source=~\"on-connect|reconcile\", instance=~\".*\"}[2h])) / sum(rate({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\"durable|changelog\", source=~\"catchup-foreground|on-connect|reconcile\", instance=~\".*\"}[2h]))"
+      - record: w1_dashboard_excluded_bytes_per_hour
+        expr: "3600 * ((sum(rate({__name__=~\"dkg_sync_attempt_request_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", source=~\"catchup-background|vm-recovery|swm-recovery|control-plane\", instance=~\".*\"}[2h])) or vector(0)) + (sum(rate({__name__=~\"dkg_sync_attempt_response_bytes(_total)?\", plane=\"durable\", phase=~\"data|meta|delta\", source=~\"catchup-background|vm-recovery|swm-recovery|control-plane\", instance=~\".*\"}[2h])) or vector(0)))"
+      - record: w1_dashboard_excluded_active_ms_per_hour
+        expr: "3600 * sum(rate({__name__=~\"dkg_sync_operation_duration_ms(_milliseconds)?_sum\", lane=~\"durable|changelog\", source=~\"catchup-background|vm-recovery|swm-recovery|control-plane\", instance=~\".*\"}[2h]))"

From a12ab44bd540573c8d929406ad802c217fd53c95 Mon Sep 17 00:00:00 2001
From: Bojan <bojan.colic131@gmail.com>
Date: Mon, 3 Aug 2026 11:45:20 +0200
Subject: [PATCH 4/5] test(observability): validate sync-cost dashboard results

---
 .github/workflows/observability-artifacts.yml |  12 +
 tools/observability/lib/w1.mjs                |   2 +-
 .../w1/w1-dashboard-rules.test.yaml           | 214 ++++++++++++++++++
 3 files changed, 227 insertions(+), 1 deletion(-)
 create mode 100644 tools/observability/w1/w1-dashboard-rules.test.yaml

diff --git a/.github/workflows/observability-artifacts.yml b/.github/workflows/observability-artifacts.yml
index 480094738c..1ed7ab46b1 100644
--- a/.github/workflows/observability-artifacts.yml
+++ b/.github/workflows/observability-artifacts.yml
@@ -147,3 +147,15 @@ jobs:
           docker run --rm -v /tmp/w1-promtool:/t:ro --entrypoint promtool \
             prom/prometheus@sha256:6559acbd5d770b15bb3c954629ce190ac3cbbdb2b7f1c30f0385c4e05104e218 \
             test rules /t/w1-rules.test.yaml
+
+      # The sync-cost fixture is expression-identical to the generated Grafana
+      # targets (verify-profile-render.mjs enforces that). Result assertions
+      # below are the second half of the contract: they pin label grouping,
+      # worker-equivalent and byte units, flame widths, histogram buckets/p95,
+      # zero-filling of request-only failures, and fail-closed evidence gates.
+      - name: Unit-test the generated sync-cost dashboard query results (pinned promtool)
+        run: |
+          set -euo pipefail
+          docker run --rm -v "${PWD}/tools/observability/w1:/t:ro" --entrypoint promtool \
+            prom/prometheus@sha256:6559acbd5d770b15bb3c954629ce190ac3cbbdb2b7f1c30f0385c4e05104e218 \
+            test rules /t/w1-dashboard-rules.test.yaml
diff --git a/tools/observability/lib/w1.mjs b/tools/observability/lib/w1.mjs
index 2305e22973..60d8e0acef 100644
--- a/tools/observability/lib/w1.mjs
+++ b/tools/observability/lib/w1.mjs
@@ -607,7 +607,7 @@ const renderRules = ({ windows }) => [
 ].join('\n');
 
 // Promtool cannot parse Grafana interpolation tokens. This fixture carries the
-// exact dashboard expressions after replacing only the five runtime macros
+// exact dashboard expressions after replacing only the six runtime macros
 // with representative safe values. verify-profile-render.mjs independently
 // performs the same substitution and compares the complete expression set,
 // so this parser target cannot drift away from the dashboard it certifies.
diff --git a/tools/observability/w1/w1-dashboard-rules.test.yaml b/tools/observability/w1/w1-dashboard-rules.test.yaml
new file mode 100644
index 0000000000..0d712ee2e2
--- /dev/null
+++ b/tools/observability/w1/w1-dashboard-rules.test.yaml
@@ -0,0 +1,214 @@
+# promtool result tests for the generated DKG Nodes — Sync Cost dashboard.
+#
+# `w1-dashboard-rules.yaml` contains the exact dashboard expressions after
+# substituting representative Grafana variables. `promtool check rules` proves
+# those expressions parse; these tests additionally prove the main panels
+# return the expected numbers from realistic counters and histogram buckets.
+# This prevents a syntactically valid grouping, unit conversion, zero-fill, or
+# histogram change from silently turning into a plausible but wrong graph.
+rule_files:
+  - w1-dashboard-rules.yaml
+
+evaluation_interval: 1m
+
+tests:
+  - interval: 1m
+    name: flame hierarchy rates worker equivalents and evidence totals preserve their units and labels
+    input_series:
+      # Foreground occupies one full worker-equivalent: 60,000 ms per minute.
+      - series: 'dkg_sync_operation_duration_ms_milliseconds_sum{lane="durable", source="catchup-foreground", outcome="resolved", instance="node-a"}'
+        values: '0+60000x120'
+      - series: 'dkg_sync_operation_duration_ms_milliseconds_count{lane="durable", source="catchup-foreground", outcome="resolved", instance="node-a"}'
+        values: '0+10x120'
+      # Recurring occupies half a worker-equivalent: 30,000 ms per minute.
+      - series: 'dkg_sync_operation_duration_ms_milliseconds_sum{lane="changelog", source="reconcile", outcome="error", instance="node-a"}'
+        values: '0+30000x120'
+      - series: 'dkg_sync_operation_duration_ms_milliseconds_count{lane="changelog", source="reconcile", outcome="error", instance="node-a"}'
+        values: '0+5x120'
+      - series: 'dkg_sync_attempt_total{transport="libp2p", plane="durable", phase="data", source="catchup-foreground", outcome="response", instance="node-a"}'
+        values: '0+60x120'
+      - series: 'dkg_sync_attempt_request_bytes_total{plane="durable", phase="data", source="catchup-foreground", instance="node-a"}'
+        values: '0+600x120'
+      - series: 'dkg_sync_attempt_response_bytes_total{plane="durable", phase="data", source="catchup-foreground", outcome="response", instance="node-a"}'
+        values: '0+300x120'
+      - series: 'dkg_sync_attempt_request_bytes_total{plane="durable", phase="delta", source="reconcile", instance="node-a"}'
+        values: '0+1200x120'
+      - series: 'dkg_sync_attempt_response_bytes_total{plane="durable", phase="delta", source="reconcile", outcome="response", instance="node-a"}'
+        values: '0+600x120'
+    promql_expr_test:
+      - expr: w1_dashboard_flame_root
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_flame_root"}'
+            value: 10800000
+      - expr: w1_dashboard_flame_source_lane_outcome
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_flame_source_lane_outcome", lane="durable", outcome="resolved", source="catchup-foreground"}'
+            value: 7200000
+          - labels: '{__name__="w1_dashboard_flame_source_lane_outcome", lane="changelog", outcome="error", source="reconcile"}'
+            value: 3600000
+      - expr: w1_dashboard_active_worker_equivalents
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_active_worker_equivalents", lane="durable", outcome="resolved", source="catchup-foreground"}'
+            value: 1
+          - labels: '{__name__="w1_dashboard_active_worker_equivalents", lane="changelog", outcome="error", source="reconcile"}'
+            value: 0.5
+      - expr: w1_dashboard_attempts_rate
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_attempts_rate", outcome="response", phase="data", plane="durable", source="catchup-foreground", transport="libp2p"}'
+            value: 1
+      - expr: w1_dashboard_request_bytes_rate
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_request_bytes_rate", source="catchup-foreground"}'
+            value: 10
+          - labels: '{__name__="w1_dashboard_request_bytes_rate", source="reconcile"}'
+            value: 20
+      - expr: w1_dashboard_response_bytes_rate
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_response_bytes_rate", outcome="response", source="catchup-foreground"}'
+            value: 5
+          - labels: '{__name__="w1_dashboard_response_bytes_rate", outcome="response", source="reconcile"}'
+            value: 10
+      - expr: w1_dashboard_gate_completed_operations
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_gate_completed_operations"}'
+            value: 1800
+      - expr: w1_dashboard_gate_payload_bytes
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_gate_payload_bytes"}'
+            value: 324000
+      - expr: w1_dashboard_all_source_active_ms_per_hour
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_all_source_active_ms_per_hour"}'
+            value: 5399999.999999999
+      - expr: w1_dashboard_all_source_bytes_per_hour
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_all_source_bytes_per_hour"}'
+            value: 162000
+      - expr: w1_dashboard_eligible_active_share
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_eligible_active_share"}'
+            value: 1
+      - expr: w1_dashboard_foreground_bytes_share
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_foreground_bytes_share"}'
+            value: 0.3333333333333333
+      - expr: w1_dashboard_recurring_bytes_share
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_recurring_bytes_share"}'
+            value: 0.6666666666666666
+      - expr: w1_dashboard_foreground_active_share
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_foreground_active_share"}'
+            value: 0.6666666666666667
+      - expr: w1_dashboard_recurring_active_share
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_recurring_active_share"}'
+            value: 0.33333333333333337
+
+  - interval: 1m
+    name: request-only failure windows keep their real byte totals instead of going blank
+    input_series:
+      - series: 'dkg_sync_attempt_request_bytes_total{plane="durable", phase="data", source="reconcile", instance="node-a"}'
+        values: '0+600x120'
+    promql_expr_test:
+      - expr: w1_dashboard_gate_payload_bytes
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_gate_payload_bytes"}'
+            value: 72000
+      - expr: w1_dashboard_all_source_bytes_per_hour
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_all_source_bytes_per_hour"}'
+            value: 36000
+
+  - interval: 1m
+    name: histogram heatmaps and catchup p95 use cumulative server-side buckets
+    input_series:
+      - series: 'dkg_sync_operation_duration_ms_milliseconds_bucket{lane="durable", source="reconcile", outcome="resolved", le="100", instance="node-a"}'
+        values: '0+30x120'
+      - series: 'dkg_sync_operation_duration_ms_milliseconds_bucket{lane="durable", source="reconcile", outcome="resolved", le="200", instance="node-a"}'
+        values: '0+60x120'
+      - series: 'dkg_sync_operation_duration_ms_milliseconds_bucket{lane="durable", source="reconcile", outcome="resolved", le="+Inf", instance="node-a"}'
+        values: '0+60x120'
+      - series: 'dkg_sync_scheduler_queue_wait_ms_milliseconds_bucket{scheduler="sync-global", lane="durable", priority_class="normal", le="100", instance="node-a"}'
+        values: '0+15x120'
+      - series: 'dkg_sync_scheduler_queue_wait_ms_milliseconds_bucket{scheduler="sync-global", lane="durable", priority_class="normal", le="+Inf", instance="node-a"}'
+        values: '0+30x120'
+      - series: 'dkg_context_graph_catchup_job_duration_ms_milliseconds_bucket{admission="walk", le="100", instance="node-a"}'
+        values: '0+30x120'
+      - series: 'dkg_context_graph_catchup_job_duration_ms_milliseconds_bucket{admission="walk", le="200", instance="node-a"}'
+        values: '0+60x120'
+      - series: 'dkg_context_graph_catchup_job_duration_ms_milliseconds_bucket{admission="walk", le="+Inf", instance="node-a"}'
+        values: '0+60x120'
+    promql_expr_test:
+      - expr: w1_dashboard_sync_operation_duration_buckets
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_sync_operation_duration_buckets", le="100"}'
+            value: 0.5
+          - labels: '{__name__="w1_dashboard_sync_operation_duration_buckets", le="200"}'
+            value: 1
+          - labels: '{__name__="w1_dashboard_sync_operation_duration_buckets", le="+Inf"}'
+            value: 1
+      - expr: w1_dashboard_queue_wait_buckets
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_queue_wait_buckets", le="100"}'
+            value: 0.25
+          - labels: '{__name__="w1_dashboard_queue_wait_buckets", le="+Inf"}'
+            value: 0.5
+      - expr: w1_dashboard_catchup_duration_buckets
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_catchup_duration_buckets", le="100"}'
+            value: 0.5
+          - labels: '{__name__="w1_dashboard_catchup_duration_buckets", le="200"}'
+            value: 1
+          - labels: '{__name__="w1_dashboard_catchup_duration_buckets", le="+Inf"}'
+            value: 1
+      - expr: w1_dashboard_catchup_duration_p95
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_catchup_duration_p95"}'
+            value: 190
+
+  - interval: 1m
+    name: cross-family joins are counted once per event and unclassified sources fail the gate
+    input_series:
+      - series: 'dkg_sync_singleflight_joins_total{scope="cg-a", owner_source="catchup-foreground", joiner_source="reconcile", instance="node-a"}'
+        values: '0+1x120'
+      - series: 'dkg_sync_singleflight_joins_total{scope="cg-a", owner_source="unspecified", joiner_source="catchup-foreground", instance="node-a"}'
+        values: '0+1x120'
+      - series: 'dkg_sync_attempt_total{transport="libp2p", plane="durable", phase="data", source="unspecified", outcome="response", instance="node-a"}'
+        values: '0+2x120'
+      - series: 'dkg_sync_operation_duration_ms_milliseconds_count{lane="durable", source="unspecified", outcome="resolved", instance="node-a"}'
+        values: '0+3x120'
+      - series: 'dkg_sync_operation_rejected_total{lane="durable", source="unspecified", reason="queue-full", instance="node-a"}'
+        values: '0+4x120'
+    promql_expr_test:
+      - expr: w1_dashboard_gate_cross_family_joins
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_gate_cross_family_joins"}'
+            value: 240
+      - expr: w1_dashboard_gate_invalidating_sources
+        eval_time: 2h
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_gate_invalidating_sources"}'
+            value: 1080

From 62e2b1e3e9ab8f08634aa12263a6d7aab00ba32b Mon Sep 17 00:00:00 2001
From: Bojan <bojan.colic131@gmail.com>
Date: Mon, 3 Aug 2026 13:11:51 +0200
Subject: [PATCH 5/5] test(observability): validate bounded LogQL queries

---
 .github/workflows/observability-artifacts.yml |  39 ++++
 docs/use-dkg/backpressure-observability.md    |   9 +-
 tools/observability/RUNBOOK.md                |   2 +
 .../grafana-dashboard-dkg-node-logs.json      |  16 +-
 tools/observability/lib/dashboards.mjs        |   7 +-
 tools/observability/lib/queries.mjs           |  12 +-
 .../verify-backpressure-logql.mjs             | 171 ++++++++++++++++++
 tools/observability/verify-profile-render.mjs |  16 +-
 .../w1/w1-dashboard-rules.test.yaml           |  41 +++++
 9 files changed, 297 insertions(+), 16 deletions(-)
 create mode 100644 tools/observability/verify-backpressure-logql.mjs

diff --git a/.github/workflows/observability-artifacts.yml b/.github/workflows/observability-artifacts.yml
index 1ed7ab46b1..cf25870a17 100644
--- a/.github/workflows/observability-artifacts.yml
+++ b/.github/workflows/observability-artifacts.yml
@@ -23,6 +23,8 @@ on:
   pull_request:
     paths:
       - 'tools/observability/**'
+      # The pinned real-Loki result test mounts this exact single-binary config.
+      - 'tools/log-collection-poc/loki-config.yaml'
       # the gate must also verify itself when the gate changes
       - '.github/workflows/observability-artifacts.yml'
       # …and it must run when the SOURCES it mirrors change, not only when the
@@ -89,6 +91,43 @@ jobs:
           node tools/observability/verify-profile-render.mjs /tmp/render \
             --prom-node-label service_instance_id --vm-uid test-vm-uid --loki-uid test-loki-uid
 
+      # The Loki queries are valid even when a nested JSON field is misspelled;
+      # they simply return no rows. Run the generated dashboard expressions
+      # against the repository's Loki 3.3.2 configuration to pin actual
+      # scheduler/lane/phase/operation labels, moved-slot peaks, empty-slot
+      # handling, and the fixed-window resource bound. The multi-arch digest is
+      # the reproducibility contract; the tag is deliberately not used.
+      - name: Evaluate worker-pressure LogQL against pinned Loki
+        run: |
+          set -euo pipefail
+          container_id="$(docker run --rm -d \
+            -p 127.0.0.1:13100:3100 \
+            -v "${PWD}/tools/log-collection-poc/loki-config.yaml:/etc/loki/loki-config.yaml:ro" \
+            grafana/loki@sha256:8af2de1abbdd7aa92b27c9bcc96f0f4140c9096b507c77921ffddf1c6ad6c48f \
+            -config.file=/etc/loki/loki-config.yaml)"
+          cleanup() {
+            status=$?
+            trap - EXIT
+            if [ "$status" -ne 0 ]; then docker logs "$container_id"; fi
+            docker stop "$container_id" >/dev/null || true
+            exit "$status"
+          }
+          trap cleanup EXIT
+          ready=0
+          for _ in $(seq 1 30); do
+            if curl --fail --silent http://127.0.0.1:13100/ready >/dev/null; then
+              ready=1
+              break
+            fi
+            sleep 1
+          done
+          if [ "$ready" -ne 1 ]; then
+            echo 'Loki did not become ready' >&2
+            exit 1
+          fi
+          LOKI_URL=http://127.0.0.1:13100 \
+            node tools/observability/verify-backpressure-logql.mjs
+
       # W1 decision queries: the semantic contract behind the numbers, checked
       # per selector rather than by sampling. Run against BOTH renders so the
       # W1 queries are proven to follow --prom-node-label like every other
diff --git a/docs/use-dkg/backpressure-observability.md b/docs/use-dkg/backpressure-observability.md
index 6e761d79e6..883717f781 100644
--- a/docs/use-dkg/backpressure-observability.md
+++ b/docs/use-dkg/backpressure-observability.md
@@ -187,7 +187,11 @@ OTLP exporters, so the records reach the Loki-backed Grafana dashboards.
 The **DKG Node — Logs** dashboard includes a **Scheduler pressure** row. Its
 flame graph expands the bounded `activeOperations` and `queuedOperations`
 arrays, groups them by scheduler, lane, and operation source, and shows the
-largest sampled `oldestAgeMs` in the selected time range.
+largest sampled `oldestAgeMs` in a fixed most-recent one-hour window. The
+lookback is intentionally independent of the Grafana-selected range: Loki 3
+splits long-range instant metric queries internally, so letting an incident
+range of 6 or 24 hours flow into all five flamegraph targets can make the
+dashboard itself create backend pressure.
 
 The two top-level branches have distinct meanings:
 
@@ -202,7 +206,8 @@ instead of a per-item event stream, so Grafana must not claim more precision
 than the log contract provides.
 
 Two heatmaps below the flame graph keep the active and queued meanings
-separate. Each heatmap groups samples by scheduler/lane and shows the
+separate and retain the full Grafana-selected historical range. Each heatmap
+groups samples by scheduler/lane and shows the
 distribution over time of the peak sampled age in each Grafana resolution
 bucket:
 
diff --git a/tools/observability/RUNBOOK.md b/tools/observability/RUNBOOK.md
index 9c10c23c91..18b5cf2844 100644
--- a/tools/observability/RUNBOOK.md
+++ b/tools/observability/RUNBOOK.md
@@ -35,6 +35,8 @@ channels): `example-alerts.md` (importable payloads: `alert-rules.provisioning.j
   split internally and fail with `maximum of series (500) reached` even at tiny
   stream counts. Use **range queries** + a Grafana reduce (`sum`/`last`) for
   totals; keep instant queries to short fixed windows (e.g. `[10m]`).
+  The worker-pressure flamegraph follows this rule with a fixed `[1h]`
+  snapshot; its adjacent heatmaps are the selected-range historical view.
 
 **To light up metrics + traces** (node side, per node): set
 `OTEL_EXPORTER_OTLP_ENDPOINT=http://<collector-host>:4318` (one env var
diff --git a/tools/observability/grafana-dashboard-dkg-node-logs.json b/tools/observability/grafana-dashboard-dkg-node-logs.json
index f78fae7bf1..bc825c881a 100644
--- a/tools/observability/grafana-dashboard-dkg-node-logs.json
+++ b/tools/observability/grafana-dashboard-dkg-node-logs.json
@@ -227,12 +227,12 @@
       "type": "text",
       "options": {
         "mode": "markdown",
-        "content": "**Worker queue diagnostics (PR #2003).** The flame graph answers **which scheduler/lane/operation produced the pressure**; the two heatmaps show **when active and queued pressure accumulated and how old it became**. All three views read the same bounded structured `[backpressure]` records from Loki. These sparse transition/summary samples describe elapsed pressure age; they are not CPU profiles, invocation counts, or exact end-to-end job durations."
+        "content": "**Worker queue diagnostics (PR #2003).** The flame graph answers **which scheduler/lane/operation produced recent pressure** using a fixed **last 1 hour** snapshot; the two heatmaps keep the full Grafana-selected range and show **when active and queued pressure accumulated and how old it became**. All three views read the same bounded structured `[backpressure]` records from Loki. These sparse transition/summary samples describe elapsed pressure age; they are not CPU profiles, invocation counts, or exact end-to-end job durations."
       },
       "transparent": true
     },
     {
-      "title": "Worker queue pressure flame graph — $node",
+      "title": "Worker queue pressure flame graph (last 1 hour) — $node",
       "gridPos": {
         "h": 14,
         "w": 24,
@@ -244,7 +244,7 @@
         "uid": "${loki}"
       },
       "type": "flamegraph",
-      "description": "Each leaf is a PR #2003 worker source. Width is its peak sampled oldest elapsed age in the selected range (milliseconds), split between admitted/active work and queued work. This is pressure age, not CPU utilization, request share, or exact completed-job runtime.",
+      "description": "Each leaf is a PR #2003 worker source. Width is its peak sampled oldest elapsed age in the fixed last 1h (milliseconds), split between admitted/active work and queued work. The fixed short lookback protects Loki from expensive long-range instant queries; the heatmaps below retain selected-range history. This is pressure age, not CPU utilization, request share, or exact completed-job runtime.",
       "options": {
         "showFlameGraphOnly": false
       },
@@ -261,7 +261,7 @@
             "uid": "${loki}"
           },
           "refId": "A",
-          "expr": "sum((max by (scheduler, lane, phase, operation) ((max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[0].operation\", age=\"activeOperations[0].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`0` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[1].operation\", age=\"activeOperations[1].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`1` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[2].operation\", age=\"activeOperations[2].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`2` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[3].operation\", age=\"activeOperations[3].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`3` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[4].operation\", age=\"activeOperations[4].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`4` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[5].operation\", age=\"activeOperations[5].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`5` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[6].operation\", age=\"activeOperations[6].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`6` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[7].operation\", age=\"activeOperations[7].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`7` | unwrap age | __error__ = `` [$__range])))) or (max by (scheduler, lane, phase, operation) ((max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[0].operation\", age=\"queuedOperations[0].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`0` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[1].operation\", age=\"queuedOperations[1].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`1` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[2].operation\", age=\"queuedOperations[2].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`2` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[3].operation\", age=\"queuedOperations[3].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`3` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[4].operation\", age=\"queuedOperations[4].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`4` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[5].operation\", age=\"queuedOperations[5].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`5` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[6].operation\", age=\"queuedOperations[6].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`6` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[7].operation\", age=\"queuedOperations[7].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`7` | unwrap age | __error__ = `` [$__range])))))",
+          "expr": "sum((max by (scheduler, lane, phase, operation) ((max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[0].operation\", age=\"activeOperations[0].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`0` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[1].operation\", age=\"activeOperations[1].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`1` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[2].operation\", age=\"activeOperations[2].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`2` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[3].operation\", age=\"activeOperations[3].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`3` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[4].operation\", age=\"activeOperations[4].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`4` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[5].operation\", age=\"activeOperations[5].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`5` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[6].operation\", age=\"activeOperations[6].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`6` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[7].operation\", age=\"activeOperations[7].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`7` | unwrap age | __error__ = `` [1h])))) or (max by (scheduler, lane, phase, operation) ((max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[0].operation\", age=\"queuedOperations[0].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`0` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[1].operation\", age=\"queuedOperations[1].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`1` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[2].operation\", age=\"queuedOperations[2].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`2` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[3].operation\", age=\"queuedOperations[3].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`3` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[4].operation\", age=\"queuedOperations[4].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`4` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[5].operation\", age=\"queuedOperations[5].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`5` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[6].operation\", age=\"queuedOperations[6].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`6` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[7].operation\", age=\"queuedOperations[7].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`7` | unwrap age | __error__ = `` [1h])))))",
           "legendFormat": "0|0|all sampled worker pressure",
           "instant": true,
           "queryType": "instant"
@@ -272,7 +272,7 @@
             "uid": "${loki}"
           },
           "refId": "B",
-          "expr": "sum(max by (scheduler, lane, phase, operation) ((max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[0].operation\", age=\"activeOperations[0].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`0` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[1].operation\", age=\"activeOperations[1].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`1` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[2].operation\", age=\"activeOperations[2].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`2` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[3].operation\", age=\"activeOperations[3].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`3` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[4].operation\", age=\"activeOperations[4].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`4` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[5].operation\", age=\"activeOperations[5].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`5` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[6].operation\", age=\"activeOperations[6].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`6` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[7].operation\", age=\"activeOperations[7].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`7` | unwrap age | __error__ = `` [$__range]))))",
+          "expr": "sum(max by (scheduler, lane, phase, operation) ((max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[0].operation\", age=\"activeOperations[0].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`0` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[1].operation\", age=\"activeOperations[1].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`1` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[2].operation\", age=\"activeOperations[2].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`2` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[3].operation\", age=\"activeOperations[3].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`3` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[4].operation\", age=\"activeOperations[4].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`4` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[5].operation\", age=\"activeOperations[5].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`5` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[6].operation\", age=\"activeOperations[6].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`6` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[7].operation\", age=\"activeOperations[7].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`7` | unwrap age | __error__ = `` [1h]))))",
           "legendFormat": "1|0|active / admitted",
           "instant": true,
           "queryType": "instant"
@@ -283,7 +283,7 @@
             "uid": "${loki}"
           },
           "refId": "C",
-          "expr": "max by (scheduler, lane, phase, operation) ((max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[0].operation\", age=\"activeOperations[0].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`0` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[1].operation\", age=\"activeOperations[1].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`1` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[2].operation\", age=\"activeOperations[2].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`2` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[3].operation\", age=\"activeOperations[3].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`3` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[4].operation\", age=\"activeOperations[4].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`4` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[5].operation\", age=\"activeOperations[5].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`5` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[6].operation\", age=\"activeOperations[6].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`6` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[7].operation\", age=\"activeOperations[7].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`7` | unwrap age | __error__ = `` [$__range])))",
+          "expr": "max by (scheduler, lane, phase, operation) ((max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[0].operation\", age=\"activeOperations[0].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`0` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[1].operation\", age=\"activeOperations[1].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`1` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[2].operation\", age=\"activeOperations[2].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`2` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[3].operation\", age=\"activeOperations[3].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`3` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[4].operation\", age=\"activeOperations[4].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`4` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[5].operation\", age=\"activeOperations[5].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`5` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[6].operation\", age=\"activeOperations[6].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`6` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"activeOperations[7].operation\", age=\"activeOperations[7].oldestAgeMs\" | operation != `` | label_format phase=`active`, slot=`7` | unwrap age | __error__ = `` [1h])))",
           "legendFormat": "2|1|{{scheduler}}/{{lane}}/{{operation}}",
           "instant": true,
           "queryType": "instant"
@@ -294,7 +294,7 @@
             "uid": "${loki}"
           },
           "refId": "D",
-          "expr": "sum(max by (scheduler, lane, phase, operation) ((max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[0].operation\", age=\"queuedOperations[0].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`0` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[1].operation\", age=\"queuedOperations[1].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`1` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[2].operation\", age=\"queuedOperations[2].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`2` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[3].operation\", age=\"queuedOperations[3].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`3` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[4].operation\", age=\"queuedOperations[4].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`4` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[5].operation\", age=\"queuedOperations[5].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`5` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[6].operation\", age=\"queuedOperations[6].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`6` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[7].operation\", age=\"queuedOperations[7].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`7` | unwrap age | __error__ = `` [$__range]))))",
+          "expr": "sum(max by (scheduler, lane, phase, operation) ((max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[0].operation\", age=\"queuedOperations[0].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`0` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[1].operation\", age=\"queuedOperations[1].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`1` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[2].operation\", age=\"queuedOperations[2].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`2` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[3].operation\", age=\"queuedOperations[3].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`3` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[4].operation\", age=\"queuedOperations[4].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`4` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[5].operation\", age=\"queuedOperations[5].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`5` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[6].operation\", age=\"queuedOperations[6].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`6` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[7].operation\", age=\"queuedOperations[7].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`7` | unwrap age | __error__ = `` [1h]))))",
           "legendFormat": "1|0|queued / waiting",
           "instant": true,
           "queryType": "instant"
@@ -305,7 +305,7 @@
             "uid": "${loki}"
           },
           "refId": "E",
-          "expr": "max by (scheduler, lane, phase, operation) ((max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[0].operation\", age=\"queuedOperations[0].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`0` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[1].operation\", age=\"queuedOperations[1].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`1` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[2].operation\", age=\"queuedOperations[2].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`2` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[3].operation\", age=\"queuedOperations[3].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`3` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[4].operation\", age=\"queuedOperations[4].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`4` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[5].operation\", age=\"queuedOperations[5].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`5` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[6].operation\", age=\"queuedOperations[6].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`6` | unwrap age | __error__ = `` [$__range])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[7].operation\", age=\"queuedOperations[7].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`7` | unwrap age | __error__ = `` [$__range])))",
+          "expr": "max by (scheduler, lane, phase, operation) ((max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[0].operation\", age=\"queuedOperations[0].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`0` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[1].operation\", age=\"queuedOperations[1].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`1` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[2].operation\", age=\"queuedOperations[2].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`2` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[3].operation\", age=\"queuedOperations[3].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`3` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[4].operation\", age=\"queuedOperations[4].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`4` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[5].operation\", age=\"queuedOperations[5].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`5` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[6].operation\", age=\"queuedOperations[6].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`6` | unwrap age | __error__ = `` [1h])) or (max_over_time({service_name=\"dkg-node\", service_instance_id=\"$node\"} |= `[backpressure]` | regexp `\\[backpressure\\] (?P<payload>\\{.*\\})` | line_format `{{.payload}}` | json scheduler, lane, operation=\"queuedOperations[7].operation\", age=\"queuedOperations[7].oldestAgeMs\" | operation != `` | label_format phase=`queued`, slot=`7` | unwrap age | __error__ = `` [1h])))",
           "legendFormat": "2|1|{{scheduler}}/{{lane}}/{{operation}}",
           "instant": true,
           "queryType": "instant"
diff --git a/tools/observability/lib/dashboards.mjs b/tools/observability/lib/dashboards.mjs
index cbadb59950..14fa659285 100644
--- a/tools/observability/lib/dashboards.mjs
+++ b/tools/observability/lib/dashboards.mjs
@@ -7,6 +7,7 @@
 import {
   backpressurePeakAgeOverTime,
   backpressurePeakAgeByOperation,
+  BACKPRESSURE_FLAME_LOOKBACK,
   dkgLogStream,
   severityIs,
   severityMatches,
@@ -105,8 +106,8 @@ const BACKPRESSURE_FLAME = (stream) => {
     def: {
       datasource: LOKI,
       type: 'flamegraph',
-      title: 'Worker queue pressure flame graph — $node',
-      description: 'Each leaf is a PR #2003 worker source. Width is its peak sampled oldest elapsed age in the selected range (milliseconds), split between admitted/active work and queued work. This is pressure age, not CPU utilization, request share, or exact completed-job runtime.',
+      title: 'Worker queue pressure flame graph (last 1 hour) — $node',
+      description: `Each leaf is a PR #2003 worker source. Width is its peak sampled oldest elapsed age in the fixed last ${BACKPRESSURE_FLAME_LOOKBACK} (milliseconds), split between admitted/active work and queued work. The fixed short lookback protects Loki from expensive long-range instant queries; the heatmaps below retain selected-range history. This is pressure age, not CPU utilization, request share, or exact completed-job runtime.`,
       options: { showFlameGraphOnly: false },
       fieldConfig: { defaults: { unit: 'ms' }, overrides: [] },
       targets: [
@@ -402,7 +403,7 @@ const buildNodeLogsDashboard = (NODE_IDENTITY) => ({
         targets: [{ datasource: LOKI, refId: 'A', expr: `sum(sum_over_time(${NB}${RPCPIPE}[$__auto]))` }] } },
     ],
     [ ROW('Scheduler pressure') ],
-    [ TEXT('**Worker queue diagnostics (PR #2003).** The flame graph answers **which scheduler/lane/operation produced the pressure**; the two heatmaps show **when active and queued pressure accumulated and how old it became**. All three views read the same bounded structured `[backpressure]` records from Loki. These sparse transition/summary samples describe elapsed pressure age; they are not CPU profiles, invocation counts, or exact end-to-end job durations.') ],
+    [ TEXT('**Worker queue diagnostics (PR #2003).** The flame graph answers **which scheduler/lane/operation produced recent pressure** using a fixed **last 1 hour** snapshot; the two heatmaps keep the full Grafana-selected range and show **when active and queued pressure accumulated and how old it became**. All three views read the same bounded structured `[backpressure]` records from Loki. These sparse transition/summary samples describe elapsed pressure age; they are not CPU profiles, invocation counts, or exact end-to-end job durations.') ],
     [ BACKPRESSURE_FLAME(NB) ],
     [ BACKPRESSURE_HEATMAP(NB, 'active'), BACKPRESSURE_HEATMAP(NB, 'queued') ],
     [ LOGSP(24, 10, 'Backpressure transitions and summaries — $node', `${NB} |= \`[backpressure]\``) ],
diff --git a/tools/observability/lib/queries.mjs b/tools/observability/lib/queries.mjs
index 38c47ecf52..8c07a609ae 100644
--- a/tools/observability/lib/queries.mjs
+++ b/tools/observability/lib/queries.mjs
@@ -24,12 +24,20 @@ export const severityMatches = (regex) => ` | severity_text=~\`${regex}\``;
  *  callers append the range selector directly: `...${RPC_USAGE_PIPELINE}[1h]`. */
 export const RPC_USAGE_PIPELINE = ' |= `rpc_usage` | logfmt | method != `` | unwrap count ';
 
+// Loki 3 internally splits long-lookback instant metric queries and can spend
+// tens of seconds of aggregate queue time on this fixed-slot expansion even
+// for a handful of records. The flamegraph is therefore an explicitly recent
+// pressure snapshot. Selected-range history remains available in the range-
+// query heatmaps below; do not replace this with Grafana's `$__range` without
+// a measured query/transform redesign.
+export const BACKPRESSURE_FLAME_LOOKBACK = '1h';
+
 /**
  * Build a Loki expression for the peak sampled age of every bounded
  * backpressure operation in a phase. PR #2003 keeps at most eight operation
  * summaries per lane, so the query expands those fixed indexes and collapses
  * them back by source. `slot` prevents LogQL's `or` from dropping an operation
- * that moved between array positions during the selected time range.
+ * that moved between array positions during the fixed recent lookback.
  *
  * The returned value is an elapsed age in milliseconds from the diagnostic
  * sample. It is not CPU time, request count, or a completed-job duration.
@@ -46,7 +54,7 @@ export const backpressurePeakAgeByOperation = (stream, phase) => {
     + ` | json scheduler, lane, operation="${field}[${index}].operation", age="${field}[${index}].oldestAgeMs"`
     + ' | operation != ``'
     + ` | label_format phase=\`${phase}\`, slot=\`${index}\``
-    + ' | unwrap age | __error__ = `` [$__range])');
+    + ` | unwrap age | __error__ = \`\` [${BACKPRESSURE_FLAME_LOOKBACK}])`);
   return `max by (scheduler, lane, phase, operation) (${entries.map((entry) => `(${entry})`).join(' or ')})`;
 };
 
diff --git a/tools/observability/verify-backpressure-logql.mjs b/tools/observability/verify-backpressure-logql.mjs
new file mode 100644
index 0000000000..76d72fc74d
--- /dev/null
+++ b/tools/observability/verify-backpressure-logql.mjs
@@ -0,0 +1,171 @@
+#!/usr/bin/env node
+// Execute the GENERATED worker-pressure flamegraph and heatmap queries against
+// the repository's pinned Loki. Structural JSON checks cannot catch valid
+// LogQL that silently returns no rows (for example a misspelled JSON age
+// field), and a parser cannot prove the scheduler/lane/operation labels or
+// peak values. This fixture covers both, plus the Loki 3 resource contract for
+// the fixed-window instant flamegraph.
+//
+// The caller owns the disposable Loki process:
+//   LOKI_URL=http://127.0.0.1:13100 node verify-backpressure-logql.mjs
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { BACKPRESSURE_FLAME_LOOKBACK } from './lib/queries.mjs';
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const LOKI = (process.env.LOKI_URL ?? 'http://127.0.0.1:13100').replace(/\/$/, '');
+const MAX_SPLITS = 0;
+const MAX_QUEUE_TIME_SECONDS = 2;
+
+const fail = (message) => { throw new Error(`backpressure LogQL verify: ${message}`); };
+const assert = (condition, message) => { if (!condition) fail(message); };
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+const request = async (endpoint, { json, params } = {}) => {
+  const response = await fetch(`${LOKI}${endpoint}`, json ? {
+    method: 'POST',
+    headers: { 'content-type': 'application/json' },
+    body: JSON.stringify(json),
+  } : {
+    method: 'POST',
+    headers: { 'content-type': 'application/x-www-form-urlencoded' },
+    body: new URLSearchParams(params),
+  });
+  const text = await response.text();
+  if (!response.ok) fail(`${endpoint} returned HTTP ${response.status}: ${text.slice(0, 800)}`);
+  return text ? JSON.parse(text) : null;
+};
+
+const node = `logql-contract-${Date.now()}`;
+const nowMs = Date.now();
+const ns = (offsetMs) => String(BigInt(nowMs + offsetMs) * 1_000_000n);
+const line = (payload) => `[backpressure] ${JSON.stringify(payload)}`;
+
+const records = [
+  [-300_000, {
+    scheduler: 'sync-global', lane: 'durable', oldestActiveAgeMs: 1800, oldestQueuedAgeMs: 3400,
+    activeOperations: [
+      { operation: 'catchup-foreground', oldestAgeMs: 1800 },
+      { operation: 'reconcile', oldestAgeMs: 1500 },
+    ],
+    queuedOperations: [
+      { operation: 'on-connect', oldestAgeMs: 3400 },
+      {},
+    ],
+  }],
+  [-120_000, {
+    scheduler: 'sync-global', lane: 'durable', oldestActiveAgeMs: 2400, oldestQueuedAgeMs: 3000,
+    // Both named operations moved slots; the empty object proves unused slots
+    // are ignored rather than becoming an unlabeled zero-valued series.
+    activeOperations: [
+      { operation: 'reconcile', oldestAgeMs: 1100 },
+      { operation: 'catchup-foreground', oldestAgeMs: 2400 },
+    ],
+    queuedOperations: [
+      {},
+      { operation: 'on-connect', oldestAgeMs: 3000 },
+    ],
+  }],
+  [-60_000, {
+    scheduler: 'sync-global', lane: 'durable', oldestActiveAgeMs: 2100, oldestQueuedAgeMs: 0,
+    activeOperations: [{}, { operation: 'catchup-foreground', oldestAgeMs: 2100 }],
+    queuedOperations: [{}],
+  }],
+];
+
+await request('/loki/api/v1/push', {
+  json: {
+    streams: [{
+      stream: { service_name: 'dkg-node', service_instance_id: node },
+      values: records.map(([offset, payload]) => [ns(offset), line(payload)]),
+    }],
+  },
+});
+
+const dashboard = JSON.parse(fs.readFileSync(path.join(HERE, 'grafana-dashboard-dkg-node-logs.json'), 'utf8'));
+const flame = dashboard.panels.find((panel) => panel.type === 'flamegraph' && panel.title?.startsWith('Worker queue pressure'));
+assert(flame, 'generated worker-pressure flamegraph is missing');
+assert(flame.title.includes('last 1 hour'), `flamegraph title does not disclose its fixed scope: ${flame.title}`);
+assert((flame.targets ?? []).map((target) => target.refId).join('') === 'ABCDE', 'flamegraph target order must remain ABCDE');
+
+const fixedWindow = `[${BACKPRESSURE_FLAME_LOOKBACK}]`;
+for (const target of flame.targets) {
+  assert(!target.expr.includes('$__range'), `target ${target.refId} still inherits the selected Grafana range`);
+  assert(target.expr.includes(fixedWindow), `target ${target.refId} does not contain fixed lookback ${fixedWindow}`);
+  assert(target.instant === true && target.queryType === 'instant', `target ${target.refId} is no longer an instant flamegraph row`);
+}
+
+const queryInstant = async (query) => request('/loki/api/v1/query', {
+  params: { query, time: String(nowMs * 1_000_000) },
+});
+const queryRange = async (query) => request('/loki/api/v1/query_range', {
+  params: {
+    query,
+    start: String((nowMs - 10 * 60_000) * 1_000_000),
+    end: String(nowMs * 1_000_000),
+    step: '30',
+  },
+});
+const withNode = (expr) => expr.replaceAll('$node', node);
+
+let instantResults;
+for (let attempt = 0; attempt < 20; attempt++) {
+  instantResults = [];
+  for (const target of flame.targets) instantResults.push([target, await queryInstant(withNode(target.expr))]);
+  if (instantResults.find(([target]) => target.refId === 'C')[1].data.result.length) break;
+  await sleep(250);
+}
+
+const byRef = Object.fromEntries(instantResults.map(([target, result]) => [target.refId, result]));
+for (const [refId, response] of Object.entries(byRef)) {
+  assert(response.status === 'success' && response.data.resultType === 'vector', `target ${refId} did not return a successful vector`);
+  const summary = response.data.stats?.summary;
+  assert(summary, `target ${refId} response omitted Loki query statistics`);
+  const splits = Number(summary.splits);
+  const queueTime = Number(summary.queueTime);
+  assert(Number.isFinite(splits) && splits <= MAX_SPLITS,
+    `target ${refId} used ${splits} internal splits (maximum ${MAX_SPLITS})`);
+  assert(Number.isFinite(queueTime) && queueTime <= MAX_QUEUE_TIME_SECONDS,
+    `target ${refId} used ${queueTime}s aggregate queue time (maximum ${MAX_QUEUE_TIME_SECONDS}s)`);
+}
+
+const scalar = (refId) => {
+  const result = byRef[refId].data.result;
+  assert(result.length === 1, `target ${refId} returned ${result.length} samples, want one`);
+  return Number(result[0].value[1]);
+};
+assert(scalar('A') === 7300, `root width is ${scalar('A')}, want 7300`);
+assert(scalar('B') === 3900, `active parent width is ${scalar('B')}, want 3900`);
+assert(scalar('D') === 3400, `queued parent width is ${scalar('D')}, want 3400`);
+
+const leafMap = (refId) => new Map(byRef[refId].data.result.map(({ metric, value }) => [
+  [metric.scheduler, metric.lane, metric.phase, metric.operation].join('|'),
+  Number(value[1]),
+]));
+const active = leafMap('C');
+const queued = leafMap('E');
+assert(active.size === 2, `active leaves=${active.size}, want 2 (empty slots must not leak)`);
+assert(active.get('sync-global|durable|active|catchup-foreground') === 2400, 'active catchup-foreground peak is not 2400 ms');
+assert(active.get('sync-global|durable|active|reconcile') === 1500, 'active reconcile peak is not 1500 ms after moving slots');
+assert(queued.size === 1, `queued leaves=${queued.size}, want 1 (empty slots must not leak)`);
+assert(queued.get('sync-global|durable|queued|on-connect') === 3400, 'queued on-connect peak is not 3400 ms after moving slots');
+
+const heatmaps = dashboard.panels.filter((panel) => panel.type === 'heatmap'
+  && (panel.title?.startsWith('Active / admitted') || panel.title?.startsWith('Queued / waiting')));
+assert(heatmaps.length === 2, `worker-pressure heatmaps=${heatmaps.length}, want 2`);
+for (const panel of heatmaps) {
+  const target = panel.targets?.[0];
+  assert(target?.queryType === 'range', `${panel.title} is not a range query`);
+  const response = await queryRange(withNode(target.expr).replaceAll('$__auto', '5m'));
+  assert(response.status === 'success' && response.data.resultType === 'matrix', `${panel.title} did not return a successful matrix`);
+  assert(response.data.result.length === 1, `${panel.title} returned ${response.data.result.length} series, want one scheduler/lane series`);
+  const [{ metric, values }] = response.data.result;
+  assert(metric.scheduler === 'sync-global' && metric.lane === 'durable', `${panel.title} lost scheduler/lane grouping`);
+  const peak = Math.max(...values.map(([, value]) => Number(value)));
+  const expected = panel.title.startsWith('Active') ? 2400 : 3400;
+  assert(peak === expected, `${panel.title} peak=${peak}, want ${expected}`);
+}
+
+const maxQueue = Math.max(...Object.values(byRef).map((response) => Number(response.data.stats.summary.queueTime)));
+console.log(`backpressure LogQL verify OK: 5 flame targets + 2 heatmaps; fixed=${fixedWindow}; max_splits=0; max_queue=${maxQueue.toFixed(6)}s`);
diff --git a/tools/observability/verify-profile-render.mjs b/tools/observability/verify-profile-render.mjs
index 4b81b52da2..e3df7a6f6b 100644
--- a/tools/observability/verify-profile-render.mjs
+++ b/tools/observability/verify-profile-render.mjs
@@ -27,6 +27,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { assertPromLabel, parseCliArgs } from './lib/cli.mjs';
+import { BACKPRESSURE_FLAME_LOOKBACK } from './lib/queries.mjs';
 
 const usage = 'usage: node verify-profile-render.mjs <dir> --prom-node-label <label> [--vm-uid <uid>] [--loki-uid <uid>]';
 const opts = parseCliArgs({
@@ -290,7 +291,7 @@ const dkgSelectorViolations = (rawExpr, rawNodeFilter) => {
   if (!flame) {
     fail('logs dashboard', 'worker-pressure flamegraph panel is missing');
   } else {
-    if (!flame.title?.includes('Worker queue pressure')) {
+    if (!flame.title?.includes('Worker queue pressure') || !flame.title?.includes('last 1 hour')) {
       fail('logs dashboard flamegraph', `unexpected title: ${JSON.stringify(flame.title)}`);
     }
     if ((flame.targets ?? []).length !== 5) {
@@ -307,6 +308,19 @@ const dkgSelectorViolations = (rawExpr, rawNodeFilter) => {
     if (!exprs.includes('|= `[backpressure]`')) {
       fail('logs dashboard flamegraph', 'queries do not select PR #2003 backpressure records');
     }
+    if (exprs.includes('$__range')) {
+      fail('logs dashboard flamegraph', 'instant queries must not inherit the arbitrary Grafana-selected range');
+    }
+    const fixedLookback = `[${BACKPRESSURE_FLAME_LOOKBACK}]`;
+    for (const target of flame.targets ?? []) {
+      if (!(target.expr ?? '').includes(fixedLookback)) {
+        fail('logs dashboard flamegraph', `target ${target.refId} does not use the fixed ${fixedLookback} Loki lookback`);
+      }
+    }
+    if (!flame.description?.includes(`fixed last ${BACKPRESSURE_FLAME_LOOKBACK}`)
+      || !flame.description?.includes('heatmaps below retain selected-range history')) {
+      fail('logs dashboard flamegraph', 'description must explain the fixed snapshot and selected-range heatmap split');
+    }
     for (const phase of ['activeOperations', 'queuedOperations']) {
       for (let index = 0; index < 8; index++) {
         if (!exprs.includes(`${phase}[${index}].operation`)) {
diff --git a/tools/observability/w1/w1-dashboard-rules.test.yaml b/tools/observability/w1/w1-dashboard-rules.test.yaml
index 0d712ee2e2..5078da387b 100644
--- a/tools/observability/w1/w1-dashboard-rules.test.yaml
+++ b/tools/observability/w1/w1-dashboard-rules.test.yaml
@@ -212,3 +212,44 @@ tests:
         exp_samples:
           - labels: '{__name__="w1_dashboard_gate_invalidating_sources"}'
             value: 1080
+
+  - interval: 1m
+    name: rejected operations and catchup panels preserve every decision label
+    input_series:
+      # Each pair shares every other grouping label and differs only in the
+      # label under test. Dropping that label merges 1 + 2 into one sample and
+      # makes this fixture fail instead of leaving a plausible-looking rate.
+      - series: 'dkg_sync_operation_rejected_total{lane="durable", source="reconcile", reason="queue-full", instance="node-a"}'
+        values: '0+60x10'
+      - series: 'dkg_sync_operation_rejected_total{lane="durable", source="reconcile", reason="timeout", instance="node-a"}'
+        values: '0+120x10'
+      - series: 'dkg_context_graph_catchup_requests_total{result="ready", include_shared_memory="true", instance="node-a"}'
+        values: '0+60x10'
+      - series: 'dkg_context_graph_catchup_requests_total{result="ready", include_shared_memory="false", instance="node-a"}'
+        values: '0+120x10'
+      - series: 'dkg_context_graph_catchup_jobs_total{status="completed", admission="walk", instance="node-a"}'
+        values: '0+60x10'
+      - series: 'dkg_context_graph_catchup_jobs_total{status="completed", admission="synthetic", instance="node-a"}'
+        values: '0+120x10'
+    promql_expr_test:
+      - expr: w1_dashboard_rejected_rate
+        eval_time: 10m
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_rejected_rate", lane="durable", reason="queue-full", source="reconcile"}'
+            value: 1
+          - labels: '{__name__="w1_dashboard_rejected_rate", lane="durable", reason="timeout", source="reconcile"}'
+            value: 2
+      - expr: w1_dashboard_catchup_requests_rate
+        eval_time: 10m
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_catchup_requests_rate", include_shared_memory="false", result="ready"}'
+            value: 2
+          - labels: '{__name__="w1_dashboard_catchup_requests_rate", include_shared_memory="true", result="ready"}'
+            value: 1
+      - expr: w1_dashboard_catchup_jobs_rate
+        eval_time: 10m
+        exp_samples:
+          - labels: '{__name__="w1_dashboard_catchup_jobs_rate", admission="synthetic", status="completed"}'
+            value: 2
+          - labels: '{__name__="w1_dashboard_catchup_jobs_rate", admission="walk", status="completed"}'
+            value: 1