feat(observe): export drain-marked members independently of held-out

CHANGELOG.md

@@ -2,6 +2,24 @@
 
 ## Unreleased
 
+- Export drain-marked cluster members as their own snapshot field and
+  Prometheus series, independent of held-out (offline) members. A full
+  online drain keeps platform health up and still fires queue and assigned
+  pages; `gha_fleet_visibility_drain_marked_members` is now the count of
+  that authorized hold instead of inferring it from health staying one.
+  Only a reason with the `drained: ` prefix counts; an empty member listing
+  is not a full drain.
+
+- Keep `fleet_health_flapping` evaluable on a healthy fleet: sum the bool
+  of `healthy < 1` over thirty one-minute steps instead of counting a
+  filter that this backend drops when health is one. Host-signal tickets
+  and dashboards use last-min of the cumulative counter, not unsigned
+  max-min, so a collector restart or series reset is not a burst.
+
+- Align queued-delivery, slow-burn and created-unbound operator copy with
+  the shipped thresholds (thirty minutes, five-to-thirty band, fifteen
+  minutes). Thresholds themselves are unchanged.
+
 - Keep platform health up when every cluster member is drain-marked,
   including an online drain. `gha_fleet_visibility_held_out_members` still
   counts only offline drained members, so queue and assigned pages keep

config/observability-dashboards.yaml

@@ -136,25 +136,25 @@ dashboards:
       - id: all_host_signals
         title: Host signals by class and member
         kind: timeseries
-        query: sum by (host_name, signal_class) (max_over_time(gha_fleet_host_signal_events{aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]))
+        query: sum by (host_name, signal_class) (last_over_time(gha_fleet_host_signal_events{aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]))
         unit: count
         description: OTEL-classified host events observed in each rolling hour without retaining duplicate raw logs.
       - id: audit_suppression
         title: Audit callback suppressions
         kind: timeseries
-        query: sum by (host_name) (max_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]))
+        query: sum by (host_name) (last_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]))
         unit: count
         description: Kernel notices that audit callbacks were suppressed during high event volume.
       - id: kernel_workqueue_hog
         title: Kernel workqueue hog events
         kind: timeseries
-        query: sum by (host_name) (max_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[24h]) - min_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[24h]))
+        query: sum by (host_name) (last_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[24h]) - min_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[24h]))
         unit: count
         description: Kernel workqueue CPU-hog signals that may precede host latency or pressure.
       - id: ufw_blocks
         title: UFW block events
         kind: timeseries
-        query: sum by (host_name) (max_over_time(gha_fleet_host_signal_events{signal_class="ufw_block",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{signal_class="ufw_block",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]))
+        query: sum by (host_name) (last_over_time(gha_fleet_host_signal_events{signal_class="ufw_block",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{signal_class="ufw_block",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]))
         unit: count
         description: Firewall block volume retained as an OTEL metric rather than repeated kernel log records.
   - id: host_utilization

config/observability-rules.yaml

@@ -72,7 +72,13 @@ rules:
     #
     # A window with no events now returns an empty result, which is the honest
     # answer to "how many events", evaluates without error, and does not fire.
-    expression: sum by (host_name) (max_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m]) - min_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m]))
+    #
+    # last_over_time - min_over_time, not max - min: a collector restart that
+    # republishes the cumulative total, or a series that resets N→0, has the
+    # same unsigned max-min span as N new events. last-min is 0 across that
+    # reset and the true delta when the counter moves up. Same treatment as
+    # host_oom_detected.
+    expression: sum by (host_name) (last_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m]) - min_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m]))
     operator: ">"
     threshold: 30
     # No outer subquery: evaluation and hold are equal, so the renderer emits
@@ -219,11 +225,19 @@ rules:
   # beside each failed one. Counting the unhealthy minutes catches the shape
   # the page is built to miss, and it is a ticket: a fleet that recovers by
   # itself every minute is not an outage, it is a defect with a heartbeat.
+  #
+  # Written as sum_over_time of a bool, not count_over_time of a filter.
+  # This backend drops `(healthy < 1)` when healthy is 1, so the inner
+  # range is empty on a quiet fleet; hold > eval then wraps that empty
+  # vector and raises error 20008, which pages as alert_evaluation_failed.
+  # `X < bool 1` returns 0 when healthy and 1 when not, so the range is
+  # never empty. sum_over_time of those 0/1 steps is the unhealthy-minute
+  # count; count_over_time would count every step, healthy or not.
   - id: fleet_health_flapping
     severity: ticket
     query_language: promql
     stream_name: gha_fleet_platform_healthy
-    expression: count_over_time((min by (host_name) (gha_fleet_platform_healthy) < 1)[30m:1m])
+    expression: sum_over_time((min by (host_name) (gha_fleet_platform_healthy) < bool 1)[30m:1m])
     operator: ">="
     threshold: 3
     evaluation_seconds: 300
@@ -486,7 +500,9 @@ rules:
     query_language: promql
     stream_name: gha_fleet_host_signal_events
     # Same `or vector(0)` removal as audit_suppression_burst; see there.
-    expression: sum by (host_name) (max_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]))
+    # last_over_time - min_over_time, not max - min: same unsigned-reset
+    # treatment as host_oom_detected and audit_suppression_burst.
+    expression: sum by (host_name) (last_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]))
     operator: ">"
     threshold: 10
     # No outer subquery: evaluation and hold are equal, so the renderer emits
@@ -548,7 +564,7 @@ rules:
     runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md
     summary: A created worker has no running job identity beyond transition grace.
     action: Verify GitHub absent/offline state before allowing the normal reaper to delete it.
-    recovery: No created lease remains unbound beyond five minutes.
+    recovery: No created lease remains unbound beyond fifteen minutes.
   - id: lifecycle_inventory_gap
     severity: page
     query_language: promql
@@ -614,9 +630,9 @@ rules:
     enabled: true
     owner: fleet-operations
     runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md
-    summary: An eligible queued job has not produced runner delivery progress within five minutes.
+    summary: An eligible queued job has not produced runner delivery progress within thirty minutes.
     action: Preserve the job and correlate queue intent, provider retry, admission, worker creation and GitHub registration before recovery.
-    recovery: Queued age remains below five minutes or the exact job advances without operator cancellation.
+    recovery: Queued wait remains below thirty minutes or the exact job advances without operator cancellation.
   - id: memory_psi_slow_burn
     severity: ticket
     query_language: promql
@@ -680,10 +696,10 @@ rules:
   #
   # A ticket must hold at least ten minutes -- that is the rule package's own
   # policy for the class, and it is right -- so the threshold has to come down
-  # to where a ten-minute hold can reach. 300 s is also the boundary this
-  # rule's own summary names: "between the two-minute objective and the
-  # five-minute page". Held ten minutes it fires once on a day like that one,
-  # which is what a slow-burn ticket should cost.
+  # to where a ten-minute hold can reach. 300 s is the slow-burn floor;
+  # lifecycle_queued_delivery_stall owns waits past thirty minutes. Held ten
+  # minutes it fires once on a day like that one, which is what a slow-burn
+  # ticket should cost.
   # The wait a developer feels, closed: first queued to runner running, over
   # the jobs that started in the last fifteen minutes. Every other wait rule
   # here reads a gauge that stops when the provider assigns the intent, so on
@@ -735,9 +751,9 @@ rules:
     enabled: true
     owner: fleet-performance
     runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md
-    summary: Queue latency is persistently between the two-minute objective and the five-minute page.
+    summary: Queue latency is persistently between five minutes and the thirty-minute page.
     action: Create a capacity or fairness ticket using phase-specific and class metrics.
-    recovery: Queue age remains below two minutes for the hold window, or a stuck job crosses five minutes and lifecycle_queued_delivery_stall owns it instead.
+    recovery: Queue age remains below five minutes for the hold window, or a stuck job crosses thirty minutes and lifecycle_queued_delivery_stall owns it instead.
   - id: snapshot_stale
     severity: page
     query_language: promql

docs/maintenance-windows.md

@@ -27,6 +27,11 @@ config, and the observer reads it there to tell maintenance from an incident.
 
 ## What the observer does during a hold
 
+Every drain-marked member, online or offline, is named on
+`gha_fleet_visibility_drain_marked_members` and the per-member
+`gha_fleet_visibility_drain_marked` rows. That series is how an authorized
+hold is observed; it does not change queue or assigned pages.
+
 While an offline member carries a `drained: ` reason, the member is **held
 out**: inventory gap counts (orphan / missing / uncovered-beyond-grace) move
 to `*_unattributable` snapshot fields, a loud listing failure moves to
@@ -37,6 +42,12 @@ ticket pages when a hold outlives half an hour, so the suppression can never
 hide a member that failed to come back. An offline member **without** a drain
 reason fails collection immediately.
 
+An **online** drain leaves the listing complete: empty on that member is the
+truth, so HeldOutMembers stays empty and queue/assigned pages still fire. If
+every reported cluster member is drain-marked, uncovered-beyond-grace does
+not fail platform health — there is no eligible placement left on purpose. A
+partial drain with an undrained sibling still fails health on that gap.
+
 ## Stopping the Incus daemon for real
 
 The daemon is socket-activated, and the pressure timer connects to the local

docs/observability-dashboards.md

@@ -29,13 +29,15 @@ The reconciler owns only dashboards carrying the versioned `managed-by:gds`
 description marker. It preserves manually created dashboards, uses OpenObserve
 hash preconditions for updates, and requires a clean post-apply read-back.
 
-The nine managed dashboards cover:
+The eleven managed dashboards cover:
 
 - capacity and pressure;
 - correlation integrity;
 - diagnostic storage retention;
 - host package, reboot, kernel and SRSO compliance;
 - classified host signals without duplicate raw-log storage;
+- host CPU, RAM, swap occupancy and major paging;
+- job lifecycle transitions;
 - lifecycle phase latency;
 - provider reliability;
 - priority/class fairness;
@@ -47,6 +49,6 @@ human decision surfaces used by alert response and CD health verification; a
 private deployment must not silently edit them.
 
 Dashboards do not prove alerts deliver. `config/observability-rules.yaml` owns
-the twenty-five-rule separately tested alert contract, and a real backend lifecycle record must
+the thirty-one-rule separately tested alert contract, and a real backend lifecycle record must
 prove Collector buffering, recovered records and explicit loss counters before
 the observability product is accepted.

docs/runbooks/fleet-alerts.md

@@ -79,10 +79,10 @@ gha-fleet reconcile-openobserve-alerts \
   `constraint=CONSTRAINT_NONE` journal line on the current boot before
   treating the page as a live kill.
 - Inventory-gap pages: a 30-second `missing_instances=1` listing blip is not
-  the page. The rule requires each of orphan, missing and uncovered-beyond-grace
-  to stay non-zero for two minutes of raw samples. Preserve all three states
-  before recovering; a member reboot without a drain marker is still a gap if
-  it lasts that long.
+  the page. The rule is a sum of two-minute mins: any of orphan, missing or
+  uncovered-beyond-grace staying non-zero for two minutes of raw samples
+  fires. Preserve all three states before recovering; a member reboot without
+  a drain marker is still a gap if it lasts that long.
 - Host-signal tickets: use `gha_fleet_host_signal_events` cumulative deltas.
   LVM activation and overlay `xino=off` are workload-volume context; audit
   suppression and workqueue-hog alerts act only on their bounded burst budget.
@@ -105,14 +105,23 @@ gha-fleet reconcile-openobserve-alerts \
   until normal successful consolidation removes or supersedes it.
 - Slow-burn tickets: inspect class/tenant percentiles and capacity evidence;
   do not page an operator for a trend without an immediate action.
-- Visibility-degraded tickets: a drain-marked member is offline, so the
-  cluster listing is partial by design. Inventory gap counts move to their
+- Platform-health pages during an authorized full-fleet drain: if every
+  reported cluster member carries a drain reason, including an online drain,
+  uncovered-beyond-grace does not zero `gha_fleet_platform_healthy`.
+  `gha_fleet_visibility_drain_marked_members` is the count. Queue and
+  assigned pages still fire because an online drain leaves the listing
+  complete (`gha_fleet_visibility_held_out_members` stays 0). A partial
+  drain with an undrained sibling still fails platform health.
+- Visibility-degraded tickets: a drain-marked member that is offline makes
+  the cluster listing partial by design. Inventory gap counts move to their
   `*_unattributable` snapshot fields and a loud listing failure moves to
   `listing_unavailable` -- suppressed with attribution, never hidden -- while
-  `gha_fleet_visibility_held_out_members` says so. The ticket fires when the
-  hold outlives half an hour: finish the maintenance and restore the member,
-  or find out why it did not come back. An offline member WITHOUT a drain
-  reason is an incident and fails the platform immediately.
+  `gha_fleet_visibility_held_out_members` says so. Online drain-marked
+  members appear only on `gha_fleet_visibility_drain_marked_members`; they
+  do not suppress gaps. The ticket fires when an offline hold outlives half
+  an hour: finish the maintenance and restore the member, or find out why it
+  did not come back. An offline member WITHOUT a drain reason is an incident
+  and fails the platform immediately.
 - Pressure-staleness pages during maintenance: a member drained through the
   drain marker keeps publishing a fresh closed state carrying
   `drained: <reason>` on every timer tick, so staleness stays silent for the

internal/fleetobserve/metrics.go

@@ -118,6 +118,11 @@ func RenderPrometheus(snapshot Snapshot, now time.Time, maxStaleness time.Durati
 	for _, member := range snapshot.HeldOutMembers {
 		metric(&output, "gha_fleet_visibility_degraded", map[string]string{"member": member}, 1)
 	}
+	gauge(&output, "gha_fleet_visibility_drain_marked_members", "Cluster members whose gate carries a drain reason, online or offline. Held-out members are the offline subset; an online drain leaves the listing complete.", float64(len(snapshot.DrainMarkedMembers)))
+	labeledGaugeHeader(&output, "gha_fleet_visibility_drain_marked", "One row per drain-marked member, for attribution.")
+	for _, member := range snapshot.DrainMarkedMembers {
+		metric(&output, "gha_fleet_visibility_drain_marked", map[string]string{"member": member}, 1)
+	}
 	gauge(&output, "gha_fleet_queue_uncovered_running_beyond_grace", "Running queue intents still uncovered after the broker reclaim window, which is the gap nothing is coming to close.", float64(snapshot.Queue.UncoveredRunningBeyondGrace))
 	gauge(&output, "gha_fleet_queue_uncovered_running_oldest_age_seconds", "Age of the oldest running queue intent whose runner holds no execution lease.", float64(snapshot.Queue.OldestUncoveredRunningAgeSeconds))
 	gauge(&output, "gha_fleet_queue_running_without_runner_identity", "Running queue intents written before or without an exact JobStarted runner identity.", float64(snapshot.Queue.RunningWithoutRunnerIdentity))