feat(obs): cover host swap and utilization

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+- Add host swap occupancy and major-paging alerts plus CPU, swap usage and
+  paging panels for the five-host OTEL fleet.
 - Spread equal-load burst placement toward the member with more remaining
   capacity and fewer pending workers instead of packing one member first.
 - Raise standard, fast and priority-standard scale-set width from 12 to 16 so

config/observability-dashboards.yaml

@@ -157,6 +157,37 @@ dashboards:
         query: sum by (host_name) (max_over_time(gha_fleet_host_signal_events{signal_class="ufw_block",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{signal_class="ufw_block",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]))
         unit: count
         description: Firewall block volume retained as an OTEL metric rather than repeated kernel log records.
+  - id: host_utilization
+    title: Fleet host utilization
+    refresh_seconds: 30
+    default_range: 6h
+    owner: fleet-performance
+    runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md
+    panels:
+      - id: host_cpu_utilization
+        title: Host CPU utilization
+        kind: timeseries
+        query: 100 * (1 - sum by (host_name) (rate(system_cpu_time{service_namespace="nddev-github-actions",state="idle"}[5m])) / clamp_min(sum by (host_name) (rate(system_cpu_time{service_namespace="nddev-github-actions"}[5m])), 0.001))
+        unit: percent
+        description: CPU use derived from cumulative OTEL host CPU time for all five fleet hosts.
+      - id: host_memory_utilization
+        title: Host memory utilization
+        kind: timeseries
+        query: 100 * max by (host_name) (system_memory_usage{service_namespace="nddev-github-actions",state="used"}) / clamp_min(sum by (host_name) (system_memory_usage{service_namespace="nddev-github-actions"}), 1)
+        unit: percent
+        description: Physical RAM use for each fleet host, independent of measured worker reservations.
+      - id: host_swap_io
+        title: Major paging by direction
+        kind: timeseries
+        query: sum by (host_name, direction) (rate(system_paging_operations{service_namespace="nddev-github-actions",type="major"}[5m]))
+        unit: bytes_per_second
+        description: Sustained swap-related major paging used to distinguish cold-page protection from thrashing.
+      - id: host_swap_usage
+        title: Emergency swap usage
+        kind: timeseries
+        query: 100 * max by (host_name) (system_paging_usage{service_namespace="nddev-github-actions",state="used"}) / clamp_min(sum by (host_name) (system_paging_usage{service_namespace="nddev-github-actions"}), 1)
+        unit: percent
+        description: Fraction of the non-schedulable four-GiB emergency swap consumed on each fleet host.
   - id: lifecycle_latency
     title: Job lifecycle latency
     refresh_seconds: 30

config/observability-rules.yaml

@@ -226,6 +226,38 @@ rules:
     summary: Standard Ubuntu package updates remain available on a fleet host.
     action: Apply the normal rolling package-maintenance procedure and prove service and runner recovery after each member.
     recovery: Every fleet host reports zero standard updates for one hour.
+  - id: host_swap_high
+    severity: ticket
+    query_language: promql
+    stream_name: system_paging_usage
+    expression: max by (host_name) (system_paging_usage{service_namespace="nddev-github-actions",state="used"}) / clamp_min(sum by (host_name) (system_paging_usage{service_namespace="nddev-github-actions"}), 1)
+    operator: ">"
+    threshold: 0.75
+    evaluation_seconds: 300
+    hold_seconds: 900
+    destination_ref: fleet_oncall
+    enabled: false
+    owner: fleet-performance
+    runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md
+    summary: A fleet host has consumed more than seventy-five percent of emergency swap.
+    action: Correlate swap occupancy with memory PSI, worker RSS and queue demand; keep swap outside schedulable capacity.
+    recovery: Every fleet host remains below seventy-five percent swap usage for fifteen minutes.
+  - id: host_swap_thrash
+    severity: page
+    query_language: promql
+    stream_name: system_paging_operations
+    expression: sum by (host_name) (rate(system_paging_operations{service_namespace="nddev-github-actions",type="major"}[5m]))
+    operator: ">"
+    threshold: 67108864
+    evaluation_seconds: 30
+    hold_seconds: 300
+    destination_ref: fleet_oncall
+    enabled: false
+    owner: fleet-performance
+    runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md
+    summary: Sustained major paging exceeds sixty-four MiB per second on a fleet host.
+    action: Close admission on the affected member and correlate paging direction with memory PSI and active worker reservations.
+    recovery: Major paging remains below sixty-four MiB per second for five minutes and memory PSI returns inside the envelope.
   - id: kernel_slab_unreclaimable
     severity: ticket
     query_language: promql

internal/observabilitydashboards/dashboards.go

@@ -127,12 +127,18 @@ func (p Panel) Validate() error {
 	if _, ok := map[string]struct{}{"stat": {}, "table": {}, "timeseries": {}}[p.Kind]; !ok {
 		return fmt.Errorf("kind is invalid")
 	}
-	if _, ok := map[string]struct{}{"bytes": {}, "count": {}, "percent": {}, "seconds": {}, "state": {}}[p.Unit]; !ok {
+	if _, ok := map[string]struct{}{"bytes": {}, "bytes_per_second": {}, "count": {}, "percent": {}, "seconds": {}, "state": {}}[p.Unit]; !ok {
 		return fmt.Errorf("unit is invalid")
 	}
-	if !strings.Contains(p.Query, "gha_fleet_") && !strings.Contains(p.Query, "gha_diagnostic_storage_") && !strings.Contains(p.Query, "otelcol_exporter_") {
+	if !strings.Contains(p.Query, "gha_fleet_") && !strings.Contains(p.Query, "gha_diagnostic_storage_") &&
+		!strings.Contains(p.Query, "otelcol_exporter_") && !strings.Contains(p.Query, "system_cpu_time") &&
+		!strings.Contains(p.Query, "system_memory_usage") && !strings.Contains(p.Query, "system_paging_") {
 		return fmt.Errorf("query does not use an owned fleet or Collector metric")
 	}
+	if (strings.Contains(p.Query, "system_cpu_time") || strings.Contains(p.Query, "system_memory_usage") || strings.Contains(p.Query, "system_paging_")) &&
+		!strings.Contains(p.Query, `service_namespace="nddev-github-actions"`) {
+		return fmt.Errorf("system host metric query is not scoped to the fleet namespace")
+	}
 	return nil
 }
 

internal/observabilitydashboards/dashboards_test.go

@@ -21,7 +21,7 @@ func TestPublishedDashboardBundleIsValidAndRenderable(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if len(bundle.Dashboards) != 9 {
+	if len(bundle.Dashboards) != 10 {
 		t.Fatalf("dashboards=%d", len(bundle.Dashboards))
 	}
 	rendered, err := Render(bundle)

internal/observabilitydashboards/openobserve.go

@@ -7,7 +7,7 @@ import (
 
 const managedDescriptionPrefix = "managed-by:gds;dashboard-contract:v1;"
 
-var metricPattern = regexp.MustCompile(`(?:gha_fleet_|gha_diagnostic_storage_|otelcol_exporter_)[a-zA-Z0-9_:]*`)
+var metricPattern = regexp.MustCompile(`(?:gha_fleet_|gha_diagnostic_storage_|otelcol_exporter_|system_cpu_time|system_memory_usage|system_paging_)[a-zA-Z0-9_:]*`)
 
 type OpenObserveDashboard struct {
 	Version                 int              `json:"version"`
@@ -136,6 +136,8 @@ func openObserveUnit(unit string) string {
 	switch unit {
 	case "bytes":
 		return "bytes"
+	case "bytes_per_second":
+		return "bytes"
 	case "percent":
 		return "percent-1"
 	case "seconds":

internal/observabilitydashboards/openobserve_test.go

@@ -14,7 +14,7 @@ func TestRenderOpenObserveV8IsDeterministicAndManaged(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if len(dashboards) != 9 {
+	if len(dashboards) != 10 {
 		t.Fatalf("dashboards=%d", len(dashboards))
 	}
 	for _, dashboard := range dashboards {
@@ -29,7 +29,8 @@ func TestRenderOpenObserveV8IsDeterministicAndManaged(t *testing.T) {
 				t.Fatalf("invalid panel %#v", panel)
 			}
 			stream := panel.Queries[0].Fields.Stream
-			if !strings.HasPrefix(stream, "gha_fleet_") && !strings.HasPrefix(stream, "gha_diagnostic_storage_") && !strings.HasPrefix(stream, "otelcol_exporter_") {
+			if !strings.HasPrefix(stream, "gha_fleet_") && !strings.HasPrefix(stream, "gha_diagnostic_storage_") &&
+				!strings.HasPrefix(stream, "otelcol_exporter_") && stream != "system_cpu_time" && stream != "system_memory_usage" && !strings.HasPrefix(stream, "system_paging_") {
 				t.Fatalf("panel %q stream is not an owned metric: %q", panel.ID, stream)
 			}
 			if panel.Layout.I != index+1 || panel.Layout.W != 96 || panel.Layout.H != 9 {

internal/observabilityrules/rules_test.go

@@ -10,8 +10,8 @@ func TestRepositoryBundleIsValid(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if len(bundle.Rules) != 25 {
-		t.Fatalf("rules = %d, want 25", len(bundle.Rules))
+	if len(bundle.Rules) != 27 {
+		t.Fatalf("rules = %d, want 27", len(bundle.Rules))
 	}
 }
 
@@ -36,6 +36,8 @@ func TestRepositoryRulesUseCurrentMetricSemantics(t *testing.T) {
 		"host_package_inventory_stale":      "gha_fleet_host_package_inventory_age_seconds",
 		"host_reboot_required":              "gha_fleet_host_reboot_required",
 		"host_standard_updates_available":   "gha_fleet_host_standard_updates_available",
+		"host_swap_high":                    `state="used"`,
+		"host_swap_thrash":                  `type="major"`,
 	}
 	seen := make(map[string]bool, len(wanted))
 	for _, rule := range bundle.Rules {