From dba890107c1435bc79d33c40a9b3fcd4377a13d6 Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Fri, 28 Aug 2026 04:51:11 +0500 Subject: [PATCH] feat(obs): cover host swap and utilization --- CHANGELOG.md | 2 ++ config/observability-dashboards.yaml | 31 ++++++++++++++++++ config/observability-rules.yaml | 32 +++++++++++++++++++ .../observabilitydashboards/dashboards.go | 10 ++++-- .../dashboards_test.go | 2 +- .../observabilitydashboards/openobserve.go | 4 ++- .../openobserve_test.go | 5 +-- internal/observabilityrules/rules_test.go | 6 ++-- 8 files changed, 84 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db30a0e2..05cb3522 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Add host swap occupancy and major-paging alerts plus CPU, swap usage and + paging panels for the five-host OTEL fleet. - Spread equal-load burst placement toward the member with more remaining capacity and fewer pending workers instead of packing one member first. - Raise standard, fast and priority-standard scale-set width from 12 to 16 so diff --git a/config/observability-dashboards.yaml b/config/observability-dashboards.yaml index 1874147a..53c39954 100644 --- a/config/observability-dashboards.yaml +++ b/config/observability-dashboards.yaml @@ -157,6 +157,37 @@ dashboards: query: sum by (host_name) (max_over_time(gha_fleet_host_signal_events{signal_class="ufw_block",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{signal_class="ufw_block",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h])) unit: count description: Firewall block volume retained as an OTEL metric rather than repeated kernel log records. + - id: host_utilization + title: Fleet host utilization + refresh_seconds: 30 + default_range: 6h + owner: fleet-performance + runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md + panels: + - id: host_cpu_utilization + title: Host CPU utilization + kind: timeseries + query: 100 * (1 - sum by (host_name) (rate(system_cpu_time{service_namespace="nddev-github-actions",state="idle"}[5m])) / clamp_min(sum by (host_name) (rate(system_cpu_time{service_namespace="nddev-github-actions"}[5m])), 0.001)) + unit: percent + description: CPU use derived from cumulative OTEL host CPU time for all five fleet hosts. + - id: host_memory_utilization + title: Host memory utilization + kind: timeseries + query: 100 * max by (host_name) (system_memory_usage{service_namespace="nddev-github-actions",state="used"}) / clamp_min(sum by (host_name) (system_memory_usage{service_namespace="nddev-github-actions"}), 1) + unit: percent + description: Physical RAM use for each fleet host, independent of measured worker reservations. + - id: host_swap_io + title: Major paging by direction + kind: timeseries + query: sum by (host_name, direction) (rate(system_paging_operations{service_namespace="nddev-github-actions",type="major"}[5m])) + unit: bytes_per_second + description: Sustained swap-related major paging used to distinguish cold-page protection from thrashing. + - id: host_swap_usage + title: Emergency swap usage + kind: timeseries + query: 100 * max by (host_name) (system_paging_usage{service_namespace="nddev-github-actions",state="used"}) / clamp_min(sum by (host_name) (system_paging_usage{service_namespace="nddev-github-actions"}), 1) + unit: percent + description: Fraction of the non-schedulable four-GiB emergency swap consumed on each fleet host. - id: lifecycle_latency title: Job lifecycle latency refresh_seconds: 30 diff --git a/config/observability-rules.yaml b/config/observability-rules.yaml index d7fa1117..a6f4b1fb 100644 --- a/config/observability-rules.yaml +++ b/config/observability-rules.yaml @@ -226,6 +226,38 @@ rules: summary: Standard Ubuntu package updates remain available on a fleet host. action: Apply the normal rolling package-maintenance procedure and prove service and runner recovery after each member. recovery: Every fleet host reports zero standard updates for one hour. + - id: host_swap_high + severity: ticket + query_language: promql + stream_name: system_paging_usage + expression: max by (host_name) (system_paging_usage{service_namespace="nddev-github-actions",state="used"}) / clamp_min(sum by (host_name) (system_paging_usage{service_namespace="nddev-github-actions"}), 1) + operator: ">" + threshold: 0.75 + evaluation_seconds: 300 + hold_seconds: 900 + destination_ref: fleet_oncall + enabled: false + owner: fleet-performance + runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md + summary: A fleet host has consumed more than seventy-five percent of emergency swap. + action: Correlate swap occupancy with memory PSI, worker RSS and queue demand; keep swap outside schedulable capacity. + recovery: Every fleet host remains below seventy-five percent swap usage for fifteen minutes. + - id: host_swap_thrash + severity: page + query_language: promql + stream_name: system_paging_operations + expression: sum by (host_name) (rate(system_paging_operations{service_namespace="nddev-github-actions",type="major"}[5m])) + operator: ">" + threshold: 67108864 + evaluation_seconds: 30 + hold_seconds: 300 + destination_ref: fleet_oncall + enabled: false + owner: fleet-performance + runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md + summary: Sustained major paging exceeds sixty-four MiB per second on a fleet host. + action: Close admission on the affected member and correlate paging direction with memory PSI and active worker reservations. + recovery: Major paging remains below sixty-four MiB per second for five minutes and memory PSI returns inside the envelope. - id: kernel_slab_unreclaimable severity: ticket query_language: promql diff --git a/internal/observabilitydashboards/dashboards.go b/internal/observabilitydashboards/dashboards.go index 4690d70d..647e96f4 100644 --- a/internal/observabilitydashboards/dashboards.go +++ b/internal/observabilitydashboards/dashboards.go @@ -127,12 +127,18 @@ func (p Panel) Validate() error { if _, ok := map[string]struct{}{"stat": {}, "table": {}, "timeseries": {}}[p.Kind]; !ok { return fmt.Errorf("kind is invalid") } - if _, ok := map[string]struct{}{"bytes": {}, "count": {}, "percent": {}, "seconds": {}, "state": {}}[p.Unit]; !ok { + if _, ok := map[string]struct{}{"bytes": {}, "bytes_per_second": {}, "count": {}, "percent": {}, "seconds": {}, "state": {}}[p.Unit]; !ok { return fmt.Errorf("unit is invalid") } - if !strings.Contains(p.Query, "gha_fleet_") && !strings.Contains(p.Query, "gha_diagnostic_storage_") && !strings.Contains(p.Query, "otelcol_exporter_") { + if !strings.Contains(p.Query, "gha_fleet_") && !strings.Contains(p.Query, "gha_diagnostic_storage_") && + !strings.Contains(p.Query, "otelcol_exporter_") && !strings.Contains(p.Query, "system_cpu_time") && + !strings.Contains(p.Query, "system_memory_usage") && !strings.Contains(p.Query, "system_paging_") { return fmt.Errorf("query does not use an owned fleet or Collector metric") } + if (strings.Contains(p.Query, "system_cpu_time") || strings.Contains(p.Query, "system_memory_usage") || strings.Contains(p.Query, "system_paging_")) && + !strings.Contains(p.Query, `service_namespace="nddev-github-actions"`) { + return fmt.Errorf("system host metric query is not scoped to the fleet namespace") + } return nil } diff --git a/internal/observabilitydashboards/dashboards_test.go b/internal/observabilitydashboards/dashboards_test.go index 3c20a01a..ea8f3104 100644 --- a/internal/observabilitydashboards/dashboards_test.go +++ b/internal/observabilitydashboards/dashboards_test.go @@ -21,7 +21,7 @@ func TestPublishedDashboardBundleIsValidAndRenderable(t *testing.T) { if err != nil { t.Fatal(err) } - if len(bundle.Dashboards) != 9 { + if len(bundle.Dashboards) != 10 { t.Fatalf("dashboards=%d", len(bundle.Dashboards)) } rendered, err := Render(bundle) diff --git a/internal/observabilitydashboards/openobserve.go b/internal/observabilitydashboards/openobserve.go index 177d420d..560cda20 100644 --- a/internal/observabilitydashboards/openobserve.go +++ b/internal/observabilitydashboards/openobserve.go @@ -7,7 +7,7 @@ import ( const managedDescriptionPrefix = "managed-by:gds;dashboard-contract:v1;" -var metricPattern = regexp.MustCompile(`(?:gha_fleet_|gha_diagnostic_storage_|otelcol_exporter_)[a-zA-Z0-9_:]*`) +var metricPattern = regexp.MustCompile(`(?:gha_fleet_|gha_diagnostic_storage_|otelcol_exporter_|system_cpu_time|system_memory_usage|system_paging_)[a-zA-Z0-9_:]*`) type OpenObserveDashboard struct { Version int `json:"version"` @@ -136,6 +136,8 @@ func openObserveUnit(unit string) string { switch unit { case "bytes": return "bytes" + case "bytes_per_second": + return "bytes" case "percent": return "percent-1" case "seconds": diff --git a/internal/observabilitydashboards/openobserve_test.go b/internal/observabilitydashboards/openobserve_test.go index a4259eb7..54049d22 100644 --- a/internal/observabilitydashboards/openobserve_test.go +++ b/internal/observabilitydashboards/openobserve_test.go @@ -14,7 +14,7 @@ func TestRenderOpenObserveV8IsDeterministicAndManaged(t *testing.T) { if err != nil { t.Fatal(err) } - if len(dashboards) != 9 { + if len(dashboards) != 10 { t.Fatalf("dashboards=%d", len(dashboards)) } for _, dashboard := range dashboards { @@ -29,7 +29,8 @@ func TestRenderOpenObserveV8IsDeterministicAndManaged(t *testing.T) { t.Fatalf("invalid panel %#v", panel) } stream := panel.Queries[0].Fields.Stream - if !strings.HasPrefix(stream, "gha_fleet_") && !strings.HasPrefix(stream, "gha_diagnostic_storage_") && !strings.HasPrefix(stream, "otelcol_exporter_") { + if !strings.HasPrefix(stream, "gha_fleet_") && !strings.HasPrefix(stream, "gha_diagnostic_storage_") && + !strings.HasPrefix(stream, "otelcol_exporter_") && stream != "system_cpu_time" && stream != "system_memory_usage" && !strings.HasPrefix(stream, "system_paging_") { t.Fatalf("panel %q stream is not an owned metric: %q", panel.ID, stream) } if panel.Layout.I != index+1 || panel.Layout.W != 96 || panel.Layout.H != 9 { diff --git a/internal/observabilityrules/rules_test.go b/internal/observabilityrules/rules_test.go index 500d7ab3..1cf25323 100644 --- a/internal/observabilityrules/rules_test.go +++ b/internal/observabilityrules/rules_test.go @@ -10,8 +10,8 @@ func TestRepositoryBundleIsValid(t *testing.T) { if err != nil { t.Fatal(err) } - if len(bundle.Rules) != 25 { - t.Fatalf("rules = %d, want 25", len(bundle.Rules)) + if len(bundle.Rules) != 27 { + t.Fatalf("rules = %d, want 27", len(bundle.Rules)) } } @@ -36,6 +36,8 @@ func TestRepositoryRulesUseCurrentMetricSemantics(t *testing.T) { "host_package_inventory_stale": "gha_fleet_host_package_inventory_age_seconds", "host_reboot_required": "gha_fleet_host_reboot_required", "host_standard_updates_available": "gha_fleet_host_standard_updates_available", + "host_swap_high": `state="used"`, + "host_swap_thrash": `type="major"`, } seen := make(map[string]bool, len(wanted)) for _, rule := range bundle.Rules {