From 4917423ef393ca247fd92867d9aab94105798e09 Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Wed, 2 Sep 2026 23:47:27 +0500 Subject: [PATCH] fix(observer): emit every Prometheus series once Completed-wait metrics were accidentally rendered inside the scale-set loop. The observer therefore emitted each aggregate ten times and the collector dropped 126 conflicting samples every scrape. Render the aggregates once and enforce global series uniqueness in the metrics test. --- internal/fleetobserve/metrics.go | 17 ++++++++--------- internal/fleetobserve/metrics_test.go | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/internal/fleetobserve/metrics.go b/internal/fleetobserve/metrics.go index 2376a1fe..59c979f3 100644 --- a/internal/fleetobserve/metrics.go +++ b/internal/fleetobserve/metrics.go @@ -169,15 +169,14 @@ func RenderPrometheus(snapshot Snapshot, now time.Time, maxStaleness time.Durati labeledGaugeHeader(&output, "gha_fleet_queue_oldest_queued_wait_seconds_by_scale_set", "Longest wait since GitHub queued a still-queued intent, per configured scale set.") for _, scaleSet := range scaleSets { metric(&output, "gha_fleet_queue_oldest_queued_wait_seconds_by_scale_set", map[string]string{"scale_set": scaleSet}, float64(snapshot.Queue.OldestQueuedWaitSecondsByScaleSet[scaleSet])) - - gauge(&output, "gha_fleet_queue_started_wait_samples", "Jobs whose runner started within the completed-wait window.", float64(snapshot.Queue.StartedWaitSamples)) - gauge(&output, "gha_fleet_queue_started_wait_median_seconds", "Median wait, first queued to runner running, over jobs that started in the window.", float64(snapshot.Queue.StartedWaitMedianSeconds)) - gauge(&output, "gha_fleet_queue_started_wait_p90_seconds", "Ninetieth-percentile wait, first queued to runner running, over jobs that started in the window.", float64(snapshot.Queue.StartedWaitP90Seconds)) - gauge(&output, "gha_fleet_queue_started_wait_max_seconds", "Longest wait, first queued to runner running, over jobs that started in the window.", float64(snapshot.Queue.StartedWaitMaxSeconds)) - labeledGaugeHeader(&output, "gha_fleet_queue_started_wait_p90_seconds_by_scale_set", "Ninetieth-percentile completed wait per scale set.") - for _, scaleSet := range scaleSets { - metric(&output, "gha_fleet_queue_started_wait_p90_seconds_by_scale_set", map[string]string{"scale_set": scaleSet}, float64(snapshot.Queue.StartedWaitP90ByScaleSet[scaleSet])) - } + } + gauge(&output, "gha_fleet_queue_started_wait_samples", "Jobs whose runner started within the completed-wait window.", float64(snapshot.Queue.StartedWaitSamples)) + gauge(&output, "gha_fleet_queue_started_wait_median_seconds", "Median wait, first queued to runner running, over jobs that started in the window.", float64(snapshot.Queue.StartedWaitMedianSeconds)) + gauge(&output, "gha_fleet_queue_started_wait_p90_seconds", "Ninetieth-percentile wait, first queued to runner running, over jobs that started in the window.", float64(snapshot.Queue.StartedWaitP90Seconds)) + gauge(&output, "gha_fleet_queue_started_wait_max_seconds", "Longest wait, first queued to runner running, over jobs that started in the window.", float64(snapshot.Queue.StartedWaitMaxSeconds)) + labeledGaugeHeader(&output, "gha_fleet_queue_started_wait_p90_seconds_by_scale_set", "Ninetieth-percentile completed wait per scale set.") + for _, scaleSet := range scaleSets { + metric(&output, "gha_fleet_queue_started_wait_p90_seconds_by_scale_set", map[string]string{"scale_set": scaleSet}, float64(snapshot.Queue.StartedWaitP90ByScaleSet[scaleSet])) } gauge(&output, "gha_fleet_incus_visible_instances", "Instances visible in the restricted Incus project.", float64(snapshot.Incus.VisibleInstances)) gauge(&output, "gha_fleet_incus_visible_maintenance_instances", "Visible exact image builder or smoke instances; observable maintenance capacity, never GitHub job runners.", float64(snapshot.Incus.VisibleMaintenanceInstances)) diff --git a/internal/fleetobserve/metrics_test.go b/internal/fleetobserve/metrics_test.go index f98bc6d8..2aaf09bf 100644 --- a/internal/fleetobserve/metrics_test.go +++ b/internal/fleetobserve/metrics_test.go @@ -88,6 +88,24 @@ func TestRenderPrometheusIsDeterministicAndBounded(t *testing.T) { } } +func TestRenderPrometheusEmitsEverySeriesOnce(t *testing.T) { + metrics := RenderPrometheus(healthyCollector(t).Collect(t.Context()), observationTime, 45*time.Second) + seen := map[string]bool{} + for line := range strings.Lines(metrics) { + if strings.HasPrefix(line, "#") || strings.TrimSpace(line) == "" { + continue + } + series, _, ok := strings.Cut(strings.TrimSpace(line), " ") + if !ok { + t.Fatalf("metric line has no value: %q", line) + } + if seen[series] { + t.Fatalf("Prometheus series emitted more than once: %s", series) + } + seen[series] = true + } +} + func TestRenderPrometheusExposesBoundedDiagnosticConvergence(t *testing.T) { collector := healthyCollector(t) collector.Diagnostics = func(time.Time) (workerdiagnostics.SpoolStats, error) {