From 12b022c9b6daee229c5cfbca890bfda8a71a2774 Mon Sep 17 00:00:00 2001 From: David Eads Date: Mon, 26 Apr 2021 09:50:11 -0400 Subject: [PATCH] allow high CPU alerts to be firing and pending --- test/extended/prometheus/prometheus.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/extended/prometheus/prometheus.go b/test/extended/prometheus/prometheus.go index a7cdc2318ca3..9526de8d06a6 100644 --- a/test/extended/prometheus/prometheus.go +++ b/test/extended/prometheus/prometheus.go @@ -70,6 +70,16 @@ var _ = g.Describe("[sig-instrumentation][Late] Alerts", func() { Text: "https://bugzilla.redhat.com/show_bug.cgi?id=1933144", }, } + allowedFiringAlerts := helper.MetricConditions{ + { + Selector: map[string]string{"alertname": "HighOverallControlPlaneCPU"}, + Text: "high CPU utilization during e2e runs is normal", + }, + { + Selector: map[string]string{"alertname": "ExtremelyHighIndividualControlPlaneCPU"}, + Text: "high CPU utilization during e2e runs is normal", + }, + } pendingAlertsWithBugs := helper.MetricConditions{} allowedPendingAlerts := helper.MetricConditions{ @@ -77,6 +87,10 @@ var _ = g.Describe("[sig-instrumentation][Late] Alerts", func() { Selector: map[string]string{"alertname": "HighOverallControlPlaneCPU"}, Text: "high CPU utilization during e2e runs is normal", }, + { + Selector: map[string]string{"alertname": "ExtremelyHighIndividualControlPlaneCPU"}, + Text: "high CPU utilization during e2e runs is normal", + }, } knownViolations := sets.NewString() @@ -109,6 +123,10 @@ count_over_time(ALERTS{alertstate="firing",severity!="info",alertname!~"Watchdog for _, series := range result.Data.Result { labels := helper.StripLabels(series.Metric, "alertname", "alertstate", "prometheus") violation := fmt.Sprintf("alert %s fired for %s seconds with labels: %s", series.Metric["alertname"], series.Value, helper.LabelsAsSelector(labels)) + if cause := allowedFiringAlerts.Matches(series); cause != nil { + debug.Insert(fmt.Sprintf("%s (allowed: %s)", violation, cause.Text)) + continue + } if cause := firingAlertsWithBugs.Matches(series); cause != nil { knownViolations.Insert(fmt.Sprintf("%s (open bug: %s)", violation, cause.Text)) } else {