Merge pull request #414 from simonpasquier/bz-1806640

Bug 1806640: fix potential errors in Prometheus alerts
openshift · May 14, 2020 · c539153 · c539153
2 parents 57aab7b + 265b5a6
commit c539153
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/manifests/0000_90_kube-controller-manager-operator_05_alert-pdb.yaml b/manifests/0000_90_kube-controller-manager-operator_05_alert-pdb.yaml
@@ -13,15 +13,15 @@ spec:
           annotations:
             message: The pod disruption budget is preventing further disruption to pods because it is at the minimum allowed level.
           expr: |
-            kube_poddisruptionbudget_status_expected_pods == on(namespace, poddisruptionbudget, service) kube_poddisruptionbudget_status_desired_healthy
+            max by(namespace, poddisruptionbudget) (kube_poddisruptionbudget_status_expected_pods == kube_poddisruptionbudget_status_desired_healthy)
           for: 15m
           labels:
             severity: warning
         - alert: PodDisruptionBudgetLimit
           annotations:
             message: The pod disruption budget is below the minimum number allowed pods.
           expr: |
-            kube_poddisruptionbudget_status_expected_pods < on(namespace, poddisruptionbudget, service) kube_poddisruptionbudget_status_desired_healthy
+            max by (namespace, poddisruptionbudget) (kube_poddisruptionbudget_status_expected_pods < kube_poddisruptionbudget_status_desired_healthy)
           for: 15m
           labels:
             severity: critical
diff --git a/test/e2e/operator_test.go b/test/e2e/operator_test.go
@@ -121,7 +121,7 @@ func TestPodDisruptionBudgetAtLimitAlert(t *testing.T) {
 	// Note: prometheus/client_golang Alerts method only works with the deprecated prometheus-k8s route.
 	// Our helper uses the thanos-querier route.  Because of this, have to pass the entire alert as a query.
 	// The thanos behavior is to error on partial response.
-	query := fmt.Sprintf("ALERTS{alertname=\"PodDisruptionBudgetAtLimit\",alertstate=\"pending\",namespace=\"%s\",poddisruptionbudget=\"%s\",prometheus=\"openshift-monitoring/k8s\",service=\"kube-state-metrics\",severity=\"warning\"}==1", name, name)
+	query := fmt.Sprintf("ALERTS{alertname=\"PodDisruptionBudgetAtLimit\",alertstate=\"pending\",namespace=\"%s\",poddisruptionbudget=\"%s\",prometheus=\"openshift-monitoring/k8s\",severity=\"warning\"}==1", name, name)
 	err = wait.PollImmediate(time.Second*3, testTimeout, func() (bool, error) {
 		response, _, err = prometheusClient.Query(context.Background(), query, time.Now())
 		if err != nil {