Avoid empty prometheus rules

globocom · Oct 17, 2019 · 4b0e1a8 · 4b0e1a8
1 parent c099d98
commit 4b0e1a8
Show file tree

Hide file tree

Showing 2 changed files with 123 additions and 9 deletions.
diff --git a/slo/slo.go b/slo/slo.go
@@ -116,7 +116,9 @@ func (slo SLO) GenerateGroupRules() []rulefmt.RuleGroup {
 			ruleGroup.Rules = append(ruleGroup.Rules, slo.generateRules(bucket)...)
 		}
 
-		rules = append(rules, ruleGroup)
+		if len(ruleGroup.Rules) > 0 {
+			rules = append(rules, ruleGroup)
+		}
 	}
 
 	return rules
@@ -167,16 +169,18 @@ func (slo SLO) generateRules(bucket string) []rulefmt.Rule {
 		}
 	}
 
-	for _, latencyBucket := range slo.Objectives.Latency {
-		latencyRateRecord := rulefmt.Rule{
-			Record: "slo:service_latency:ratio_rate_" + bucket,
-			Expr:   slo.LatencyRecord.ComputeExpr(bucket, latencyBucket.LE),
-			Labels: slo.labels(),
-		}
+	if slo.LatencyRecord.Expr != "" {
+		for _, latencyBucket := range slo.Objectives.Latency {
+			latencyRateRecord := rulefmt.Rule{
+				Record: "slo:service_latency:ratio_rate_" + bucket,
+				Expr:   slo.LatencyRecord.ComputeExpr(bucket, latencyBucket.LE),
+				Labels: slo.labels(),
+			}
 
-		latencyRateRecord.Labels["le"] = latencyBucket.LE
+			latencyRateRecord.Labels["le"] = latencyBucket.LE
 
-		rules = append(rules, latencyRateRecord)
+			rules = append(rules, latencyRateRecord)
+		}
 	}
 
 	return rules

diff --git a/slo/slo_test.go b/slo/slo_test.go
@@ -696,3 +696,113 @@ func TestSLOGenerateAlertRules(t *testing.T) {
 		Annotations: slo.Annotations,
 	})
 }
+
+func TestSLOGenerateAlertRulesWithoutExpressions(t *testing.T) {
+	slo := &SLO{
+		Name: "my-team.my-service.payment",
+		Objectives: Objectives{
+			Availability: 99.9,
+			Latency: []methods.LatencyTarget{
+				{
+					LE:     "0.1",
+					Target: 95,
+				},
+				{
+					LE:     "0.5",
+					Target: 99,
+				},
+			},
+		},
+		ErrorRateRecord: ExprBlock{
+			AlertMethod: "multi-window",
+		},
+		LatencyRecord: ExprBlock{
+			AlertMethod: "multi-window",
+		},
+		Labels: map[string]string{
+			"channel": "my-channel",
+		},
+		Annotations: map[string]string{
+			"message":   "Service A has lower SLI",
+			"link":      "http://wiki.ops/1234",
+			"dashboard": "http://grafana.globo.com",
+		},
+	}
+
+	alertRules := slo.GenerateAlertRules()
+	assert.Len(t, alertRules, 4)
+
+	assert.Equal(t, alertRules[0], rulefmt.Rule{
+		Alert: "slo:my-team.my-service.payment.errors.page",
+		Expr:  "(slo:service_errors_total:ratio_rate_1h{service=\"my-team.my-service.payment\"} > (14.4 * 0.001) and slo:service_errors_total:ratio_rate_5m{service=\"my-team.my-service.payment\"} > (14.4 * 0.001)) or (slo:service_errors_total:ratio_rate_6h{service=\"my-team.my-service.payment\"} > (6 * 0.001) and slo:service_errors_total:ratio_rate_30m{service=\"my-team.my-service.payment\"} > (6 * 0.001))",
+		Labels: map[string]string{
+			"channel":  "my-channel",
+			"severity": "page",
+		},
+		Annotations: slo.Annotations,
+	})
+
+	assert.Equal(t, alertRules[1], rulefmt.Rule{
+		Alert: "slo:my-team.my-service.payment.errors.ticket",
+		Expr:  "(slo:service_errors_total:ratio_rate_1d{service=\"my-team.my-service.payment\"} > (3 * 0.001) and slo:service_errors_total:ratio_rate_2h{service=\"my-team.my-service.payment\"} > (3 * 0.001)) or (slo:service_errors_total:ratio_rate_3d{service=\"my-team.my-service.payment\"} > (1 * 0.001) and slo:service_errors_total:ratio_rate_6h{service=\"my-team.my-service.payment\"} > (1 * 0.001))",
+		Labels: map[string]string{
+			"channel":  "my-channel",
+			"severity": "ticket",
+		},
+		Annotations: slo.Annotations,
+	})
+
+	assert.Equal(t, alertRules[2], rulefmt.Rule{
+		Alert: "slo:my-team.my-service.payment.latency.page",
+		Expr: ("(" +
+			"slo:service_latency:ratio_rate_1h{le=\"0.1\", service=\"my-team.my-service.payment\"} < 0.28" +
+			" and " +
+			"slo:service_latency:ratio_rate_5m{le=\"0.1\", service=\"my-team.my-service.payment\"} < 0.28" +
+			") or (" +
+			"slo:service_latency:ratio_rate_6h{le=\"0.1\", service=\"my-team.my-service.payment\"} < 0.7" +
+			" and " +
+			"slo:service_latency:ratio_rate_30m{le=\"0.1\", service=\"my-team.my-service.payment\"} < 0.7" +
+			") or (" +
+			"slo:service_latency:ratio_rate_1h{le=\"0.5\", service=\"my-team.my-service.payment\"} < 0.856" +
+			" and " +
+			"slo:service_latency:ratio_rate_5m{le=\"0.5\", service=\"my-team.my-service.payment\"} < 0.856" +
+			") or (" +
+			"slo:service_latency:ratio_rate_6h{le=\"0.5\", service=\"my-team.my-service.payment\"} < 0.94" +
+			" and " +
+			"slo:service_latency:ratio_rate_30m{le=\"0.5\", service=\"my-team.my-service.payment\"} < 0.94" +
+			")"),
+
+		Labels: map[string]string{
+			"channel":  "my-channel",
+			"severity": "page",
+		},
+		Annotations: slo.Annotations,
+	})
+
+	assert.Equal(t, alertRules[3], rulefmt.Rule{
+		Alert: "slo:my-team.my-service.payment.latency.ticket",
+		Expr: ("(" +
+			"slo:service_latency:ratio_rate_1d{le=\"0.1\", service=\"my-team.my-service.payment\"} < 0.85" +
+			" and " +
+			"slo:service_latency:ratio_rate_2h{le=\"0.1\", service=\"my-team.my-service.payment\"} < 0.85" +
+			") or (" +
+			"slo:service_latency:ratio_rate_3d{le=\"0.1\", service=\"my-team.my-service.payment\"} < 0.95" +
+			" and " +
+			"slo:service_latency:ratio_rate_6h{le=\"0.1\", service=\"my-team.my-service.payment\"} < 0.95" +
+			") or (" +
+			"slo:service_latency:ratio_rate_1d{le=\"0.5\", service=\"my-team.my-service.payment\"} < 0.97" +
+			" and " +
+			"slo:service_latency:ratio_rate_2h{le=\"0.5\", service=\"my-team.my-service.payment\"} < 0.97" +
+			") or (" +
+			"slo:service_latency:ratio_rate_3d{le=\"0.5\", service=\"my-team.my-service.payment\"} < 0.99" +
+			" and " +
+			"slo:service_latency:ratio_rate_6h{le=\"0.5\", service=\"my-team.my-service.payment\"} < 0.99" +
+			")"),
+
+		Labels: map[string]string{
+			"channel":  "my-channel",
+			"severity": "ticket",
+		},
+		Annotations: slo.Annotations,
+	})
+}