From 8336ef3be373366fd0528098d2db4a655769688a Mon Sep 17 00:00:00 2001 From: Assaf Admi Date: Wed, 28 Feb 2024 11:37:32 +0200 Subject: [PATCH] Add additional alerts validations Signed-off-by: assafad --- pkg/testutil/alert_custom_validation_test.go | 116 +++++++++++++++++++ pkg/testutil/alert_custom_validations.go | 72 ++++++++++++ pkg/testutil/alert_validation_test.go | 2 +- 3 files changed, 189 insertions(+), 1 deletion(-) create mode 100644 pkg/testutil/alert_custom_validation_test.go create mode 100644 pkg/testutil/alert_custom_validations.go diff --git a/pkg/testutil/alert_custom_validation_test.go b/pkg/testutil/alert_custom_validation_test.go new file mode 100644 index 0000000..1ce3075 --- /dev/null +++ b/pkg/testutil/alert_custom_validation_test.go @@ -0,0 +1,116 @@ +package testutil + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/util/intstr" + + promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" +) + +var _ = Describe("Custom Validators", func() { + var linter *Linter + + BeforeEach(func() { + linter = New() + }) + + Context("Custom Alert Validations", func() { + It("should not return error if all custom validations were added and all exist", func() { + alert := &promv1.Rule{ + Alert: "ExampleAlert", + Expr: intstr.FromString("sum(rate(http_requests_total[5m]))"), + Labels: map[string]string{ + "severity": "critical", + "operator_health_impact": "critical", + "kubernetes_operator_part_of": "example_part_of", + "kubernetes_operator_component": "example_component", + }, + Annotations: map[string]string{ + "summary": "Example summary", + "description": "Example description", + "runbook_url": "example/runbook/url", + }, + } + linter.AddCustomAlertValidations(ValidateAlertNameLength, ValidateAlertRunbookURLAnnotation, + ValidateHealthImpactLabel, ValidateAlertPartOfAndComponentLabels) + problems := linter.LintAlert(alert) + Expect(problems).To(BeEmpty()) + }) + + It("should return error if alert name length custom validation was added and alert name is too long", func() { + alert := &promv1.Rule{ + Alert: "ExampleAlertWithVeryLongNameExtendedToMeetRequiredLength", + Expr: intstr.FromString("sum(rate(http_requests_total[5m]))"), + Labels: map[string]string{ + "severity": "critical", + }, + Annotations: map[string]string{ + "summary": "Example summary", + "description": "Example description", + }, + } + linter.AddCustomAlertValidations(ValidateAlertNameLength) + problems := linter.LintAlert(alert) + Expect(problems).To(HaveLen(1)) + Expect(problems[0].Description).To(ContainSubstring("alert name exceeds 50 characters")) + }) + + It("should return error if runbook_url custom validation was added and is missing", func() { + alert := &promv1.Rule{ + Alert: "ExampleAlert", + Expr: intstr.FromString("sum(rate(http_requests_total[5m]))"), + Labels: map[string]string{ + "severity": "critical", + }, + Annotations: map[string]string{ + "summary": "Example summary", + "description": "Example description", + }, + } + linter.AddCustomAlertValidations(ValidateAlertRunbookURLAnnotation) + problems := linter.LintAlert(alert) + Expect(problems).To(HaveLen(1)) + Expect(problems[0].Description).To(ContainSubstring("alert must have a runbook_url annotation")) + }) + + It("should return error if operator_health_impact custom validation was added and is missing or invalid", func() { + alert := &promv1.Rule{ + Alert: "ExampleAlert", + Expr: intstr.FromString("sum(rate(http_requests_total[5m]))"), + Labels: map[string]string{ + "severity": "critical", + "operator_health_impact": "invalid_operator_health_impact", + }, + Annotations: map[string]string{ + "summary": "Example summary", + "description": "Example description", + }, + } + linter.AddCustomAlertValidations(ValidateHealthImpactLabel) + problems := linter.LintAlert(alert) + Expect(problems).To(HaveLen(1)) + Expect(problems[0].Description).To(ContainSubstring("alert must have a operator_health_impact label with value critical, warning, or none")) + }) + + It("should return error if operator_part_of and operator_component custom validation was added and both are missing", func() { + alert := &promv1.Rule{ + Alert: "ExampleAlert", + Expr: intstr.FromString("sum(rate(http_requests_total[5m]))"), + Labels: map[string]string{ + "severity": "critical", + }, + Annotations: map[string]string{ + "summary": "Example summary", + "description": "Example description", + }, + } + linter.AddCustomAlertValidations(ValidateAlertPartOfAndComponentLabels) + problems := linter.LintAlert(alert) + Expect(problems).To(HaveLen(2)) + Expect(problems[0].Description).To(ContainSubstring("alert must have a kubernetes_operator_part_of label")) + Expect(problems[1].Description).To(ContainSubstring("alert must have a kubernetes_operator_component label")) + }) + }) +}) diff --git a/pkg/testutil/alert_custom_validations.go b/pkg/testutil/alert_custom_validations.go new file mode 100644 index 0000000..0513e8a --- /dev/null +++ b/pkg/testutil/alert_custom_validations.go @@ -0,0 +1,72 @@ +package testutil + +import ( + promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" +) + +func ValidateAlertNameLength(alert *promv1.Rule) []Problem { + var result []Problem + + if len(alert.Alert) > 50 { + result = append(result, Problem{ + ResourceName: alert.Alert, + Description: "alert name exceeds 50 characters", + }) + } + + return result +} + +func ValidateAlertRunbookURLAnnotation(alert *promv1.Rule) []Problem { + var result []Problem + + runbookURL := alert.Annotations["runbook_url"] + if runbookURL == "" { + result = append(result, Problem{ + ResourceName: alert.Alert, + Description: "alert must have a runbook_url annotation", + }) + } + + return result +} + +func ValidateHealthImpactLabel(alert *promv1.Rule) []Problem { + var result []Problem + + healthImpact := alert.Labels["operator_health_impact"] + if !isValidHealthImpact(healthImpact) { + result = append(result, Problem{ + ResourceName: alert.Alert, + Description: "alert must have a operator_health_impact label with value critical, warning, or none", + }) + } + + return result +} + +func ValidateAlertPartOfAndComponentLabels(alert *promv1.Rule) []Problem { + var result []Problem + + partOf := alert.Labels["kubernetes_operator_part_of"] + if partOf == "" { + result = append(result, Problem{ + ResourceName: alert.Alert, + Description: "alert must have a kubernetes_operator_part_of label", + }) + } + + component := alert.Labels["kubernetes_operator_component"] + if component == "" { + result = append(result, Problem{ + ResourceName: alert.Alert, + Description: "alert must have a kubernetes_operator_component label", + }) + } + + return result +} + +func isValidHealthImpact(healthImpact string) bool { + return healthImpact == "critical" || healthImpact == "warning" || healthImpact == "none" +} diff --git a/pkg/testutil/alert_validation_test.go b/pkg/testutil/alert_validation_test.go index e451c06..4b9c7d3 100644 --- a/pkg/testutil/alert_validation_test.go +++ b/pkg/testutil/alert_validation_test.go @@ -9,7 +9,7 @@ import ( promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" ) -var _ = Describe("Validators", func() { +var _ = Describe("Default Validators", func() { var linter *Linter BeforeEach(func() {