From 63585cd346297d2d284c4214a3057d9f43c940e4 Mon Sep 17 00:00:00 2001 From: Assaf Admi Date: Wed, 28 Feb 2024 11:37:32 +0200 Subject: [PATCH] Add additional alerts validations Signed-off-by: assafad --- pkg/testutil/alert_custom_validation_test.go | 115 +++++++++++++++++++ pkg/testutil/alert_custom_validations.go | 72 ++++++++++++ pkg/testutil/alert_validation.go | 6 +- pkg/testutil/alert_validation_test.go | 5 +- 4 files changed, 194 insertions(+), 4 deletions(-) create mode 100644 pkg/testutil/alert_custom_validation_test.go create mode 100644 pkg/testutil/alert_custom_validations.go diff --git a/pkg/testutil/alert_custom_validation_test.go b/pkg/testutil/alert_custom_validation_test.go new file mode 100644 index 0000000..c54c6f2 --- /dev/null +++ b/pkg/testutil/alert_custom_validation_test.go @@ -0,0 +1,115 @@ +package testutil + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +var _ = Describe("Custom Validators", func() { + var linter *Linter + + BeforeEach(func() { + linter = New() + }) + + Context("Custom Alert Validations", func() { + It("should not return error if all custom validations were added and all exist", func() { + alert := &promv1.Rule{ + Alert: "ExampleAlert", + Expr: intstr.FromString("sum(rate(http_requests_total[5m]))"), + Labels: map[string]string{ + "severity": "critical", + "operator_health_impact": "critical", + "kubernetes_operator_part_of": "example_part_of", + "kubernetes_operator_component": "example_component", + }, + Annotations: map[string]string{ + "summary": "Example summary", + "description": "Example description", + "runbook_url": "example/runbook/url", + }, + } + linter.AddCustomAlertValidations(ValidateAlertNameLength, ValidateAlertRunbookURLAnnotation, + ValidateAlertHealthImpactLabel, ValidateAlertPartOfAndComponentLabels) + problems := linter.LintAlert(alert) + Expect(problems).To(BeEmpty()) + }) + + It("should return error if alert name length custom validation was added and alert name is too long", func() { + alert := &promv1.Rule{ + Alert: "ExampleAlertWithVeryLongNameExtendedToMeetRequiredLength", + Expr: intstr.FromString("sum(rate(http_requests_total[5m]))"), + Labels: map[string]string{ + "severity": "critical", + }, + Annotations: map[string]string{ + "summary": "Example summary", + "description": "Example description", + }, + } + linter.AddCustomAlertValidations(ValidateAlertNameLength) + problems := linter.LintAlert(alert) + Expect(problems).To(HaveLen(1)) + Expect(problems[0].Description).To(ContainSubstring("alert name exceeds 50 characters")) + }) + + It("should return error if runbook_url custom validation was added and is missing", func() { + alert := &promv1.Rule{ + Alert: "ExampleAlert", + Expr: intstr.FromString("sum(rate(http_requests_total[5m]))"), + Labels: map[string]string{ + "severity": "critical", + }, + Annotations: map[string]string{ + "summary": "Example summary", + "description": "Example description", + }, + } + linter.AddCustomAlertValidations(ValidateAlertRunbookURLAnnotation) + problems := linter.LintAlert(alert) + Expect(problems).To(HaveLen(1)) + Expect(problems[0].Description).To(ContainSubstring("alert must have a runbook_url annotation")) + }) + + It("should return error if operator_health_impact custom validation was added and is missing or invalid", func() { + alert := &promv1.Rule{ + Alert: "ExampleAlert", + Expr: intstr.FromString("sum(rate(http_requests_total[5m]))"), + Labels: map[string]string{ + "severity": "critical", + "operator_health_impact": "invalid_operator_health_impact", + }, + Annotations: map[string]string{ + "summary": "Example summary", + "description": "Example description", + }, + } + linter.AddCustomAlertValidations(ValidateAlertHealthImpactLabel) + problems := linter.LintAlert(alert) + Expect(problems).To(HaveLen(1)) + Expect(problems[0].Description).To(ContainSubstring("alert must have a operator_health_impact label with value critical, warning, or none")) + }) + + It("should return error if operator_part_of and operator_component custom validation was added and both are missing", func() { + alert := &promv1.Rule{ + Alert: "ExampleAlert", + Expr: intstr.FromString("sum(rate(http_requests_total[5m]))"), + Labels: map[string]string{ + "severity": "critical", + }, + Annotations: map[string]string{ + "summary": "Example summary", + "description": "Example description", + }, + } + linter.AddCustomAlertValidations(ValidateAlertPartOfAndComponentLabels) + problems := linter.LintAlert(alert) + Expect(problems).To(HaveLen(2)) + Expect(problems[0].Description).To(ContainSubstring("alert must have a kubernetes_operator_part_of label")) + Expect(problems[1].Description).To(ContainSubstring("alert must have a kubernetes_operator_component label")) + }) + }) +}) diff --git a/pkg/testutil/alert_custom_validations.go b/pkg/testutil/alert_custom_validations.go new file mode 100644 index 0000000..ec09026 --- /dev/null +++ b/pkg/testutil/alert_custom_validations.go @@ -0,0 +1,72 @@ +package testutil + +import ( + promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" +) + +func ValidateAlertNameLength(alert *promv1.Rule) []Problem { + var result []Problem + + if len(alert.Alert) > 50 { + result = append(result, Problem{ + ResourceName: alert.Alert, + Description: "alert name exceeds 50 characters", + }) + } + + return result +} + +func ValidateAlertRunbookURLAnnotation(alert *promv1.Rule) []Problem { + var result []Problem + + runbookURL := alert.Annotations["runbook_url"] + if runbookURL == "" { + result = append(result, Problem{ + ResourceName: alert.Alert, + Description: "alert must have a runbook_url annotation", + }) + } + + return result +} + +func ValidateAlertHealthImpactLabel(alert *promv1.Rule) []Problem { + var result []Problem + + healthImpact := alert.Labels["operator_health_impact"] + if !isValidHealthImpact(healthImpact) { + result = append(result, Problem{ + ResourceName: alert.Alert, + Description: "alert must have a operator_health_impact label with value critical, warning, or none", + }) + } + + return result +} + +func ValidateAlertPartOfAndComponentLabels(alert *promv1.Rule) []Problem { + var result []Problem + + partOf := alert.Labels["kubernetes_operator_part_of"] + if partOf == "" { + result = append(result, Problem{ + ResourceName: alert.Alert, + Description: "alert must have a kubernetes_operator_part_of label", + }) + } + + component := alert.Labels["kubernetes_operator_component"] + if component == "" { + result = append(result, Problem{ + ResourceName: alert.Alert, + Description: "alert must have a kubernetes_operator_component label", + }) + } + + return result +} + +func isValidHealthImpact(healthImpact string) bool { + return healthImpact == "critical" || healthImpact == "warning" || healthImpact == "none" +} diff --git a/pkg/testutil/alert_validation.go b/pkg/testutil/alert_validation.go index 54c2941..e1bf438 100644 --- a/pkg/testutil/alert_validation.go +++ b/pkg/testutil/alert_validation.go @@ -46,7 +46,7 @@ func validateAlertHasSeverityLabel(alert *promv1.Rule) []Problem { var result []Problem severity := alert.Labels["severity"] - if severity == "" || (severity != "critical" && severity != "warning" && severity != "info") { + if !isValidSeverity(severity) { result = append(result, Problem{ ResourceName: alert.Alert, Description: "alert must have a severity label with value critical, warning, or info", @@ -89,3 +89,7 @@ func isPascalCase(s string) bool { pascalCaseRegex := regexp.MustCompile(pascalCasePattern) return pascalCaseRegex.MatchString(s) } + +func isValidSeverity(severity string) bool { + return severity == "critical" || severity == "warning" || severity == "info" +} diff --git a/pkg/testutil/alert_validation_test.go b/pkg/testutil/alert_validation_test.go index e451c06..009750c 100644 --- a/pkg/testutil/alert_validation_test.go +++ b/pkg/testutil/alert_validation_test.go @@ -4,12 +4,11 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/apimachinery/pkg/util/intstr" - promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) -var _ = Describe("Validators", func() { +var _ = Describe("Default Validators", func() { var linter *Linter BeforeEach(func() {