-
Notifications
You must be signed in to change notification settings - Fork 153
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Ensure PodDisruptionBudgetAtLimit alert is silenced
Signed-off-by: machadovilaca <machadovilaca@gmail.com>
- Loading branch information
1 parent
8e008fe
commit 54b08c2
Showing
11 changed files
with
330 additions
and
69 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
32 changes: 0 additions & 32 deletions
32
controllers/observability/observability_controller_test.go
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
107 changes: 107 additions & 0 deletions
107
controllers/observability/pod_disruption_budget_at_limit.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,107 @@ | ||
package observability | ||
|
||
import ( | ||
"crypto/tls" | ||
"crypto/x509" | ||
"fmt" | ||
"net/http" | ||
"os" | ||
"time" | ||
|
||
"github.com/kubevirt/hyperconverged-cluster-operator/pkg/alertmanager" | ||
) | ||
|
||
const ( | ||
alertmanagerSvcHost = "alertmanager-main.openshift-monitoring.svc.cluster.local:9094" | ||
tlsCertPath = "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt" | ||
) | ||
|
||
func (r *Reconciler) ensurePodDisruptionBudgetAtLimitIsSilenced() error { | ||
if r.amApi == nil { | ||
var err error | ||
r.amApi, err = r.NewAlertmanagerApi() | ||
if err != nil { | ||
return fmt.Errorf("failed to initialize alertmanager api: %w", err) | ||
} | ||
} | ||
|
||
amSilences, err := r.amApi.ListSilences() | ||
if err != nil { | ||
return fmt.Errorf("failed to list alertmanager silences: %w", err) | ||
} | ||
|
||
if FindPodDisruptionBudgetAtLimitSilence(amSilences) != nil { | ||
log.Info("KubeVirt PodDisruptionBudgetAtLimit alerts are already silenced") | ||
return nil | ||
} | ||
|
||
silence := alertmanager.Silence{ | ||
Comment: "Silence KubeVirt PodDisruptionBudgetAtLimit alerts", | ||
CreatedBy: "hyperconverged-cluster-operator", | ||
EndsAt: "3000-01-01T00:00:00Z", | ||
Matchers: []alertmanager.Matcher{ | ||
{ | ||
IsEqual: true, | ||
Name: "alertname", | ||
Value: "PodDisruptionBudgetAtLimit", | ||
}, | ||
{ | ||
IsRegex: true, | ||
Name: "poddisruptionbudget", | ||
Value: "kubevirt-disruption-budget-.*", | ||
}, | ||
}, | ||
StartsAt: time.Now().Format(time.RFC3339), | ||
} | ||
|
||
if err := r.amApi.CreateSilence(silence); err != nil { | ||
return fmt.Errorf("failed to create alertmanager silence: %w", err) | ||
} | ||
log.Info("Silenced PodDisruptionBudgetAtLimit alerts") | ||
|
||
return nil | ||
} | ||
|
||
func (r *Reconciler) NewAlertmanagerApi() (*alertmanager.Api, error) { | ||
caCert, err := os.ReadFile(tlsCertPath) | ||
if err != nil { | ||
return nil, fmt.Errorf("failed to read ca cert: %w", err) | ||
} | ||
|
||
caCertPool := x509.NewCertPool() | ||
caCertPool.AppendCertsFromPEM(caCert) | ||
|
||
httpClient := http.Client{} | ||
httpClient.Transport = &http.Transport{ | ||
TLSClientConfig: &tls.Config{RootCAs: caCertPool}, | ||
} | ||
|
||
return alertmanager.NewAPI(httpClient, alertmanagerSvcHost, r.config.BearerToken), nil | ||
} | ||
|
||
func FindPodDisruptionBudgetAtLimitSilence(amSilences []alertmanager.Silence) *alertmanager.Silence { | ||
for _, silence := range amSilences { | ||
if silence.Status.State != "active" { | ||
continue | ||
} | ||
|
||
var isPDBSilence bool | ||
var isKubeVirtPDBSilence bool | ||
|
||
for _, matcher := range silence.Matchers { | ||
if matcher.Name == "alertname" && matcher.Value == "PodDisruptionBudgetAtLimit" && matcher.IsEqual { | ||
isPDBSilence = true | ||
} | ||
|
||
if matcher.Name == "poddisruptionbudget" && matcher.IsRegex && matcher.Value == "kubevirt-disruption-budget-.*" { | ||
isKubeVirtPDBSilence = true | ||
} | ||
} | ||
|
||
if isPDBSilence && isKubeVirtPDBSilence { | ||
return &silence | ||
} | ||
} | ||
|
||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.