From 8bfbf5833cec56816bd15bb9ef11ebedf3d2c1e7 Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Tue, 25 Apr 2023 13:33:58 -0400 Subject: [PATCH 1/2] ci: improve ems alert logging and event generation --- integration/test/alert/alert.go | 41 ++++++++++++---------------- integration/test/bookend_ems_test.go | 37 +++++++++++++------------ 2 files changed, 38 insertions(+), 40 deletions(-) diff --git a/integration/test/alert/alert.go b/integration/test/alert/alert.go index cbf99d6eb..90485582c 100644 --- a/integration/test/alert/alert.go +++ b/integration/test/alert/alert.go @@ -2,6 +2,7 @@ package promAlerts import ( "fmt" + "github.com/Netapp/harvest-automation/test/installer" "github.com/Netapp/harvest-automation/test/utils" "github.com/netapp/harvest/v2/pkg/conf" "github.com/netapp/harvest/v2/pkg/tree" @@ -13,7 +14,7 @@ import ( const PrometheusAlertURL string = "http://localhost:9090/api/v1/alerts" const TestClusterName = "umeng-aff300-05-06" const TestNodeName = "umeng-aff300-06" -const User = "admin" +const Admin = "admin" var volumeArwState = []string{ `"disable-in-progress"`, @@ -34,6 +35,7 @@ type PromAlert struct { } func GetAlerts() (map[string]int, int) { + now := time.Now() alertsData := make(map[string]int) totalAlerts := 0 @@ -56,6 +58,11 @@ func GetAlerts() (map[string]int, int) { } } } + defer log.Info(). + Int("alertsData", len(alertsData)). + Int("totalAlerts", totalAlerts). + Str("dur", time.Since(now).Round(time.Millisecond).String()). + Msg("Get Prometheus alerts") return alertsData, totalAlerts } @@ -93,8 +100,13 @@ func GetEmsAlerts(dir string, fileName string) ([]string, []string, []string) { func GenerateEvents(emsNames []string, nodeScopedEms []string) []string { supportedEms := make([]string, 0) var jsonValue []byte - addr, user, pass, nodeName := GetPollerDetail() - url := "https://" + addr + "/api/private/cli/event/generate" + err := conf.LoadHarvestConfig(installer.HarvestConfigFile) + poller, err2 := conf.PollerNamed(TestClusterName) + dc1, err3 := conf.PollerNamed("dc1") + if err != nil && err2 != nil && err3 != nil { + log.Fatal().Errs("errors", []error{err, err2, err3}).Msg("Failed to load config") + } + url := "https://" + poller.Addr + "/api/private/cli/event/generate" method := "POST" volumeArwCount := 0 @@ -112,13 +124,13 @@ func GenerateEvents(emsNames []string, nodeScopedEms []string) []string { // Handle for node-scoped ems, Passing node-name as input if utils.Contains(nodeScopedEms, ems) { - jsonValue = []byte(fmt.Sprintf(`{"message-name": "%s", "values": [%s,2,3,4,5,6,7,8,9], "node": "%s"}`, ems, value, nodeName)) + jsonValue = []byte(fmt.Sprintf(`{"message-name": "%s", "values": [%s,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20], "node": "%s"}`, ems, value, TestNodeName)) } else { - jsonValue = []byte(fmt.Sprintf(`{"message-name": "%s", "values": [%s,2,3,4,5,6,7,8,9]}`, ems, value)) + jsonValue = []byte(fmt.Sprintf(`{"message-name": "%s", "values": [%s,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]}`, ems, value)) } var data map[string]interface{} - data = utils.SendPostReqAndGetRes(url, method, jsonValue, user, pass) + data = utils.SendPostReqAndGetRes(url, method, jsonValue, Admin, dc1.Password) if response := data["error"]; response != nil { errorDetail := response.(map[string]interface{}) code := errorDetail["code"].(string) @@ -133,20 +145,3 @@ func GenerateEvents(emsNames []string, nodeScopedEms []string) []string { return supportedEms } - -func GetPollerDetail() (string, string, string, string) { - var ( - err error - poller *conf.Poller - ) - - if err = conf.LoadHarvestConfig(utils.GetConfigDir() + "/harvest.yml"); err != nil { - utils.PanicIfNotNil(err) - } - - if poller, err = conf.PollerNamed(TestClusterName); err != nil { - utils.PanicIfNotNil(err) - } - - return poller.Addr, User, poller.Password, TestNodeName -} diff --git a/integration/test/bookend_ems_test.go b/integration/test/bookend_ems_test.go index bdd649301..ced0c34c4 100644 --- a/integration/test/bookend_ems_test.go +++ b/integration/test/bookend_ems_test.go @@ -5,6 +5,7 @@ import ( "github.com/Netapp/harvest-automation/test/utils" "github.com/rs/zerolog/log" "testing" + "time" ) var issuingEmsNames []string @@ -31,7 +32,7 @@ var nodeScopedResolvingEmsList = []string{ } func setupAlerts() { - totalAlerts := 0 + numAlerts := 0 emsConfigDir := utils.GetHarvestRootDir() + "/conf/ems/9.6.0" log.Info().Str("EmsConfigDir", emsConfigDir).Msg("Directory path") @@ -39,22 +40,30 @@ func setupAlerts() { _, issuingEmsNames, resolvingEmsNames = promAlerts.GetEmsAlerts(emsConfigDir, "ems.yaml") // Identify supported issuing ems names for the given cluster + now := time.Now() supportedIssuingEms = promAlerts.GenerateEvents(issuingEmsNames, nodeScopedIssuingEmsList) - log.Info().Msgf("Total supported issuing ems: %d", len(supportedIssuingEms)) + log.Info(). + Int("supportedIssuingEms", len(supportedIssuingEms)). + Str("dur", time.Since(now).Round(time.Millisecond).String()). + Msg("Supported issuing ems") // Fetch previous prometheus alerts oldAlertsData, _ = promAlerts.GetAlerts() // Identify supported ems names for the given cluster + now = time.Now() supportedResolvingEms = promAlerts.GenerateEvents(resolvingEmsNames, nodeScopedResolvingEmsList) - log.Info().Msgf("Total supported resolving ems:%d", len(supportedResolvingEms)) + log.Info(). + Int("supportedResolvingEms", len(supportedResolvingEms)). + Str("dur", time.Since(now).Round(time.Millisecond).String()). + Msg("Supported resolving ems") // Fetch current prometheus alerts - newAlertsData, totalAlerts = promAlerts.GetAlerts() - if totalAlerts == 0 { + newAlertsData, numAlerts = promAlerts.GetAlerts() + if numAlerts == 0 { log.Info().Msg("No alerts found in prometheus") } - log.Info().Msgf("Total firing alerts %d", totalAlerts) + log.Info().Int("numAlerts", numAlerts).Msg("Firing alerts") } func TestEmsTestSuite(t *testing.T) { @@ -62,21 +71,15 @@ func TestEmsTestSuite(t *testing.T) { setupAlerts() // Evaluate bookend active ems events - foundBookendEms := make([]string, 0) - for _, issuingEms := range supportedIssuingEms { - // If the issuingEms did not exit before, then ignore the test-case. + // If the issuingEms did not exist before, then ignore the test-case. if oldAlertsData[issuingEms] > 0 { - v := oldAlertsData[issuingEms] - newAlertsData[issuingEms] - if v < 1 { - foundBookendEms = append(foundBookendEms, issuingEms) + v := newAlertsData[issuingEms] - oldAlertsData[issuingEms] + if v > 1 { + t.Errorf("Extra bookend ems alerts raised event=%s, count=%d", issuingEms, v) } } else { - log.Info().Str("issuingEms", issuingEms).Msg("There is no active issuingEms") + log.Info().Str("issuingEms", issuingEms).Msg("Ignore. Did not exist before") } } - if len(foundBookendEms) > 0 { - log.Error().Strs("foundBookendEms", foundBookendEms).Msg("Unexpected bookendEms found") - t.Errorf("One or more extra bookend ems alerts %s have been raised", foundBookendEms) - } } From b6b33060037855cd1cf007e512af1261ac5a3bdc Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Tue, 25 Apr 2023 15:03:02 -0400 Subject: [PATCH 2/2] ci: improve ems alert logging and event generation --- integration/test/alert/alert.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/test/alert/alert.go b/integration/test/alert/alert.go index 90485582c..262a591ff 100644 --- a/integration/test/alert/alert.go +++ b/integration/test/alert/alert.go @@ -58,7 +58,7 @@ func GetAlerts() (map[string]int, int) { } } } - defer log.Info(). + log.Info(). Int("alertsData", len(alertsData)). Int("totalAlerts", totalAlerts). Str("dur", time.Since(now).Round(time.Millisecond).String()).