From b7af6e01115a596e9533ddf1e836b4d296913f3d Mon Sep 17 00:00:00 2001 From: Tobias Ulrich Date: Mon, 19 Dec 2022 13:35:23 -0300 Subject: [PATCH] Add logs to reporter and watcher --- metadata/version.go | 2 +- reporters/dogstatsd.go | 11 +++++++---- reporters/dogstatsd/handlers.go | 6 ++---- reporters/reporter.go | 11 +++++++++-- watcher/watcher.go | 30 +++++++++++++++++++++++------- 5 files changed, 42 insertions(+), 18 deletions(-) diff --git a/metadata/version.go b/metadata/version.go index 38a59a8ed..d85b76e9b 100644 --- a/metadata/version.go +++ b/metadata/version.go @@ -8,7 +8,7 @@ package metadata //Version of Maestro -var Version = "9.6.1" +var Version = "9.15.1" //KubeVersion is the desired Kubernetes version var KubeVersion = "v1.13.9" diff --git a/reporters/dogstatsd.go b/reporters/dogstatsd.go index 1a5bdd182..4f70e613c 100644 --- a/reporters/dogstatsd.go +++ b/reporters/dogstatsd.go @@ -49,15 +49,15 @@ func (d *DogStatsD) Report(event string, opts map[string]interface{}) error { defer d.mutex.RUnlock() handlerI, prs := handlers.Find(event) if prs == false { - return fmt.Errorf("reportHandler for %s doesn't exist", event) + return fmt.Errorf("DogStatsD: reportHandler for %s doesn't exist", event) } opts[constants.TagRegion] = d.region handler := handlerI.(func(dogstatsd.Client, string, map[string]string) error) err := handler(d.client, event, toMapStringString(opts)) if err != nil { - d.logger.Error(err) + return fmt.Errorf("DogStatsD: failed to report event '%s': %w", event, err) } - return err + return nil } // MakeDogStatsD adds a DogStatsD struct to the Reporters' singleton @@ -116,13 +116,15 @@ func (d *DogStatsD) restartTicker() { for range d.ticker.C { d.mutex.Lock() if err := d.restartDogStatsdClient(); err != nil { - d.logger.Errorf("DogStatsD: failed to close statsd connection during restart: %s", err.Error()) + d.logger.WithError(err).Errorf("DogStatsD: failed to close statsd connection during restart") } d.mutex.Unlock() } } func (d *DogStatsD) restartDogStatsdClient() error { + d.logger.Info("Trying to restart the dogstatsd client") + err := d.statsdClient.Close() if err != nil { return fmt.Errorf("failed to close statsd connection: %w", err) @@ -133,6 +135,7 @@ func (d *DogStatsD) restartDogStatsdClient() error { return fmt.Errorf("failed to recreate dogstatsd client: %w", err) } + d.logger.Info("DogStatsD was restarted successfully") d.statsdClient = c.Client.(*statsd.Client) d.client = c return nil diff --git a/reporters/dogstatsd/handlers.go b/reporters/dogstatsd/handlers.go index fabaec7a5..0a62053ed 100644 --- a/reporters/dogstatsd/handlers.go +++ b/reporters/dogstatsd/handlers.go @@ -71,8 +71,7 @@ func GruStatusHandler(c dogstatsd.Client, event string, if err != nil { return err } - c.Gauge(fmt.Sprintf("gru.%s", opts["status"]), gauge, tags, 1) - return nil + return c.Gauge(fmt.Sprintf("gru.%s", opts["status"]), gauge, tags, 1) } // GruTimingHandler calls dogstatsd.Client.Timing with tags formatted as key:value @@ -135,6 +134,5 @@ func GaugeHandler( if err != nil { return err } - c.Gauge(event, gauge, tags, 1) - return nil + return c.Gauge(event, gauge, tags, 1) } diff --git a/reporters/reporter.go b/reporters/reporter.go index b5443f40f..626e80b54 100644 --- a/reporters/reporter.go +++ b/reporters/reporter.go @@ -52,8 +52,15 @@ func copyOpts(src map[string]interface{}) map[string]interface{} { // Report is Reporters' implementation of the Reporter interface func (r *Reporters) Report(event string, opts map[string]interface{}) error { + var aggregatedErrors []error for _, reporter := range r.reporters { - reporter.Report(event, copyOpts(opts)) + if err := reporter.Report(event, copyOpts(opts)); err != nil { + aggregatedErrors = append(aggregatedErrors, err) + } + } + + if len(aggregatedErrors) > 0 { + return fmt.Errorf("failed to report '%s' event: %v", event, aggregatedErrors) } return nil } @@ -80,7 +87,7 @@ func MakeReporters(config *viper.Viper, logger *logrus.Logger) { for k := range GetInstance().reporters { correctlySet = append(correctlySet, k) } - logger.Info(fmt.Sprintf("Active reporters: %s", strings.Join(correctlySet, ", "))) + logger.Infof("Active reporters: %s", strings.Join(correctlySet, ", ")) } // NewReporters ctor diff --git a/watcher/watcher.go b/watcher/watcher.go index 767b9ec62..cfd15c248 100644 --- a/watcher/watcher.go +++ b/watcher/watcher.go @@ -286,7 +286,9 @@ func (w *Watcher) reportRoomsStatusesRoutine() { case <-podStateCountTicker.C: w.PodStatesCount() case <-roomStatusTicker.C: - _ = w.ReportRoomsStatuses() + if err := w.ReportRoomsStatuses(); err != nil { + w.Logger.WithError(err).Error("failed to report room status") + } } } } @@ -421,6 +423,8 @@ func (w *Watcher) ReportRoomsStatuses() error { return nil } + w.Logger.Info("Start to report RoomsStatuses") + var roomCountByStatus *models.RoomsStatusCount err := w.MetricsReporter.WithSegment(models.SegmentGroupBy, func() error { var err error @@ -461,14 +465,27 @@ func (w *Watcher) ReportRoomsStatuses() error { } for _, r := range roomDataSlice { - _ = reporters.Report(reportersConstants.EventGruStatus, map[string]interface{}{ + w.Logger. + WithField("gauge", r.Gauge). + Infof("Start reporting %s", r.Status) + + err := reporters.Report(reportersConstants.EventGruStatus, map[string]interface{}{ reportersConstants.TagGame: w.GameName, reportersConstants.TagScheduler: w.SchedulerName, "status": r.Status, "gauge": r.Gauge, }) + if err != nil { + w.Logger. + WithError(err). + Errorf("Finished to report status %s with error", r.Status) + } else { + w.Logger.Infof("Finished to report status %s", r.Status) + } } + w.Logger.Info("Finished to report RoomsStatuses") + return nil } @@ -741,8 +758,8 @@ func (w *Watcher) RemoveDeadRooms() error { podsToDelete := w.filterPodsByName(logger, pods, append(roomsNoPingSince, roomsOnOccupiedTimeout...)) startedEvent := models.NewSchedulerEvent( - models.StartRemoveDeadRoomsEventName, - w.SchedulerName, + models.StartRemoveDeadRoomsEventName, + w.SchedulerName, map[string]interface{}{ "amount": len(podsToDelete), }, @@ -806,8 +823,8 @@ func (w *Watcher) RemoveDeadRooms() error { } finishedEvent := models.NewSchedulerEvent( - models.FinishedRemoveDeadRoomsEventName, - w.SchedulerName, + models.FinishedRemoveDeadRoomsEventName, + w.SchedulerName, map[string]interface{}{ models.SuccessMetadataName: err == nil, }) @@ -818,7 +835,6 @@ func (w *Watcher) RemoveDeadRooms() error { } logger.Info("finish check of dead rooms") - return nil }