From 1c12af84860800e9599257aa6886662c273033e2 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 12:18:12 +0100 Subject: [PATCH 01/36] GRAPHITE_BASE -> GraphiteBase --- alert.go | 6 +++--- alertscollection.go | 4 ++-- hound.go | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/alert.go b/alert.go index d4cc3fb..d098e7d 100644 --- a/alert.go +++ b/alert.go @@ -64,11 +64,11 @@ func cleanMetric(metric string) string { } func (a Alert) Url() string { - return GRAPHITE_BASE + "?target=keepLastValue(" + a.Metric + ")&format=raw&from=-" + WINDOW + return GraphiteBase + "?target=keepLastValue(" + a.Metric + ")&format=raw&from=-" + WINDOW } func (a Alert) DailyGraphUrl() string { - return GRAPHITE_BASE + "?target=" + + return GraphiteBase + "?target=" + a.Metric + "&target=threshold(" + fmt.Sprintf("%f", a.Threshold) + ")&width=" + fmt.Sprintf("%d", GRAPH_WIDTH) + @@ -79,7 +79,7 @@ func (a Alert) DailyGraphUrl() string { } func (a Alert) WeeklyGraphUrl() string { - return GRAPHITE_BASE + "?target=" + + return GraphiteBase + "?target=" + a.Metric + "&target=threshold(" + fmt.Sprintf("%f", a.Threshold) + ")&width=" + fmt.Sprintf("%d", GRAPH_WIDTH) + diff --git a/alertscollection.go b/alertscollection.go index 9bf9c99..19d972e 100644 --- a/alertscollection.go +++ b/alertscollection.go @@ -138,7 +138,7 @@ func (ac *AlertsCollection) DisplayAll() { } func (ac *AlertsCollection) MakePageResponse() PageResponse { - pr := PageResponse{GraphiteBase: GRAPHITE_BASE, + pr := PageResponse{GraphiteBase: GraphiteBase, MetricBase: METRIC_BASE} for _, a := range ac.Alerts { pr.Alerts = append(pr.Alerts, a) @@ -147,7 +147,7 @@ func (ac *AlertsCollection) MakePageResponse() PageResponse { } func (ac *AlertsCollection) MakeIndivPageResponse(idx string) IndivPageResponse { - return IndivPageResponse{GraphiteBase: GRAPHITE_BASE, + return IndivPageResponse{GraphiteBase: GraphiteBase, MetricBase: METRIC_BASE, Alert: ac.ByHash(idx)} } diff --git a/hound.go b/hound.go index 5f87228..9a48f3c 100644 --- a/hound.go +++ b/hound.go @@ -16,7 +16,7 @@ import ( ) var ( - GRAPHITE_BASE string + GraphiteBase string CARBON_BASE string METRIC_BASE string EMAIL_FROM string @@ -104,7 +104,7 @@ func main() { log.Info("running on ", c.HTTPPort) // set global values - GRAPHITE_BASE = c.GraphiteBase + GraphiteBase = c.GraphiteBase CARBON_BASE = c.CarbonBase METRIC_BASE = c.MetricBase EMAIL_FROM = c.EmailFrom From ac3b432ddd1ca26cc1ecb9415e32106247e00eb0 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 12:19:55 +0100 Subject: [PATCH 02/36] email_to -> emailTo --- alert.go | 4 ++-- alertscollection_test.go | 6 +++--- emailer.go | 12 ++++++------ hound.go | 8 ++++---- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/alert.go b/alert.go index d098e7d..b4d3159 100644 --- a/alert.go +++ b/alert.go @@ -46,14 +46,14 @@ var WEEKLY_BGCOLOR = "EEEEEE" var WEEKLY_COLORLIST = "%23cccccc,%236699cc" func NewAlert(name string, metric string, atype string, threshold float64, - direction string, fetcher Fetcher, email_to string, runbook_link string) *Alert { + direction string, fetcher Fetcher, emailTo string, runbook_link string) *Alert { if atype == "" { atype = "Alert" } return &Alert{Name: name, Type: atype, Metric: cleanMetric(metric), Threshold: threshold, Direction: direction, Backoff: 0, LastAlerted: time.Now(), Status: "OK", Message: "", - PreviousStatus: "OK", Fetcher: fetcher, EmailTo: email_to, + PreviousStatus: "OK", Fetcher: fetcher, EmailTo: emailTo, Value: 0.0, RunBookLink: runbook_link, } } diff --git a/alertscollection_test.go b/alertscollection_test.go index f7f914a..99303c4 100644 --- a/alertscollection_test.go +++ b/alertscollection_test.go @@ -15,9 +15,9 @@ func Test_intmin(t *testing.T) { type DummyEmailer struct{} -func (d DummyEmailer) Throttled(failures, global_throttle int, email_to string) {} -func (d DummyEmailer) RecoveryThrottled(recoveries_sent, global_throttle int, email_to string) {} -func (d DummyEmailer) EncounteredErrors(errors int, email_to string) {} +func (d DummyEmailer) Throttled(failures, global_throttle int, emailTo string) {} +func (d DummyEmailer) RecoveryThrottled(recoveries_sent, global_throttle int, emailTo string) {} +func (d DummyEmailer) EncounteredErrors(errors int, emailTo string) {} func Test_emptyAlertsCollection(t *testing.T) { ac := NewAlertsCollection(DummyEmailer{}) diff --git a/emailer.go b/emailer.go index 42ba99d..0edc209 100644 --- a/emailer.go +++ b/emailer.go @@ -12,36 +12,36 @@ type Emailer interface { type SMTPEmailer struct{} -func (e SMTPEmailer) Throttled(failures, global_throttle int, email_to string) { +func (e SMTPEmailer) Throttled(failures, global_throttle int, emailTo string) { simpleSendMail( EMAIL_FROM, - email_to, + emailTo, "[ALERT] Hound is throttled", fmt.Sprintf("%d metrics were not OK.\nHound stopped sending messages after %d.\n"+ "This probably indicates an infrastructure problem (network, graphite, etc)", failures, global_throttle)) } -func (e SMTPEmailer) RecoveryThrottled(recoveries_sent, global_throttle int, email_to string) { +func (e SMTPEmailer) RecoveryThrottled(recoveries_sent, global_throttle int, emailTo string) { if !EMAIL_ON_ERROR { return } simpleSendMail( EMAIL_FROM, - email_to, + emailTo, "[ALERT] Hound is recovered", fmt.Sprintf("%d metrics recovered.\nHound stopped sending individual messages after %d.\n", recoveries_sent, global_throttle)) } -func (e SMTPEmailer) EncounteredErrors(errors int, email_to string) { +func (e SMTPEmailer) EncounteredErrors(errors int, emailTo string) { if !EMAIL_ON_ERROR { return } simpleSendMail( EMAIL_FROM, - email_to, + emailTo, "[ERROR] Hound encountered errors", fmt.Sprintf("%d metrics had errors. If this is more than a couple, it usually "+ "means that Graphite has fallen behind. It doesn't necessarily mean "+ diff --git a/hound.go b/hound.go index 9a48f3c..80311e6 100644 --- a/hound.go +++ b/hound.go @@ -143,11 +143,11 @@ func main() { // initialize all the alerts ac := NewAlertsCollection(SMTPEmailer{}) for _, a := range f.Alerts { - email_to := a.EmailTo - if email_to == "" { - email_to = c.EmailTo + emailTo := a.EmailTo + if emailTo == "" { + emailTo = c.EmailTo } - ac.AddAlert(NewAlert(a.Name, a.Metric, a.Type, a.Threshold, a.Direction, HTTPFetcher{}, email_to, a.RunBookLink)) + ac.AddAlert(NewAlert(a.Name, a.Metric, a.Type, a.Threshold, a.Direction, HTTPFetcher{}, emailTo, a.RunBookLink)) } // kick it off in the background From a519e3f92a0a767e2beea369789a9c22b250af0c Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 12:20:50 +0100 Subject: [PATCH 03/36] CARBON_BASE -> CarbonBase --- alertscollection.go | 2 +- hound.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/alertscollection.go b/alertscollection.go index 19d972e..f6924c8 100644 --- a/alertscollection.go +++ b/alertscollection.go @@ -99,7 +99,7 @@ func (ac *AlertsCollection) HandleErrors(errors int) { func LogToGraphite(alerts_sent, recoveries_sent, failures, errors, successes int) { var clientGraphite net.Conn - clientGraphite, err := net.Dial("tcp", CARBON_BASE) + clientGraphite, err := net.Dial("tcp", CarbonBase) if err != nil || clientGraphite == nil { return } diff --git a/hound.go b/hound.go index 80311e6..52060ad 100644 --- a/hound.go +++ b/hound.go @@ -17,7 +17,7 @@ import ( var ( GraphiteBase string - CARBON_BASE string + CarbonBase string METRIC_BASE string EMAIL_FROM string EMAIL_TO string @@ -105,7 +105,7 @@ func main() { log.Info("running on ", c.HTTPPort) // set global values GraphiteBase = c.GraphiteBase - CARBON_BASE = c.CarbonBase + CarbonBase = c.CarbonBase METRIC_BASE = c.MetricBase EMAIL_FROM = c.EmailFrom EMAIL_TO = c.EmailTo From 3250d89a22ab72eba18e57d36fe8d26fbf37ec21 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 12:21:59 +0100 Subject: [PATCH 04/36] METRIC_BASE -> MetricBase --- alertscollection.go | 16 ++++++++-------- hound.go | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/alertscollection.go b/alertscollection.go index f6924c8..b3b99e9 100644 --- a/alertscollection.go +++ b/alertscollection.go @@ -107,12 +107,12 @@ func LogToGraphite(alerts_sent, recoveries_sent, failures, errors, successes int now := int32(time.Now().Unix()) buffer := bytes.NewBufferString("") - fmt.Fprintf(buffer, "%salerts_sent %d %d\n", METRIC_BASE, alerts_sent, now) - fmt.Fprintf(buffer, "%srecoveries_sent %d %d\n", METRIC_BASE, recoveries_sent, now) - fmt.Fprintf(buffer, "%sfailures %d %d\n", METRIC_BASE, failures, now) - fmt.Fprintf(buffer, "%serrors %d %d\n", METRIC_BASE, errors, now) - fmt.Fprintf(buffer, "%ssuccesses %d %d\n", METRIC_BASE, successes, now) - fmt.Fprintf(buffer, "%sglobal_backoff %d %d\n", METRIC_BASE, GLOBAL_BACKOFF, now) + fmt.Fprintf(buffer, "%salerts_sent %d %d\n", MetricBase, alerts_sent, now) + fmt.Fprintf(buffer, "%srecoveries_sent %d %d\n", MetricBase, recoveries_sent, now) + fmt.Fprintf(buffer, "%sfailures %d %d\n", MetricBase, failures, now) + fmt.Fprintf(buffer, "%serrors %d %d\n", MetricBase, errors, now) + fmt.Fprintf(buffer, "%ssuccesses %d %d\n", MetricBase, successes, now) + fmt.Fprintf(buffer, "%sglobal_backoff %d %d\n", MetricBase, GLOBAL_BACKOFF, now) clientGraphite.Write(buffer.Bytes()) } @@ -139,7 +139,7 @@ func (ac *AlertsCollection) DisplayAll() { func (ac *AlertsCollection) MakePageResponse() PageResponse { pr := PageResponse{GraphiteBase: GraphiteBase, - MetricBase: METRIC_BASE} + MetricBase: MetricBase} for _, a := range ac.Alerts { pr.Alerts = append(pr.Alerts, a) } @@ -148,6 +148,6 @@ func (ac *AlertsCollection) MakePageResponse() PageResponse { func (ac *AlertsCollection) MakeIndivPageResponse(idx string) IndivPageResponse { return IndivPageResponse{GraphiteBase: GraphiteBase, - MetricBase: METRIC_BASE, + MetricBase: MetricBase, Alert: ac.ByHash(idx)} } diff --git a/hound.go b/hound.go index 52060ad..79d0bb3 100644 --- a/hound.go +++ b/hound.go @@ -18,7 +18,7 @@ import ( var ( GraphiteBase string CarbonBase string - METRIC_BASE string + MetricBase string EMAIL_FROM string EMAIL_TO string CHECK_INTERVAL int @@ -106,7 +106,7 @@ func main() { // set global values GraphiteBase = c.GraphiteBase CarbonBase = c.CarbonBase - METRIC_BASE = c.MetricBase + MetricBase = c.MetricBase EMAIL_FROM = c.EmailFrom EMAIL_TO = c.EmailTo CHECK_INTERVAL = c.CheckInterval From fdd2029dcf784489196137afb43edce701228bd2 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 12:22:51 +0100 Subject: [PATCH 05/36] EMAIL_FROM -> EmailFrom --- alert.go | 4 ++-- emailer.go | 6 +++--- hound.go | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/alert.go b/alert.go index b4d3159..745f65f 100644 --- a/alert.go +++ b/alert.go @@ -202,7 +202,7 @@ func (a *Alert) SendRecoveryMessage() { "name": a.Name, }, ).Debug("sending Recovery Message") - simpleSendMail(EMAIL_FROM, + simpleSendMail(EmailFrom, a.EmailTo, a.RecoveryEmailSubject(), a.RecoveryEmailBody()) @@ -239,7 +239,7 @@ func (a *Alert) SendAlert() { "name": a.Name, }, ).Debug("Sending Alert") - simpleSendMail(EMAIL_FROM, + simpleSendMail(EmailFrom, a.EmailTo, a.AlertEmailSubject(), a.AlertEmailBody()) diff --git a/emailer.go b/emailer.go index 0edc209..dc1983b 100644 --- a/emailer.go +++ b/emailer.go @@ -14,7 +14,7 @@ type SMTPEmailer struct{} func (e SMTPEmailer) Throttled(failures, global_throttle int, emailTo string) { simpleSendMail( - EMAIL_FROM, + EmailFrom, emailTo, "[ALERT] Hound is throttled", fmt.Sprintf("%d metrics were not OK.\nHound stopped sending messages after %d.\n"+ @@ -27,7 +27,7 @@ func (e SMTPEmailer) RecoveryThrottled(recoveries_sent, global_throttle int, ema return } simpleSendMail( - EMAIL_FROM, + EmailFrom, emailTo, "[ALERT] Hound is recovered", fmt.Sprintf("%d metrics recovered.\nHound stopped sending individual messages after %d.\n", @@ -40,7 +40,7 @@ func (e SMTPEmailer) EncounteredErrors(errors int, emailTo string) { return } simpleSendMail( - EMAIL_FROM, + EmailFrom, emailTo, "[ERROR] Hound encountered errors", fmt.Sprintf("%d metrics had errors. If this is more than a couple, it usually "+ diff --git a/hound.go b/hound.go index 79d0bb3..4986a6f 100644 --- a/hound.go +++ b/hound.go @@ -19,7 +19,7 @@ var ( GraphiteBase string CarbonBase string MetricBase string - EMAIL_FROM string + EmailFrom string EMAIL_TO string CHECK_INTERVAL int GLOBAL_THROTTLE int @@ -107,7 +107,7 @@ func main() { GraphiteBase = c.GraphiteBase CarbonBase = c.CarbonBase MetricBase = c.MetricBase - EMAIL_FROM = c.EmailFrom + EmailFrom = c.EmailFrom EMAIL_TO = c.EmailTo CHECK_INTERVAL = c.CheckInterval GLOBAL_THROTTLE = c.GlobalThrottle From ee08be7a6aa307dc34538551bae62ac1d16a6288 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 14:47:41 +0100 Subject: [PATCH 06/36] EMAIL_TO -> EmailTo --- alertscollection.go | 6 +++--- hound.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/alertscollection.go b/alertscollection.go index b3b99e9..f377c48 100644 --- a/alertscollection.go +++ b/alertscollection.go @@ -64,11 +64,11 @@ func (ac *AlertsCollection) ProcessAll() { alerts_sent = alerts_sent + as } if alerts_sent >= GLOBAL_THROTTLE { - ac.Emailer.Throttled(failures, GLOBAL_THROTTLE, EMAIL_TO) + ac.Emailer.Throttled(failures, GLOBAL_THROTTLE, EmailTo) } if recoveries_sent >= GLOBAL_THROTTLE { - ac.Emailer.RecoveryThrottled(recoveries_sent, GLOBAL_THROTTLE, EMAIL_TO) + ac.Emailer.RecoveryThrottled(recoveries_sent, GLOBAL_THROTTLE, EmailTo) } ac.HandleErrors(errors) LogToGraphite(alerts_sent, recoveries_sent, failures, errors, successes) @@ -88,7 +88,7 @@ func (ac *AlertsCollection) HandleErrors(errors int) { d := backoff_time(GLOBAL_BACKOFF) window := LAST_ERROR_EMAIL.Add(d) if time.Now().After(window) { - ac.Emailer.EncounteredErrors(errors, EMAIL_TO) + ac.Emailer.EncounteredErrors(errors, EmailTo) LAST_ERROR_EMAIL = time.Now() GLOBAL_BACKOFF = intmin(GLOBAL_BACKOFF+1, len(BACKOFF_DURATIONS)) } diff --git a/hound.go b/hound.go index 4986a6f..a5e1678 100644 --- a/hound.go +++ b/hound.go @@ -20,7 +20,7 @@ var ( CarbonBase string MetricBase string EmailFrom string - EMAIL_TO string + EmailTo string CHECK_INTERVAL int GLOBAL_THROTTLE int GLOBAL_BACKOFF int @@ -108,7 +108,7 @@ func main() { CarbonBase = c.CarbonBase MetricBase = c.MetricBase EmailFrom = c.EmailFrom - EMAIL_TO = c.EmailTo + EmailTo = c.EmailTo CHECK_INTERVAL = c.CheckInterval GLOBAL_THROTTLE = c.GlobalThrottle GLOBAL_BACKOFF = 0 From a0a614c6b7c5a7e8bca91801fddc3786698de573 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 14:48:25 +0100 Subject: [PATCH 07/36] CHECK_INTERVAL -> CheckInterval --- alertscollection.go | 2 +- hound.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/alertscollection.go b/alertscollection.go index f377c48..7dc925d 100644 --- a/alertscollection.go +++ b/alertscollection.go @@ -127,7 +127,7 @@ func (ac *AlertsCollection) Run() { for { ac.ProcessAll() ac.DisplayAll() - time.Sleep(time.Duration(CHECK_INTERVAL) * time.Minute) + time.Sleep(time.Duration(CheckInterval) * time.Minute) } } diff --git a/hound.go b/hound.go index a5e1678..2c8415e 100644 --- a/hound.go +++ b/hound.go @@ -21,7 +21,7 @@ var ( MetricBase string EmailFrom string EmailTo string - CHECK_INTERVAL int + CheckInterval int GLOBAL_THROTTLE int GLOBAL_BACKOFF int LAST_ERROR_EMAIL time.Time @@ -109,7 +109,7 @@ func main() { MetricBase = c.MetricBase EmailFrom = c.EmailFrom EmailTo = c.EmailTo - CHECK_INTERVAL = c.CheckInterval + CheckInterval = c.CheckInterval GLOBAL_THROTTLE = c.GlobalThrottle GLOBAL_BACKOFF = 0 EMAIL_ON_ERROR = c.EmailOnError From 956b06da820b6ec03f3b3d9b59fc0c53aeb02446 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 14:50:01 +0100 Subject: [PATCH 08/36] GLOBAL_THROTTLE -> GlobalThrottle --- alert.go | 4 ++-- alertscollection.go | 10 +++++----- hound.go | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/alert.go b/alert.go index 745f65f..40201dc 100644 --- a/alert.go +++ b/alert.go @@ -272,7 +272,7 @@ func (a *Alert) JustRecovered() bool { } func (a *Alert) SendRecoveryMessageIfNeeded(recoveries_sent int) { - if a.JustRecovered() && recoveries_sent < GLOBAL_THROTTLE { + if a.JustRecovered() && recoveries_sent < GlobalThrottle { a.SendRecoveryMessage() } } @@ -306,7 +306,7 @@ func (a *Alert) UpdateState(recoveries_sent int) (int, int, int, int, int) { }, ).Debug("throttled") } else { - if a.Status == "Failed" && alerts_sent < GLOBAL_THROTTLE { + if a.Status == "Failed" && alerts_sent < GlobalThrottle { a.SendAlert() alerts_sent++ } diff --git a/alertscollection.go b/alertscollection.go index 7dc925d..19c11ef 100644 --- a/alertscollection.go +++ b/alertscollection.go @@ -63,12 +63,12 @@ func (ac *AlertsCollection) ProcessAll() { failures = failures + f alerts_sent = alerts_sent + as } - if alerts_sent >= GLOBAL_THROTTLE { - ac.Emailer.Throttled(failures, GLOBAL_THROTTLE, EmailTo) + if alerts_sent >= GlobalThrottle { + ac.Emailer.Throttled(failures, GlobalThrottle, EmailTo) } - if recoveries_sent >= GLOBAL_THROTTLE { - ac.Emailer.RecoveryThrottled(recoveries_sent, GLOBAL_THROTTLE, EmailTo) + if recoveries_sent >= GlobalThrottle { + ac.Emailer.RecoveryThrottled(recoveries_sent, GlobalThrottle, EmailTo) } ac.HandleErrors(errors) LogToGraphite(alerts_sent, recoveries_sent, failures, errors, successes) @@ -79,7 +79,7 @@ func ExposeVars(failures, errors, successes int) { EXP_FAILED.Set(int64(failures)) EXP_ERRORS.Set(int64(errors)) EXP_PASSED.Set(int64(successes)) - EXP_GLOBAL_THROTTLE.Set(int64(GLOBAL_THROTTLE)) + EXP_GLOBAL_THROTTLE.Set(int64(GlobalThrottle)) EXP_GLOBAL_BACKOFF.Set(int64(GLOBAL_BACKOFF)) } diff --git a/hound.go b/hound.go index 2c8415e..e32111f 100644 --- a/hound.go +++ b/hound.go @@ -22,7 +22,7 @@ var ( EmailFrom string EmailTo string CheckInterval int - GLOBAL_THROTTLE int + GlobalThrottle int GLOBAL_BACKOFF int LAST_ERROR_EMAIL time.Time EMAIL_ON_ERROR bool @@ -49,7 +49,7 @@ type config struct { EmailFrom string `envconfig:"EMAIL_FROM"` EmailTo string `envconfig:"EMAIL_TO"` CheckInterval int `envconfig:"CHECK_INTERVAL"` - GlobalThrottle int `envconfig:"GLOBAL_THROTTLE"` + GlobalThrottle int `envconfig:"GlobalThrottle"` HTTPPort string `envconfig:"HTTP_PORT"` TemplateFile string `envconfig:"TEMPLATE_FILE"` AlertTemplateFile string `envconfig:"ALERT_TEMPLATE_FILE"` @@ -110,7 +110,7 @@ func main() { EmailFrom = c.EmailFrom EmailTo = c.EmailTo CheckInterval = c.CheckInterval - GLOBAL_THROTTLE = c.GlobalThrottle + GlobalThrottle = c.GlobalThrottle GLOBAL_BACKOFF = 0 EMAIL_ON_ERROR = c.EmailOnError SMTP_SERVER = c.SMTPServer From b59c4a45bffa7fad095903fae16360896566ed76 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 14:51:27 +0100 Subject: [PATCH 09/36] LAST_ERROR_EMAIL -> LastErrorEmail --- alertscollection.go | 4 ++-- hound.go | 32 ++++++++++++++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/alertscollection.go b/alertscollection.go index 19c11ef..ba4168b 100644 --- a/alertscollection.go +++ b/alertscollection.go @@ -86,10 +86,10 @@ func ExposeVars(failures, errors, successes int) { func (ac *AlertsCollection) HandleErrors(errors int) { if errors > 0 { d := backoff_time(GLOBAL_BACKOFF) - window := LAST_ERROR_EMAIL.Add(d) + window := LastErrorEmail.Add(d) if time.Now().After(window) { ac.Emailer.EncounteredErrors(errors, EmailTo) - LAST_ERROR_EMAIL = time.Now() + LastErrorEmail = time.Now() GLOBAL_BACKOFF = intmin(GLOBAL_BACKOFF+1, len(BACKOFF_DURATIONS)) } } else { diff --git a/hound.go b/hound.go index e32111f..4c7fc69 100644 --- a/hound.go +++ b/hound.go @@ -16,21 +16,21 @@ import ( ) var ( - GraphiteBase string - CarbonBase string - MetricBase string - EmailFrom string - EmailTo string - CheckInterval int - GlobalThrottle int - GLOBAL_BACKOFF int - LAST_ERROR_EMAIL time.Time - EMAIL_ON_ERROR bool - SMTP_SERVER string - SMTP_PORT int - SMTP_USER string - SMTP_PASSWORD string - WINDOW string + GraphiteBase string + CarbonBase string + MetricBase string + EmailFrom string + EmailTo string + CheckInterval int + GlobalThrottle int + GLOBAL_BACKOFF int + LastErrorEmail time.Time + EMAIL_ON_ERROR bool + SMTP_SERVER string + SMTP_PORT int + SMTP_USER string + SMTP_PASSWORD string + WINDOW string ) var ( @@ -130,7 +130,7 @@ func main() { WINDOW = "10mins" } - LAST_ERROR_EMAIL = time.Now() + LastErrorEmail = time.Now() go func() { // update uptime From 6f7af4b5f1ec63bcc4fb58aead3505f365fe765d Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 14:57:18 +0100 Subject: [PATCH 10/36] EMAIL_ON_ERROR -> EmailOnError --- emailer.go | 4 ++-- hound.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/emailer.go b/emailer.go index dc1983b..a29632d 100644 --- a/emailer.go +++ b/emailer.go @@ -23,7 +23,7 @@ func (e SMTPEmailer) Throttled(failures, global_throttle int, emailTo string) { } func (e SMTPEmailer) RecoveryThrottled(recoveries_sent, global_throttle int, emailTo string) { - if !EMAIL_ON_ERROR { + if !EmailOnError { return } simpleSendMail( @@ -36,7 +36,7 @@ func (e SMTPEmailer) RecoveryThrottled(recoveries_sent, global_throttle int, ema } func (e SMTPEmailer) EncounteredErrors(errors int, emailTo string) { - if !EMAIL_ON_ERROR { + if !EmailOnError { return } simpleSendMail( diff --git a/hound.go b/hound.go index 4c7fc69..b053f7d 100644 --- a/hound.go +++ b/hound.go @@ -25,7 +25,7 @@ var ( GlobalThrottle int GLOBAL_BACKOFF int LastErrorEmail time.Time - EMAIL_ON_ERROR bool + EmailOnError bool SMTP_SERVER string SMTP_PORT int SMTP_USER string @@ -112,7 +112,7 @@ func main() { CheckInterval = c.CheckInterval GlobalThrottle = c.GlobalThrottle GLOBAL_BACKOFF = 0 - EMAIL_ON_ERROR = c.EmailOnError + EmailOnError = c.EmailOnError SMTP_SERVER = c.SMTPServer SMTP_PORT = c.SMTPPort SMTP_USER = c.SMTPUser From b088cf46b1d54c54247d24a3bc78ec552667c811 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 14:58:12 +0100 Subject: [PATCH 11/36] SMTP_SERVER -> SMTPServer --- alert.go | 10 +++++----- hound.go | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/alert.go b/alert.go index 40201dc..a64931e 100644 --- a/alert.go +++ b/alert.go @@ -355,8 +355,8 @@ func simpleSendMail(from, to, subject string, body string) error { message += fmt.Sprintf("%s: %s\r\n", k, v) } message += "\r\n" + base64.StdEncoding.EncodeToString([]byte(body)) - s := fmt.Sprintf("%s:%d", SMTP_SERVER, SMTP_PORT) - auth := smtp.PlainAuth("", SMTP_USER, SMTP_PASSWORD, SMTP_SERVER) + s := fmt.Sprintf("%s:%d", SMTPServer, SMTP_PORT) + auth := smtp.PlainAuth("", SMTP_USER, SMTP_PASSWORD, SMTPServer) if SMTP_PORT == 25 { err := SendMail(s, auth, from, []string{to}, []byte(message)) @@ -372,7 +372,7 @@ func simpleSendMail(from, to, subject string, body string) error { } else { tlsconfig := &tls.Config{ InsecureSkipVerify: true, - ServerName: SMTP_SERVER, + ServerName: SMTPServer, } conn, err := tls.Dial("tcp", s, tlsconfig) @@ -381,7 +381,7 @@ func simpleSendMail(from, to, subject string, body string) error { return err } - c, err := smtp.NewClient(conn, SMTP_SERVER) + c, err := smtp.NewClient(conn, SMTPServer) if err != nil { log.WithFields(log.Fields{"err": err}).Error("smtp.NewClient failed") return err @@ -394,7 +394,7 @@ func simpleSendMail(from, to, subject string, body string) error { "err": err, "SMTP_USER": SMTP_USER, "SMTP_PASSWORD": SMTP_PASSWORD, - "SMTP_SERVER": SMTP_SERVER, + "SMTP_SERVER": SMTPServer, }).Error("auth failed") return err } diff --git a/hound.go b/hound.go index b053f7d..c8cfd3e 100644 --- a/hound.go +++ b/hound.go @@ -26,7 +26,7 @@ var ( GLOBAL_BACKOFF int LastErrorEmail time.Time EmailOnError bool - SMTP_SERVER string + SMTPServer string SMTP_PORT int SMTP_USER string SMTP_PASSWORD string @@ -113,7 +113,7 @@ func main() { GlobalThrottle = c.GlobalThrottle GLOBAL_BACKOFF = 0 EmailOnError = c.EmailOnError - SMTP_SERVER = c.SMTPServer + SMTPServer = c.SMTPServer SMTP_PORT = c.SMTPPort SMTP_USER = c.SMTPUser SMTP_PASSWORD = c.SMTPPassword From 836c47e37d5cd3340b1ed0a7c5605a4d299fb8dc Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 14:58:49 +0100 Subject: [PATCH 12/36] SMTP_PORT -> SMTPPort --- alert.go | 4 ++-- hound.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/alert.go b/alert.go index a64931e..f09a9b4 100644 --- a/alert.go +++ b/alert.go @@ -355,10 +355,10 @@ func simpleSendMail(from, to, subject string, body string) error { message += fmt.Sprintf("%s: %s\r\n", k, v) } message += "\r\n" + base64.StdEncoding.EncodeToString([]byte(body)) - s := fmt.Sprintf("%s:%d", SMTPServer, SMTP_PORT) + s := fmt.Sprintf("%s:%d", SMTPServer, SMTPPort) auth := smtp.PlainAuth("", SMTP_USER, SMTP_PASSWORD, SMTPServer) - if SMTP_PORT == 25 { + if SMTPPort == 25 { err := SendMail(s, auth, from, []string{to}, []byte(message)) if err != nil { log.WithFields( diff --git a/hound.go b/hound.go index c8cfd3e..9785d34 100644 --- a/hound.go +++ b/hound.go @@ -27,7 +27,7 @@ var ( LastErrorEmail time.Time EmailOnError bool SMTPServer string - SMTP_PORT int + SMTPPort int SMTP_USER string SMTP_PASSWORD string WINDOW string @@ -114,7 +114,7 @@ func main() { GLOBAL_BACKOFF = 0 EmailOnError = c.EmailOnError SMTPServer = c.SMTPServer - SMTP_PORT = c.SMTPPort + SMTPPort = c.SMTPPort SMTP_USER = c.SMTPUser SMTP_PASSWORD = c.SMTPPassword WINDOW = c.Window From 7045e8cdac33bf0682abad2cdc3ee6b373c041ea Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 14:59:33 +0100 Subject: [PATCH 13/36] SMTP_USER -> SMTPUser --- alert.go | 4 ++-- hound.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/alert.go b/alert.go index f09a9b4..f33c562 100644 --- a/alert.go +++ b/alert.go @@ -356,7 +356,7 @@ func simpleSendMail(from, to, subject string, body string) error { } message += "\r\n" + base64.StdEncoding.EncodeToString([]byte(body)) s := fmt.Sprintf("%s:%d", SMTPServer, SMTPPort) - auth := smtp.PlainAuth("", SMTP_USER, SMTP_PASSWORD, SMTPServer) + auth := smtp.PlainAuth("", SMTPUser, SMTP_PASSWORD, SMTPServer) if SMTPPort == 25 { err := SendMail(s, auth, from, []string{to}, []byte(message)) @@ -392,7 +392,7 @@ func simpleSendMail(from, to, subject string, body string) error { log.WithFields( log.Fields{ "err": err, - "SMTP_USER": SMTP_USER, + "SMTP_USER": SMTPUser, "SMTP_PASSWORD": SMTP_PASSWORD, "SMTP_SERVER": SMTPServer, }).Error("auth failed") diff --git a/hound.go b/hound.go index 9785d34..1617bbf 100644 --- a/hound.go +++ b/hound.go @@ -28,7 +28,7 @@ var ( EmailOnError bool SMTPServer string SMTPPort int - SMTP_USER string + SMTPUser string SMTP_PASSWORD string WINDOW string ) @@ -115,7 +115,7 @@ func main() { EmailOnError = c.EmailOnError SMTPServer = c.SMTPServer SMTPPort = c.SMTPPort - SMTP_USER = c.SMTPUser + SMTPUser = c.SMTPUser SMTP_PASSWORD = c.SMTPPassword WINDOW = c.Window From 68e4d535e410d9027f0ac8b18c25b7abad0d273c Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:00:11 +0100 Subject: [PATCH 14/36] SMTP_PASSWORD -> SMTPPassword --- alert.go | 4 ++-- hound.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/alert.go b/alert.go index f33c562..4f26c11 100644 --- a/alert.go +++ b/alert.go @@ -356,7 +356,7 @@ func simpleSendMail(from, to, subject string, body string) error { } message += "\r\n" + base64.StdEncoding.EncodeToString([]byte(body)) s := fmt.Sprintf("%s:%d", SMTPServer, SMTPPort) - auth := smtp.PlainAuth("", SMTPUser, SMTP_PASSWORD, SMTPServer) + auth := smtp.PlainAuth("", SMTPUser, SMTPPassword, SMTPServer) if SMTPPort == 25 { err := SendMail(s, auth, from, []string{to}, []byte(message)) @@ -393,7 +393,7 @@ func simpleSendMail(from, to, subject string, body string) error { log.Fields{ "err": err, "SMTP_USER": SMTPUser, - "SMTP_PASSWORD": SMTP_PASSWORD, + "SMTP_PASSWORD": SMTPPassword, "SMTP_SERVER": SMTPServer, }).Error("auth failed") return err diff --git a/hound.go b/hound.go index 1617bbf..e9622d0 100644 --- a/hound.go +++ b/hound.go @@ -29,7 +29,7 @@ var ( SMTPServer string SMTPPort int SMTPUser string - SMTP_PASSWORD string + SMTPPassword string WINDOW string ) @@ -116,7 +116,7 @@ func main() { SMTPServer = c.SMTPServer SMTPPort = c.SMTPPort SMTPUser = c.SMTPUser - SMTP_PASSWORD = c.SMTPPassword + SMTPPassword = c.SMTPPassword WINDOW = c.Window // some defaults From 5e612e61a08ba1f11894be3f0815077a58c15253 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:01:33 +0100 Subject: [PATCH 15/36] WINDOW -> Window --- alert.go | 2 +- alert_test.go | 2 +- hound.go | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/alert.go b/alert.go index 4f26c11..5e86cdf 100644 --- a/alert.go +++ b/alert.go @@ -64,7 +64,7 @@ func cleanMetric(metric string) string { } func (a Alert) Url() string { - return GraphiteBase + "?target=keepLastValue(" + a.Metric + ")&format=raw&from=-" + WINDOW + return GraphiteBase + "?target=keepLastValue(" + a.Metric + ")&format=raw&from=-" + Window } func (a Alert) DailyGraphUrl() string { diff --git a/alert_test.go b/alert_test.go index 5e7ee80..936cf76 100644 --- a/alert_test.go +++ b/alert_test.go @@ -32,7 +32,7 @@ func Test_StringWhiteSpaceRemoval(t *testing.T) { func Test_Url(t *testing.T) { a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") - if a.Url() != "?target=keepLastValue(foo)&format=raw&from=-"+WINDOW { + if a.Url() != "?target=keepLastValue(foo)&format=raw&from=-"+Window { t.Error(fmt.Sprintf("wrong value: %s", a.Url())) } } diff --git a/hound.go b/hound.go index e9622d0..6d871ef 100644 --- a/hound.go +++ b/hound.go @@ -30,7 +30,7 @@ var ( SMTPPort int SMTPUser string SMTPPassword string - WINDOW string + Window string ) var ( @@ -117,7 +117,7 @@ func main() { SMTPPort = c.SMTPPort SMTPUser = c.SMTPUser SMTPPassword = c.SMTPPassword - WINDOW = c.Window + Window = c.Window // some defaults if c.ReadTimeout == 0 { @@ -126,8 +126,8 @@ func main() { if c.WriteTimeout == 0 { c.WriteTimeout = 10 } - if WINDOW == "" { - WINDOW = "10mins" + if Window == "" { + Window = "10mins" } LastErrorEmail = time.Now() From 02350d2e25b9a41168729227b93c630792f661ab Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:02:23 +0100 Subject: [PATCH 16/36] GLOBAL_BACKOFF -> GlobalBackoff --- alertscollection.go | 10 +++++----- hound.go | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/alertscollection.go b/alertscollection.go index ba4168b..47157fa 100644 --- a/alertscollection.go +++ b/alertscollection.go @@ -80,20 +80,20 @@ func ExposeVars(failures, errors, successes int) { EXP_ERRORS.Set(int64(errors)) EXP_PASSED.Set(int64(successes)) EXP_GLOBAL_THROTTLE.Set(int64(GlobalThrottle)) - EXP_GLOBAL_BACKOFF.Set(int64(GLOBAL_BACKOFF)) + EXP_GLOBAL_BACKOFF.Set(int64(GlobalBackoff)) } func (ac *AlertsCollection) HandleErrors(errors int) { if errors > 0 { - d := backoff_time(GLOBAL_BACKOFF) + d := backoff_time(GlobalBackoff) window := LastErrorEmail.Add(d) if time.Now().After(window) { ac.Emailer.EncounteredErrors(errors, EmailTo) LastErrorEmail = time.Now() - GLOBAL_BACKOFF = intmin(GLOBAL_BACKOFF+1, len(BACKOFF_DURATIONS)) + GlobalBackoff = intmin(GlobalBackoff+1, len(BACKOFF_DURATIONS)) } } else { - GLOBAL_BACKOFF = 0 + GlobalBackoff = 0 } } @@ -112,7 +112,7 @@ func LogToGraphite(alerts_sent, recoveries_sent, failures, errors, successes int fmt.Fprintf(buffer, "%sfailures %d %d\n", MetricBase, failures, now) fmt.Fprintf(buffer, "%serrors %d %d\n", MetricBase, errors, now) fmt.Fprintf(buffer, "%ssuccesses %d %d\n", MetricBase, successes, now) - fmt.Fprintf(buffer, "%sglobal_backoff %d %d\n", MetricBase, GLOBAL_BACKOFF, now) + fmt.Fprintf(buffer, "%sglobal_backoff %d %d\n", MetricBase, GlobalBackoff, now) clientGraphite.Write(buffer.Bytes()) } diff --git a/hound.go b/hound.go index 6d871ef..21e1f81 100644 --- a/hound.go +++ b/hound.go @@ -23,7 +23,7 @@ var ( EmailTo string CheckInterval int GlobalThrottle int - GLOBAL_BACKOFF int + GlobalBackoff int LastErrorEmail time.Time EmailOnError bool SMTPServer string @@ -111,7 +111,7 @@ func main() { EmailTo = c.EmailTo CheckInterval = c.CheckInterval GlobalThrottle = c.GlobalThrottle - GLOBAL_BACKOFF = 0 + GlobalBackoff = 0 EmailOnError = c.EmailOnError SMTPServer = c.SMTPServer SMTPPort = c.SMTPPort From 9d09f57c96b680c044a98400c03936d501009853 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:03:20 +0100 Subject: [PATCH 17/36] EXP_FAILED -> ExpFailed --- alertscollection.go | 2 +- hound.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/alertscollection.go b/alertscollection.go index 47157fa..dc03f39 100644 --- a/alertscollection.go +++ b/alertscollection.go @@ -76,7 +76,7 @@ func (ac *AlertsCollection) ProcessAll() { } func ExposeVars(failures, errors, successes int) { - EXP_FAILED.Set(int64(failures)) + ExpFailed.Set(int64(failures)) EXP_ERRORS.Set(int64(errors)) EXP_PASSED.Set(int64(successes)) EXP_GLOBAL_THROTTLE.Set(int64(GlobalThrottle)) diff --git a/hound.go b/hound.go index 21e1f81..bd6304a 100644 --- a/hound.go +++ b/hound.go @@ -34,7 +34,7 @@ var ( ) var ( - EXP_FAILED = expvar.NewInt("failed") + ExpFailed = expvar.NewInt("failed") EXP_PASSED = expvar.NewInt("passed") EXP_ERRORS = expvar.NewInt("errors") EXP_GLOBAL_THROTTLE = expvar.NewInt("throttle") From 00dd7ac017fc855d193772e36a09a2063ef0bf5b Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:03:49 +0100 Subject: [PATCH 18/36] EXP_PASSED -> ExpPassed --- alertscollection.go | 2 +- hound.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/alertscollection.go b/alertscollection.go index dc03f39..8a1b48d 100644 --- a/alertscollection.go +++ b/alertscollection.go @@ -78,7 +78,7 @@ func (ac *AlertsCollection) ProcessAll() { func ExposeVars(failures, errors, successes int) { ExpFailed.Set(int64(failures)) EXP_ERRORS.Set(int64(errors)) - EXP_PASSED.Set(int64(successes)) + ExpPassed.Set(int64(successes)) EXP_GLOBAL_THROTTLE.Set(int64(GlobalThrottle)) EXP_GLOBAL_BACKOFF.Set(int64(GlobalBackoff)) } diff --git a/hound.go b/hound.go index bd6304a..336b193 100644 --- a/hound.go +++ b/hound.go @@ -35,7 +35,7 @@ var ( var ( ExpFailed = expvar.NewInt("failed") - EXP_PASSED = expvar.NewInt("passed") + ExpPassed = expvar.NewInt("passed") EXP_ERRORS = expvar.NewInt("errors") EXP_GLOBAL_THROTTLE = expvar.NewInt("throttle") EXP_GLOBAL_BACKOFF = expvar.NewInt("backoff") From 91d116dc3d15f4496dce5b7b882c46969d13cf06 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:04:58 +0100 Subject: [PATCH 19/36] EXP_ERRORS -> ExpErrors --- alertscollection.go | 2 +- hound.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/alertscollection.go b/alertscollection.go index 8a1b48d..27b3df4 100644 --- a/alertscollection.go +++ b/alertscollection.go @@ -77,7 +77,7 @@ func (ac *AlertsCollection) ProcessAll() { func ExposeVars(failures, errors, successes int) { ExpFailed.Set(int64(failures)) - EXP_ERRORS.Set(int64(errors)) + ExpErrors.Set(int64(errors)) ExpPassed.Set(int64(successes)) EXP_GLOBAL_THROTTLE.Set(int64(GlobalThrottle)) EXP_GLOBAL_BACKOFF.Set(int64(GlobalBackoff)) diff --git a/hound.go b/hound.go index 336b193..c786547 100644 --- a/hound.go +++ b/hound.go @@ -36,7 +36,7 @@ var ( var ( ExpFailed = expvar.NewInt("failed") ExpPassed = expvar.NewInt("passed") - EXP_ERRORS = expvar.NewInt("errors") + ExpErrors = expvar.NewInt("errors") EXP_GLOBAL_THROTTLE = expvar.NewInt("throttle") EXP_GLOBAL_BACKOFF = expvar.NewInt("backoff") EXP_UPTIME = expvar.NewInt("uptime") From 74af32e5156e00afbf647c85010dd40ef227934b Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:05:27 +0100 Subject: [PATCH 20/36] EXP_GLOBAL_THROTTLE -> ExpGlobalThrottle --- alertscollection.go | 2 +- hound.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/alertscollection.go b/alertscollection.go index 27b3df4..bd3cbfa 100644 --- a/alertscollection.go +++ b/alertscollection.go @@ -79,7 +79,7 @@ func ExposeVars(failures, errors, successes int) { ExpFailed.Set(int64(failures)) ExpErrors.Set(int64(errors)) ExpPassed.Set(int64(successes)) - EXP_GLOBAL_THROTTLE.Set(int64(GlobalThrottle)) + ExpGlobalThrottle.Set(int64(GlobalThrottle)) EXP_GLOBAL_BACKOFF.Set(int64(GlobalBackoff)) } diff --git a/hound.go b/hound.go index c786547..9ca17ed 100644 --- a/hound.go +++ b/hound.go @@ -34,12 +34,12 @@ var ( ) var ( - ExpFailed = expvar.NewInt("failed") - ExpPassed = expvar.NewInt("passed") - ExpErrors = expvar.NewInt("errors") - EXP_GLOBAL_THROTTLE = expvar.NewInt("throttle") - EXP_GLOBAL_BACKOFF = expvar.NewInt("backoff") - EXP_UPTIME = expvar.NewInt("uptime") + ExpFailed = expvar.NewInt("failed") + ExpPassed = expvar.NewInt("passed") + ExpErrors = expvar.NewInt("errors") + ExpGlobalThrottle = expvar.NewInt("throttle") + EXP_GLOBAL_BACKOFF = expvar.NewInt("backoff") + EXP_UPTIME = expvar.NewInt("uptime") ) type config struct { From 57f15dd74ff6497a832184755d25a6d8a380bb28 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:06:03 +0100 Subject: [PATCH 21/36] EXP_GLOBAL_BACKOFF -> ExpGlobalBackoff --- alertscollection.go | 2 +- hound.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/alertscollection.go b/alertscollection.go index bd3cbfa..8637461 100644 --- a/alertscollection.go +++ b/alertscollection.go @@ -80,7 +80,7 @@ func ExposeVars(failures, errors, successes int) { ExpErrors.Set(int64(errors)) ExpPassed.Set(int64(successes)) ExpGlobalThrottle.Set(int64(GlobalThrottle)) - EXP_GLOBAL_BACKOFF.Set(int64(GlobalBackoff)) + ExpGlobalBackoff.Set(int64(GlobalBackoff)) } func (ac *AlertsCollection) HandleErrors(errors int) { diff --git a/hound.go b/hound.go index 9ca17ed..abbd5c8 100644 --- a/hound.go +++ b/hound.go @@ -34,12 +34,12 @@ var ( ) var ( - ExpFailed = expvar.NewInt("failed") - ExpPassed = expvar.NewInt("passed") - ExpErrors = expvar.NewInt("errors") - ExpGlobalThrottle = expvar.NewInt("throttle") - EXP_GLOBAL_BACKOFF = expvar.NewInt("backoff") - EXP_UPTIME = expvar.NewInt("uptime") + ExpFailed = expvar.NewInt("failed") + ExpPassed = expvar.NewInt("passed") + ExpErrors = expvar.NewInt("errors") + ExpGlobalThrottle = expvar.NewInt("throttle") + ExpGlobalBackoff = expvar.NewInt("backoff") + EXP_UPTIME = expvar.NewInt("uptime") ) type config struct { From 29b717d689b2ec21e585e7ff7ee8cfac7dc56a4d Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:06:43 +0100 Subject: [PATCH 22/36] EXP_UPTIME -> ExpUptime --- hound.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hound.go b/hound.go index abbd5c8..049b00b 100644 --- a/hound.go +++ b/hound.go @@ -39,7 +39,7 @@ var ( ExpErrors = expvar.NewInt("errors") ExpGlobalThrottle = expvar.NewInt("throttle") ExpGlobalBackoff = expvar.NewInt("backoff") - EXP_UPTIME = expvar.NewInt("uptime") + ExpUptime = expvar.NewInt("uptime") ) type config struct { @@ -136,7 +136,7 @@ func main() { // update uptime for { time.Sleep(1 * time.Second) - EXP_UPTIME.Add(1) + ExpUptime.Add(1) } }() From 1f4df4a4a60ce9acf0d0bf0503bb58a3c251c490 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:08:22 +0100 Subject: [PATCH 23/36] global_throttle -> globalThrottle --- alertscollection_test.go | 6 +++--- emailer.go | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/alertscollection_test.go b/alertscollection_test.go index 99303c4..489ab01 100644 --- a/alertscollection_test.go +++ b/alertscollection_test.go @@ -15,9 +15,9 @@ func Test_intmin(t *testing.T) { type DummyEmailer struct{} -func (d DummyEmailer) Throttled(failures, global_throttle int, emailTo string) {} -func (d DummyEmailer) RecoveryThrottled(recoveries_sent, global_throttle int, emailTo string) {} -func (d DummyEmailer) EncounteredErrors(errors int, emailTo string) {} +func (d DummyEmailer) Throttled(failures, globalThrottle int, emailTo string) {} +func (d DummyEmailer) RecoveryThrottled(recoveries_sent, globalThrottle int, emailTo string) {} +func (d DummyEmailer) EncounteredErrors(errors int, emailTo string) {} func Test_emptyAlertsCollection(t *testing.T) { ac := NewAlertsCollection(DummyEmailer{}) diff --git a/emailer.go b/emailer.go index a29632d..e4f0242 100644 --- a/emailer.go +++ b/emailer.go @@ -12,17 +12,17 @@ type Emailer interface { type SMTPEmailer struct{} -func (e SMTPEmailer) Throttled(failures, global_throttle int, emailTo string) { +func (e SMTPEmailer) Throttled(failures, globalThrottle int, emailTo string) { simpleSendMail( EmailFrom, emailTo, "[ALERT] Hound is throttled", fmt.Sprintf("%d metrics were not OK.\nHound stopped sending messages after %d.\n"+ "This probably indicates an infrastructure problem (network, graphite, etc)", failures, - global_throttle)) + globalThrottle)) } -func (e SMTPEmailer) RecoveryThrottled(recoveries_sent, global_throttle int, emailTo string) { +func (e SMTPEmailer) RecoveryThrottled(recoveries_sent, globalThrottle int, emailTo string) { if !EmailOnError { return } @@ -32,7 +32,7 @@ func (e SMTPEmailer) RecoveryThrottled(recoveries_sent, global_throttle int, ema "[ALERT] Hound is recovered", fmt.Sprintf("%d metrics recovered.\nHound stopped sending individual messages after %d.\n", recoveries_sent, - global_throttle)) + globalThrottle)) } func (e SMTPEmailer) EncounteredErrors(errors int, emailTo string) { From c4cd95655063adc6dadc88c67fae408cfc0eb411 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:09:37 +0100 Subject: [PATCH 24/36] recoveries_sent -> recoveriesSent --- alert.go | 14 +++++++------- alertscollection.go | 16 ++++++++-------- alertscollection_test.go | 6 +++--- emailer.go | 4 ++-- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/alert.go b/alert.go index 5e86cdf..1287906 100644 --- a/alert.go +++ b/alert.go @@ -271,13 +271,13 @@ func (a *Alert) JustRecovered() bool { return a.PreviousStatus == "Failed" || a.PreviousStatus == "Error" } -func (a *Alert) SendRecoveryMessageIfNeeded(recoveries_sent int) { - if a.JustRecovered() && recoveries_sent < GlobalThrottle { +func (a *Alert) SendRecoveryMessageIfNeeded(recoveriesSent int) { + if a.JustRecovered() && recoveriesSent < GlobalThrottle { a.SendRecoveryMessage() } } -func (a *Alert) UpdateState(recoveries_sent int) (int, int, int, int, int) { +func (a *Alert) UpdateState(recoveriesSent int) (int, int, int, int, int) { successes := 0 errors := 0 failures := 0 @@ -285,9 +285,9 @@ func (a *Alert) UpdateState(recoveries_sent int) (int, int, int, int, int) { if a.Status == "OK" { successes++ - a.SendRecoveryMessageIfNeeded(recoveries_sent) + a.SendRecoveryMessageIfNeeded(recoveriesSent) if a.JustRecovered() { - recoveries_sent++ + recoveriesSent++ } a.Backoff = 0 } else { @@ -302,7 +302,7 @@ func (a *Alert) UpdateState(recoveries_sent int) (int, int, int, int, int) { // wait for the throttling to expire log.WithFields( log.Fields{ - "recoveries_sent": recoveries_sent, + "recoveriesSent": recoveriesSent, }, ).Debug("throttled") } else { @@ -316,7 +316,7 @@ func (a *Alert) UpdateState(recoveries_sent int) (int, int, int, int, int) { } // cycle the previous status a.PreviousStatus = a.Status - return successes, recoveries_sent, errors, failures, alerts_sent + return successes, recoveriesSent, errors, failures, alerts_sent } func (a Alert) Hash() string { diff --git a/alertscollection.go b/alertscollection.go index 8637461..03abe88 100644 --- a/alertscollection.go +++ b/alertscollection.go @@ -50,15 +50,15 @@ func (ac *AlertsCollection) ProcessAll() { // fetch/calculate new status for all ac.CheckAll() alerts_sent := 0 - recoveries_sent := 0 + recoveriesSent := 0 errors := 0 failures := 0 successes := 0 for _, a := range ac.Alerts { - s, rs, e, f, as := a.UpdateState(recoveries_sent) + s, rs, e, f, as := a.UpdateState(recoveriesSent) successes = successes + s - recoveries_sent = recoveries_sent + rs + recoveriesSent = recoveriesSent + rs errors = errors + e failures = failures + f alerts_sent = alerts_sent + as @@ -67,11 +67,11 @@ func (ac *AlertsCollection) ProcessAll() { ac.Emailer.Throttled(failures, GlobalThrottle, EmailTo) } - if recoveries_sent >= GlobalThrottle { - ac.Emailer.RecoveryThrottled(recoveries_sent, GlobalThrottle, EmailTo) + if recoveriesSent >= GlobalThrottle { + ac.Emailer.RecoveryThrottled(recoveriesSent, GlobalThrottle, EmailTo) } ac.HandleErrors(errors) - LogToGraphite(alerts_sent, recoveries_sent, failures, errors, successes) + LogToGraphite(alerts_sent, recoveriesSent, failures, errors, successes) ExposeVars(failures, errors, successes) } @@ -97,7 +97,7 @@ func (ac *AlertsCollection) HandleErrors(errors int) { } } -func LogToGraphite(alerts_sent, recoveries_sent, failures, errors, successes int) { +func LogToGraphite(alerts_sent, recoveriesSent, failures, errors, successes int) { var clientGraphite net.Conn clientGraphite, err := net.Dial("tcp", CarbonBase) if err != nil || clientGraphite == nil { @@ -108,7 +108,7 @@ func LogToGraphite(alerts_sent, recoveries_sent, failures, errors, successes int buffer := bytes.NewBufferString("") fmt.Fprintf(buffer, "%salerts_sent %d %d\n", MetricBase, alerts_sent, now) - fmt.Fprintf(buffer, "%srecoveries_sent %d %d\n", MetricBase, recoveries_sent, now) + fmt.Fprintf(buffer, "%srecoveries_sent %d %d\n", MetricBase, recoveriesSent, now) fmt.Fprintf(buffer, "%sfailures %d %d\n", MetricBase, failures, now) fmt.Fprintf(buffer, "%serrors %d %d\n", MetricBase, errors, now) fmt.Fprintf(buffer, "%ssuccesses %d %d\n", MetricBase, successes, now) diff --git a/alertscollection_test.go b/alertscollection_test.go index 489ab01..8982f8e 100644 --- a/alertscollection_test.go +++ b/alertscollection_test.go @@ -15,9 +15,9 @@ func Test_intmin(t *testing.T) { type DummyEmailer struct{} -func (d DummyEmailer) Throttled(failures, globalThrottle int, emailTo string) {} -func (d DummyEmailer) RecoveryThrottled(recoveries_sent, globalThrottle int, emailTo string) {} -func (d DummyEmailer) EncounteredErrors(errors int, emailTo string) {} +func (d DummyEmailer) Throttled(failures, globalThrottle int, emailTo string) {} +func (d DummyEmailer) RecoveryThrottled(recoveriesSent, globalThrottle int, emailTo string) {} +func (d DummyEmailer) EncounteredErrors(errors int, emailTo string) {} func Test_emptyAlertsCollection(t *testing.T) { ac := NewAlertsCollection(DummyEmailer{}) diff --git a/emailer.go b/emailer.go index e4f0242..6d1196c 100644 --- a/emailer.go +++ b/emailer.go @@ -22,7 +22,7 @@ func (e SMTPEmailer) Throttled(failures, globalThrottle int, emailTo string) { globalThrottle)) } -func (e SMTPEmailer) RecoveryThrottled(recoveries_sent, globalThrottle int, emailTo string) { +func (e SMTPEmailer) RecoveryThrottled(recoveriesSent, globalThrottle int, emailTo string) { if !EmailOnError { return } @@ -31,7 +31,7 @@ func (e SMTPEmailer) RecoveryThrottled(recoveries_sent, globalThrottle int, emai emailTo, "[ALERT] Hound is recovered", fmt.Sprintf("%d metrics recovered.\nHound stopped sending individual messages after %d.\n", - recoveries_sent, + recoveriesSent, globalThrottle)) } From 9f81779aa0c5e6b086bdcdf1bae2823309cc9120 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:13:29 +0100 Subject: [PATCH 25/36] don't export SMTPEmailer --- emailer.go | 8 ++++---- hound.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/emailer.go b/emailer.go index 6d1196c..95c3c8a 100644 --- a/emailer.go +++ b/emailer.go @@ -10,9 +10,9 @@ type Emailer interface { Throttled(int, int, string) } -type SMTPEmailer struct{} +type smtpEmailer struct{} -func (e SMTPEmailer) Throttled(failures, globalThrottle int, emailTo string) { +func (e smtpEmailer) Throttled(failures, globalThrottle int, emailTo string) { simpleSendMail( EmailFrom, emailTo, @@ -22,7 +22,7 @@ func (e SMTPEmailer) Throttled(failures, globalThrottle int, emailTo string) { globalThrottle)) } -func (e SMTPEmailer) RecoveryThrottled(recoveriesSent, globalThrottle int, emailTo string) { +func (e smtpEmailer) RecoveryThrottled(recoveriesSent, globalThrottle int, emailTo string) { if !EmailOnError { return } @@ -35,7 +35,7 @@ func (e SMTPEmailer) RecoveryThrottled(recoveriesSent, globalThrottle int, email globalThrottle)) } -func (e SMTPEmailer) EncounteredErrors(errors int, emailTo string) { +func (e smtpEmailer) EncounteredErrors(errors int, emailTo string) { if !EmailOnError { return } diff --git a/hound.go b/hound.go index 049b00b..bf8eba9 100644 --- a/hound.go +++ b/hound.go @@ -141,7 +141,7 @@ func main() { }() // initialize all the alerts - ac := NewAlertsCollection(SMTPEmailer{}) + ac := NewAlertsCollection(smtpEmailer{}) for _, a := range f.Alerts { emailTo := a.EmailTo if emailTo == "" { From 5b45e34eb769ab95379f8722e97feaceac907944 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:15:10 +0100 Subject: [PATCH 26/36] don't export expvar variables --- alertscollection.go | 10 +++++----- hound.go | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/alertscollection.go b/alertscollection.go index 03abe88..b1bf868 100644 --- a/alertscollection.go +++ b/alertscollection.go @@ -76,11 +76,11 @@ func (ac *AlertsCollection) ProcessAll() { } func ExposeVars(failures, errors, successes int) { - ExpFailed.Set(int64(failures)) - ExpErrors.Set(int64(errors)) - ExpPassed.Set(int64(successes)) - ExpGlobalThrottle.Set(int64(GlobalThrottle)) - ExpGlobalBackoff.Set(int64(GlobalBackoff)) + expFailed.Set(int64(failures)) + expErrors.Set(int64(errors)) + expPassed.Set(int64(successes)) + expGlobalThrottle.Set(int64(GlobalThrottle)) + expGlobalBackoff.Set(int64(GlobalBackoff)) } func (ac *AlertsCollection) HandleErrors(errors int) { diff --git a/hound.go b/hound.go index bf8eba9..e1d60b5 100644 --- a/hound.go +++ b/hound.go @@ -34,12 +34,12 @@ var ( ) var ( - ExpFailed = expvar.NewInt("failed") - ExpPassed = expvar.NewInt("passed") - ExpErrors = expvar.NewInt("errors") - ExpGlobalThrottle = expvar.NewInt("throttle") - ExpGlobalBackoff = expvar.NewInt("backoff") - ExpUptime = expvar.NewInt("uptime") + expFailed = expvar.NewInt("failed") + expPassed = expvar.NewInt("passed") + expErrors = expvar.NewInt("errors") + expGlobalThrottle = expvar.NewInt("throttle") + expGlobalBackoff = expvar.NewInt("backoff") + expUptime = expvar.NewInt("uptime") ) type config struct { @@ -136,7 +136,7 @@ func main() { // update uptime for { time.Sleep(1 * time.Second) - ExpUptime.Add(1) + expUptime.Add(1) } }() From 24877c7e8bc54f152429bad3a7a6776c70b9a366 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:16:17 +0100 Subject: [PATCH 27/36] runbook_link -> runbookLink --- alert.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alert.go b/alert.go index 1287906..0c2eb6f 100644 --- a/alert.go +++ b/alert.go @@ -46,7 +46,7 @@ var WEEKLY_BGCOLOR = "EEEEEE" var WEEKLY_COLORLIST = "%23cccccc,%236699cc" func NewAlert(name string, metric string, atype string, threshold float64, - direction string, fetcher Fetcher, emailTo string, runbook_link string) *Alert { + direction string, fetcher Fetcher, emailTo string, runbookLink string) *Alert { if atype == "" { atype = "Alert" } @@ -54,7 +54,7 @@ func NewAlert(name string, metric string, atype string, threshold float64, Metric: cleanMetric(metric), Threshold: threshold, Direction: direction, Backoff: 0, LastAlerted: time.Now(), Status: "OK", Message: "", PreviousStatus: "OK", Fetcher: fetcher, EmailTo: emailTo, - Value: 0.0, RunBookLink: runbook_link, + Value: 0.0, RunBookLink: runbookLink, } } From 8645b11ab3f23e3b3104a1c1ac515f6ea0fdcc6e Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:16:57 +0100 Subject: [PATCH 28/36] alerts_sent -> alertsSent --- alert.go | 8 ++++---- alertscollection.go | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/alert.go b/alert.go index 0c2eb6f..3cbbc44 100644 --- a/alert.go +++ b/alert.go @@ -281,7 +281,7 @@ func (a *Alert) UpdateState(recoveriesSent int) (int, int, int, int, int) { successes := 0 errors := 0 failures := 0 - alerts_sent := 0 + alertsSent := 0 if a.Status == "OK" { successes++ @@ -306,9 +306,9 @@ func (a *Alert) UpdateState(recoveriesSent int) (int, int, int, int, int) { }, ).Debug("throttled") } else { - if a.Status == "Failed" && alerts_sent < GlobalThrottle { + if a.Status == "Failed" && alertsSent < GlobalThrottle { a.SendAlert() - alerts_sent++ + alertsSent++ } a.Backoff = intmin(a.Backoff+1, len(BACKOFF_DURATIONS)) a.LastAlerted = time.Now() @@ -316,7 +316,7 @@ func (a *Alert) UpdateState(recoveriesSent int) (int, int, int, int, int) { } // cycle the previous status a.PreviousStatus = a.Status - return successes, recoveriesSent, errors, failures, alerts_sent + return successes, recoveriesSent, errors, failures, alertsSent } func (a Alert) Hash() string { diff --git a/alertscollection.go b/alertscollection.go index b1bf868..24ce639 100644 --- a/alertscollection.go +++ b/alertscollection.go @@ -49,7 +49,7 @@ func (ac *AlertsCollection) CheckAll() { func (ac *AlertsCollection) ProcessAll() { // fetch/calculate new status for all ac.CheckAll() - alerts_sent := 0 + alertsSent := 0 recoveriesSent := 0 errors := 0 failures := 0 @@ -61,9 +61,9 @@ func (ac *AlertsCollection) ProcessAll() { recoveriesSent = recoveriesSent + rs errors = errors + e failures = failures + f - alerts_sent = alerts_sent + as + alertsSent = alertsSent + as } - if alerts_sent >= GlobalThrottle { + if alertsSent >= GlobalThrottle { ac.Emailer.Throttled(failures, GlobalThrottle, EmailTo) } @@ -71,7 +71,7 @@ func (ac *AlertsCollection) ProcessAll() { ac.Emailer.RecoveryThrottled(recoveriesSent, GlobalThrottle, EmailTo) } ac.HandleErrors(errors) - LogToGraphite(alerts_sent, recoveriesSent, failures, errors, successes) + LogToGraphite(alertsSent, recoveriesSent, failures, errors, successes) ExposeVars(failures, errors, successes) } @@ -97,7 +97,7 @@ func (ac *AlertsCollection) HandleErrors(errors int) { } } -func LogToGraphite(alerts_sent, recoveriesSent, failures, errors, successes int) { +func LogToGraphite(alertsSent, recoveriesSent, failures, errors, successes int) { var clientGraphite net.Conn clientGraphite, err := net.Dial("tcp", CarbonBase) if err != nil || clientGraphite == nil { @@ -107,7 +107,7 @@ func LogToGraphite(alerts_sent, recoveriesSent, failures, errors, successes int) now := int32(time.Now().Unix()) buffer := bytes.NewBufferString("") - fmt.Fprintf(buffer, "%salerts_sent %d %d\n", MetricBase, alerts_sent, now) + fmt.Fprintf(buffer, "%salerts_sent %d %d\n", MetricBase, alertsSent, now) fmt.Fprintf(buffer, "%srecoveries_sent %d %d\n", MetricBase, recoveriesSent, now) fmt.Fprintf(buffer, "%sfailures %d %d\n", MetricBase, failures, now) fmt.Fprintf(buffer, "%serrors %d %d\n", MetricBase, errors, now) From 5c48e7738222a19a0d6ee4aef7a26994e1722a9c Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:18:47 +0100 Subject: [PATCH 29/36] raw_response -> rawResponse --- alert.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alert.go b/alert.go index 3cbbc44..dfe4416 100644 --- a/alert.go +++ b/alert.go @@ -328,9 +328,9 @@ func (a Alert) Hash() string { return fmt.Sprintf("%x", h.Sum(nil))[0:10] } -func extractLastValue(raw_response string) (float64, error) { +func extractLastValue(rawResponse string) (float64, error) { // just take the most recent value - parts := strings.Split(strings.Trim(raw_response, "\n\t "), ",") + parts := strings.Split(strings.Trim(rawResponse, "\n\t "), ",") return strconv.ParseFloat(parts[len(parts)-1], 64) } From bca927b041e0cea9e2e23e75bc22ee2f2a663bab Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:19:42 +0100 Subject: [PATCH 30/36] backoff_time -> backoffTime --- alert.go | 4 ++-- alertscollection.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/alert.go b/alert.go index dfe4416..ee548cd 100644 --- a/alert.go +++ b/alert.go @@ -228,7 +228,7 @@ func (a *Alert) Throttled() bool { if a.Backoff == 0 { return false } - d := backoff_time(a.Backoff) + d := backoffTime(a.Backoff) window := a.LastAlerted.Add(d) return time.Now().Before(window) } @@ -451,6 +451,6 @@ var BACKOFF_DURATIONS = []time.Duration{ time.Duration(24) * time.Hour, } -func backoff_time(level int) time.Duration { +func backoffTime(level int) time.Duration { return BACKOFF_DURATIONS[level] } diff --git a/alertscollection.go b/alertscollection.go index 24ce639..698dbf7 100644 --- a/alertscollection.go +++ b/alertscollection.go @@ -85,7 +85,7 @@ func ExposeVars(failures, errors, successes int) { func (ac *AlertsCollection) HandleErrors(errors int) { if errors > 0 { - d := backoff_time(GlobalBackoff) + d := backoffTime(GlobalBackoff) window := LastErrorEmail.Add(d) if time.Now().After(window) { ac.Emailer.EncounteredErrors(errors, EmailTo) From 5a2628f63f6d205607a294a7ee279b536eccee62 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:20:49 +0100 Subject: [PATCH 31/36] GRAPH_WIDTH -> graphWidth --- alert.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/alert.go b/alert.go index ee548cd..f8c64b9 100644 --- a/alert.go +++ b/alert.go @@ -36,7 +36,7 @@ type Alert struct { RunBookLink string } -var GRAPH_WIDTH = 800 +var graphWidth = 800 var DAILY_GRAPH_HEIGHT = 150 var WEEKLY_GRAPH_HEIGHT = 75 var FGCOLOR = "000000" @@ -71,7 +71,7 @@ func (a Alert) DailyGraphUrl() string { return GraphiteBase + "?target=" + a.Metric + "&target=threshold(" + fmt.Sprintf("%f", a.Threshold) + - ")&width=" + fmt.Sprintf("%d", GRAPH_WIDTH) + + ")&width=" + fmt.Sprintf("%d", graphWidth) + "&height=" + fmt.Sprintf("%d", DAILY_GRAPH_HEIGHT) + "&bgcolor=" + DAILY_BGCOLOR + "&fgcolor=" + FGCOLOR + "&hideGrid=true&colorList=" + @@ -82,7 +82,7 @@ func (a Alert) WeeklyGraphUrl() string { return GraphiteBase + "?target=" + a.Metric + "&target=threshold(" + fmt.Sprintf("%f", a.Threshold) + - ")&width=" + fmt.Sprintf("%d", GRAPH_WIDTH) + + ")&width=" + fmt.Sprintf("%d", graphWidth) + "&height=" + fmt.Sprintf("%d", WEEKLY_GRAPH_HEIGHT) + "&hideGrid=true&hideLegend=true&graphOnly=true&hideAxes=true&bgcolor=" + WEEKLY_BGCOLOR + "&fgcolor=" + FGCOLOR + From bf00319eb90074f070254751ce21c28bd36455e2 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:22:25 +0100 Subject: [PATCH 32/36] DAILY_GRAPH_HEIGHT -> dailyGraphHeight etc --- alert.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/alert.go b/alert.go index f8c64b9..02e271c 100644 --- a/alert.go +++ b/alert.go @@ -37,13 +37,13 @@ type Alert struct { } var graphWidth = 800 -var DAILY_GRAPH_HEIGHT = 150 -var WEEKLY_GRAPH_HEIGHT = 75 -var FGCOLOR = "000000" -var DAILY_BGCOLOR = "FFFFFF" -var DAILY_COLORLIST = "%23999999,%23006699" -var WEEKLY_BGCOLOR = "EEEEEE" -var WEEKLY_COLORLIST = "%23cccccc,%236699cc" +var dailyGraphHeight = 150 +var weeklyGraphHeight = 75 +var fgColor = "000000" +var dailyBgColor = "FFFFFF" +var dailyColorlist = "%23999999,%23006699" +var weeklyBgColor = "EEEEEE" +var weeklyColorlist = "%23cccccc,%236699cc" func NewAlert(name string, metric string, atype string, threshold float64, direction string, fetcher Fetcher, emailTo string, runbookLink string) *Alert { @@ -72,10 +72,10 @@ func (a Alert) DailyGraphUrl() string { a.Metric + "&target=threshold(" + fmt.Sprintf("%f", a.Threshold) + ")&width=" + fmt.Sprintf("%d", graphWidth) + - "&height=" + fmt.Sprintf("%d", DAILY_GRAPH_HEIGHT) + - "&bgcolor=" + DAILY_BGCOLOR + - "&fgcolor=" + FGCOLOR + "&hideGrid=true&colorList=" + - DAILY_COLORLIST + "&from=-24hours" + "&height=" + fmt.Sprintf("%d", dailyGraphHeight) + + "&bgcolor=" + dailyBgColor + + "&fgcolor=" + fgColor + "&hideGrid=true&colorList=" + + dailyColorlist + "&from=-24hours" } func (a Alert) WeeklyGraphUrl() string { @@ -83,10 +83,10 @@ func (a Alert) WeeklyGraphUrl() string { a.Metric + "&target=threshold(" + fmt.Sprintf("%f", a.Threshold) + ")&width=" + fmt.Sprintf("%d", graphWidth) + - "&height=" + fmt.Sprintf("%d", WEEKLY_GRAPH_HEIGHT) + + "&height=" + fmt.Sprintf("%d", weeklyGraphHeight) + "&hideGrid=true&hideLegend=true&graphOnly=true&hideAxes=true&bgcolor=" + - WEEKLY_BGCOLOR + "&fgcolor=" + FGCOLOR + - "&hideGrid=true&colorList=" + WEEKLY_COLORLIST + "&from=-7days" + weeklyBgColor + "&fgcolor=" + fgColor + + "&hideGrid=true&colorList=" + weeklyColorlist + "&from=-7days" } type Fetcher interface { From c732d9988d86bdf6c23633cc51809cf2b058939e Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:23:21 +0100 Subject: [PATCH 33/36] BACKOFF_DURATIONS -> backoffDurations --- alert.go | 6 +++--- alertscollection.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/alert.go b/alert.go index 02e271c..0842870 100644 --- a/alert.go +++ b/alert.go @@ -310,7 +310,7 @@ func (a *Alert) UpdateState(recoveriesSent int) (int, int, int, int, int) { a.SendAlert() alertsSent++ } - a.Backoff = intmin(a.Backoff+1, len(BACKOFF_DURATIONS)) + a.Backoff = intmin(a.Backoff+1, len(backoffDurations)) a.LastAlerted = time.Now() } } @@ -441,7 +441,7 @@ func encodeRFC2047(String string) string { return strings.Trim(addr.String(), " <>") } -var BACKOFF_DURATIONS = []time.Duration{ +var backoffDurations = []time.Duration{ time.Duration(5) * time.Minute, time.Duration(30) * time.Minute, time.Duration(1) * time.Hour, @@ -452,5 +452,5 @@ var BACKOFF_DURATIONS = []time.Duration{ } func backoffTime(level int) time.Duration { - return BACKOFF_DURATIONS[level] + return backoffDurations[level] } diff --git a/alertscollection.go b/alertscollection.go index 698dbf7..7263d4f 100644 --- a/alertscollection.go +++ b/alertscollection.go @@ -90,7 +90,7 @@ func (ac *AlertsCollection) HandleErrors(errors int) { if time.Now().After(window) { ac.Emailer.EncounteredErrors(errors, EmailTo) LastErrorEmail = time.Now() - GlobalBackoff = intmin(GlobalBackoff+1, len(BACKOFF_DURATIONS)) + GlobalBackoff = intmin(GlobalBackoff+1, len(backoffDurations)) } } else { GlobalBackoff = 0 From 5047ae574e40ba3d81c98bfb4b1d14c1036d8d07 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:28:19 +0100 Subject: [PATCH 34/36] Url -> URL --- alert.go | 10 +++++----- alert.html | 4 ++-- alert_test.go | 18 +++++++++--------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/alert.go b/alert.go index 0842870..41d7ec1 100644 --- a/alert.go +++ b/alert.go @@ -63,11 +63,11 @@ func cleanMetric(metric string) string { return re.ReplaceAllString(metric, "") } -func (a Alert) Url() string { +func (a Alert) URL() string { return GraphiteBase + "?target=keepLastValue(" + a.Metric + ")&format=raw&from=-" + Window } -func (a Alert) DailyGraphUrl() string { +func (a Alert) DailyGraphURL() string { return GraphiteBase + "?target=" + a.Metric + "&target=threshold(" + fmt.Sprintf("%f", a.Threshold) + @@ -78,7 +78,7 @@ func (a Alert) DailyGraphUrl() string { dailyColorlist + "&from=-24hours" } -func (a Alert) WeeklyGraphUrl() string { +func (a Alert) WeeklyGraphURL() string { return GraphiteBase + "?target=" + a.Metric + "&target=threshold(" + fmt.Sprintf("%f", a.Threshold) + @@ -100,7 +100,7 @@ func (h HTTPFetcher) Get(url string) (*http.Response, error) { } func (a *Alert) Fetch() (float64, error) { - resp, err := a.Fetcher.Get(a.Url()) + resp, err := a.Fetcher.Get(a.URL()) if err != nil { a.Status = "Error" a.Message = "graphite request failed" @@ -262,7 +262,7 @@ func (a *Alert) IncludeRunBookLink() string { func (a *Alert) AlertEmailBody() string { return fmt.Sprintf("%s [%s] has triggered an alert\nStatus:\t%s\nMessage:\t%s\n\nDaily Graph: <%s>\nWeekly Graph: <%s>%s\n", - a.Name, a.Metric, a.Status, a.Message, a.DailyGraphUrl(), a.WeeklyGraphUrl(), a.IncludeRunBookLink()) + a.Name, a.Metric, a.Status, a.Message, a.DailyGraphURL(), a.WeeklyGraphURL(), a.IncludeRunBookLink()) } // did this alert just return to a healthy state? diff --git a/alert.html b/alert.html index 98a0322..1c433c3 100644 --- a/alert.html +++ b/alert.html @@ -37,10 +37,10 @@

Hound:

Daily Graph

- +

Weekly Graph

- + {{ if $element.RunBookLink }} diff --git a/alert_test.go b/alert_test.go index 936cf76..4a9aa24 100644 --- a/alert_test.go +++ b/alert_test.go @@ -30,24 +30,24 @@ func Test_StringWhiteSpaceRemoval(t *testing.T) { } } -func Test_Url(t *testing.T) { +func Test_URL(t *testing.T) { a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") - if a.Url() != "?target=keepLastValue(foo)&format=raw&from=-"+Window { - t.Error(fmt.Sprintf("wrong value: %s", a.Url())) + if a.URL() != "?target=keepLastValue(foo)&format=raw&from=-"+Window { + t.Error(fmt.Sprintf("wrong value: %s", a.URL())) } } -func Test_DailyGraphUrl(t *testing.T) { +func Test_DailyGraphURL(t *testing.T) { a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") - if a.DailyGraphUrl() != "?target=foo&target=threshold(10.000000)&width=800&height=150&bgcolor=FFFFFF&fgcolor=000000&hideGrid=true&colorList=%23999999,%23006699&from=-24hours" { - t.Error(fmt.Sprintf("wrong value: %s", a.DailyGraphUrl())) + if a.DailyGraphURL() != "?target=foo&target=threshold(10.000000)&width=800&height=150&bgcolor=FFFFFF&fgcolor=000000&hideGrid=true&colorList=%23999999,%23006699&from=-24hours" { + t.Error(fmt.Sprintf("wrong value: %s", a.DailyGraphURL())) } } -func Test_WeeklyGraphUrl(t *testing.T) { +func Test_WeeklyGraphURL(t *testing.T) { a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") - if a.WeeklyGraphUrl() != "?target=foo&target=threshold(10.000000)&width=800&height=75&hideGrid=true&hideLegend=true&graphOnly=true&hideAxes=true&bgcolor=EEEEEE&fgcolor=000000&hideGrid=true&colorList=%23cccccc,%236699cc&from=-7days" { - t.Error(fmt.Sprintf("wrong value: %s", a.WeeklyGraphUrl())) + if a.WeeklyGraphURL() != "?target=foo&target=threshold(10.000000)&width=800&height=75&hideGrid=true&hideLegend=true&graphOnly=true&hideAxes=true&bgcolor=EEEEEE&fgcolor=000000&hideGrid=true&colorList=%23cccccc,%236699cc&from=-7days" { + t.Error(fmt.Sprintf("wrong value: %s", a.WeeklyGraphURL())) } } From b2927e0ea54a3b781aa973f05dc4625b67afeca5 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 15:30:58 +0100 Subject: [PATCH 35/36] convert elses to guards --- alert.go | 132 ++++++++++++++++++++++++++----------------------------- 1 file changed, 63 insertions(+), 69 deletions(-) diff --git a/alert.go b/alert.go index 41d7ec1..67f0adb 100644 --- a/alert.go +++ b/alert.go @@ -157,25 +157,22 @@ func (a *Alert) UpdateStatus(lv float64) { func (a Alert) String() string { if a.Status == "OK" { return fmt.Sprintf("%s\t%s [%s]", a.Status, a.Name, a.Metric) - } else { - return fmt.Sprintf("%s\t%s [%s]: %s (%s)", a.Status, a.Name, a.Metric, a.Message, a.LastAlerted) } + return fmt.Sprintf("%s\t%s [%s]: %s (%s)", a.Status, a.Name, a.Metric, a.Message, a.LastAlerted) } func (a Alert) RenderDirection() string { if a.Status == "OK" { if a.Direction == "above" { return "<" - } else { - return ">" - } - } else { - if a.Direction == "above" { - return ">" - } else { - return "<" } + return ">" } + if a.Direction == "above" { + return ">" + } + return "<" + } func (a Alert) BootstrapStatus() string { @@ -191,9 +188,9 @@ func (a Alert) BootstrapStatus() string { func (a Alert) GlyphIcon() string { if a.Type == "Notice" { return "glyphicon-info-sign" - } else { - return "glyphicon-warning-sign" } + return "glyphicon-warning-sign" + } func (a *Alert) SendRecoveryMessage() { @@ -219,9 +216,8 @@ func (a *Alert) RecoveryEmailBody() string { func invertDirection(d string) string { if d == "above" { return "below" - } else { - return "above" } + return "above" } func (a *Alert) Throttled() bool { @@ -248,9 +244,8 @@ func (a *Alert) SendAlert() { func (a *Alert) AlertEmailSubject() string { if a.Type == "Alert" { return fmt.Sprintf("[ALERT] %s", a.Name) - } else { - return fmt.Sprintf("[NOTICE] %s", a.Name) } + return fmt.Sprintf("[NOTICE] %s", a.Name) } func (a *Alert) IncludeRunBookLink() string { @@ -369,70 +364,69 @@ func simpleSendMail(from, to, subject string, body string) error { ).Error("error sending mail") } return err - } else { - tlsconfig := &tls.Config{ - InsecureSkipVerify: true, - ServerName: SMTPServer, - } - - conn, err := tls.Dial("tcp", s, tlsconfig) - if err != nil { - log.WithFields(log.Fields{"err": err}).Error("tls.Dial failed") - return err - } + } + tlsconfig := &tls.Config{ + InsecureSkipVerify: true, + ServerName: SMTPServer, + } - c, err := smtp.NewClient(conn, SMTPServer) - if err != nil { - log.WithFields(log.Fields{"err": err}).Error("smtp.NewClient failed") - return err - } + conn, err := tls.Dial("tcp", s, tlsconfig) + if err != nil { + log.WithFields(log.Fields{"err": err}).Error("tls.Dial failed") + return err + } - // Auth - if err = c.Auth(auth); err != nil { - log.WithFields( - log.Fields{ - "err": err, - "SMTP_USER": SMTPUser, - "SMTP_PASSWORD": SMTPPassword, - "SMTP_SERVER": SMTPServer, - }).Error("auth failed") - return err - } + c, err := smtp.NewClient(conn, SMTPServer) + if err != nil { + log.WithFields(log.Fields{"err": err}).Error("smtp.NewClient failed") + return err + } - // To && From - if err = c.Mail(from); err != nil { - log.WithFields(log.Fields{"err": err, "from": from}).Error("from address failed") - return err - } + // Auth + if err = c.Auth(auth); err != nil { + log.WithFields( + log.Fields{ + "err": err, + "SMTP_USER": SMTPUser, + "SMTP_PASSWORD": SMTPPassword, + "SMTP_SERVER": SMTPServer, + }).Error("auth failed") + return err + } - if err = c.Rcpt(to); err != nil { - log.WithFields(log.Fields{"err": err}).Error("to address failed") - return err - } + // To && From + if err = c.Mail(from); err != nil { + log.WithFields(log.Fields{"err": err, "from": from}).Error("from address failed") + return err + } - // Data - w, err := c.Data() - if err != nil { - log.WithFields(log.Fields{"err": err}).Error("smtp Data() failed") - return err - } + if err = c.Rcpt(to); err != nil { + log.WithFields(log.Fields{"err": err}).Error("to address failed") + return err + } - _, err = w.Write([]byte(message)) - if err != nil { - log.WithFields(log.Fields{"err": err}).Error("smtp Write failed") - return err - } + // Data + w, err := c.Data() + if err != nil { + log.WithFields(log.Fields{"err": err}).Error("smtp Data() failed") + return err + } - err = w.Close() - if err != nil { - log.WithFields(log.Fields{"err": err}).Error("smtp close failed") - return err - } + _, err = w.Write([]byte(message)) + if err != nil { + log.WithFields(log.Fields{"err": err}).Error("smtp Write failed") + return err + } - c.Quit() + err = w.Close() + if err != nil { + log.WithFields(log.Fields{"err": err}).Error("smtp close failed") return err } + c.Quit() + return err + } func encodeRFC2047(String string) string { From 0e7ff78c1266be35c5c9997a5f3c3958b8d4b543 Mon Sep 17 00:00:00 2001 From: Anders Pearson Date: Sat, 29 Jul 2017 16:16:25 +0100 Subject: [PATCH 36/36] don't export as many variables --- alert.go | 98 ++++++++++++++++----------------- alert_test.go | 60 ++++++++++---------- alertscollection.go | 116 +++++++++++++++++++-------------------- alertscollection_test.go | 20 +++---- config.go | 6 +- emailer.go | 12 ++-- hound.go | 74 ++++++++++++------------- 7 files changed, 193 insertions(+), 193 deletions(-) diff --git a/alert.go b/alert.go index 67f0adb..0edc68d 100644 --- a/alert.go +++ b/alert.go @@ -19,7 +19,7 @@ import ( log "github.com/Sirupsen/logrus" ) -type Alert struct { +type alert struct { Name string Metric string Type string @@ -30,7 +30,7 @@ type Alert struct { Status string Message string PreviousStatus string - Fetcher Fetcher + fetcher fetcher EmailTo string Value float64 RunBookLink string @@ -45,15 +45,15 @@ var dailyColorlist = "%23999999,%23006699" var weeklyBgColor = "EEEEEE" var weeklyColorlist = "%23cccccc,%236699cc" -func NewAlert(name string, metric string, atype string, threshold float64, - direction string, fetcher Fetcher, emailTo string, runbookLink string) *Alert { +func newAlert(name string, metric string, atype string, threshold float64, + direction string, fetcher fetcher, emailTo string, runbookLink string) *alert { if atype == "" { atype = "Alert" } - return &Alert{Name: name, Type: atype, + return &alert{Name: name, Type: atype, Metric: cleanMetric(metric), Threshold: threshold, Direction: direction, Backoff: 0, LastAlerted: time.Now(), Status: "OK", Message: "", - PreviousStatus: "OK", Fetcher: fetcher, EmailTo: emailTo, + PreviousStatus: "OK", fetcher: fetcher, EmailTo: emailTo, Value: 0.0, RunBookLink: runbookLink, } } @@ -63,12 +63,12 @@ func cleanMetric(metric string) string { return re.ReplaceAllString(metric, "") } -func (a Alert) URL() string { - return GraphiteBase + "?target=keepLastValue(" + a.Metric + ")&format=raw&from=-" + Window +func (a alert) URL() string { + return graphiteBase + "?target=keepLastValue(" + a.Metric + ")&format=raw&from=-" + window } -func (a Alert) DailyGraphURL() string { - return GraphiteBase + "?target=" + +func (a alert) DailyGraphURL() string { + return graphiteBase + "?target=" + a.Metric + "&target=threshold(" + fmt.Sprintf("%f", a.Threshold) + ")&width=" + fmt.Sprintf("%d", graphWidth) + @@ -78,8 +78,8 @@ func (a Alert) DailyGraphURL() string { dailyColorlist + "&from=-24hours" } -func (a Alert) WeeklyGraphURL() string { - return GraphiteBase + "?target=" + +func (a alert) WeeklyGraphURL() string { + return graphiteBase + "?target=" + a.Metric + "&target=threshold(" + fmt.Sprintf("%f", a.Threshold) + ")&width=" + fmt.Sprintf("%d", graphWidth) + @@ -89,18 +89,18 @@ func (a Alert) WeeklyGraphURL() string { "&hideGrid=true&colorList=" + weeklyColorlist + "&from=-7days" } -type Fetcher interface { +type fetcher interface { Get(string) (*http.Response, error) } -type HTTPFetcher struct{} +type httpFetcher struct{} -func (h HTTPFetcher) Get(url string) (*http.Response, error) { +func (h httpFetcher) Get(url string) (*http.Response, error) { return http.Get(url) } -func (a *Alert) Fetch() (float64, error) { - resp, err := a.Fetcher.Get(a.URL()) +func (a *alert) Fetch() (float64, error) { + resp, err := a.fetcher.Get(a.URL()) if err != nil { a.Status = "Error" a.Message = "graphite request failed" @@ -121,7 +121,7 @@ func (a *Alert) Fetch() (float64, error) { return lv, err } -func (a *Alert) CheckMetric() bool { +func (a *alert) CheckMetric() bool { lv, err := a.Fetch() if err != nil { return false @@ -131,7 +131,7 @@ func (a *Alert) CheckMetric() bool { } -func (a *Alert) UpdateStatus(lv float64) { +func (a *alert) UpdateStatus(lv float64) { a.Value = lv if a.Direction == "above" { // pass if metric is below the threshold @@ -154,14 +154,14 @@ func (a *Alert) UpdateStatus(lv float64) { } } -func (a Alert) String() string { +func (a alert) String() string { if a.Status == "OK" { return fmt.Sprintf("%s\t%s [%s]", a.Status, a.Name, a.Metric) } return fmt.Sprintf("%s\t%s [%s]: %s (%s)", a.Status, a.Name, a.Metric, a.Message, a.LastAlerted) } -func (a Alert) RenderDirection() string { +func (a alert) RenderDirection() string { if a.Status == "OK" { if a.Direction == "above" { return "<" @@ -175,7 +175,7 @@ func (a Alert) RenderDirection() string { } -func (a Alert) BootstrapStatus() string { +func (a alert) BootstrapStatus() string { if a.Status == "OK" { return "OK" } @@ -185,7 +185,7 @@ func (a Alert) BootstrapStatus() string { return "warning" } -func (a Alert) GlyphIcon() string { +func (a alert) GlyphIcon() string { if a.Type == "Notice" { return "glyphicon-info-sign" } @@ -193,23 +193,23 @@ func (a Alert) GlyphIcon() string { } -func (a *Alert) SendRecoveryMessage() { +func (a *alert) SendRecoveryMessage() { log.WithFields( log.Fields{ "name": a.Name, }, ).Debug("sending Recovery Message") - simpleSendMail(EmailFrom, + simpleSendMail(emailFrom, a.EmailTo, a.RecoveryEmailSubject(), a.RecoveryEmailBody()) } -func (a *Alert) RecoveryEmailSubject() string { +func (a *alert) RecoveryEmailSubject() string { return fmt.Sprintf("[RECOVERED] %s", a.Name) } -func (a *Alert) RecoveryEmailBody() string { +func (a *alert) RecoveryEmailBody() string { return fmt.Sprintf("%s [%s] has returned %s %f", a.Name, a.Metric, invertDirection(a.Direction), a.Threshold) } @@ -220,7 +220,7 @@ func invertDirection(d string) string { return "above" } -func (a *Alert) Throttled() bool { +func (a *alert) Throttled() bool { if a.Backoff == 0 { return false } @@ -229,50 +229,50 @@ func (a *Alert) Throttled() bool { return time.Now().Before(window) } -func (a *Alert) SendAlert() { +func (a *alert) SendAlert() { log.WithFields( log.Fields{ "name": a.Name, }, ).Debug("Sending Alert") - simpleSendMail(EmailFrom, + simpleSendMail(emailFrom, a.EmailTo, - a.AlertEmailSubject(), - a.AlertEmailBody()) + a.alertEmailSubject(), + a.alertEmailBody()) } -func (a *Alert) AlertEmailSubject() string { +func (a *alert) alertEmailSubject() string { if a.Type == "Alert" { return fmt.Sprintf("[ALERT] %s", a.Name) } return fmt.Sprintf("[NOTICE] %s", a.Name) } -func (a *Alert) IncludeRunBookLink() string { +func (a *alert) IncludeRunBookLink() string { if a.RunBookLink == "" { return "" } return fmt.Sprintf("\n\nRunbook link:\n%s\n", a.RunBookLink) } -func (a *Alert) AlertEmailBody() string { +func (a *alert) alertEmailBody() string { return fmt.Sprintf("%s [%s] has triggered an alert\nStatus:\t%s\nMessage:\t%s\n\nDaily Graph: <%s>\nWeekly Graph: <%s>%s\n", a.Name, a.Metric, a.Status, a.Message, a.DailyGraphURL(), a.WeeklyGraphURL(), a.IncludeRunBookLink()) } // did this alert just return to a healthy state? // returns 1 if just recovered, 0 otherwise -func (a *Alert) JustRecovered() bool { +func (a *alert) JustRecovered() bool { return a.PreviousStatus == "Failed" || a.PreviousStatus == "Error" } -func (a *Alert) SendRecoveryMessageIfNeeded(recoveriesSent int) { - if a.JustRecovered() && recoveriesSent < GlobalThrottle { +func (a *alert) SendRecoveryMessageIfNeeded(recoveriesSent int) { + if a.JustRecovered() && recoveriesSent < globalThrottle { a.SendRecoveryMessage() } } -func (a *Alert) UpdateState(recoveriesSent int) (int, int, int, int, int) { +func (a *alert) UpdateState(recoveriesSent int) (int, int, int, int, int) { successes := 0 errors := 0 failures := 0 @@ -301,7 +301,7 @@ func (a *Alert) UpdateState(recoveriesSent int) (int, int, int, int, int) { }, ).Debug("throttled") } else { - if a.Status == "Failed" && alertsSent < GlobalThrottle { + if a.Status == "Failed" && alertsSent < globalThrottle { a.SendAlert() alertsSent++ } @@ -314,7 +314,7 @@ func (a *Alert) UpdateState(recoveriesSent int) (int, int, int, int, int) { return successes, recoveriesSent, errors, failures, alertsSent } -func (a Alert) Hash() string { +func (a alert) Hash() string { h := sha1.New() io.WriteString(h, fmt.Sprintf("metric: %s", a.Metric)) io.WriteString(h, fmt.Sprintf("direction: %s", a.Direction)) @@ -350,10 +350,10 @@ func simpleSendMail(from, to, subject string, body string) error { message += fmt.Sprintf("%s: %s\r\n", k, v) } message += "\r\n" + base64.StdEncoding.EncodeToString([]byte(body)) - s := fmt.Sprintf("%s:%d", SMTPServer, SMTPPort) - auth := smtp.PlainAuth("", SMTPUser, SMTPPassword, SMTPServer) + s := fmt.Sprintf("%s:%d", smtpServer, smtpPort) + auth := smtp.PlainAuth("", smtpUser, smtpPassword, smtpServer) - if SMTPPort == 25 { + if smtpPort == 25 { err := SendMail(s, auth, from, []string{to}, []byte(message)) if err != nil { log.WithFields( @@ -367,7 +367,7 @@ func simpleSendMail(from, to, subject string, body string) error { } tlsconfig := &tls.Config{ InsecureSkipVerify: true, - ServerName: SMTPServer, + ServerName: smtpServer, } conn, err := tls.Dial("tcp", s, tlsconfig) @@ -376,7 +376,7 @@ func simpleSendMail(from, to, subject string, body string) error { return err } - c, err := smtp.NewClient(conn, SMTPServer) + c, err := smtp.NewClient(conn, smtpServer) if err != nil { log.WithFields(log.Fields{"err": err}).Error("smtp.NewClient failed") return err @@ -387,9 +387,9 @@ func simpleSendMail(from, to, subject string, body string) error { log.WithFields( log.Fields{ "err": err, - "SMTP_USER": SMTPUser, - "SMTP_PASSWORD": SMTPPassword, - "SMTP_SERVER": SMTPServer, + "SMTP_USER": smtpUser, + "SMTP_PASSWORD": smtpPassword, + "SMTP_SERVER": smtpServer, }).Error("auth failed") return err } diff --git a/alert_test.go b/alert_test.go index 4a9aa24..6473ef4 100644 --- a/alert_test.go +++ b/alert_test.go @@ -12,7 +12,7 @@ type DummyFetcher struct{} func (d DummyFetcher) Get(url string) (*http.Response, error) { return nil, nil } func Test_String(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") if a.String() != "OK\tfoo [foo]" { t.Error("wrong value") } @@ -24,53 +24,53 @@ func Test_String(t *testing.T) { } func Test_StringWhiteSpaceRemoval(t *testing.T) { - a := NewAlert("foo", " foo\n\n \t \r", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", " foo\n\n \t \r", "", 10, "above", DummyFetcher{}, "test@example.com", "") if a.Metric != "foo" { t.Error("whitespace not removed from metric") } } func Test_URL(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") - if a.URL() != "?target=keepLastValue(foo)&format=raw&from=-"+Window { + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + if a.URL() != "?target=keepLastValue(foo)&format=raw&from=-"+window { t.Error(fmt.Sprintf("wrong value: %s", a.URL())) } } func Test_DailyGraphURL(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") if a.DailyGraphURL() != "?target=foo&target=threshold(10.000000)&width=800&height=150&bgcolor=FFFFFF&fgcolor=000000&hideGrid=true&colorList=%23999999,%23006699&from=-24hours" { t.Error(fmt.Sprintf("wrong value: %s", a.DailyGraphURL())) } } func Test_WeeklyGraphURL(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") if a.WeeklyGraphURL() != "?target=foo&target=threshold(10.000000)&width=800&height=75&hideGrid=true&hideLegend=true&graphOnly=true&hideAxes=true&bgcolor=EEEEEE&fgcolor=000000&hideGrid=true&colorList=%23cccccc,%236699cc&from=-7days" { t.Error(fmt.Sprintf("wrong value: %s", a.WeeklyGraphURL())) } } func Test_RecoveryEmailSubject(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") if a.RecoveryEmailSubject() != "[RECOVERED] foo" { t.Error(fmt.Sprintf("wrong value: %s", a.RecoveryEmailSubject())) } } func Test_AlertEmailSubject(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") - if a.AlertEmailSubject() != "[ALERT] foo" { - t.Error(fmt.Sprintf("wrong value: %s", a.AlertEmailSubject())) + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + if a.alertEmailSubject() != "[ALERT] foo" { + t.Error(fmt.Sprintf("wrong value: %s", a.alertEmailSubject())) } a.Type = "Notice" - if a.AlertEmailSubject() != "[NOTICE] foo" { - t.Error(fmt.Sprintf("wrong value: %s", a.AlertEmailSubject())) + if a.alertEmailSubject() != "[NOTICE] foo" { + t.Error(fmt.Sprintf("wrong value: %s", a.alertEmailSubject())) } } func Test_GlyphIcon(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") if a.GlyphIcon() != "glyphicon-warning-sign" { t.Error(fmt.Sprintf("wrong value: %s", a.GlyphIcon())) } @@ -80,26 +80,26 @@ func Test_GlyphIcon(t *testing.T) { } } -func Test_AlertEmailBody(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") - if !strings.HasPrefix(a.AlertEmailBody(), "foo [foo] has triggered an alert") { - t.Error(fmt.Sprintf("wrong value: %s", a.AlertEmailBody())) +func Test_alertEmailBody(t *testing.T) { + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + if !strings.HasPrefix(a.alertEmailBody(), "foo [foo] has triggered an alert") { + t.Error(fmt.Sprintf("wrong value: %s", a.alertEmailBody())) } - a = NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "runbooklinkfoo") - if !strings.Contains(a.AlertEmailBody(), "runbooklinkfoo") { - t.Error(fmt.Sprintf("wrong value: %s", a.AlertEmailBody())) + a = newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "runbooklinkfoo") + if !strings.Contains(a.alertEmailBody(), "runbooklinkfoo") { + t.Error(fmt.Sprintf("wrong value: %s", a.alertEmailBody())) } } func Test_RecoveryEmailBody(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") if !strings.HasPrefix(a.RecoveryEmailBody(), "foo [foo] has returned below 10.000000") { t.Error(fmt.Sprintf("wrong value: %s", a.RecoveryEmailBody())) } } func Test_UpdateState(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") s, rs, e, f, as := a.UpdateState(0) if s != 1 { t.Error("s is wrong") @@ -176,7 +176,7 @@ func Test_extractLastValue(t *testing.T) { } func Test_UpdateStatus(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") a.UpdateStatus(11.0) if a.Status != "Failed" { t.Error("should've failed") @@ -197,23 +197,23 @@ func Test_UpdateStatus(t *testing.T) { } func Test_RenderDirection(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") if a.RenderDirection() != "<" { t.Error("(OK, above) expected <, got >") } - a = NewAlert("foo", "foo", "", 10, "below", DummyFetcher{}, "test@example.com", "") + a = newAlert("foo", "foo", "", 10, "below", DummyFetcher{}, "test@example.com", "") if a.RenderDirection() != ">" { t.Error("(OK, below) expected <, got >") } - a = NewAlert("foo", "foo", "", 10, "below", DummyFetcher{}, "test@example.com", "") + a = newAlert("foo", "foo", "", 10, "below", DummyFetcher{}, "test@example.com", "") a.Status = "Failed" if a.RenderDirection() != "<" { t.Error("(Failed, below) expected >, got <") } - a = NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a = newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") a.Status = "Failed" if a.RenderDirection() != ">" { t.Error("(Failed, above) expected <, got >") @@ -221,7 +221,7 @@ func Test_RenderDirection(t *testing.T) { } func Test_BootstrapStatus(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") if a.BootstrapStatus() != "OK" { t.Error("bootstrap status OK expected OK") } @@ -236,7 +236,7 @@ func Test_BootstrapStatus(t *testing.T) { } func Test_JustRecovered(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") a.PreviousStatus = "Failed" if !a.JustRecovered() { t.Error("JustRecovered expected true") @@ -252,7 +252,7 @@ func Test_JustRecovered(t *testing.T) { } func Test_Hash(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") expected := "22138d2e6b" result := a.Hash() if result != expected { diff --git a/alertscollection.go b/alertscollection.go index 7263d4f..f5598a4 100644 --- a/alertscollection.go +++ b/alertscollection.go @@ -9,53 +9,53 @@ import ( log "github.com/Sirupsen/logrus" ) -type PageResponse struct { +type pageResponse struct { GraphiteBase string MetricBase string - Alerts []*Alert + Alerts []*alert } -type IndivPageResponse struct { +type indivPageResponse struct { GraphiteBase string MetricBase string - Alert *Alert + Alert *alert } -type AlertsCollection struct { - Alerts []*Alert - AlertsByHash map[string]*Alert - Emailer Emailer +type alertsCollection struct { + alerts []*alert + alertsByHash map[string]*alert + emailer emailer } -func NewAlertsCollection(e Emailer) *AlertsCollection { - return &AlertsCollection{Emailer: e, AlertsByHash: make(map[string]*Alert)} +func newAlertsCollection(e emailer) *alertsCollection { + return &alertsCollection{emailer: e, alertsByHash: make(map[string]*alert)} } -func (ac *AlertsCollection) AddAlert(a *Alert) { - ac.Alerts = append(ac.Alerts, a) - ac.AlertsByHash[a.Hash()] = a +func (ac *alertsCollection) addAlert(a *alert) { + ac.alerts = append(ac.alerts, a) + ac.alertsByHash[a.Hash()] = a } -func (ac *AlertsCollection) ByHash(s string) *Alert { - return ac.AlertsByHash[s] +func (ac *alertsCollection) byHash(s string) *alert { + return ac.alertsByHash[s] } -func (ac *AlertsCollection) CheckAll() { - for _, a := range ac.Alerts { +func (ac *alertsCollection) checkAll() { + for _, a := range ac.alerts { a.CheckMetric() } } -func (ac *AlertsCollection) ProcessAll() { +func (ac *alertsCollection) processAll() { // fetch/calculate new status for all - ac.CheckAll() + ac.checkAll() alertsSent := 0 recoveriesSent := 0 errors := 0 failures := 0 successes := 0 - for _, a := range ac.Alerts { + for _, a := range ac.alerts { s, rs, e, f, as := a.UpdateState(recoveriesSent) successes = successes + s recoveriesSent = recoveriesSent + rs @@ -63,43 +63,43 @@ func (ac *AlertsCollection) ProcessAll() { failures = failures + f alertsSent = alertsSent + as } - if alertsSent >= GlobalThrottle { - ac.Emailer.Throttled(failures, GlobalThrottle, EmailTo) + if alertsSent >= globalThrottle { + ac.emailer.Throttled(failures, globalThrottle, emailTo) } - if recoveriesSent >= GlobalThrottle { - ac.Emailer.RecoveryThrottled(recoveriesSent, GlobalThrottle, EmailTo) + if recoveriesSent >= globalThrottle { + ac.emailer.RecoveryThrottled(recoveriesSent, globalThrottle, emailTo) } - ac.HandleErrors(errors) - LogToGraphite(alertsSent, recoveriesSent, failures, errors, successes) - ExposeVars(failures, errors, successes) + ac.handleErrors(errors) + logToGraphite(alertsSent, recoveriesSent, failures, errors, successes) + exposeVars(failures, errors, successes) } -func ExposeVars(failures, errors, successes int) { +func exposeVars(failures, errors, successes int) { expFailed.Set(int64(failures)) expErrors.Set(int64(errors)) expPassed.Set(int64(successes)) - expGlobalThrottle.Set(int64(GlobalThrottle)) - expGlobalBackoff.Set(int64(GlobalBackoff)) + expGlobalThrottle.Set(int64(globalThrottle)) + expGlobalBackoff.Set(int64(globalBackoff)) } -func (ac *AlertsCollection) HandleErrors(errors int) { +func (ac *alertsCollection) handleErrors(errors int) { if errors > 0 { - d := backoffTime(GlobalBackoff) - window := LastErrorEmail.Add(d) + d := backoffTime(globalBackoff) + window := lastErrorEmail.Add(d) if time.Now().After(window) { - ac.Emailer.EncounteredErrors(errors, EmailTo) - LastErrorEmail = time.Now() - GlobalBackoff = intmin(GlobalBackoff+1, len(backoffDurations)) + ac.emailer.EncounteredErrors(errors, emailTo) + lastErrorEmail = time.Now() + globalBackoff = intmin(globalBackoff+1, len(backoffDurations)) } } else { - GlobalBackoff = 0 + globalBackoff = 0 } } -func LogToGraphite(alertsSent, recoveriesSent, failures, errors, successes int) { +func logToGraphite(alertsSent, recoveriesSent, failures, errors, successes int) { var clientGraphite net.Conn - clientGraphite, err := net.Dial("tcp", CarbonBase) + clientGraphite, err := net.Dial("tcp", carbonBase) if err != nil || clientGraphite == nil { return } @@ -107,12 +107,12 @@ func LogToGraphite(alertsSent, recoveriesSent, failures, errors, successes int) now := int32(time.Now().Unix()) buffer := bytes.NewBufferString("") - fmt.Fprintf(buffer, "%salerts_sent %d %d\n", MetricBase, alertsSent, now) - fmt.Fprintf(buffer, "%srecoveries_sent %d %d\n", MetricBase, recoveriesSent, now) - fmt.Fprintf(buffer, "%sfailures %d %d\n", MetricBase, failures, now) - fmt.Fprintf(buffer, "%serrors %d %d\n", MetricBase, errors, now) - fmt.Fprintf(buffer, "%ssuccesses %d %d\n", MetricBase, successes, now) - fmt.Fprintf(buffer, "%sglobal_backoff %d %d\n", MetricBase, GlobalBackoff, now) + fmt.Fprintf(buffer, "%salerts_sent %d %d\n", metricBase, alertsSent, now) + fmt.Fprintf(buffer, "%srecoveries_sent %d %d\n", metricBase, recoveriesSent, now) + fmt.Fprintf(buffer, "%sfailures %d %d\n", metricBase, failures, now) + fmt.Fprintf(buffer, "%serrors %d %d\n", metricBase, errors, now) + fmt.Fprintf(buffer, "%ssuccesses %d %d\n", metricBase, successes, now) + fmt.Fprintf(buffer, "%sglobal_backoff %d %d\n", metricBase, globalBackoff, now) clientGraphite.Write(buffer.Bytes()) } @@ -123,31 +123,31 @@ func intmin(a, b int) int { return b } -func (ac *AlertsCollection) Run() { +func (ac *alertsCollection) Run() { for { - ac.ProcessAll() + ac.processAll() ac.DisplayAll() - time.Sleep(time.Duration(CheckInterval) * time.Minute) + time.Sleep(time.Duration(checkInterval) * time.Minute) } } -func (ac *AlertsCollection) DisplayAll() { - for _, a := range ac.Alerts { +func (ac *alertsCollection) DisplayAll() { + for _, a := range ac.alerts { log.Debug(a) } } -func (ac *AlertsCollection) MakePageResponse() PageResponse { - pr := PageResponse{GraphiteBase: GraphiteBase, - MetricBase: MetricBase} - for _, a := range ac.Alerts { +func (ac *alertsCollection) MakePageResponse() pageResponse { + pr := pageResponse{GraphiteBase: graphiteBase, + MetricBase: metricBase} + for _, a := range ac.alerts { pr.Alerts = append(pr.Alerts, a) } return pr } -func (ac *AlertsCollection) MakeIndivPageResponse(idx string) IndivPageResponse { - return IndivPageResponse{GraphiteBase: GraphiteBase, - MetricBase: MetricBase, - Alert: ac.ByHash(idx)} +func (ac *alertsCollection) MakeindivPageResponse(idx string) indivPageResponse { + return indivPageResponse{GraphiteBase: graphiteBase, + MetricBase: metricBase, + Alert: ac.byHash(idx)} } diff --git a/alertscollection_test.go b/alertscollection_test.go index 8982f8e..9e636d3 100644 --- a/alertscollection_test.go +++ b/alertscollection_test.go @@ -20,26 +20,26 @@ func (d DummyEmailer) RecoveryThrottled(recoveriesSent, globalThrottle int, emai func (d DummyEmailer) EncounteredErrors(errors int, emailTo string) {} func Test_emptyAlertsCollection(t *testing.T) { - ac := NewAlertsCollection(DummyEmailer{}) - ac.ProcessAll() + ac := newAlertsCollection(DummyEmailer{}) + ac.processAll() ac.DisplayAll() ac.MakePageResponse() } -func Test_HandleErrors(t *testing.T) { - ac := NewAlertsCollection(DummyEmailer{}) - ac.HandleErrors(1) +func Test_handleErrors(t *testing.T) { + ac := newAlertsCollection(DummyEmailer{}) + ac.handleErrors(1) } -func Test_AddAlert(t *testing.T) { - ac := NewAlertsCollection(DummyEmailer{}) - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") - ac.AddAlert(a) +func Test_addAlert(t *testing.T) { + ac := newAlertsCollection(DummyEmailer{}) + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + ac.addAlert(a) ac.DisplayAll() ac.MakePageResponse() - retrieved := ac.ByHash(a.Hash()) + retrieved := ac.byHash(a.Hash()) if retrieved != a { t.Error("failed to retrieve alert") } diff --git a/config.go b/config.go index 70595d8..f7bc4ee 100644 --- a/config.go +++ b/config.go @@ -1,6 +1,6 @@ package main -type AlertData struct { +type alertData struct { Name string Metric string Type string @@ -10,6 +10,6 @@ type AlertData struct { RunBookLink string } -type ConfigData struct { - Alerts []AlertData +type configData struct { + Alerts []alertData } diff --git a/emailer.go b/emailer.go index 95c3c8a..a7acef5 100644 --- a/emailer.go +++ b/emailer.go @@ -4,7 +4,7 @@ import ( "fmt" ) -type Emailer interface { +type emailer interface { EncounteredErrors(int, string) RecoveryThrottled(int, int, string) Throttled(int, int, string) @@ -14,7 +14,7 @@ type smtpEmailer struct{} func (e smtpEmailer) Throttled(failures, globalThrottle int, emailTo string) { simpleSendMail( - EmailFrom, + emailFrom, emailTo, "[ALERT] Hound is throttled", fmt.Sprintf("%d metrics were not OK.\nHound stopped sending messages after %d.\n"+ @@ -23,11 +23,11 @@ func (e smtpEmailer) Throttled(failures, globalThrottle int, emailTo string) { } func (e smtpEmailer) RecoveryThrottled(recoveriesSent, globalThrottle int, emailTo string) { - if !EmailOnError { + if !emailOnError { return } simpleSendMail( - EmailFrom, + emailFrom, emailTo, "[ALERT] Hound is recovered", fmt.Sprintf("%d metrics recovered.\nHound stopped sending individual messages after %d.\n", @@ -36,11 +36,11 @@ func (e smtpEmailer) RecoveryThrottled(recoveriesSent, globalThrottle int, email } func (e smtpEmailer) EncounteredErrors(errors int, emailTo string) { - if !EmailOnError { + if !emailOnError { return } simpleSendMail( - EmailFrom, + emailFrom, emailTo, "[ERROR] Hound encountered errors", fmt.Sprintf("%d metrics had errors. If this is more than a couple, it usually "+ diff --git a/hound.go b/hound.go index e1d60b5..f2299b6 100644 --- a/hound.go +++ b/hound.go @@ -16,21 +16,21 @@ import ( ) var ( - GraphiteBase string - CarbonBase string - MetricBase string - EmailFrom string - EmailTo string - CheckInterval int - GlobalThrottle int - GlobalBackoff int - LastErrorEmail time.Time - EmailOnError bool - SMTPServer string - SMTPPort int - SMTPUser string - SMTPPassword string - Window string + graphiteBase string + carbonBase string + metricBase string + emailFrom string + emailTo string + checkInterval int + globalThrottle int + globalBackoff int + lastErrorEmail time.Time + emailOnError bool + smtpServer string + smtpPort int + smtpUser string + smtpPassword string + window string ) var ( @@ -49,7 +49,7 @@ type config struct { EmailFrom string `envconfig:"EMAIL_FROM"` EmailTo string `envconfig:"EMAIL_TO"` CheckInterval int `envconfig:"CHECK_INTERVAL"` - GlobalThrottle int `envconfig:"GlobalThrottle"` + GlobalThrottle int `envconfig:"GLOBAL_THROTTLE"` HTTPPort string `envconfig:"HTTP_PORT"` TemplateFile string `envconfig:"TEMPLATE_FILE"` AlertTemplateFile string `envconfig:"ALERT_TEMPLATE_FILE"` @@ -76,7 +76,7 @@ func main() { log.Fatal(err) } - f := ConfigData{} + f := configData{} err = json.Unmarshal(file, &f) if err != nil { log.Fatal(err) @@ -104,20 +104,20 @@ func main() { log.Info("running on ", c.HTTPPort) // set global values - GraphiteBase = c.GraphiteBase - CarbonBase = c.CarbonBase - MetricBase = c.MetricBase - EmailFrom = c.EmailFrom - EmailTo = c.EmailTo - CheckInterval = c.CheckInterval - GlobalThrottle = c.GlobalThrottle - GlobalBackoff = 0 - EmailOnError = c.EmailOnError - SMTPServer = c.SMTPServer - SMTPPort = c.SMTPPort - SMTPUser = c.SMTPUser - SMTPPassword = c.SMTPPassword - Window = c.Window + graphiteBase = c.GraphiteBase + carbonBase = c.CarbonBase + metricBase = c.MetricBase + emailFrom = c.EmailFrom + emailTo = c.EmailTo + checkInterval = c.CheckInterval + globalThrottle = c.GlobalThrottle + globalBackoff = 0 + emailOnError = c.EmailOnError + smtpServer = c.SMTPServer + smtpPort = c.SMTPPort + smtpUser = c.SMTPUser + smtpPassword = c.SMTPPassword + window = c.Window // some defaults if c.ReadTimeout == 0 { @@ -126,11 +126,11 @@ func main() { if c.WriteTimeout == 0 { c.WriteTimeout = 10 } - if Window == "" { - Window = "10mins" + if window == "" { + window = "10mins" } - LastErrorEmail = time.Now() + lastErrorEmail = time.Now() go func() { // update uptime @@ -141,13 +141,13 @@ func main() { }() // initialize all the alerts - ac := NewAlertsCollection(smtpEmailer{}) + ac := newAlertsCollection(smtpEmailer{}) for _, a := range f.Alerts { emailTo := a.EmailTo if emailTo == "" { emailTo = c.EmailTo } - ac.AddAlert(NewAlert(a.Name, a.Metric, a.Type, a.Threshold, a.Direction, HTTPFetcher{}, emailTo, a.RunBookLink)) + ac.addAlert(newAlert(a.Name, a.Metric, a.Type, a.Threshold, a.Direction, httpFetcher{}, emailTo, a.RunBookLink)) } // kick it off in the background @@ -169,7 +169,7 @@ func main() { http.HandleFunc("/alert/", func(w http.ResponseWriter, r *http.Request) { stringIdx := strings.Split(r.URL.String(), "/")[2] - pr := ac.MakeIndivPageResponse(stringIdx) + pr := ac.MakeindivPageResponse(stringIdx) if c.AlertTemplateFile == "" { // default to same location as index.html