diff --git a/alert.go b/alert.go index d4cc3fb..0edc68d 100644 --- a/alert.go +++ b/alert.go @@ -19,7 +19,7 @@ import ( log "github.com/Sirupsen/logrus" ) -type Alert struct { +type alert struct { Name string Metric string Type string @@ -30,31 +30,31 @@ type Alert struct { Status string Message string PreviousStatus string - Fetcher Fetcher + fetcher fetcher EmailTo string Value float64 RunBookLink string } -var GRAPH_WIDTH = 800 -var DAILY_GRAPH_HEIGHT = 150 -var WEEKLY_GRAPH_HEIGHT = 75 -var FGCOLOR = "000000" -var DAILY_BGCOLOR = "FFFFFF" -var DAILY_COLORLIST = "%23999999,%23006699" -var WEEKLY_BGCOLOR = "EEEEEE" -var WEEKLY_COLORLIST = "%23cccccc,%236699cc" +var graphWidth = 800 +var dailyGraphHeight = 150 +var weeklyGraphHeight = 75 +var fgColor = "000000" +var dailyBgColor = "FFFFFF" +var dailyColorlist = "%23999999,%23006699" +var weeklyBgColor = "EEEEEE" +var weeklyColorlist = "%23cccccc,%236699cc" -func NewAlert(name string, metric string, atype string, threshold float64, - direction string, fetcher Fetcher, email_to string, runbook_link string) *Alert { +func newAlert(name string, metric string, atype string, threshold float64, + direction string, fetcher fetcher, emailTo string, runbookLink string) *alert { if atype == "" { atype = "Alert" } - return &Alert{Name: name, Type: atype, + return &alert{Name: name, Type: atype, Metric: cleanMetric(metric), Threshold: threshold, Direction: direction, Backoff: 0, LastAlerted: time.Now(), Status: "OK", Message: "", - PreviousStatus: "OK", Fetcher: fetcher, EmailTo: email_to, - Value: 0.0, RunBookLink: runbook_link, + PreviousStatus: "OK", fetcher: fetcher, EmailTo: emailTo, + Value: 0.0, RunBookLink: runbookLink, } } @@ -63,44 +63,44 @@ func cleanMetric(metric string) string { return re.ReplaceAllString(metric, "") } -func (a Alert) Url() string { - return GRAPHITE_BASE + "?target=keepLastValue(" + a.Metric + ")&format=raw&from=-" + WINDOW +func (a alert) URL() string { + return graphiteBase + "?target=keepLastValue(" + a.Metric + ")&format=raw&from=-" + window } -func (a Alert) DailyGraphUrl() string { - return GRAPHITE_BASE + "?target=" + +func (a alert) DailyGraphURL() string { + return graphiteBase + "?target=" + a.Metric + "&target=threshold(" + fmt.Sprintf("%f", a.Threshold) + - ")&width=" + fmt.Sprintf("%d", GRAPH_WIDTH) + - "&height=" + fmt.Sprintf("%d", DAILY_GRAPH_HEIGHT) + - "&bgcolor=" + DAILY_BGCOLOR + - "&fgcolor=" + FGCOLOR + "&hideGrid=true&colorList=" + - DAILY_COLORLIST + "&from=-24hours" + ")&width=" + fmt.Sprintf("%d", graphWidth) + + "&height=" + fmt.Sprintf("%d", dailyGraphHeight) + + "&bgcolor=" + dailyBgColor + + "&fgcolor=" + fgColor + "&hideGrid=true&colorList=" + + dailyColorlist + "&from=-24hours" } -func (a Alert) WeeklyGraphUrl() string { - return GRAPHITE_BASE + "?target=" + +func (a alert) WeeklyGraphURL() string { + return graphiteBase + "?target=" + a.Metric + "&target=threshold(" + fmt.Sprintf("%f", a.Threshold) + - ")&width=" + fmt.Sprintf("%d", GRAPH_WIDTH) + - "&height=" + fmt.Sprintf("%d", WEEKLY_GRAPH_HEIGHT) + + ")&width=" + fmt.Sprintf("%d", graphWidth) + + "&height=" + fmt.Sprintf("%d", weeklyGraphHeight) + "&hideGrid=true&hideLegend=true&graphOnly=true&hideAxes=true&bgcolor=" + - WEEKLY_BGCOLOR + "&fgcolor=" + FGCOLOR + - "&hideGrid=true&colorList=" + WEEKLY_COLORLIST + "&from=-7days" + weeklyBgColor + "&fgcolor=" + fgColor + + "&hideGrid=true&colorList=" + weeklyColorlist + "&from=-7days" } -type Fetcher interface { +type fetcher interface { Get(string) (*http.Response, error) } -type HTTPFetcher struct{} +type httpFetcher struct{} -func (h HTTPFetcher) Get(url string) (*http.Response, error) { +func (h httpFetcher) Get(url string) (*http.Response, error) { return http.Get(url) } -func (a *Alert) Fetch() (float64, error) { - resp, err := a.Fetcher.Get(a.Url()) +func (a *alert) Fetch() (float64, error) { + resp, err := a.fetcher.Get(a.URL()) if err != nil { a.Status = "Error" a.Message = "graphite request failed" @@ -121,7 +121,7 @@ func (a *Alert) Fetch() (float64, error) { return lv, err } -func (a *Alert) CheckMetric() bool { +func (a *alert) CheckMetric() bool { lv, err := a.Fetch() if err != nil { return false @@ -131,7 +131,7 @@ func (a *Alert) CheckMetric() bool { } -func (a *Alert) UpdateStatus(lv float64) { +func (a *alert) UpdateStatus(lv float64) { a.Value = lv if a.Direction == "above" { // pass if metric is below the threshold @@ -154,31 +154,28 @@ func (a *Alert) UpdateStatus(lv float64) { } } -func (a Alert) String() string { +func (a alert) String() string { if a.Status == "OK" { return fmt.Sprintf("%s\t%s [%s]", a.Status, a.Name, a.Metric) - } else { - return fmt.Sprintf("%s\t%s [%s]: %s (%s)", a.Status, a.Name, a.Metric, a.Message, a.LastAlerted) } + return fmt.Sprintf("%s\t%s [%s]: %s (%s)", a.Status, a.Name, a.Metric, a.Message, a.LastAlerted) } -func (a Alert) RenderDirection() string { +func (a alert) RenderDirection() string { if a.Status == "OK" { if a.Direction == "above" { return "<" - } else { - return ">" - } - } else { - if a.Direction == "above" { - return ">" - } else { - return "<" } + return ">" } + if a.Direction == "above" { + return ">" + } + return "<" + } -func (a Alert) BootstrapStatus() string { +func (a alert) BootstrapStatus() string { if a.Status == "OK" { return "OK" } @@ -188,106 +185,104 @@ func (a Alert) BootstrapStatus() string { return "warning" } -func (a Alert) GlyphIcon() string { +func (a alert) GlyphIcon() string { if a.Type == "Notice" { return "glyphicon-info-sign" - } else { - return "glyphicon-warning-sign" } + return "glyphicon-warning-sign" + } -func (a *Alert) SendRecoveryMessage() { +func (a *alert) SendRecoveryMessage() { log.WithFields( log.Fields{ "name": a.Name, }, ).Debug("sending Recovery Message") - simpleSendMail(EMAIL_FROM, + simpleSendMail(emailFrom, a.EmailTo, a.RecoveryEmailSubject(), a.RecoveryEmailBody()) } -func (a *Alert) RecoveryEmailSubject() string { +func (a *alert) RecoveryEmailSubject() string { return fmt.Sprintf("[RECOVERED] %s", a.Name) } -func (a *Alert) RecoveryEmailBody() string { +func (a *alert) RecoveryEmailBody() string { return fmt.Sprintf("%s [%s] has returned %s %f", a.Name, a.Metric, invertDirection(a.Direction), a.Threshold) } func invertDirection(d string) string { if d == "above" { return "below" - } else { - return "above" } + return "above" } -func (a *Alert) Throttled() bool { +func (a *alert) Throttled() bool { if a.Backoff == 0 { return false } - d := backoff_time(a.Backoff) + d := backoffTime(a.Backoff) window := a.LastAlerted.Add(d) return time.Now().Before(window) } -func (a *Alert) SendAlert() { +func (a *alert) SendAlert() { log.WithFields( log.Fields{ "name": a.Name, }, ).Debug("Sending Alert") - simpleSendMail(EMAIL_FROM, + simpleSendMail(emailFrom, a.EmailTo, - a.AlertEmailSubject(), - a.AlertEmailBody()) + a.alertEmailSubject(), + a.alertEmailBody()) } -func (a *Alert) AlertEmailSubject() string { +func (a *alert) alertEmailSubject() string { if a.Type == "Alert" { return fmt.Sprintf("[ALERT] %s", a.Name) - } else { - return fmt.Sprintf("[NOTICE] %s", a.Name) } + return fmt.Sprintf("[NOTICE] %s", a.Name) } -func (a *Alert) IncludeRunBookLink() string { +func (a *alert) IncludeRunBookLink() string { if a.RunBookLink == "" { return "" } return fmt.Sprintf("\n\nRunbook link:\n%s\n", a.RunBookLink) } -func (a *Alert) AlertEmailBody() string { +func (a *alert) alertEmailBody() string { return fmt.Sprintf("%s [%s] has triggered an alert\nStatus:\t%s\nMessage:\t%s\n\nDaily Graph: <%s>\nWeekly Graph: <%s>%s\n", - a.Name, a.Metric, a.Status, a.Message, a.DailyGraphUrl(), a.WeeklyGraphUrl(), a.IncludeRunBookLink()) + a.Name, a.Metric, a.Status, a.Message, a.DailyGraphURL(), a.WeeklyGraphURL(), a.IncludeRunBookLink()) } // did this alert just return to a healthy state? // returns 1 if just recovered, 0 otherwise -func (a *Alert) JustRecovered() bool { +func (a *alert) JustRecovered() bool { return a.PreviousStatus == "Failed" || a.PreviousStatus == "Error" } -func (a *Alert) SendRecoveryMessageIfNeeded(recoveries_sent int) { - if a.JustRecovered() && recoveries_sent < GLOBAL_THROTTLE { +func (a *alert) SendRecoveryMessageIfNeeded(recoveriesSent int) { + if a.JustRecovered() && recoveriesSent < globalThrottle { a.SendRecoveryMessage() } } -func (a *Alert) UpdateState(recoveries_sent int) (int, int, int, int, int) { +func (a *alert) UpdateState(recoveriesSent int) (int, int, int, int, int) { successes := 0 errors := 0 failures := 0 - alerts_sent := 0 + alertsSent := 0 if a.Status == "OK" { successes++ - a.SendRecoveryMessageIfNeeded(recoveries_sent) + a.SendRecoveryMessageIfNeeded(recoveriesSent) if a.JustRecovered() { - recoveries_sent++ + recoveriesSent++ } a.Backoff = 0 } else { @@ -302,24 +297,24 @@ func (a *Alert) UpdateState(recoveries_sent int) (int, int, int, int, int) { // wait for the throttling to expire log.WithFields( log.Fields{ - "recoveries_sent": recoveries_sent, + "recoveriesSent": recoveriesSent, }, ).Debug("throttled") } else { - if a.Status == "Failed" && alerts_sent < GLOBAL_THROTTLE { + if a.Status == "Failed" && alertsSent < globalThrottle { a.SendAlert() - alerts_sent++ + alertsSent++ } - a.Backoff = intmin(a.Backoff+1, len(BACKOFF_DURATIONS)) + a.Backoff = intmin(a.Backoff+1, len(backoffDurations)) a.LastAlerted = time.Now() } } // cycle the previous status a.PreviousStatus = a.Status - return successes, recoveries_sent, errors, failures, alerts_sent + return successes, recoveriesSent, errors, failures, alertsSent } -func (a Alert) Hash() string { +func (a alert) Hash() string { h := sha1.New() io.WriteString(h, fmt.Sprintf("metric: %s", a.Metric)) io.WriteString(h, fmt.Sprintf("direction: %s", a.Direction)) @@ -328,9 +323,9 @@ func (a Alert) Hash() string { return fmt.Sprintf("%x", h.Sum(nil))[0:10] } -func extractLastValue(raw_response string) (float64, error) { +func extractLastValue(rawResponse string) (float64, error) { // just take the most recent value - parts := strings.Split(strings.Trim(raw_response, "\n\t "), ",") + parts := strings.Split(strings.Trim(rawResponse, "\n\t "), ",") return strconv.ParseFloat(parts[len(parts)-1], 64) } @@ -355,10 +350,10 @@ func simpleSendMail(from, to, subject string, body string) error { message += fmt.Sprintf("%s: %s\r\n", k, v) } message += "\r\n" + base64.StdEncoding.EncodeToString([]byte(body)) - s := fmt.Sprintf("%s:%d", SMTP_SERVER, SMTP_PORT) - auth := smtp.PlainAuth("", SMTP_USER, SMTP_PASSWORD, SMTP_SERVER) + s := fmt.Sprintf("%s:%d", smtpServer, smtpPort) + auth := smtp.PlainAuth("", smtpUser, smtpPassword, smtpServer) - if SMTP_PORT == 25 { + if smtpPort == 25 { err := SendMail(s, auth, from, []string{to}, []byte(message)) if err != nil { log.WithFields( @@ -369,70 +364,69 @@ func simpleSendMail(from, to, subject string, body string) error { ).Error("error sending mail") } return err - } else { - tlsconfig := &tls.Config{ - InsecureSkipVerify: true, - ServerName: SMTP_SERVER, - } - - conn, err := tls.Dial("tcp", s, tlsconfig) - if err != nil { - log.WithFields(log.Fields{"err": err}).Error("tls.Dial failed") - return err - } + } + tlsconfig := &tls.Config{ + InsecureSkipVerify: true, + ServerName: smtpServer, + } - c, err := smtp.NewClient(conn, SMTP_SERVER) - if err != nil { - log.WithFields(log.Fields{"err": err}).Error("smtp.NewClient failed") - return err - } + conn, err := tls.Dial("tcp", s, tlsconfig) + if err != nil { + log.WithFields(log.Fields{"err": err}).Error("tls.Dial failed") + return err + } - // Auth - if err = c.Auth(auth); err != nil { - log.WithFields( - log.Fields{ - "err": err, - "SMTP_USER": SMTP_USER, - "SMTP_PASSWORD": SMTP_PASSWORD, - "SMTP_SERVER": SMTP_SERVER, - }).Error("auth failed") - return err - } + c, err := smtp.NewClient(conn, smtpServer) + if err != nil { + log.WithFields(log.Fields{"err": err}).Error("smtp.NewClient failed") + return err + } - // To && From - if err = c.Mail(from); err != nil { - log.WithFields(log.Fields{"err": err, "from": from}).Error("from address failed") - return err - } + // Auth + if err = c.Auth(auth); err != nil { + log.WithFields( + log.Fields{ + "err": err, + "SMTP_USER": smtpUser, + "SMTP_PASSWORD": smtpPassword, + "SMTP_SERVER": smtpServer, + }).Error("auth failed") + return err + } - if err = c.Rcpt(to); err != nil { - log.WithFields(log.Fields{"err": err}).Error("to address failed") - return err - } + // To && From + if err = c.Mail(from); err != nil { + log.WithFields(log.Fields{"err": err, "from": from}).Error("from address failed") + return err + } - // Data - w, err := c.Data() - if err != nil { - log.WithFields(log.Fields{"err": err}).Error("smtp Data() failed") - return err - } + if err = c.Rcpt(to); err != nil { + log.WithFields(log.Fields{"err": err}).Error("to address failed") + return err + } - _, err = w.Write([]byte(message)) - if err != nil { - log.WithFields(log.Fields{"err": err}).Error("smtp Write failed") - return err - } + // Data + w, err := c.Data() + if err != nil { + log.WithFields(log.Fields{"err": err}).Error("smtp Data() failed") + return err + } - err = w.Close() - if err != nil { - log.WithFields(log.Fields{"err": err}).Error("smtp close failed") - return err - } + _, err = w.Write([]byte(message)) + if err != nil { + log.WithFields(log.Fields{"err": err}).Error("smtp Write failed") + return err + } - c.Quit() + err = w.Close() + if err != nil { + log.WithFields(log.Fields{"err": err}).Error("smtp close failed") return err } + c.Quit() + return err + } func encodeRFC2047(String string) string { @@ -441,7 +435,7 @@ func encodeRFC2047(String string) string { return strings.Trim(addr.String(), " <>") } -var BACKOFF_DURATIONS = []time.Duration{ +var backoffDurations = []time.Duration{ time.Duration(5) * time.Minute, time.Duration(30) * time.Minute, time.Duration(1) * time.Hour, @@ -451,6 +445,6 @@ var BACKOFF_DURATIONS = []time.Duration{ time.Duration(24) * time.Hour, } -func backoff_time(level int) time.Duration { - return BACKOFF_DURATIONS[level] +func backoffTime(level int) time.Duration { + return backoffDurations[level] } diff --git a/alert.html b/alert.html index 98a0322..1c433c3 100644 --- a/alert.html +++ b/alert.html @@ -37,10 +37,10 @@

Hound:

Daily Graph

- +

Weekly Graph

- + {{ if $element.RunBookLink }} diff --git a/alert_test.go b/alert_test.go index 5e7ee80..6473ef4 100644 --- a/alert_test.go +++ b/alert_test.go @@ -12,7 +12,7 @@ type DummyFetcher struct{} func (d DummyFetcher) Get(url string) (*http.Response, error) { return nil, nil } func Test_String(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") if a.String() != "OK\tfoo [foo]" { t.Error("wrong value") } @@ -24,53 +24,53 @@ func Test_String(t *testing.T) { } func Test_StringWhiteSpaceRemoval(t *testing.T) { - a := NewAlert("foo", " foo\n\n \t \r", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", " foo\n\n \t \r", "", 10, "above", DummyFetcher{}, "test@example.com", "") if a.Metric != "foo" { t.Error("whitespace not removed from metric") } } -func Test_Url(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") - if a.Url() != "?target=keepLastValue(foo)&format=raw&from=-"+WINDOW { - t.Error(fmt.Sprintf("wrong value: %s", a.Url())) +func Test_URL(t *testing.T) { + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + if a.URL() != "?target=keepLastValue(foo)&format=raw&from=-"+window { + t.Error(fmt.Sprintf("wrong value: %s", a.URL())) } } -func Test_DailyGraphUrl(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") - if a.DailyGraphUrl() != "?target=foo&target=threshold(10.000000)&width=800&height=150&bgcolor=FFFFFF&fgcolor=000000&hideGrid=true&colorList=%23999999,%23006699&from=-24hours" { - t.Error(fmt.Sprintf("wrong value: %s", a.DailyGraphUrl())) +func Test_DailyGraphURL(t *testing.T) { + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + if a.DailyGraphURL() != "?target=foo&target=threshold(10.000000)&width=800&height=150&bgcolor=FFFFFF&fgcolor=000000&hideGrid=true&colorList=%23999999,%23006699&from=-24hours" { + t.Error(fmt.Sprintf("wrong value: %s", a.DailyGraphURL())) } } -func Test_WeeklyGraphUrl(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") - if a.WeeklyGraphUrl() != "?target=foo&target=threshold(10.000000)&width=800&height=75&hideGrid=true&hideLegend=true&graphOnly=true&hideAxes=true&bgcolor=EEEEEE&fgcolor=000000&hideGrid=true&colorList=%23cccccc,%236699cc&from=-7days" { - t.Error(fmt.Sprintf("wrong value: %s", a.WeeklyGraphUrl())) +func Test_WeeklyGraphURL(t *testing.T) { + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + if a.WeeklyGraphURL() != "?target=foo&target=threshold(10.000000)&width=800&height=75&hideGrid=true&hideLegend=true&graphOnly=true&hideAxes=true&bgcolor=EEEEEE&fgcolor=000000&hideGrid=true&colorList=%23cccccc,%236699cc&from=-7days" { + t.Error(fmt.Sprintf("wrong value: %s", a.WeeklyGraphURL())) } } func Test_RecoveryEmailSubject(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") if a.RecoveryEmailSubject() != "[RECOVERED] foo" { t.Error(fmt.Sprintf("wrong value: %s", a.RecoveryEmailSubject())) } } func Test_AlertEmailSubject(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") - if a.AlertEmailSubject() != "[ALERT] foo" { - t.Error(fmt.Sprintf("wrong value: %s", a.AlertEmailSubject())) + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + if a.alertEmailSubject() != "[ALERT] foo" { + t.Error(fmt.Sprintf("wrong value: %s", a.alertEmailSubject())) } a.Type = "Notice" - if a.AlertEmailSubject() != "[NOTICE] foo" { - t.Error(fmt.Sprintf("wrong value: %s", a.AlertEmailSubject())) + if a.alertEmailSubject() != "[NOTICE] foo" { + t.Error(fmt.Sprintf("wrong value: %s", a.alertEmailSubject())) } } func Test_GlyphIcon(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") if a.GlyphIcon() != "glyphicon-warning-sign" { t.Error(fmt.Sprintf("wrong value: %s", a.GlyphIcon())) } @@ -80,26 +80,26 @@ func Test_GlyphIcon(t *testing.T) { } } -func Test_AlertEmailBody(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") - if !strings.HasPrefix(a.AlertEmailBody(), "foo [foo] has triggered an alert") { - t.Error(fmt.Sprintf("wrong value: %s", a.AlertEmailBody())) +func Test_alertEmailBody(t *testing.T) { + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + if !strings.HasPrefix(a.alertEmailBody(), "foo [foo] has triggered an alert") { + t.Error(fmt.Sprintf("wrong value: %s", a.alertEmailBody())) } - a = NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "runbooklinkfoo") - if !strings.Contains(a.AlertEmailBody(), "runbooklinkfoo") { - t.Error(fmt.Sprintf("wrong value: %s", a.AlertEmailBody())) + a = newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "runbooklinkfoo") + if !strings.Contains(a.alertEmailBody(), "runbooklinkfoo") { + t.Error(fmt.Sprintf("wrong value: %s", a.alertEmailBody())) } } func Test_RecoveryEmailBody(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") if !strings.HasPrefix(a.RecoveryEmailBody(), "foo [foo] has returned below 10.000000") { t.Error(fmt.Sprintf("wrong value: %s", a.RecoveryEmailBody())) } } func Test_UpdateState(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") s, rs, e, f, as := a.UpdateState(0) if s != 1 { t.Error("s is wrong") @@ -176,7 +176,7 @@ func Test_extractLastValue(t *testing.T) { } func Test_UpdateStatus(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") a.UpdateStatus(11.0) if a.Status != "Failed" { t.Error("should've failed") @@ -197,23 +197,23 @@ func Test_UpdateStatus(t *testing.T) { } func Test_RenderDirection(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") if a.RenderDirection() != "<" { t.Error("(OK, above) expected <, got >") } - a = NewAlert("foo", "foo", "", 10, "below", DummyFetcher{}, "test@example.com", "") + a = newAlert("foo", "foo", "", 10, "below", DummyFetcher{}, "test@example.com", "") if a.RenderDirection() != ">" { t.Error("(OK, below) expected <, got >") } - a = NewAlert("foo", "foo", "", 10, "below", DummyFetcher{}, "test@example.com", "") + a = newAlert("foo", "foo", "", 10, "below", DummyFetcher{}, "test@example.com", "") a.Status = "Failed" if a.RenderDirection() != "<" { t.Error("(Failed, below) expected >, got <") } - a = NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a = newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") a.Status = "Failed" if a.RenderDirection() != ">" { t.Error("(Failed, above) expected <, got >") @@ -221,7 +221,7 @@ func Test_RenderDirection(t *testing.T) { } func Test_BootstrapStatus(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") if a.BootstrapStatus() != "OK" { t.Error("bootstrap status OK expected OK") } @@ -236,7 +236,7 @@ func Test_BootstrapStatus(t *testing.T) { } func Test_JustRecovered(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") a.PreviousStatus = "Failed" if !a.JustRecovered() { t.Error("JustRecovered expected true") @@ -252,7 +252,7 @@ func Test_JustRecovered(t *testing.T) { } func Test_Hash(t *testing.T) { - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") expected := "22138d2e6b" result := a.Hash() if result != expected { diff --git a/alertscollection.go b/alertscollection.go index 9bf9c99..f5598a4 100644 --- a/alertscollection.go +++ b/alertscollection.go @@ -9,97 +9,97 @@ import ( log "github.com/Sirupsen/logrus" ) -type PageResponse struct { +type pageResponse struct { GraphiteBase string MetricBase string - Alerts []*Alert + Alerts []*alert } -type IndivPageResponse struct { +type indivPageResponse struct { GraphiteBase string MetricBase string - Alert *Alert + Alert *alert } -type AlertsCollection struct { - Alerts []*Alert - AlertsByHash map[string]*Alert - Emailer Emailer +type alertsCollection struct { + alerts []*alert + alertsByHash map[string]*alert + emailer emailer } -func NewAlertsCollection(e Emailer) *AlertsCollection { - return &AlertsCollection{Emailer: e, AlertsByHash: make(map[string]*Alert)} +func newAlertsCollection(e emailer) *alertsCollection { + return &alertsCollection{emailer: e, alertsByHash: make(map[string]*alert)} } -func (ac *AlertsCollection) AddAlert(a *Alert) { - ac.Alerts = append(ac.Alerts, a) - ac.AlertsByHash[a.Hash()] = a +func (ac *alertsCollection) addAlert(a *alert) { + ac.alerts = append(ac.alerts, a) + ac.alertsByHash[a.Hash()] = a } -func (ac *AlertsCollection) ByHash(s string) *Alert { - return ac.AlertsByHash[s] +func (ac *alertsCollection) byHash(s string) *alert { + return ac.alertsByHash[s] } -func (ac *AlertsCollection) CheckAll() { - for _, a := range ac.Alerts { +func (ac *alertsCollection) checkAll() { + for _, a := range ac.alerts { a.CheckMetric() } } -func (ac *AlertsCollection) ProcessAll() { +func (ac *alertsCollection) processAll() { // fetch/calculate new status for all - ac.CheckAll() - alerts_sent := 0 - recoveries_sent := 0 + ac.checkAll() + alertsSent := 0 + recoveriesSent := 0 errors := 0 failures := 0 successes := 0 - for _, a := range ac.Alerts { - s, rs, e, f, as := a.UpdateState(recoveries_sent) + for _, a := range ac.alerts { + s, rs, e, f, as := a.UpdateState(recoveriesSent) successes = successes + s - recoveries_sent = recoveries_sent + rs + recoveriesSent = recoveriesSent + rs errors = errors + e failures = failures + f - alerts_sent = alerts_sent + as + alertsSent = alertsSent + as } - if alerts_sent >= GLOBAL_THROTTLE { - ac.Emailer.Throttled(failures, GLOBAL_THROTTLE, EMAIL_TO) + if alertsSent >= globalThrottle { + ac.emailer.Throttled(failures, globalThrottle, emailTo) } - if recoveries_sent >= GLOBAL_THROTTLE { - ac.Emailer.RecoveryThrottled(recoveries_sent, GLOBAL_THROTTLE, EMAIL_TO) + if recoveriesSent >= globalThrottle { + ac.emailer.RecoveryThrottled(recoveriesSent, globalThrottle, emailTo) } - ac.HandleErrors(errors) - LogToGraphite(alerts_sent, recoveries_sent, failures, errors, successes) - ExposeVars(failures, errors, successes) + ac.handleErrors(errors) + logToGraphite(alertsSent, recoveriesSent, failures, errors, successes) + exposeVars(failures, errors, successes) } -func ExposeVars(failures, errors, successes int) { - EXP_FAILED.Set(int64(failures)) - EXP_ERRORS.Set(int64(errors)) - EXP_PASSED.Set(int64(successes)) - EXP_GLOBAL_THROTTLE.Set(int64(GLOBAL_THROTTLE)) - EXP_GLOBAL_BACKOFF.Set(int64(GLOBAL_BACKOFF)) +func exposeVars(failures, errors, successes int) { + expFailed.Set(int64(failures)) + expErrors.Set(int64(errors)) + expPassed.Set(int64(successes)) + expGlobalThrottle.Set(int64(globalThrottle)) + expGlobalBackoff.Set(int64(globalBackoff)) } -func (ac *AlertsCollection) HandleErrors(errors int) { +func (ac *alertsCollection) handleErrors(errors int) { if errors > 0 { - d := backoff_time(GLOBAL_BACKOFF) - window := LAST_ERROR_EMAIL.Add(d) + d := backoffTime(globalBackoff) + window := lastErrorEmail.Add(d) if time.Now().After(window) { - ac.Emailer.EncounteredErrors(errors, EMAIL_TO) - LAST_ERROR_EMAIL = time.Now() - GLOBAL_BACKOFF = intmin(GLOBAL_BACKOFF+1, len(BACKOFF_DURATIONS)) + ac.emailer.EncounteredErrors(errors, emailTo) + lastErrorEmail = time.Now() + globalBackoff = intmin(globalBackoff+1, len(backoffDurations)) } } else { - GLOBAL_BACKOFF = 0 + globalBackoff = 0 } } -func LogToGraphite(alerts_sent, recoveries_sent, failures, errors, successes int) { +func logToGraphite(alertsSent, recoveriesSent, failures, errors, successes int) { var clientGraphite net.Conn - clientGraphite, err := net.Dial("tcp", CARBON_BASE) + clientGraphite, err := net.Dial("tcp", carbonBase) if err != nil || clientGraphite == nil { return } @@ -107,12 +107,12 @@ func LogToGraphite(alerts_sent, recoveries_sent, failures, errors, successes int now := int32(time.Now().Unix()) buffer := bytes.NewBufferString("") - fmt.Fprintf(buffer, "%salerts_sent %d %d\n", METRIC_BASE, alerts_sent, now) - fmt.Fprintf(buffer, "%srecoveries_sent %d %d\n", METRIC_BASE, recoveries_sent, now) - fmt.Fprintf(buffer, "%sfailures %d %d\n", METRIC_BASE, failures, now) - fmt.Fprintf(buffer, "%serrors %d %d\n", METRIC_BASE, errors, now) - fmt.Fprintf(buffer, "%ssuccesses %d %d\n", METRIC_BASE, successes, now) - fmt.Fprintf(buffer, "%sglobal_backoff %d %d\n", METRIC_BASE, GLOBAL_BACKOFF, now) + fmt.Fprintf(buffer, "%salerts_sent %d %d\n", metricBase, alertsSent, now) + fmt.Fprintf(buffer, "%srecoveries_sent %d %d\n", metricBase, recoveriesSent, now) + fmt.Fprintf(buffer, "%sfailures %d %d\n", metricBase, failures, now) + fmt.Fprintf(buffer, "%serrors %d %d\n", metricBase, errors, now) + fmt.Fprintf(buffer, "%ssuccesses %d %d\n", metricBase, successes, now) + fmt.Fprintf(buffer, "%sglobal_backoff %d %d\n", metricBase, globalBackoff, now) clientGraphite.Write(buffer.Bytes()) } @@ -123,31 +123,31 @@ func intmin(a, b int) int { return b } -func (ac *AlertsCollection) Run() { +func (ac *alertsCollection) Run() { for { - ac.ProcessAll() + ac.processAll() ac.DisplayAll() - time.Sleep(time.Duration(CHECK_INTERVAL) * time.Minute) + time.Sleep(time.Duration(checkInterval) * time.Minute) } } -func (ac *AlertsCollection) DisplayAll() { - for _, a := range ac.Alerts { +func (ac *alertsCollection) DisplayAll() { + for _, a := range ac.alerts { log.Debug(a) } } -func (ac *AlertsCollection) MakePageResponse() PageResponse { - pr := PageResponse{GraphiteBase: GRAPHITE_BASE, - MetricBase: METRIC_BASE} - for _, a := range ac.Alerts { +func (ac *alertsCollection) MakePageResponse() pageResponse { + pr := pageResponse{GraphiteBase: graphiteBase, + MetricBase: metricBase} + for _, a := range ac.alerts { pr.Alerts = append(pr.Alerts, a) } return pr } -func (ac *AlertsCollection) MakeIndivPageResponse(idx string) IndivPageResponse { - return IndivPageResponse{GraphiteBase: GRAPHITE_BASE, - MetricBase: METRIC_BASE, - Alert: ac.ByHash(idx)} +func (ac *alertsCollection) MakeindivPageResponse(idx string) indivPageResponse { + return indivPageResponse{GraphiteBase: graphiteBase, + MetricBase: metricBase, + Alert: ac.byHash(idx)} } diff --git a/alertscollection_test.go b/alertscollection_test.go index f7f914a..9e636d3 100644 --- a/alertscollection_test.go +++ b/alertscollection_test.go @@ -15,31 +15,31 @@ func Test_intmin(t *testing.T) { type DummyEmailer struct{} -func (d DummyEmailer) Throttled(failures, global_throttle int, email_to string) {} -func (d DummyEmailer) RecoveryThrottled(recoveries_sent, global_throttle int, email_to string) {} -func (d DummyEmailer) EncounteredErrors(errors int, email_to string) {} +func (d DummyEmailer) Throttled(failures, globalThrottle int, emailTo string) {} +func (d DummyEmailer) RecoveryThrottled(recoveriesSent, globalThrottle int, emailTo string) {} +func (d DummyEmailer) EncounteredErrors(errors int, emailTo string) {} func Test_emptyAlertsCollection(t *testing.T) { - ac := NewAlertsCollection(DummyEmailer{}) - ac.ProcessAll() + ac := newAlertsCollection(DummyEmailer{}) + ac.processAll() ac.DisplayAll() ac.MakePageResponse() } -func Test_HandleErrors(t *testing.T) { - ac := NewAlertsCollection(DummyEmailer{}) - ac.HandleErrors(1) +func Test_handleErrors(t *testing.T) { + ac := newAlertsCollection(DummyEmailer{}) + ac.handleErrors(1) } -func Test_AddAlert(t *testing.T) { - ac := NewAlertsCollection(DummyEmailer{}) - a := NewAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") - ac.AddAlert(a) +func Test_addAlert(t *testing.T) { + ac := newAlertsCollection(DummyEmailer{}) + a := newAlert("foo", "foo", "", 10, "above", DummyFetcher{}, "test@example.com", "") + ac.addAlert(a) ac.DisplayAll() ac.MakePageResponse() - retrieved := ac.ByHash(a.Hash()) + retrieved := ac.byHash(a.Hash()) if retrieved != a { t.Error("failed to retrieve alert") } diff --git a/config.go b/config.go index 70595d8..f7bc4ee 100644 --- a/config.go +++ b/config.go @@ -1,6 +1,6 @@ package main -type AlertData struct { +type alertData struct { Name string Metric string Type string @@ -10,6 +10,6 @@ type AlertData struct { RunBookLink string } -type ConfigData struct { - Alerts []AlertData +type configData struct { + Alerts []alertData } diff --git a/emailer.go b/emailer.go index 42ba99d..a7acef5 100644 --- a/emailer.go +++ b/emailer.go @@ -4,44 +4,44 @@ import ( "fmt" ) -type Emailer interface { +type emailer interface { EncounteredErrors(int, string) RecoveryThrottled(int, int, string) Throttled(int, int, string) } -type SMTPEmailer struct{} +type smtpEmailer struct{} -func (e SMTPEmailer) Throttled(failures, global_throttle int, email_to string) { +func (e smtpEmailer) Throttled(failures, globalThrottle int, emailTo string) { simpleSendMail( - EMAIL_FROM, - email_to, + emailFrom, + emailTo, "[ALERT] Hound is throttled", fmt.Sprintf("%d metrics were not OK.\nHound stopped sending messages after %d.\n"+ "This probably indicates an infrastructure problem (network, graphite, etc)", failures, - global_throttle)) + globalThrottle)) } -func (e SMTPEmailer) RecoveryThrottled(recoveries_sent, global_throttle int, email_to string) { - if !EMAIL_ON_ERROR { +func (e smtpEmailer) RecoveryThrottled(recoveriesSent, globalThrottle int, emailTo string) { + if !emailOnError { return } simpleSendMail( - EMAIL_FROM, - email_to, + emailFrom, + emailTo, "[ALERT] Hound is recovered", fmt.Sprintf("%d metrics recovered.\nHound stopped sending individual messages after %d.\n", - recoveries_sent, - global_throttle)) + recoveriesSent, + globalThrottle)) } -func (e SMTPEmailer) EncounteredErrors(errors int, email_to string) { - if !EMAIL_ON_ERROR { +func (e smtpEmailer) EncounteredErrors(errors int, emailTo string) { + if !emailOnError { return } simpleSendMail( - EMAIL_FROM, - email_to, + emailFrom, + emailTo, "[ERROR] Hound encountered errors", fmt.Sprintf("%d metrics had errors. If this is more than a couple, it usually "+ "means that Graphite has fallen behind. It doesn't necessarily mean "+ diff --git a/hound.go b/hound.go index 5f87228..f2299b6 100644 --- a/hound.go +++ b/hound.go @@ -16,30 +16,30 @@ import ( ) var ( - GRAPHITE_BASE string - CARBON_BASE string - METRIC_BASE string - EMAIL_FROM string - EMAIL_TO string - CHECK_INTERVAL int - GLOBAL_THROTTLE int - GLOBAL_BACKOFF int - LAST_ERROR_EMAIL time.Time - EMAIL_ON_ERROR bool - SMTP_SERVER string - SMTP_PORT int - SMTP_USER string - SMTP_PASSWORD string - WINDOW string + graphiteBase string + carbonBase string + metricBase string + emailFrom string + emailTo string + checkInterval int + globalThrottle int + globalBackoff int + lastErrorEmail time.Time + emailOnError bool + smtpServer string + smtpPort int + smtpUser string + smtpPassword string + window string ) var ( - EXP_FAILED = expvar.NewInt("failed") - EXP_PASSED = expvar.NewInt("passed") - EXP_ERRORS = expvar.NewInt("errors") - EXP_GLOBAL_THROTTLE = expvar.NewInt("throttle") - EXP_GLOBAL_BACKOFF = expvar.NewInt("backoff") - EXP_UPTIME = expvar.NewInt("uptime") + expFailed = expvar.NewInt("failed") + expPassed = expvar.NewInt("passed") + expErrors = expvar.NewInt("errors") + expGlobalThrottle = expvar.NewInt("throttle") + expGlobalBackoff = expvar.NewInt("backoff") + expUptime = expvar.NewInt("uptime") ) type config struct { @@ -76,7 +76,7 @@ func main() { log.Fatal(err) } - f := ConfigData{} + f := configData{} err = json.Unmarshal(file, &f) if err != nil { log.Fatal(err) @@ -104,20 +104,20 @@ func main() { log.Info("running on ", c.HTTPPort) // set global values - GRAPHITE_BASE = c.GraphiteBase - CARBON_BASE = c.CarbonBase - METRIC_BASE = c.MetricBase - EMAIL_FROM = c.EmailFrom - EMAIL_TO = c.EmailTo - CHECK_INTERVAL = c.CheckInterval - GLOBAL_THROTTLE = c.GlobalThrottle - GLOBAL_BACKOFF = 0 - EMAIL_ON_ERROR = c.EmailOnError - SMTP_SERVER = c.SMTPServer - SMTP_PORT = c.SMTPPort - SMTP_USER = c.SMTPUser - SMTP_PASSWORD = c.SMTPPassword - WINDOW = c.Window + graphiteBase = c.GraphiteBase + carbonBase = c.CarbonBase + metricBase = c.MetricBase + emailFrom = c.EmailFrom + emailTo = c.EmailTo + checkInterval = c.CheckInterval + globalThrottle = c.GlobalThrottle + globalBackoff = 0 + emailOnError = c.EmailOnError + smtpServer = c.SMTPServer + smtpPort = c.SMTPPort + smtpUser = c.SMTPUser + smtpPassword = c.SMTPPassword + window = c.Window // some defaults if c.ReadTimeout == 0 { @@ -126,28 +126,28 @@ func main() { if c.WriteTimeout == 0 { c.WriteTimeout = 10 } - if WINDOW == "" { - WINDOW = "10mins" + if window == "" { + window = "10mins" } - LAST_ERROR_EMAIL = time.Now() + lastErrorEmail = time.Now() go func() { // update uptime for { time.Sleep(1 * time.Second) - EXP_UPTIME.Add(1) + expUptime.Add(1) } }() // initialize all the alerts - ac := NewAlertsCollection(SMTPEmailer{}) + ac := newAlertsCollection(smtpEmailer{}) for _, a := range f.Alerts { - email_to := a.EmailTo - if email_to == "" { - email_to = c.EmailTo + emailTo := a.EmailTo + if emailTo == "" { + emailTo = c.EmailTo } - ac.AddAlert(NewAlert(a.Name, a.Metric, a.Type, a.Threshold, a.Direction, HTTPFetcher{}, email_to, a.RunBookLink)) + ac.addAlert(newAlert(a.Name, a.Metric, a.Type, a.Threshold, a.Direction, httpFetcher{}, emailTo, a.RunBookLink)) } // kick it off in the background @@ -169,7 +169,7 @@ func main() { http.HandleFunc("/alert/", func(w http.ResponseWriter, r *http.Request) { stringIdx := strings.Split(r.URL.String(), "/")[2] - pr := ac.MakeIndivPageResponse(stringIdx) + pr := ac.MakeindivPageResponse(stringIdx) if c.AlertTemplateFile == "" { // default to same location as index.html