Skip to content

Commit

Permalink
Fix #19: add formatting and subjects to emails
Browse files Browse the repository at this point in the history
- the underlying alert struct needs to be refactored to store the raw data and let the
alerters figure out how to do the formatting.

- until it can be refactored, simple message parsing and formatting has been added to
emails.

- Fixed a few errant typos/idioms
  • Loading branch information
jeffwillette committed Nov 1, 2017
1 parent 8e7cd73 commit 9bf3a7c
Show file tree
Hide file tree
Showing 6 changed files with 75 additions and 34 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ background process as per your operating system.

### As A Systemd Service (for Linux systems with systemd)

If you have a systemd based system then you can refer to [docker-alertd.service.example](https://github.com/deltaskelta/docker-alertd/blob/master/docker-alertd.service.example)
If you have a systemd based system then you can refer to [docker-alertd.service.example](https://github.com/deltaskelta/docker-alertd/blob/master/docker-alertd.service.example)
the example systemd service file and this [tutorial](https://www.digitalocean.com/community/tutorials/how-to-use-systemctl-to-manage-systemd-services-and-units)

### With Launchd (MacOS)
Expand Down
26 changes: 14 additions & 12 deletions cmd/alertd_container.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ type AlertdContainer struct {
func (c *AlertdContainer) CheckMetrics(s *types.Stats, e error) {
switch {
case e != nil:
c.Alert.Add(e, nil, "Received an unknown error")
c.Alert.Add(e, nil, "Received an unknown error", "")
default:
if c.CPUCheck.Limit != nil {
c.CheckCPUUsage(s)
Expand Down Expand Up @@ -124,17 +124,17 @@ func (c *AlertdContainer) CheckExists(e error) {
switch {
case c.IsUnknown(e) && !c.ExistenceCheck.AlertActive:
// if the alert is not active I need to alert and make it active
c.Alert.Add(e, ErrExistCheckFail, fmt.Sprintf("%s", c.Name))
c.Alert.Add(e, ErrExistCheckFail, fmt.Sprintf("%s", c.Name), ErrExistCheckFail.Error())
c.ExistenceCheck.ToggleAlertActive()

case c.IsUnknown(e) && c.ExistenceCheck.AlertActive:
// do nothing
case c.HasErrored(e):
// if there is some other error besides an existence check error
c.Alert.Add(e, ErrUnknown, fmt.Sprintf("%s", c.Name))
c.Alert.Add(e, ErrUnknown, fmt.Sprintf("%s", c.Name), "")

case c.HasBecomeKnown(e):
c.Alert.Add(ErrExistCheckRecovered, nil, fmt.Sprintf("%s", c.Name))
c.Alert.Add(ErrExistCheckRecovered, nil, fmt.Sprintf("%s", c.Name), ErrExistCheckRecovered.Error())
c.ExistenceCheck.ToggleAlertActive()
default:
return // nothing is wrong, just keep going
Expand All @@ -152,13 +152,15 @@ func (c *AlertdContainer) CheckRunning(j *types.ContainerJSON) {
switch {
case c.ShouldAlertRunning(j) && !c.RunningCheck.AlertActive:
c.Alert.Add(ErrRunningCheckFail, nil, fmt.Sprintf("%s: expected running state: "+
"%t, current running state: %t", c.Name, *c.RunningCheck.Expected, j.State.Running))
"%t, current running state: %t", c.Name, *c.RunningCheck.Expected, j.State.Running),
ErrRunningCheckFail.Error())

c.RunningCheck.ToggleAlertActive()

case !c.ShouldAlertRunning(j) && c.RunningCheck.AlertActive:
c.Alert.Add(ErrRunningCheckRecovered, nil, fmt.Sprintf("%s: expected running state: "+
"%t, current running state: %t", c.Name, *c.RunningCheck.Expected, j.State.Running))
"%t, current running state: %t", c.Name, *c.RunningCheck.Expected, j.State.Running),
ErrRunningCheckRecovered.Error())

c.RunningCheck.ToggleAlertActive()
}
Expand Down Expand Up @@ -189,13 +191,13 @@ func (c *AlertdContainer) CheckCPUUsage(s *types.Stats) {
switch {
case a && !c.CPUCheck.AlertActive:
c.Alert.Add(ErrCPUCheckFail, nil, fmt.Sprintf("%s: CPU limit: %d, current usage: %d",
c.Name, c.CPUCheck.Limit, u))
c.Name, c.CPUCheck.Limit, u), ErrCPUCheckFail.Error())

c.CPUCheck.ToggleAlertActive()

case !a && c.CPUCheck.AlertActive:
c.Alert.Add(ErrCPUCheckRecovered, nil, fmt.Sprintf("%s: CPU limit: %d, current usage %d",
c.Name, c.CPUCheck.Limit, u))
c.Name, c.CPUCheck.Limit, u), ErrCPUCheckRecovered.Error())

c.CPUCheck.ToggleAlertActive()
}
Expand All @@ -215,13 +217,13 @@ func (c *AlertdContainer) CheckMinPids(s *types.Stats) {
// do nothing because the check is disabled
case a && !c.PIDCheck.AlertActive:
c.Alert.Add(ErrMinPIDCheckFail, nil, fmt.Sprintf("%s: minimum PIDs: %d, current PIDs: %d",
c.Name, c.PIDCheck.Limit, s.PidsStats.Current))
c.Name, c.PIDCheck.Limit, s.PidsStats.Current), ErrMinPIDCheckFail.Error())

c.PIDCheck.ToggleAlertActive()

case !a && c.PIDCheck.AlertActive:
c.Alert.Add(ErrMinPIDCheckRecovered, nil, fmt.Sprintf("%s: minimum PIDs: %d, current PIDs: %d",
c.Name, c.PIDCheck.Limit, s.PidsStats.Current))
c.Name, c.PIDCheck.Limit, s.PidsStats.Current), ErrMinPIDCheckRecovered.Error())

c.PIDCheck.ToggleAlertActive()
}
Expand Down Expand Up @@ -251,13 +253,13 @@ func (c *AlertdContainer) CheckMemory(s *types.Stats) {
// do nothing because the check is disabled
case a && !c.MemCheck.AlertActive:
c.Alert.Add(ErrMemCheckFail, nil, fmt.Sprintf("%s: Memory limit: %d, current usage: %d",
c.Name, c.MemCheck.Limit, u))
c.Name, c.MemCheck.Limit, u), ErrMemCheckFail.Error())

c.MemCheck.ToggleAlertActive()

case !a && c.MemCheck.AlertActive:
c.Alert.Add(ErrMemCheckRecovered, nil, fmt.Sprintf("%s: Memory limit: %d, current usage: %d",
c.Name, c.MemCheck.Limit, u))
c.Name, c.MemCheck.Limit, u), ErrMemCheckRecovered.Error())

c.MemCheck.ToggleAlertActive()
}
Expand Down
36 changes: 23 additions & 13 deletions cmd/alerters.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,20 @@ type Email struct {
// Alert sends an email alert
func (e Email) Alert(a *Alert) error {
// alerts in string form
alerts := a.Dump()
alerts := a.DumpEmail()

subject := e.Subject + ": "
for i := range a.SubjectAddendums {
// add addendums to the subject
subject += fmt.Sprintf("%s ", a.SubjectAddendums[i])
if i == 2 { // subjects cannot be too long, stop if it is at position 3
subject += fmt.Sprintf("...")
}
}

// The email message formatted properly
formattedMsg := []byte(fmt.Sprintf("To: %s\r\nSubject: %s\r\n\r\n%s\r\n",
e.To, e.Subject, alerts))
e.To, subject, alerts))

// Set up authentication/address information
auth := smtp.PlainAuth("", e.From, e.Password, e.SMTP)
Expand Down Expand Up @@ -140,9 +149,9 @@ func (s Slack) Alert(a *Alert) error {

// Pushover contains all info needed to push a notification to Pushover api
type Pushover struct {
ApiToken string
UserKey string
ApiURL string
APIToken string
UserKey string
APIURL string
}

// Valid returns an error if pushover settings are invalid
Expand All @@ -153,16 +162,16 @@ func (p Pushover) Valid() error {
return nil // assume that pushover was omitted
}

if p.ApiToken == "" {
errString = append(errString, ErrPushoverApiToken.Error())
if p.APIToken == "" {
errString = append(errString, ErrPushoverAPIToken.Error())
}

if p.UserKey == "" {
errString = append(errString, ErrPushoverUserKey.Error())
}

if p.ApiURL == "" {
errString = append(errString, ErrPushoverApiURL.Error())
if p.APIURL == "" {
errString = append(errString, ErrPushoverAPIURL.Error())
}

if len(errString) == 0 {
Expand All @@ -176,13 +185,14 @@ func (p Pushover) Valid() error {
}

// Alert sends the alert to Pushover API
func (s Pushover) Alert(a *Alert) error {
func (p Pushover) Alert(a *Alert) error {
alerts := a.Dump()

parsed_body := fmt.Sprintf("token=%s&user=%s&message=%s", s.ApiToken, s.UserKey, url.QueryEscape(alerts))
body := bytes.NewBufferString(parsed_body)
parsedBody := fmt.Sprintf("token=%s&user=%s&message=%s", p.APIToken, p.UserKey,
url.QueryEscape(alerts))
body := bytes.NewBufferString(parsedBody)

resp, err := http.Post(s.ApiURL, "application/x-www-form-urlencoded", body)
resp, err := http.Post(p.APIURL, "application/x-www-form-urlencoded", body)
if err != nil {
return err
}
Expand Down
10 changes: 5 additions & 5 deletions cmd/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,15 @@ import (

// these errors are for the purpose of being able to compare them later
var (
ErrEmptyConfig = errors.New("The configuration is completely empty (check config file)")
ErrEmptyConfig = errors.New("the configuration is completely empty (check config file)")
ErrEmailNoSMTP = errors.New("no email SMTP server")
ErrEmailNoTo = errors.New("no email to addresses")
ErrEmailNoFrom = errors.New("no email from addresses")
ErrEmailNoPass = errors.New("no email password")
ErrEmailNoPort = errors.New("no email port")
ErrEmailNoSubject = errors.New("no email subject")
ErrSlackNoWebHookURL = errors.New("no slack webhook url")
ErrNoContainers = errors.New("There were no containers found in the configuration file")
ErrNoContainers = errors.New("there were no containers found in the configuration file")
ErrExistCheckFail = errors.New("Existence check failure")
ErrExistCheckRecovered = errors.New("Existence check recovered")
ErrRunningCheckFail = errors.New("Running check failure")
Expand All @@ -30,9 +30,9 @@ var (
ErrMaxPIDCheckFail = errors.New("Max PID check Failure")
ErrMaxPIDCheckRecovered = errors.New("Max PID check recovered")
ErrUnknown = errors.New("Received an unknown error")
ErrPushoverApiToken = errors.New("no pushover api token")
ErrPushoverUserKey = errors.New("no pushover user key")
ErrPushoverApiURL = errors.New("no pushover api url")
ErrPushoverAPIToken = errors.New("no pushover api token")
ErrPushoverUserKey = errors.New("no pushover user key")
ErrPushoverAPIURL = errors.New("no pushover api url")
)

// ErrContainsErr returns true if the error string contains the message
Expand Down
33 changes: 31 additions & 2 deletions cmd/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package cmd
import (
"fmt"
"log"
"strings"

"github.com/pkg/errors"
)
Expand All @@ -16,7 +17,8 @@ type Evaluator interface {
// Alert is the struct that stores information about alerts and its methods satisfy the
// Alerter interface
type Alert struct {
Messages []error
Messages []error
SubjectAddendums []string
}

// ShouldSend returns true if there is an alert message to be sent
Expand All @@ -37,7 +39,9 @@ func (a *Alert) Len() int {
}

// Add should take in an error and wrap it
func (a *Alert) Add(e1, e2 error, s string) {
func (a *Alert) Add(e1, e2 error, s, subAddendum string) {

a.SubjectAddendums = append(a.SubjectAddendums, subAddendum)

e := e1
if e2 != nil {
Expand All @@ -54,6 +58,10 @@ func (a *Alert) Concat(b ...*Alert) {
for _, msg := range v.Messages {
a.Messages = append(a.Messages, msg)
}

for _, addendum := range v.SubjectAddendums {
a.SubjectAddendums = append(a.SubjectAddendums, addendum)
}
}
}

Expand All @@ -68,6 +76,7 @@ func (a *Alert) Log() {
// Clear will reset the alert to an empty string
func (a *Alert) Clear() {
a.Messages = []error{}
a.SubjectAddendums = []string{}
}

// Dump takes the slice of alerts and dumps them to a single string
Expand All @@ -79,6 +88,26 @@ func (a *Alert) Dump() string {
return s
}

// DumpEmail behaves like dump, but formats them for email by splitting on ":" and adding
// \n\t (newline and tab) for the first two segments and joining the last segment. This
// should result in an email that is formatted as follows...
// [containerName]:
// [alertName]:
// Error: [errString]
func (a *Alert) DumpEmail() (s string) {
for _, e := range a.Messages {
errString := e.Error()
splitErr := strings.SplitN(errString, ":", 3)

for _, v := range splitErr {
s += fmt.Sprintf("%s\n\t", v)
}
s += fmt.Sprintf("\n\n")

}
return s
}

// Send is for sending out alerts to syslog and to alerts that are active in conf
func (a *Alert) Send(b []Alerter) {
a.Log()
Expand Down
2 changes: 1 addition & 1 deletion cmd/version.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
"github.com/spf13/cobra"
)

var version = "Version 0.4.3"
var version = "Version 0.4.4"

// versionCmd represents the version command
var versionCmd = &cobra.Command{
Expand Down

0 comments on commit 9bf3a7c

Please sign in to comment.