Skip to content

Commit

Permalink
added extra info to POST notifications error log which eases the noti…
Browse files Browse the repository at this point in the history 
…fication failure debugging

	- cmd/bosun/conf/[actionNotify.go, unknownNotify.go]: modified NotificationDetails struct to include temaplte key map and action string
        - cmd/bosun/conf/notify.go: modified PrepareAlert func and notifiaction failure message
  • Loading branch information
Pradeep Mishra committed Feb 21, 2018
1 parent 6a41655 commit 6c53990
Show file tree
Hide file tree
Showing 3 changed files with 132 additions and 24 deletions.
29 changes: 28 additions & 1 deletion cmd/bosun/conf/actionNotify.go
View file
Expand Up @@ -103,6 +103,33 @@ func (n *Notification) PrepareAction(at models.ActionType, t *Template, c System
}
return buf.String(), nil
}
n.prepareFromTemplateKeys(pn, *tks, render, actionDefaults)

ak := map[string][]string{
"alert_key": {},
}

for i := range states {
contain := func(key string, array []string) bool {
for _, value := range array {
if value == key {
return true
}
}
return false
}

if contain(fmt.Sprint(states[i].AlertKey), ak["alert_key"]) != true {
ak["alert_key"] = append(ak["alert_key"], fmt.Sprint(states[i].AlertKey))
}
}

details := &NotificationDetails{
Ak: ak["alert_key"],
At: at.String(),
NotifyName: n.Name,
TemplateKey: tks.BodyTemplate,
}

n.prepareFromTemplateKeys(pn, *tks, render, actionDefaults, details)
return pn
}
74 changes: 56 additions & 18 deletions cmd/bosun/conf/notify.go
View file
Expand Up @@ -78,7 +78,13 @@ func (n *Notification) PrepareAlert(rt *models.RenderedTemplates, ak string, att
url = rt.Get(n.PostTemplate)
}
body := rt.GetDefault(n.BodyTemplate, "subject")
pn.HTTP = append(pn.HTTP, n.PrepHttp("POST", url, body, ak))
details := &NotificationDetails{
Ak: []string{ak},
NotifyName: n.Name,
TemplateKey: n.BodyTemplate,
NotifyType: 1,
}
pn.HTTP = append(pn.HTTP, n.PrepHttp("POST", url, body, details))
}
if n.Get != nil || n.GetTemplate != "" {
url := ""
Expand All @@ -87,7 +93,13 @@ func (n *Notification) PrepareAlert(rt *models.RenderedTemplates, ak string, att
} else {
url = rt.Get(n.GetTemplate)
}
pn.HTTP = append(pn.HTTP, n.PrepHttp("GET", url, "", ak))
details := &NotificationDetails{
Ak: []string{ak},
NotifyName: n.Name,
TemplateKey: n.BodyTemplate,
NotifyType: 1,
}
pn.HTTP = append(pn.HTTP, n.PrepHttp("GET", url, "", details))
}
if n.Print {
if n.BodyTemplate != "" {
Expand All @@ -105,12 +117,25 @@ func (n *Notification) NotifyAlert(rt *models.RenderedTemplates, c SystemConfPro
}

type PreparedHttp struct {
URL string
Method string
Headers map[string]string `json:",omitempty"`
Body string
AK string
NotifyName string
URL string
Method string
Headers map[string]string `json:",omitempty"`
Body string
Details *NotificationDetails
}

const (
alert = iota + 1
unknown
multiunknown
)

type NotificationDetails struct {
Ak []string // alert key
At string // action type
NotifyName string // notification name
TemplateKey string // template key
NotifyType int // notifications type e.g alert, unknown etc
}

func (p *PreparedHttp) Send() (int, error) {
Expand All @@ -136,19 +161,31 @@ func (p *PreparedHttp) Send() (int, error) {
}
if resp.StatusCode >= 300 {
collect.Add("post.sent_failed", nil, 1)
return resp.StatusCode, fmt.Errorf("bad response on notification with name %s for alert %s method %s: %d", p.NotifyName, p.AK, p.Method, resp.StatusCode)
switch p.Details.NotifyType {
case alert:
return resp.StatusCode, fmt.Errorf("bad response for '%s' alert notification using template key '%s' for alert keys %v method %s: %d",
p.Details.NotifyName, p.Details.TemplateKey, strings.Join(p.Details.Ak, ","), p.Method, resp.StatusCode)
case unknown:
return resp.StatusCode, fmt.Errorf("bad response for '%s' unknown notification using template key '%s' for alert keys %v method %s: %d",
p.Details.NotifyName, p.Details.TemplateKey, strings.Join(p.Details.Ak, ","), p.Method, resp.StatusCode)
case multiunknown:
return resp.StatusCode, fmt.Errorf("bad response for '%s' multi-unknown notification using template key '%s' for alert keys %v method %s: %d",
p.Details.NotifyName, p.Details.TemplateKey, strings.Join(p.Details.Ak, ","), p.Method, resp.StatusCode)
default:
return resp.StatusCode, fmt.Errorf("bad response for '%s' action '%s' notification using template key '%s' for alert keys %v method %s: %d",
p.Details.NotifyName, p.Details.At, p.Details.TemplateKey, strings.Join(p.Details.Ak, ","), p.Method, resp.StatusCode)
}
}
collect.Add("post.sent", nil, 1)
return resp.StatusCode, nil
}

func (n *Notification) PrepHttp(method string, url string, body string, ak string) *PreparedHttp {
func (n *Notification) PrepHttp(method string, url string, body string, alertDetails *NotificationDetails) *PreparedHttp {
prep := &PreparedHttp{
Method: method,
URL: url,
Headers: map[string]string{},
AK: ak,
NotifyName: n.Name,
Method: method,
URL: url,
Headers: map[string]string{},
Details: alertDetails,
}
if method == http.MethodPost {
prep.Body = body
Expand All @@ -157,13 +194,14 @@ func (n *Notification) PrepHttp(method string, url string, body string, ak strin
return prep
}

func (n *Notification) SendHttp(method string, url string, body string, ak string) {
p := n.PrepHttp(method, url, body, ak)
func (n *Notification) SendHttp(method string, url string, body string) {
details := &NotificationDetails{}
p := n.PrepHttp(method, url, body, details)
stat, err := p.Send()
if err != nil {
slog.Errorf("Sending http notification: %s", err)
}
slog.Infof("%s notification successful for alert %s. Status: %d", method, ak, stat)
slog.Infof("%s notification successful for alert %s. Status: %d", method, details.Ak, stat)
}

type PreparedEmail struct {
Expand Down
53 changes: 48 additions & 5 deletions cmd/bosun/conf/unknownNotify.go
View file
Expand Up @@ -27,6 +27,7 @@ var defaultUnknownTemplate = &Template{
`)),
Subject: template.Must(template.New("subject").Parse(`{{.Name}}: {{.Group | len}} unknown alerts`)),
}

var unknownDefaults defaultTemplates

func init() {
Expand Down Expand Up @@ -68,8 +69,35 @@ func (n *Notification) PrepareUnknown(t *Template, c SystemConfProvider, name st
}
return buf.String(), nil
}

tks := n.UnknownTemplateKeys
n.prepareFromTemplateKeys(pn, tks, render, unknownDefaults)

ak := map[string][]string{
"alert_key": {},
}

for i := range aks {
contain := func(key string, array []string) bool {
for _, value := range array {
if value == key {
return true
}
}
return false
}
if contain(fmt.Sprint(aks[i]), ak["alert_key"]) != true {
ak["alert_key"] = append(ak["alert_key"], fmt.Sprint(aks[i]))
}
}

details := &NotificationDetails{
NotifyName: n.Name,
TemplateKey: tks.BodyTemplate,
Ak: ak["alert_key"],
NotifyType: 2,
}

n.prepareFromTemplateKeys(pn, tks, render, unknownDefaults, details)
return pn
}

Expand Down Expand Up @@ -132,8 +160,23 @@ func (n *Notification) PrepareMultipleUnknowns(t *Template, c SystemConfProvider
}
return buf.String(), nil
}

tks := n.UnknownMultiTemplateKeys
n.prepareFromTemplateKeys(pn, tks, render, unknownMultiDefaults)

ak := []string{}

for _, v := range groups {
ak = append(ak, fmt.Sprint(v))
}

details := &NotificationDetails{
NotifyName: n.Name,
TemplateKey: tks.BodyTemplate,
Ak: ak,
NotifyType: 3,
}

n.prepareFromTemplateKeys(pn, tks, render, unknownMultiDefaults, details)
return pn
}

Expand All @@ -142,7 +185,7 @@ func (n *Notification) NotifyMultipleUnknowns(t *Template, c SystemConfProvider,
}

// code common to PrepareAction / PrepareUnknown / PrepareMultipleUnknowns
func (n *Notification) prepareFromTemplateKeys(pn *PreparedNotifications, tks NotificationTemplateKeys, render func(string, *template.Template) (string, error), defaults defaultTemplates) {
func (n *Notification) prepareFromTemplateKeys(pn *PreparedNotifications, tks NotificationTemplateKeys, render func(string, *template.Template) (string, error), defaults defaultTemplates, alertDetails *NotificationDetails) {

if len(n.Email) > 0 || n.Post != nil || tks.PostTemplate != "" {
body, _ := render(tks.BodyTemplate, defaults.body)
Expand All @@ -168,9 +211,9 @@ func (n *Notification) prepareFromTemplateKeys(pn *PreparedNotifications, tks No
}
if postURL != "" {
body, _ := render(tks.BodyTemplate, defaults.subject)
pn.HTTP = append(pn.HTTP, n.PrepHttp("POST", postURL, body, ""))
pn.HTTP = append(pn.HTTP, n.PrepHttp("POST", postURL, body, alertDetails))
}
if getURL != "" {
pn.HTTP = append(pn.HTTP, n.PrepHttp("GET", getURL, "", ""))
pn.HTTP = append(pn.HTTP, n.PrepHttp("GET", getURL, "", alertDetails))
}
}

0 comments on commit 6c53990

Please sign in to comment.