Merge pull request #70 from prometheus/webhook

Add a generic webhook notifier.

README.md

@@ -16,6 +16,7 @@ following aspects:
 * aggregating alerts by labelset
 * handling notification repeats
 * sending alert notifications via external services (currently email,
+generic web hook,
 [PagerDuty](http://www.pagerduty.com/),
 [HipChat](http://www.hipchat.com/),
 [Slack](http://www.slack.com/),

config/config.proto

@@ -58,7 +58,7 @@ message HipChatConfig {
 
 // Configuration for notification via Slack.
 message SlackConfig {
-  // Slack webhook url, (https://api.slack.com/incoming-webhooks).
+  // Slack webhook URL, (https://api.slack.com/incoming-webhooks).
   optional string webhook_url = 1;
   // Slack channel override, (like #other-channel or @username).
   optional string channel = 2;
@@ -70,6 +70,7 @@ message SlackConfig {
   optional bool send_resolved = 5 [default = false];
 }
 
+// Configuration for notification via Flowdock.
 message FlowdockConfig {
   // Flowdock flow API token.
   optional string api_token = 1;
@@ -81,6 +82,14 @@ message FlowdockConfig {
   optional bool send_resolved = 4 [default = false];
 }
 
+// Configuration for notification via generic webhook.
+message WebhookConfig {
+  // URL to send POST request to.
+  optional string url = 1;
+  // Notify when resolved.
+  optional bool send_resolved = 2 [default = false];
+}
+
 // Notification configuration definition.
 message NotificationConfig {
   // Name of this NotificationConfig. Referenced from AggregationRule.
@@ -97,6 +106,8 @@ message NotificationConfig {
   repeated SlackConfig slack_config = 6;
   // Zero or more Flowdock notification configurations.
   repeated FlowdockConfig flowdock_config = 7;
+  // Zero or more generic web hook notification configurations.
+  repeated WebhookConfig webhook_config = 8;
 }
 
 // A regex-based label filter used in aggregations.

main.go

@@ -57,7 +57,7 @@ func main() {
 	}
 	saveSilencesTicker := time.NewTicker(10 * time.Second)
 	go func() {
-		for _ = range saveSilencesTicker.C {
+		for range saveSilencesTicker.C {
 			if err := silencer.SaveToFile(*silencesFile); err != nil {
 				log.Error("Error saving silences to file: ", err)
 			}

manager/alert.go

@@ -33,14 +33,14 @@ type Alerts []*Alert
 // Alert models an action triggered by Prometheus.
 type Alert struct {
 	// Short summary of alert.
-	Summary string
+	Summary string `json:"summary"`
 	// Long description of alert.
-	Description string
+	Description string `json:"description"`
 	// Label value pairs for purpose of aggregation, matching, and disposition
 	// dispatching. This must minimally include an "alertname" label.
-	Labels AlertLabelSet
+	Labels AlertLabelSet `json:"labels"`
 	// Extra key/value information which is not used for aggregation.
-	Payload AlertPayload
+	Payload AlertPayload `json:"payload"`
 }
 
 func (a *Alert) Name() string {

manager/manager.go

@@ -403,7 +403,7 @@ func (s *memoryAlertManager) runIteration() {
 // Run the memoryAlertManager's main dispatcher loop.
 func (s *memoryAlertManager) Run() {
 	iterationTicker := time.NewTicker(time.Second)
-	for _ = range iterationTicker.C {
+	for range iterationTicker.C {
 		s.checkSanity()
 		s.runIteration()
 	}

manager/notifier.go

@@ -374,6 +374,40 @@ func newFlowdockMessage(op notificationOp, config *pb.FlowdockConfig, a *Alert)
 	return msg
 }
 
+type webhookMessage struct {
+	Version string  `json:"version"`
+	Status  string  `json:"status"`
+	Alerts  []Alert `json:"alert"`
+}
+
+func (n *notifier) sendWebhookNotification(op notificationOp, config *pb.WebhookConfig, a *Alert) error {
+	status := ""
+	switch op {
+	case notificationOpTrigger:
+		status = "firing"
+	case notificationOpResolve:
+		status = "resolved"
+	}
+
+	msg := &webhookMessage{
+		Version: "1",
+		Status:  status,
+		Alerts:  []Alert{*a},
+	}
+	jsonMessage, err := json.Marshal(msg)
+	if err != nil {
+		return err
+	}
+	httpResponse, err := postJSON(jsonMessage, config.GetUrl())
+	if err != nil {
+		return err
+	}
+	if err := processResponse(httpResponse, "Webhook", a); err != nil {
+		return err
+	}
+	return nil
+}
+
 func postJSON(jsonMessage []byte, url string) (*http.Response, error) {
 	timeout := time.Duration(5 * time.Second)
 	client := http.Client{
@@ -582,6 +616,14 @@ func (n *notifier) handleNotification(a *Alert, op notificationOp, config *pb.No
 			log.Errorln("Error sending Flowdock notification:", err)
 		}
 	}
+	for _, whConfig := range config.WebhookConfig {
+		if op == notificationOpResolve && !whConfig.GetSendResolved() {
+			continue
+		}
+		if err := n.sendWebhookNotification(op, whConfig, a); err != nil {
+			log.Errorln("Error sending Webhook notification:", err)
+		}
+	}
 }
 
 func (n *notifier) Dispatch() {

manager/notifier_test.go

@@ -16,10 +16,17 @@ package manager
 import (
 	"bytes"
 	"crypto/tls"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
 	"os"
+	"reflect"
 	"testing"
 	"time"
+
+	pb "github.com/prometheus/alertmanager/config/generated"
 )
 
 func TestWriteEmailBody(t *testing.T) {
@@ -169,3 +176,48 @@ func TestGetSMTPAuth(t *testing.T) {
 		t.Errorf("PLAIN auth with bad host-port: expected error but got %T, %v", auth, cfg)
 	}
 }
+
+func TestSendWebhookNotification(t *testing.T) {
+	var body []byte
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var err error
+		body, err = ioutil.ReadAll(r.Body)
+		if err != nil {
+			t.Errorf("error reading webhook notification: %s", err)
+		}
+	}))
+	defer ts.Close()
+
+	config := &pb.WebhookConfig{
+		Url: &ts.URL,
+	}
+	alert := &Alert{
+		Summary:     "Testsummary",
+		Description: "Test alert description, something went wrong here.",
+		Labels: AlertLabelSet{
+			"alertname": "TestAlert",
+		},
+		Payload: AlertPayload{
+			"payload_label1": "payload_value1",
+		},
+	}
+	n := &notifier{}
+	err := n.sendWebhookNotification(notificationOpTrigger, config, alert)
+	if err != nil {
+		t.Errorf("error sending webhook notification: %s", err)
+	}
+
+	var msg webhookMessage
+	err = json.Unmarshal(body, &msg)
+	if err != nil {
+		t.Errorf("error unmarshalling webhook notification: %s", err)
+	}
+	expected := webhookMessage{
+		Version: "1",
+		Status:  "firing",
+		Alerts:  []Alert{*alert},
+	}
+	if !reflect.DeepEqual(msg, expected) {
+		t.Errorf("incorrect webhook notification: Expected: %s Actual: %s", expected, msg)
+	}
+}

web/api/api.go

@@ -25,19 +25,19 @@ import (
 )
 
 type AlertManagerService struct {
-	Manager  manager.AlertManager
-	Silencer *manager.Silencer
+	Manager    manager.AlertManager
+	Silencer   *manager.Silencer
 	PathPrefix string
 }
 
 func (s AlertManagerService) Handler() http.Handler {
 	r := httprouter.New()
-	r.POST(s.PathPrefix + "api/alerts", s.addAlerts)
-	r.GET(s.PathPrefix + "api/silences", s.silenceSummary)
-	r.POST(s.PathPrefix + "api/silences", s.addSilence)
-	r.GET(s.PathPrefix + "api/silences/:id", s.getSilence)
-	r.POST(s.PathPrefix + "api/silences/:id", s.updateSilence)
-	r.DELETE(s.PathPrefix + "api/silences/:id", s.deleteSilence)
+	r.POST(s.PathPrefix+"api/alerts", s.addAlerts)
+	r.GET(s.PathPrefix+"api/silences", s.silenceSummary)
+	r.POST(s.PathPrefix+"api/silences", s.addSilence)
+	r.GET(s.PathPrefix+"api/silences/:id", s.getSilence)
+	r.POST(s.PathPrefix+"api/silences/:id", s.updateSilence)
+	r.DELETE(s.PathPrefix+"api/silences/:id", s.deleteSilence)
 
 	return r
 }

web/silences.go

@@ -24,7 +24,7 @@ type SilenceStatus struct {
 }
 
 type SilencesHandler struct {
-	Silencer *manager.Silencer
+	Silencer   *manager.Silencer
 	PathPrefix string
 }