forked from AcalephStorage/consul-alerts
/
check-handler.go
110 lines (94 loc) · 2.42 KB
/
check-handler.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
package main
import (
"bytes"
"time"
"encoding/json"
"net/http"
"os/exec"
"github.com/AcalephStorage/consul-alerts/consul"
"github.com/AcalephStorage/consul-alerts/notifier"
log "github.com/AcalephStorage/consul-alerts/Godeps/_workspace/src/github.com/Sirupsen/logrus"
)
var checksChannel = make(chan []consul.Check, 1)
var firstCheckRun = true
func checkHandler(w http.ResponseWriter, r *http.Request) {
consulClient.LoadConfig()
if firstCheckRun {
log.Println("Now watching for health changes.")
firstCheckRun = false
w.WriteHeader(200)
return
}
if !consulClient.ChecksEnabled() {
log.Println("Checks handling disabled. Checks ignored.")
w.WriteHeader(200)
return
}
if len(checksChannel) == 1 {
log.Println("replacing health check request queue.")
<-checksChannel
}
var checks []consul.Check
toWatchObject(r.Body, &checks)
go startProcess(checks)
w.WriteHeader(200)
}
func startProcess(checks []consul.Check) {
checksChannel <- checks
}
func processChecks() {
for {
<-checksChannel
log.Println("Running health check.")
changeThreshold := consulClient.CheckChangeThreshold()
for elapsed := 0; elapsed < changeThreshold; elapsed += 10 {
consulClient.UpdateCheckData()
time.Sleep(10 * time.Second)
}
consulClient.UpdateCheckData()
log.Println("Processing health checks for notification.")
alerts := consulClient.NewAlerts()
if len(alerts) > 0 {
notify(alerts)
}
}
}
func notify(alerts []consul.Check) {
messages := make([]notifier.Message, len(alerts))
for i, alert := range alerts {
messages[i] = notifier.Message{
Node: alert.Node,
Service: alert.ServiceName,
Check: alert.Name,
Status: alert.Status,
Output: alert.Output,
Notes: alert.Notes,
Timestamp: time.Now(),
}
}
for _, n := range builtinNotifiers() {
n.Notify(messages)
}
for _, n := range consulClient.CustomNotifiers() {
executeHealthNotifier(messages, n)
}
}
func executeHealthNotifier(messages []notifier.Message, notifCmd string) {
data, err := json.Marshal(&messages)
if err != nil {
log.Println("Unable to read messages: ", err)
return
}
input := bytes.NewReader(data)
output := new(bytes.Buffer)
cmd := exec.Command(notifCmd)
cmd.Stdin = input
cmd.Stdout = output
cmd.Stderr = output
if err := cmd.Run(); err != nil {
log.Println("error running notifier: ", err)
} else {
log.Println(">>> notification sent to:", notifCmd)
}
log.Println(output)
}