forked from rancher/host-api
-
Notifications
You must be signed in to change notification settings - Fork 0
/
healthcheck.go
123 lines (99 loc) · 2.49 KB
/
healthcheck.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
package healthcheck
import (
"fmt"
"strings"
"time"
"github.com/Sirupsen/logrus"
"github.com/rancherio/go-rancher/client"
"github.com/rancherio/host-api/pkg/haproxy"
"github.com/rancherio/host-api/util"
)
var PREFIX = "cattle-"
var SERVER_NAME = "svname"
var STATUS = "status"
var LOOPS = (60 * 60) / 2
func Poll() error {
client, err := util.GetRancherClient()
if err != nil {
return err
}
if client == nil {
return fmt.Errorf("Can not create RancherClient, No credentials found")
}
m := &Monitor{
client: client,
reportedStatus: map[string]string{},
}
for stat := range m.getStats() {
m.processStat(stat)
}
return nil
}
type Monitor struct {
client *client.RancherClient
reportedStatus map[string]string
}
func (m *Monitor) getStats() <-chan haproxy.Stat {
c := make(chan haproxy.Stat)
go m.readStats(c)
return c
}
func (m *Monitor) readStats(c chan<- haproxy.Stat) {
defer close(c)
count := 0
h := &haproxy.Monitor{
SocketPath: haproxy.HAPROXY_SOCK,
}
for i := 0; i < LOOPS; i++ {
// Sleep up front. This way if this program gets restarted really fast we don't spam cattle
time.Sleep(2 * time.Second)
stats, err := h.Stats()
currentCount := 0
if err != nil {
logrus.Errorf("Failed to read stats: %v", err)
continue
}
for _, stat := range stats {
if strings.HasPrefix(stat[SERVER_NAME], PREFIX) {
currentCount++
c <- stat
}
}
if currentCount != count {
count = currentCount
logrus.Infof("Monitoring %d backends", count)
}
}
}
func (m *Monitor) processStat(stat haproxy.Stat) {
serverName := strings.TrimPrefix(stat[SERVER_NAME], PREFIX)
currentStatus := stat[STATUS]
previousStatus := m.reportedStatus[serverName]
if strings.HasPrefix(currentStatus, "UP ") {
// do nothing on partial UP
return
}
if currentStatus == "UP" && previousStatus != "UP" && previousStatus != "INIT" {
currentStatus = "INIT"
}
if previousStatus != currentStatus {
err := m.reportStatus(serverName, currentStatus)
if err != nil {
logrus.Errorf("Failed to report status %s=%s: %v", serverName, currentStatus, err)
} else {
m.reportedStatus[serverName] = currentStatus
}
}
}
func (m *Monitor) reportStatus(serverName, currentStatus string) error {
_, err := m.client.ServiceEvent.Create(&client.ServiceEvent{
HealthcheckUuid: serverName,
ReportedHealth: currentStatus,
ExternalTimestamp: time.Now().Unix(),
})
if err != nil {
return err
}
logrus.Infof("%s=%s", serverName, currentStatus)
return nil
}