forked from rancher/healthcheck
/
healthcheck.go
143 lines (115 loc) · 2.85 KB
/
healthcheck.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
package main
import (
"fmt"
"strings"
"time"
"github.com/Sirupsen/logrus"
"github.com/patrickmn/go-cache"
"github.com/rancher/go-rancher/client"
"github.com/rancher/healthcheck/metadata"
"github.com/rancher/healthcheck/pkg/haproxy"
"github.com/rancher/healthcheck/util"
)
var Prefix = "cattle-"
var ServerName = "svname"
var Status = "status"
func Poll() error {
client, err := util.GetRancherClient()
if err != nil {
return err
}
if client == nil {
return fmt.Errorf("Can not create RancherClient, No credentials found")
}
metadataPoller := &metadata.Poller{}
if err = metadataPoller.Init(); err != nil {
return err
}
configUpdater := &haproxy.Provider{
Poller: metadataPoller,
}
if err = configUpdater.Start(); err != nil {
return err
}
go configUpdater.Run()
c := cache.New(1*time.Hour, 1*time.Minute)
m := &Monitor{
client: client,
reportedStatus: c,
}
for stat := range m.getStats() {
m.processStat(stat)
}
return nil
}
type Monitor struct {
client *client.RancherClient
reportedStatus *cache.Cache
}
func (m *Monitor) getStats() <-chan haproxy.Stat {
c := make(chan haproxy.Stat)
go m.readStats(c)
return c
}
func (m *Monitor) readStats(c chan<- haproxy.Stat) {
defer close(c)
count := 0
h := &haproxy.Monitor{
SocketPath: haproxy.HaproxySock,
}
for {
// Sleep up front. This way if this program gets restarted really fast we don't spam cattle
time.Sleep(2 * time.Second)
stats, err := h.Stats()
currentCount := 0
if err != nil {
logrus.Errorf("Failed to read stats: %v", err)
continue
}
for _, stat := range stats {
if strings.HasPrefix(stat[ServerName], Prefix) {
currentCount++
c <- stat
}
}
if currentCount != count {
count = currentCount
logrus.Infof("Monitoring %d backends", count)
}
}
}
func (m *Monitor) processStat(stat haproxy.Stat) {
serverName := strings.TrimPrefix(stat[ServerName], Prefix)
currentStatus := stat[Status]
previousStatus, _ := m.reportedStatus.Get(serverName)
if strings.HasPrefix(currentStatus, "UP ") {
// do nothing on partial UP
return
}
if currentStatus == "UP" && previousStatus != "UP" && previousStatus != "INIT" {
currentStatus = "INIT"
}
update := true
if previousStatus != currentStatus {
err := m.reportStatus(serverName, currentStatus)
if err != nil {
logrus.Errorf("Failed to report status %s=%s: %v", serverName, currentStatus, err)
update = false
}
}
if update {
m.reportedStatus.Set(serverName, currentStatus, cache.DefaultExpiration)
}
}
func (m *Monitor) reportStatus(serverName, currentStatus string) error {
_, err := m.client.ServiceEvent.Create(&client.ServiceEvent{
HealthcheckUuid: serverName,
ReportedHealth: currentStatus,
ExternalTimestamp: time.Now().Unix(),
})
if err != nil {
return err
}
logrus.Infof("%s=%s", serverName, currentStatus)
return nil
}