-
Notifications
You must be signed in to change notification settings - Fork 13
/
health_checker.go
108 lines (87 loc) · 2.09 KB
/
health_checker.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
package executor
import (
"sync"
"time"
"kubesphere.io/alert/pkg/logger"
"kubesphere.io/alert/pkg/models"
rs "kubesphere.io/alert/pkg/services/executor/resource_control"
)
type UpdateRequest struct {
sync.RWMutex
RequestCount int64
}
type HealthChecker struct {
executor *Executor
UpdateCh chan string
RequestStatus UpdateRequest
}
func NewHealthChecker() *HealthChecker {
hc := &HealthChecker{
UpdateCh: make(chan string, 10000),
}
hc.RequestStatus.RequestCount = 0
return hc
}
func (hc *HealthChecker) SetExecutor(executor *Executor) {
hc.executor = executor
}
func (hc *HealthChecker) update() {
runners := hc.executor.GetRunners()
//Update runner status
rs.UpdateAlertStatus(runners, hc.executor.GetName())
}
func (hc *HealthChecker) checkAndUpdate() {
runners := hc.executor.GetRunners()
//Update runner status
rs.UpdateAlertStatus(runners, hc.executor.GetName())
//Check wild runners
alerts := rs.QueryAlerts(hc.executor.GetName(), "running")
wildRunners := difference(runners, alerts)
if len(wildRunners) > 0 {
logger.Error(nil, "Wild Runners %v", wildRunners)
}
for _, alertId := range wildRunners {
hc.executor.TerminateRunner(alertId)
}
}
func (hc *HealthChecker) HealthCheck() {
timer := time.NewTicker(time.Second * 30)
defer timer.Stop()
for {
select {
case <-timer.C:
hc.checkAndUpdate()
case <-hc.UpdateCh:
hc.RequestStatus.Lock()
hc.RequestStatus.RequestCount = hc.RequestStatus.RequestCount + 1
hc.RequestStatus.Unlock()
}
}
}
func (hc *HealthChecker) UpdateLoop() {
for {
hc.RequestStatus.Lock()
count := hc.RequestStatus.RequestCount
hc.RequestStatus.Unlock()
if count > 0 {
hc.update()
}
hc.RequestStatus.Lock()
hc.RequestStatus.RequestCount = 0
hc.RequestStatus.Unlock()
time.Sleep(time.Second)
}
}
func difference(runners []rs.RunnerInfo, alerts []models.Alert) []string {
alertmap := map[string]bool{}
for _, x := range alerts {
alertmap[x.AlertId] = true
}
diff := []string{}
for _, x := range runners {
if _, ok := alertmap[x.AlertId]; !ok {
diff = append(diff, x.AlertId)
}
}
return diff
}