forked from coreos/fleet
/
monitor.go
62 lines (53 loc) · 1.37 KB
/
monitor.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
package heart
import (
"errors"
"time"
log "github.com/coreos/fleet/Godeps/_workspace/src/github.com/golang/glog"
)
func NewMonitor(ttl time.Duration) *Monitor {
return &Monitor{ttl, ttl / 2}
}
type Monitor struct {
TTL time.Duration
ival time.Duration
}
// Monitor ensures a Heart is still beating until a channel is closed, returning
// an error if the heartbeats fail.
func (m *Monitor) Monitor(hrt Heart, stop chan bool) error {
ticker := time.Tick(m.ival)
for {
select {
case <-stop:
log.V(1).Info("Monitor exiting due to stop signal")
return nil
case <-ticker:
if _, err := m.check(hrt); err != nil {
return err
}
}
}
}
// check attempts to beat a Heart several times within a timeout, returning the
// log index at which the beat succeeded or an error
func (m *Monitor) check(hrt Heart) (idx uint64, err error) {
// time out after a third of the machine presence TTL, attempting
// the heartbeat up to four times
timeout := m.TTL / 3
interval := timeout / 4
tchan := time.After(timeout)
next := time.After(0)
for idx == 0 {
select {
case <-tchan:
err = errors.New("Monitor timed out before successful heartbeat")
return
case <-next:
idx, err = hrt.Beat(m.TTL)
if err != nil {
log.V(1).Infof("Monitor heartbeat function returned err, retrying in %v: %v", interval, err)
}
next = time.After(interval)
}
}
return
}