-
Notifications
You must be signed in to change notification settings - Fork 0
/
sentinel.go
240 lines (203 loc) · 6.12 KB
/
sentinel.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
// Copyright 2016 ljy2010a
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package sentinel
import (
"sync"
"sync/atomic"
"github.com/garyburd/redigo/redis"
)
// Sentinel maintains a pool of master *redigo.Poll.
// The application calls the Pool method to get the pool.
// NOTICE : as the the switch-master signal tick Sentinel will try to reconnect
// the new master
// you should always get the pool from Sentinel , do not keep the pool for
// your own
// Example like redigo
// The following example shows how to use a Sentinel in application. The
// application creates a Sentinel at application startup and makes it available
// to
// request handlers using a global variable.
type Sentinel struct {
//keep the sentinel addrs , when pub `+sentinel` will change
SentinelAddrs []string
//sentinel mastername
MasterName string
// SentinelDial is an application supplied function for creating and
// configuring a
// connection.
//
// The connection returned from Dial must not be in a special state
// (subscribed to pubsub channel, transaction started, ...).
SentinelDial func(addr string) (redis.Conn, error)
// A concurrent Map to save the sentinel pool
sentinelPools *poolMap
// This dial is for the master pool like redigo , but as the sentinel model
// , the master addr will be filled after sentinel get the addr by name
//
// The connection returned from Dial must not be in a special state
// (subscribed to pubsub channel, transaction started, ...).
PoolDial func(addr string) (redis.Conn, error)
// Keep the master *redigo.Pool
MasterPool *redis.Pool
// SlavesDial is an application supplied function for creating and
// configuring a
// connection.
//
// The connection returned from Dial must not be in a special state
// (subscribed to pubsub channel, transaction started, ...).
SlavesDial func(addr string) (redis.Conn, error)
// A concurrent Map to save the sentinel pool
slavesPools *poolMap
// Slaves Pool templete
SlavesPoolTe *redis.Pool
//to save the last addr for master addr
lastMasterAddr string
// enable to get the slaves pool
EnableSlaves bool
// when master odwon switch a slaves for readonly
AutoSwitchSlaves bool
//
eventSrv *eventServer
//
closed int64
//
wg sync.WaitGroup
}
// Return the sentinel's master addr
func (s *Sentinel) LastMasterAddr() string {
return s.lastMasterAddr
}
// Get the master pool from sentinel.
// The application should run the Load() func below first
// NOTICE : When the master ODOWN , you will get a bad conn from the pool
// you should not keep the pool for your own,please get pool from sentinel
// always
func (s *Sentinel) Pool() *redis.Pool {
return s.MasterPool
}
// Get the sentinelsAddrs snapshot
// do not make sure all the addr available
func (s *Sentinel) SentinelsAddrs() []string {
s.SentinelAddrs = s.sentinelPools.keys()
return s.SentinelAddrs
}
// Synchronize
func (s *Sentinel) wrap(f func()) {
s.wg.Add(1)
func() {
if atomic.LoadInt64(&s.closed) == 0 {
f()
}
s.wg.Done()
}()
}
// Close all the pool
func (s *Sentinel) Close() {
// s.closed = true
atomic.StoreInt64(&s.closed, 1)
logger.Debug("close sentinel begin wait all proc stop")
s.wg.Wait()
logger.Debug("all proc stop")
s.eventSrv.Close()
sentinelAddrs := s.sentinelPools.keys()
for _, addr := range sentinelAddrs {
if pool := s.sentinelPools.get(addr); pool != nil {
pool.Close()
}
s.sentinelPools.del(addr)
}
if s.MasterPool != nil {
s.MasterPool.Close()
}
logger.Debug("close sentinel done ")
}
// Begin to run the Sentinel. Here is the Process below
// 1. Connect the sentinels , add it to sentinelPools , start sentry() to
// subscribe the news from sentinel-server
// 2. Get the master addr from sentinel
// 3. Start the monitors to keep the sentinel available
func (s *Sentinel) Load() error {
s.sentinelPools = newPoolMap()
s.eventSrv = newEventServer()
s.eventSrv.Add(cmd_switch_master, s.switchMaster)
s.eventSrv.Add(cmd_sentinel, s.addSentinel)
// s.closed = false
logger.Debugf("sentinel begin to conn %v \n", s.SentinelAddrs)
// connect the sentinel user offer
s.refreshSentinels(s.SentinelAddrs)
logger.Debugf("sentinel has to conn %v \n", s.sentinelPools.keys())
// search for the other sentinel
sentinelAddrs, err := s.sentinelAddrs()
if err != nil {
return err
}
// connect the left over sentinel
s.refreshSentinels(sentinelAddrs)
//reset the connected sentinelAddrs
s.SentinelAddrs = s.sentinelPools.keys()
// get the master addr form sentinel
masterAddr, err := s.masterAddr()
if err != nil {
return err
}
s.lastMasterAddr = masterAddr
logger.Debugf("sentinel load master %v ", s.lastMasterAddr)
s.MasterPool.Dial = func() (redis.Conn, error) {
return s.PoolDial(s.lastMasterAddr)
}
go s.taskRefreshSentinel()
if s.EnableSlaves {
s.slavesPools = newPoolMap()
slaveAddrs, err := s.slavesAddrs()
if err != nil {
return err
}
s.refreshSlaves(slaveAddrs)
go s.taskRefreshSlaves()
}
return err
}
// Sentinel sentry for sub events
func (s *Sentinel) sentry(addr string, pool *redis.Pool) error {
var failTimes int64
RESTART:
if atomic.LoadInt64(&s.closed) != 0 {
return nil
}
conn := pool.Get()
// notify := make(chan bool)
// if atomic.LoadInt64(&failTimes) > 0 {
// time.AfterFunc(
// time.Duration(atomic.LoadInt64(&failTimes))*5*time.Second,
// func() {
// notify <- false
// })
// } else {
// notify <- true
// }
// <-notify
err := s.subscribeSentinel(conn)
if err != nil {
atomic.AddInt64(&failTimes, 1)
}
if atomic.LoadInt64(&failTimes) > 3 {
if pool := s.sentinelPools.get(addr); pool != nil {
pool.Close()
}
s.sentinelPools.del(addr)
} else {
goto RESTART
}
return err
}