-
Notifications
You must be signed in to change notification settings - Fork 0
/
checkpoint.go
291 lines (271 loc) · 9.41 KB
/
checkpoint.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
package collector
import (
"errors"
"fmt"
nimo "github.com/gugemichael/nimo4go"
LOG "github.com/vinllen/log4go"
"github.com/vinllen/mgo"
"github.com/vinllen/mgo/bson"
"math"
"mongoshake/collector/configure"
utils "mongoshake/common"
"sort"
"sync"
"sync/atomic"
"time"
)
const (
StorageTypeAPI = "api"
StorageTypeDB = "database"
CheckpointName = "name"
CheckpointAckTs = "ackTs"
CheckpointSyncTs = "syncTs"
CheckpointMoveChunkIntervalMS = 5000
)
type Persist interface {
Load() error
Flush() error
}
type CheckpointManager struct {
syncMap map[string]*OplogSyncer
mutex sync.RWMutex
FlushChan chan bool
url string
db string
table string
startPosition int64
conn *utils.MongoConn
persistList []Persist
}
func NewCheckpointManager(startPosition int64) *CheckpointManager {
// we can't insert Timestamp(0, 0) that will be treat as Now() inserted
// into mongo. so we use Timestamp(0, 1)
startPosition = int64(math.Max(float64(startPosition), 1))
manager := &CheckpointManager{
syncMap: make(map[string]*OplogSyncer),
FlushChan: make(chan bool),
url: conf.Options.ContextStorageUrl,
db: utils.AppDatabase(),
table: conf.Options.ContextStorageCollection,
startPosition: startPosition,
}
return manager
}
func (manager *CheckpointManager) addOplogSyncer(syncer *OplogSyncer) {
manager.syncMap[syncer.replset] = syncer
}
func (manager *CheckpointManager) registerPersis(persist Persist) {
manager.persistList = append(manager.persistList, persist)
}
func (manager *CheckpointManager) start() {
startTime := time.Now()
checkTime := time.Now()
var intervalMs int64
if conf.Options.MoveChunkEnable && conf.Options.CheckpointInterval < CheckpointMoveChunkIntervalMS {
intervalMs = CheckpointMoveChunkIntervalMS
} else {
intervalMs = conf.Options.CheckpointInterval
}
nimo.GoRoutineInLoop(func() {
now := time.Now()
select {
case <-manager.FlushChan:
LOG.Info("CheckpointManager flush immediately begin")
manager.mutex.Lock()
if err := manager.Flush(); err != nil {
LOG.Warn("CheckpointManager flush immediately failed. %v", err)
} else {
LOG.Info("CheckpointManager flush immediately successful")
}
manager.mutex.Unlock()
case <-time.After(time.Second):
// update checkpoint ackts each second
// we delayed a few minutes to tolerate the receiver's flush buffer
// in AckRequired() tunnel. such as "rpc". While collector is restarted,
// we can't get the correct worker ack offset since collector have lost
// the unack offset...
if conf.Options.Tunnel != "direct" && now.Before(startTime.Add(3*time.Minute)) {
return
}
if now.Before(checkTime.Add(time.Duration(intervalMs) * time.Millisecond)) {
return
}
LOG.Info("CheckpointManager flush periodically begin")
manager.mutex.Lock()
if err := manager.Flush(); err != nil {
LOG.Warn("CheckpointManager flush periodically failed. %v", err)
} else {
LOG.Info("CheckpointManager flush periodically successful")
}
manager.mutex.Unlock()
checkTime = time.Now()
}
})
}
func (manager *CheckpointManager) Get(replset string) bson.MongoTimestamp {
ackTs, err := calculateSyncerAckTs(manager.syncMap[replset])
if err != nil {
LOG.Info("CheckpointManager get ackTs for sycner %v failed. %v", replset, err)
}
return ackTs
}
// firstly load checkpoit info to CheckpointManager without concurrent access
func (manager *CheckpointManager) Load() error {
if !manager.ensureNetwork() {
return fmt.Errorf("CheckpointManager connect to %v failed", manager.url)
}
manager.mutex.Lock()
defer manager.mutex.Unlock()
iter := manager.conn.Session.DB(manager.db).C(manager.table).Find(bson.M{}).Iter()
ckptDoc := make(map[string]interface{})
for iter.Next(ckptDoc) {
replset, ok1 := ckptDoc[CheckpointName].(string)
ackTs, ok2 := ckptDoc[CheckpointAckTs].(bson.MongoTimestamp)
syncTs, ok3 := ckptDoc[CheckpointSyncTs].(bson.MongoTimestamp)
if !ok1 || !ok2 || !ok3 {
return fmt.Errorf("CheckpointManager load checkpoint illegal record %v. ok1[%v] ok2[%v] ok3[%v]",
ckptDoc, ok1, ok2, ok3)
} else if syncer, ok := manager.syncMap[replset]; !ok {
fmt.Errorf("CheckpointManager load checkpoint unknown replset %v", ckptDoc)
} else {
syncer.batcher.syncTs = syncTs
syncer.batcher.unsyncTs = syncTs
for _, worker := range syncer.batcher.workerGroup {
worker.unack = int64(ackTs)
worker.ack = int64(ackTs)
}
LOG.Info("CheckpointManager load checkpoint set replset[%v] checkpoint to exist ackTs[%v] syncTs[%v]",
replset, utils.TimestampToLog(ackTs), utils.TimestampToLog(syncTs))
}
}
if err := iter.Close(); err != nil {
LOG.Critical("CheckpointManager close iterator failed. %v", err)
}
for replset, syncer := range manager.syncMap {
// there is no checkpoint before or this is a new node
if syncer.batcher.syncTs == 0 {
startTs := manager.startPosition << 32
syncer.batcher.syncTs = bson.MongoTimestamp(startTs)
syncer.batcher.unsyncTs = bson.MongoTimestamp(startTs)
for _, worker := range syncer.batcher.workerGroup {
worker.unack = startTs
worker.ack = startTs
}
LOG.Info("CheckpointManager load checkpoint set replset[%v] checkpoint to start position %v",
replset, utils.TimestampToLog(startTs))
}
}
for _, persist := range manager.persistList {
if err := persist.Load(); err != nil {
LOG.Critical("CheckpointManager flush persist object error %v", err)
manager.conn.Close()
manager.conn = nil
return err
}
}
return nil
}
func (manager *CheckpointManager) Flush() error {
if !manager.ensureNetwork() {
return fmt.Errorf("CheckpointManager connect to %v failed", manager.url)
}
for replset, syncer := range manager.syncMap {
ackTs := manager.Get(replset)
syncTs := syncer.batcher.syncTs
unsyncTs := syncer.batcher.unsyncTs
nimo.AssertTrue(syncTs == unsyncTs, "should panic when syncTs ¡= unsyncTs")
for _, worker := range syncer.batcher.workerGroup {
ack := bson.MongoTimestamp(atomic.LoadInt64(&worker.ack))
unack := bson.MongoTimestamp(atomic.LoadInt64(&worker.unack))
LOG.Info("syncer %v worker ack[%v] unack[%v] syncTs[%v]", replset,
utils.TimestampToLog(ack), utils.TimestampToLog(unack), utils.TimestampToLog(syncTs))
}
syncer.replMetric.AddCheckpoint(1)
syncer.replMetric.SetLSNCheckpoint(int64(ackTs))
ckptDoc := map[string]interface{}{
CheckpointName: replset,
CheckpointAckTs: ackTs,
CheckpointSyncTs: syncTs,
}
if _, err := manager.conn.Session.DB(manager.db).C(manager.table).
Upsert(bson.M{CheckpointName: replset}, ckptDoc); err != nil {
manager.conn.Close()
manager.conn = nil
return fmt.Errorf("CheckpointManager upsert %v error. %v", ckptDoc, err)
}
}
for _, persist := range manager.persistList {
if err := persist.Flush(); err != nil {
manager.conn.Close()
manager.conn = nil
return err
}
}
return nil
}
func (manager *CheckpointManager) ensureNetwork() bool {
// make connection if we don't already established
if manager.conn == nil {
if conn, err := utils.NewMongoConn(manager.url, utils.ConnectModePrimary, true); err == nil {
manager.conn = conn
} else {
LOG.Warn("CheckpointManager connect to %v failed. %v", manager.url, err)
manager.conn = nil
return false
}
}
// set WriteMajority while checkpoint is writing to ConfigServer
if conf.Options.IsShardCluster() {
manager.conn.Session.EnsureSafe(&mgo.Safe{WMode: utils.MajorityWriteConcern})
}
return true
}
func calculateSyncerAckTs(sync *OplogSyncer) (v bson.MongoTimestamp, err error) {
// no need to lock and eventually consistence is acceptable
allAcked := true
candidates := make([]int64, 0, len(sync.batcher.workerGroup))
allAckValues := make([]int64, 0, len(sync.batcher.workerGroup))
for _, worker := range sync.batcher.workerGroup {
// read ack value first because of we don't wanna
// a result of ack > unack. There wouldn't be cpu
// reorder under atomic !
ack := atomic.LoadInt64(&worker.ack)
unack := atomic.LoadInt64(&worker.unack)
if ack == 0 && unack == 0 {
// have no oplogs synced in this worker. skip
} else if ack == unack || worker.IsAllAcked() {
// all oplogs have been acked for right now or previous status
worker.AllAcked(true)
allAckValues = append(allAckValues, ack)
} else if unack > ack {
// most likely. partial oplogs acked (0 is possible)
candidates = append(candidates, ack)
allAcked = false
} else if unack < ack && unack == 0 {
// collector restarts. receiver unack value if from buffer
// this is rarely happened. However we have delayed for
// a bit log time. so we could use it
allAcked = false
} else if unack < ack && unack != 0 {
// we should wait the bigger unack follows up the ack
// they (unack and ack) will be equivalent soon !
return 0, fmt.Errorf("candidates should follow up unack[%d] ack[%d]", unack, ack)
}
}
if allAcked && len(allAckValues) != 0 {
// free to choose the maximum value. ascend order
// the last one is the biggest
sort.Sort(utils.Int64Slice(allAckValues))
return bson.MongoTimestamp(allAckValues[len(allAckValues)-1]), nil
}
if len(candidates) == 0 {
return 0, errors.New("no candidates ack values found")
}
// ascend order. first is the smallest
sort.Sort(utils.Int64Slice(candidates))
if candidates[0] == 0 {
return 0, errors.New("smallest candidates is zero")
}
LOG.Info("worker offset %v use lowest %d", candidates, candidates[0])
return bson.MongoTimestamp(candidates[0]), nil
}