-
Notifications
You must be signed in to change notification settings - Fork 269
/
member.go
87 lines (78 loc) · 2.24 KB
/
member.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"fmt"
"time"
"github.com/pingcap/tiflow/dm/master/scheduler"
"github.com/pingcap/tiflow/dm/pb"
)
const (
checkMemberTimes = 5
checkMemberInterval = 10 * time.Second
)
// checkMembersReadyLoop checks whether all DM-master and DM-worker members have been ready.
// NOTE: in this chaos case, we ensure 3 DM-master and 3 DM-worker started.
func checkMembersReadyLoop(ctx context.Context, cli pb.MasterClient, masterCount, workerCount int) (err error) {
for i := 0; i < checkMemberTimes; i++ {
select {
case <-ctx.Done():
return nil
case <-time.After(checkMemberInterval):
err = checkMembersReady(ctx, cli, masterCount, workerCount)
if err == nil {
return nil
}
}
}
return err
}
func checkMembersReady(ctx context.Context, cli pb.MasterClient, masterCount, workerCount int) error {
resp, err := cli.ListMember(ctx, &pb.ListMemberRequest{})
if err != nil {
return err
} else if !resp.Result {
return fmt.Errorf("fail to list member: %s", resp.Msg)
}
var (
hasLeader bool
allMasterAlive bool
allWorkerOnline bool
)
for _, m := range resp.Members {
if m.GetLeader() != nil {
hasLeader = true
} else if lm := m.GetMaster(); lm != nil {
var aliveCount int
for _, master := range lm.Masters {
if master.Alive {
aliveCount++
}
}
allMasterAlive = aliveCount == masterCount
} else if lw := m.GetWorker(); lw != nil {
var onlineCount int
for _, worker := range lw.Workers {
if worker.Stage != string(scheduler.WorkerOffline) {
onlineCount++
}
}
allWorkerOnline = onlineCount == workerCount
}
}
if !hasLeader || !allMasterAlive || !allWorkerOnline {
return fmt.Errorf("not all members are ready: %s", resp.String())
}
return nil
}