-
Notifications
You must be signed in to change notification settings - Fork 2k
/
group.go
375 lines (344 loc) · 11.2 KB
/
group.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
/*
Copyright 2021 The Vitess Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controller
import (
"fmt"
"sort"
"strings"
"sync"
"vitess.io/vitess/go/stats"
"vitess.io/vitess/go/vt/log"
"vitess.io/vitess/go/vt/orchestrator/inst"
"vitess.io/vitess/go/vt/vtgr/db"
)
var (
groupOnlineSize = stats.NewGaugesWithMultiLabels("MysqlGroupOnlineSize", "Online MySQL server in the group", []string{"Keyspace", "Shard"})
isLostQuorum = stats.NewGaugesWithMultiLabels("MysqlGroupLostQuorum", "If MySQL group lost quorum", []string{"Keyspace", "Shard"})
)
// SQLGroup contains views from all the nodes within the shard
type SQLGroup struct {
views []*db.GroupView
resolvedView *ResolvedView
size int
singlePrimary bool
statsTags []string
sync.Mutex
}
// NewSQLGroup creates a new SQLGroup
func NewSQLGroup(size int, singlePrimary bool, keyspace, shard string) *SQLGroup {
return &SQLGroup{size: size, singlePrimary: singlePrimary, statsTags: []string{keyspace, shard}}
}
// ResolvedView is the resolved view
type ResolvedView struct {
groupName string
view map[inst.InstanceKey]db.GroupMember
}
// recordView adds a view to the group
func (group *SQLGroup) recordView(view *db.GroupView) {
group.Lock()
defer group.Unlock()
group.views = append(group.views, view)
}
// overrideView overrides a view to the group
func (group *SQLGroup) overrideView(views []*db.GroupView) {
group.Lock()
defer group.Unlock()
group.views = views
group.resolveLocked()
}
// clear reset the views
func (group *SQLGroup) clear() {
group.Lock()
defer group.Unlock()
group.views = nil
group.resolvedView = nil
}
// GetViews returns views from everyone in the group
func (group *SQLGroup) GetViews() []*db.GroupView {
group.Lock()
defer group.Unlock()
return group.views
}
// GetGroupName returns the group name
func (group *SQLGroup) GetGroupName() string {
group.Lock()
defer group.Unlock()
rv := group.resolvedView
return rv.groupName
}
// GetOnlineGroupInfo returns number of online members in the group and also if the primary is read only
func (group *SQLGroup) GetOnlineGroupInfo() (int, bool) {
group.Lock()
defer group.Unlock()
rv := group.resolvedView
view := rv.view
onlineSize := 0
isPrimaryReadOnly := false
for _, status := range view {
if status.State == db.ONLINE {
onlineSize++
}
if status.Role == db.PRIMARY {
isPrimaryReadOnly = isPrimaryReadOnly || status.ReadOnly
}
}
return onlineSize, isPrimaryReadOnly
}
// IsUnconnectedReplica checks if the node is connected to a group
func (group *SQLGroup) IsUnconnectedReplica(instanceKey *inst.InstanceKey) bool {
if instanceKey == nil {
return false
}
group.Lock()
defer group.Unlock()
rv := group.resolvedView
view := rv.view
status, ok := view[*instanceKey]
if !ok {
return true
}
return status.State != db.ONLINE && status.State != db.RECOVERING
}
// IsAllOfflineOrError returns true if all the nodes are in offline mode
func (group *SQLGroup) IsAllOfflineOrError() bool {
group.Lock()
defer group.Unlock()
rv := group.resolvedView
view := rv.view
for _, status := range view {
if status.State != db.OFFLINE && status.State != db.ERROR {
return false
}
}
return true
}
// GetStatus returns GroupMember status for given a host
func (group *SQLGroup) GetStatus(instanceKey *inst.InstanceKey) *db.GroupMember {
if instanceKey == nil {
return nil
}
group.Lock()
defer group.Unlock()
rv := group.resolvedView
view := rv.view
status, ok := view[*instanceKey]
if !ok {
return nil
}
return &status
}
// IsSafeToBootstrap checks if it is safe to bootstrap a mysql group
func (group *SQLGroup) IsSafeToBootstrap() bool {
group.Lock()
defer group.Unlock()
// for bootstrap we require group at least has quorum number of views
// this is to make sure we don't bootstrap a group improperly
if len(group.views) < group.size {
log.Errorf("[sql_group] cannot bootstrap because we only have %v views | expected %v", len(group.views), group.size)
return false
}
// we think it is safe to bootstrap a group if all the views don't have a primary host
host, port, _ := group.getPrimaryLocked()
if host != "" || port != 0 {
log.Warningf("not safe to bootstrap sql group because %v/%v might already be primary", host, port)
}
return host == "" && port == 0
}
// GetPrimary returns the hostname, port of the primary that everyone agreed on
// isActive bool indicates if there is any node in the group whose primary is "ONLINE"
func (group *SQLGroup) GetPrimary() (string, int, bool) {
group.Lock()
defer group.Unlock()
return group.getPrimaryLocked()
}
func (group *SQLGroup) getPrimaryLocked() (string, int, bool) {
rv := group.resolvedView
view := rv.view
for instance, status := range view {
if status.Role == db.PRIMARY {
return instance.Hostname, instance.Port, status.State == db.ONLINE
}
}
return "", 0, false
}
// Resolve merges the views into a map
func (group *SQLGroup) Resolve() error {
group.Lock()
defer group.Unlock()
return group.resolveLocked()
}
func (group *SQLGroup) resolveLocked() error {
rv := &ResolvedView{}
group.resolvedView = rv
m := make(map[inst.InstanceKey]db.GroupMember)
for _, view := range group.views {
if rv.groupName == "" && view.GroupName != "" {
rv.groupName = view.GroupName
}
if view.GroupName != "" && rv.groupName != view.GroupName {
log.Errorf("previous group name %v found %v", rv.groupName, view.GroupName)
return db.ErrGroupSplitBrain
}
for _, member := range view.UnresolvedMembers {
instance := view.CreateInstanceKey(member)
memberState := member.State
memberRole := member.Role
isReadOnly := member.ReadOnly
st, ok := m[instance]
if !ok {
m[instance] = db.GroupMember{
HostName: instance.Hostname,
Port: instance.Port,
State: memberState,
Role: memberRole,
ReadOnly: isReadOnly,
}
continue
}
if st.State == memberState && st.Role == memberRole && st.ReadOnly == isReadOnly {
continue
}
m[instance] = db.GroupMember{
HostName: instance.Hostname,
Port: instance.Port,
State: group.mergeState(st.State, memberState),
Role: group.mergeRole(st.Role, memberRole),
ReadOnly: st.ReadOnly || isReadOnly,
}
}
}
rv.view = m
return group.resolvedView.validate(group.singlePrimary, group.statsTags)
}
func (rv *ResolvedView) validate(singlePrimary bool, statsTags []string) error {
if !rv.hasGroup() {
log.Info("Resolved view does not have a group")
return nil
}
hasPrimary := false
primaryState := db.UNKNOWNSTATE
var onlineCount, recoveringCount, unreachableCount, offlineCount, errorCount int
for _, status := range rv.view {
if status.Role == db.PRIMARY {
if singlePrimary && hasPrimary {
log.Errorf("Found more than one primary in the group")
return db.ErrGroupSplitBrain
}
hasPrimary = true
primaryState = status.State
if status.State != db.ONLINE {
log.Warningf("Found a PRIMARY not ONLINE (%v)", status.State)
}
}
switch status.State {
case db.ONLINE:
onlineCount++
case db.UNREACHABLE:
unreachableCount++
case db.OFFLINE:
offlineCount++
case db.ERROR:
errorCount++
case db.RECOVERING:
recoveringCount++
}
}
groupOnlineSize.Set(statsTags, int64(onlineCount))
if unreachableCount > 0 || errorCount > 0 || offlineCount > 0 {
log.Warningf("Some of nodes are unconnected in the group. hasPrimary=%v (%v), online_count=%v, recovering_count=%v, unreachable_count=%v, offline_count=%v, error_count=%v", hasPrimary, primaryState, onlineCount, recoveringCount, unreachableCount, offlineCount, errorCount)
}
if unreachableCount >= len(rv.view)/2+1 {
log.Errorf("Backoff error by quorum unreachable: found %v number of UNREACHABLE nodes while quorum is %v", unreachableCount, len(rv.view)/2+1)
isLostQuorum.Set(statsTags, 1)
} else {
isLostQuorum.Set(statsTags, 0)
}
// In theory there should be no UNREACHABLE nodes
// raise ErrGroupBackoffError to backoff and wait
// If we lost quorum, then the group is not writable
// If we still have a functioning group, we can backoff and wait
// the unreachable node should either be expelled or we have a frozen view
// Note: this means we should set group_replication_unreachable_majority_timeout
// greater than 0. Otherwise VTGR can see all nodes are ONLINE when a single node
// is partitioned and end up doing nothing.
if unreachableCount > 0 {
return db.ErrGroupBackoffError
}
// Ongoing bootstrap, we should backoff and wait
if recoveringCount == 1 && (offlineCount+recoveringCount == len(rv.view)) {
log.Warningf("Group has one recovery node with all others in offline mode")
return db.ErrGroupOngoingBootstrap
}
// We don't have quorum number of unreachable, but the primary is not online
// This most likely means there is a failover in the group we should back off and wait
if hasPrimary && primaryState != db.ONLINE {
log.Warningf("Found a PRIMARY that is not ONLINE (%v)", primaryState)
return db.ErrGroupBackoffError
}
// If all the node in view are OFFLINE or ERROR, it is an inactive group
// It is expected to have no primary in this case
if !hasPrimary && (offlineCount+errorCount != len(rv.view)) {
log.Warningf("Group is NOT all offline or error without a primary node")
return db.ErrGroupBackoffError
}
return nil
}
func (rv *ResolvedView) hasGroup() bool {
return rv.groupName != ""
}
func (group *SQLGroup) mergeState(s1, s2 db.MemberState) db.MemberState {
return db.MemberState(group.maxStatus(int(s1), int(s2)))
}
func (group *SQLGroup) mergeRole(r1, r2 db.MemberRole) db.MemberRole {
return db.MemberRole(group.maxStatus(int(r1), int(r2)))
}
func (group *SQLGroup) maxStatus(a, b int) int {
if a > b {
return a
}
return b
}
// ToString returns a string representatino of the sql group
func (group *SQLGroup) ToString() string {
group.Lock()
defer group.Unlock()
var sb strings.Builder
views := group.views
for _, view := range views {
sb.WriteString(fmt.Sprintf("[%s] SQLGroup group=%s", view.TabletAlias, view.GroupName))
for _, member := range view.UnresolvedMembers {
sb.WriteString(fmt.Sprintf(" | %s %s %s readonly=%v", member.HostName, member.Role, member.State, member.ReadOnly))
}
sb.WriteString("\n")
}
rv := group.resolvedView
if rv != nil {
sb.WriteString("[resolved_view]\n")
sb.WriteString(fmt.Sprintf("group_name=%v\n", rv.groupName))
keys := make([]inst.InstanceKey, 0, len(rv.view))
for k := range rv.view {
keys = append(keys, k)
}
sort.Slice(keys, func(i, j int) bool {
return keys[i].Hostname < keys[j].Hostname
})
for _, instance := range keys {
status := rv.view[instance]
sb.WriteString(fmt.Sprintf("[%s] state=%v role=%v readonly=%v\n", instance.Hostname, status.State, status.Role, status.ReadOnly))
}
}
return sb.String()
}
func (group *SQLGroup) quorum() int {
return group.size/2 + 1
}