-
Notifications
You must be signed in to change notification settings - Fork 66
/
check.go
256 lines (235 loc) · 8.95 KB
/
check.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
package service
import (
"errors"
"fmt"
"net"
"time"
"github.com/go-logr/logr"
goredis "github.com/go-redis/redis"
corev1 "k8s.io/api/core/v1"
redisv1beta1 "github.com/ucloud/redis-operator/pkg/apis/redis/v1beta1"
"github.com/ucloud/redis-operator/pkg/client/k8s"
"github.com/ucloud/redis-operator/pkg/client/redis"
"github.com/ucloud/redis-operator/pkg/util"
)
// RedisClusterCheck defines the intercace able to check the correct status of a redis cluster
type RedisClusterCheck interface {
CheckRedisNumber(redisCluster *redisv1beta1.RedisCluster) error
CheckSentinelNumber(redisCluster *redisv1beta1.RedisCluster) error
CheckSentinelReadyReplicas(redisCluster *redisv1beta1.RedisCluster) error
CheckAllSlavesFromMaster(master string, redisCluster *redisv1beta1.RedisCluster, auth *util.AuthConfig) error
CheckSentinelNumberInMemory(sentinel string, redisCluster *redisv1beta1.RedisCluster, auth *util.AuthConfig) error
CheckSentinelSlavesNumberInMemory(sentinel string, redisCluster *redisv1beta1.RedisCluster, auth *util.AuthConfig) error
CheckSentinelMonitor(sentinel string, monitor string, auth *util.AuthConfig) error
GetMasterIP(redisCluster *redisv1beta1.RedisCluster, auth *util.AuthConfig) (string, error)
GetNumberMasters(redisCluster *redisv1beta1.RedisCluster, auth *util.AuthConfig) (int, error)
GetRedisesIPs(redisCluster *redisv1beta1.RedisCluster, auth *util.AuthConfig) ([]string, error)
GetSentinelsIPs(redisCluster *redisv1beta1.RedisCluster) ([]string, error)
GetMinimumRedisPodTime(redisCluster *redisv1beta1.RedisCluster) (time.Duration, error)
CheckRedisConfig(redisCluster *redisv1beta1.RedisCluster, addr string, auth *util.AuthConfig) error
}
// RedisClusterChecker is our implementation of RedisClusterCheck intercace
type RedisClusterChecker struct {
k8sService k8s.Services
redisClient redis.Client
logger logr.Logger
}
// NewRedisClusterChecker creates an object of the RedisClusterChecker struct
func NewRedisClusterChecker(k8sService k8s.Services, redisClient redis.Client, logger logr.Logger) *RedisClusterChecker {
return &RedisClusterChecker{
k8sService: k8sService,
redisClient: redisClient,
logger: logger,
}
}
// CheckRedisConfig check current redis config is same as custom config
func (r *RedisClusterChecker) CheckRedisConfig(redisCluster *redisv1beta1.RedisCluster, addr string, auth *util.AuthConfig) error {
client := goredis.NewClient(&goredis.Options{
Addr: net.JoinHostPort(addr, "6379"),
Password: auth.Password,
DB: 0,
})
defer client.Close()
configs, err := r.redisClient.GetAllRedisConfig(client)
if err != nil {
return err
}
// TODO when custom config use unit like mb gb, will return configs conflict
for key, value := range redisCluster.Spec.Config {
if value != configs[key] {
return fmt.Errorf("%s configs conflict, expect: %s, current: %s", key, value, configs[key])
}
}
return nil
}
// CheckRedisNumber controls that the number of deployed redis is the same than the requested on the spec
func (r *RedisClusterChecker) CheckRedisNumber(rc *redisv1beta1.RedisCluster) error {
ss, err := r.k8sService.GetStatefulSet(rc.Namespace, util.GetRedisName(rc))
if err != nil {
return err
}
if rc.Spec.Size != *ss.Spec.Replicas {
return errors.New("number of redis pods differ from specification")
}
if rc.Spec.Size != ss.Status.ReadyReplicas {
return errors.New("waiting all of redis pods become ready")
}
return nil
}
// CheckSentinelNumber controls that the number of deployed sentinel is the same than the requested on the spec
func (r *RedisClusterChecker) CheckSentinelNumber(rc *redisv1beta1.RedisCluster) error {
d, err := r.k8sService.GetStatefulSet(rc.Namespace, util.GetSentinelName(rc))
if err != nil {
return err
}
if rc.Spec.Sentinel.Replicas != *d.Spec.Replicas {
return errors.New("number of sentinel pods differ from specification")
}
return nil
}
// CheckSentinelReadyReplicas controls that the number of deployed sentinel ready pod is the same than the requested on the spec
func (r *RedisClusterChecker) CheckSentinelReadyReplicas(rc *redisv1beta1.RedisCluster) error {
d, err := r.k8sService.GetStatefulSet(rc.Namespace, util.GetSentinelName(rc))
if err != nil {
return err
}
if rc.Spec.Sentinel.Replicas != d.Status.ReadyReplicas {
return errors.New("waiting all of sentinel pods become ready")
}
return nil
}
// CheckAllSlavesFromMaster controls that all slaves have the same master (the real one)
func (r *RedisClusterChecker) CheckAllSlavesFromMaster(master string, rc *redisv1beta1.RedisCluster, auth *util.AuthConfig) error {
rips, err := r.GetRedisesIPs(rc, auth)
if err != nil {
return err
}
for _, rip := range rips {
slave, err := r.redisClient.GetSlaveMasterIP(rip, auth)
if err != nil {
return err
}
if slave != "" && slave != master {
return fmt.Errorf("slave %s don't have the master %s, has %s", rip, master, slave)
}
}
return nil
}
// CheckSentinelNumberInMemory controls that sentinels have only the living sentinels on its memory.
func (r *RedisClusterChecker) CheckSentinelNumberInMemory(sentinel string, rc *redisv1beta1.RedisCluster, auth *util.AuthConfig) error {
nSentinels, err := r.redisClient.GetNumberSentinelsInMemory(sentinel, auth)
if err != nil {
return err
} else if nSentinels != rc.Spec.Sentinel.Replicas {
return errors.New("sentinels in memory mismatch")
}
return nil
}
// CheckSentinelSlavesNumberInMemory controls that sentinels have only the spected slaves number.
func (r *RedisClusterChecker) CheckSentinelSlavesNumberInMemory(sentinel string, rc *redisv1beta1.RedisCluster, auth *util.AuthConfig) error {
nSlaves, err := r.redisClient.GetNumberSentinelSlavesInMemory(sentinel, auth)
if err != nil {
return err
} else if nSlaves != rc.Spec.Size-1 {
return errors.New("sentinel's slaves in memory mismatch")
}
return nil
}
// CheckSentinelMonitor controls if the sentinels are monitoring the expected master
func (r *RedisClusterChecker) CheckSentinelMonitor(sentinel string, monitor string, auth *util.AuthConfig) error {
actualMonitorIP, err := r.redisClient.GetSentinelMonitor(sentinel, auth)
if err != nil {
return err
}
if actualMonitorIP != monitor {
return errors.New("the monitor on the sentinel config does not match with the expected one")
}
return nil
}
// GetMasterIP connects to all redis and returns the master of the redis cluster
func (r *RedisClusterChecker) GetMasterIP(rc *redisv1beta1.RedisCluster, auth *util.AuthConfig) (string, error) {
rips, err := r.GetRedisesIPs(rc, auth)
if err != nil {
return "", err
}
masters := []string{}
for _, rip := range rips {
master, err := r.redisClient.IsMaster(rip, auth)
if err != nil {
return "", err
}
if master {
masters = append(masters, rip)
}
}
if len(masters) != 1 {
return "", errors.New("number of redis nodes known as master is different than 1")
}
return masters[0], nil
}
// GetNumberMasters returns the number of redis nodes that are working as a master
func (r *RedisClusterChecker) GetNumberMasters(rc *redisv1beta1.RedisCluster, auth *util.AuthConfig) (int, error) {
nMasters := 0
rips, err := r.GetRedisesIPs(rc, auth)
if err != nil {
return nMasters, err
}
for _, rip := range rips {
master, err := r.redisClient.IsMaster(rip, auth)
if err != nil {
return nMasters, err
}
if master {
nMasters++
}
}
return nMasters, nil
}
// GetRedisesIPs returns the IPs of the Redis nodes
func (r *RedisClusterChecker) GetRedisesIPs(rc *redisv1beta1.RedisCluster, auth *util.AuthConfig) ([]string, error) {
redises := []string{}
rps, err := r.k8sService.GetStatefulSetPods(rc.Namespace, util.GetRedisName(rc))
if err != nil {
return nil, err
}
for _, rp := range rps.Items {
if rp.Status.Phase == corev1.PodRunning { // Only work with running pods
redises = append(redises, rp.Status.PodIP)
}
}
return redises, nil
}
// GetSentinelsIPs returns the IPs of the Sentinel nodes
func (r *RedisClusterChecker) GetSentinelsIPs(rc *redisv1beta1.RedisCluster) ([]string, error) {
sentinels := []string{}
rps, err := r.k8sService.GetStatefulSetPods(rc.Namespace, util.GetSentinelName(rc))
if err != nil {
return nil, err
}
for _, sp := range rps.Items {
if sp.Status.Phase == corev1.PodRunning { // Only work with running pods
sentinels = append(sentinels, sp.Status.PodIP)
}
}
return sentinels, nil
}
// GetMinimumRedisPodTime returns the minimum time a pod is alive
func (r *RedisClusterChecker) GetMinimumRedisPodTime(rc *redisv1beta1.RedisCluster) (time.Duration, error) {
minTime := 100000 * time.Hour // More than ten years
rps, err := r.k8sService.GetStatefulSetPods(rc.Namespace, util.GetRedisName(rc))
if err != nil {
return minTime, err
}
for _, redisNode := range rps.Items {
if redisNode.Status.StartTime == nil {
continue
}
start := redisNode.Status.StartTime.Round(time.Second)
alive := time.Now().Sub(start)
r.logger.V(2).Info(fmt.Sprintf("pod %s has been alive for %.f seconds", redisNode.Status.PodIP, alive.Seconds()))
if alive < minTime {
minTime = alive
}
}
return minTime, nil
}