-
Notifications
You must be signed in to change notification settings - Fork 358
/
heal.go
158 lines (135 loc) · 5.3 KB
/
heal.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
package service
import (
"errors"
"sort"
"strconv"
redisfailoverv1 "github.com/spotahome/redis-operator/api/redisfailover/v1"
"github.com/spotahome/redis-operator/log"
"github.com/spotahome/redis-operator/service/k8s"
"github.com/spotahome/redis-operator/service/redis"
)
// RedisFailoverHeal defines the interface able to fix the problems on the redis failovers
type RedisFailoverHeal interface {
MakeMaster(ip string, rFailover *redisfailoverv1.RedisFailover) error
SetOldestAsMaster(rFailover *redisfailoverv1.RedisFailover) error
SetMasterOnAll(masterIP string, rFailover *redisfailoverv1.RedisFailover) error
NewSentinelMonitor(ip string, monitor string, rFailover *redisfailoverv1.RedisFailover) error
RestoreSentinel(ip string) error
SetSentinelCustomConfig(ip string, rFailover *redisfailoverv1.RedisFailover) error
SetRedisCustomConfig(ip string, rFailover *redisfailoverv1.RedisFailover) error
DeletePod(podName string, rFailover *redisfailoverv1.RedisFailover) error
}
// RedisFailoverHealer is our implementation of RedisFailoverCheck interface
type RedisFailoverHealer struct {
k8sService k8s.Services
redisClient redis.Client
logger log.Logger
}
// NewRedisFailoverHealer creates an object of the RedisFailoverChecker struct
func NewRedisFailoverHealer(k8sService k8s.Services, redisClient redis.Client, logger log.Logger) *RedisFailoverHealer {
return &RedisFailoverHealer{
k8sService: k8sService,
redisClient: redisClient,
logger: logger,
}
}
func (r *RedisFailoverHealer) MakeMaster(ip string, rf *redisfailoverv1.RedisFailover) error {
password, err := k8s.GetRedisPassword(r.k8sService, rf)
if err != nil {
return err
}
return r.redisClient.MakeMaster(ip, password)
}
// SetOldestAsMaster puts all redis to the same master, choosen by order of appearance
func (r *RedisFailoverHealer) SetOldestAsMaster(rf *redisfailoverv1.RedisFailover) error {
ssp, err := r.k8sService.GetStatefulSetPods(rf.Namespace, GetRedisName(rf))
if err != nil {
return err
}
if len(ssp.Items) < 1 {
return errors.New("number of redis pods are 0")
}
// Order the pods so we start by the oldest one
sort.Slice(ssp.Items, func(i, j int) bool {
return ssp.Items[i].CreationTimestamp.Before(&ssp.Items[j].CreationTimestamp)
})
password, err := k8s.GetRedisPassword(r.k8sService, rf)
if err != nil {
return err
}
newMasterIP := ""
for _, pod := range ssp.Items {
if newMasterIP == "" {
newMasterIP = pod.Status.PodIP
r.logger.Debugf("New master is %s with ip %s", pod.Name, newMasterIP)
if err := r.redisClient.MakeMaster(newMasterIP, password); err != nil {
return err
}
} else {
r.logger.Debugf("Making pod %s slave of %s", pod.Name, newMasterIP)
if err := r.redisClient.MakeSlaveOf(pod.Status.PodIP, newMasterIP, password); err != nil {
return err
}
}
}
return nil
}
// SetMasterOnAll puts all redis nodes as a slave of a given master
func (r *RedisFailoverHealer) SetMasterOnAll(masterIP string, rf *redisfailoverv1.RedisFailover) error {
ssp, err := r.k8sService.GetStatefulSetPods(rf.Namespace, GetRedisName(rf))
if err != nil {
return err
}
password, err := k8s.GetRedisPassword(r.k8sService, rf)
if err != nil {
return err
}
for _, pod := range ssp.Items {
if pod.Status.PodIP == masterIP {
r.logger.Debugf("Ensure pod %s is master", pod.Name)
if err := r.redisClient.MakeMaster(masterIP, password); err != nil {
return err
}
} else {
r.logger.Debugf("Making pod %s slave of %s", pod.Name, masterIP)
if err := r.redisClient.MakeSlaveOf(pod.Status.PodIP, masterIP, password); err != nil {
return err
}
}
}
return nil
}
// NewSentinelMonitor changes the master that Sentinel has to monitor
func (r *RedisFailoverHealer) NewSentinelMonitor(ip string, monitor string, rf *redisfailoverv1.RedisFailover) error {
r.logger.Debug("Sentinel is not monitoring the correct master, changing...")
quorum := strconv.Itoa(int(getQuorum(rf)))
password, err := k8s.GetRedisPassword(r.k8sService, rf)
if err != nil {
return err
}
return r.redisClient.MonitorRedis(ip, monitor, quorum, password)
}
// RestoreSentinel clear the number of sentinels on memory
func (r *RedisFailoverHealer) RestoreSentinel(ip string) error {
r.logger.Debugf("Restoring sentinel %s...", ip)
return r.redisClient.ResetSentinel(ip)
}
// SetSentinelCustomConfig will call sentinel to set the configuration given in config
func (r *RedisFailoverHealer) SetSentinelCustomConfig(ip string, rf *redisfailoverv1.RedisFailover) error {
r.logger.Debugf("Setting the custom config on sentinel %s...", ip)
return r.redisClient.SetCustomSentinelConfig(ip, rf.Spec.Sentinel.CustomConfig)
}
// SetRedisCustomConfig will call redis to set the configuration given in config
func (r *RedisFailoverHealer) SetRedisCustomConfig(ip string, rf *redisfailoverv1.RedisFailover) error {
r.logger.Debugf("Setting the custom config on redis %s...", ip)
password, err := k8s.GetRedisPassword(r.k8sService, rf)
if err != nil {
return err
}
return r.redisClient.SetCustomRedisConfig(ip, rf.Spec.Redis.CustomConfig, password)
}
//DeletePod delete a failing pod so kubernetes relaunch it again
func (r *RedisFailoverHealer) DeletePod(podName string, rFailover *redisfailoverv1.RedisFailover) error {
r.logger.Debugf("Deleting pods %s...", podName)
return r.k8sService.DeletePod(rFailover.Namespace, podName)
}