diff --git a/Dockerfile b/Dockerfile index 6bde876f2..d52ed3f6c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ ARG GIT_SHA=0000000 WORKDIR /src -COPY go.mod ./ go.sum ./ +COPY go.mod go.sum ./ RUN go mod download COPY pkg ./ cmd ./ version ./ diff --git a/pkg/apis/redis/v1alpha1/default.go b/pkg/apis/redis/v1alpha1/default.go index 49e96095d..32f4dfed7 100644 --- a/pkg/apis/redis/v1alpha1/default.go +++ b/pkg/apis/redis/v1alpha1/default.go @@ -12,9 +12,10 @@ import ( ) const ( - minMasterSize = 3 - minClusterReplicas = 1 - defaultRedisImage = "redis:5.0.4-alpine" + minMasterSize = 3 + minClusterReplicas = 1 + defaultRedisImage = "redis:5.0.4-alpine" + defaultMonitorImage = "oliver006/redis_exporter:latest" ) func (in *DistributedRedisCluster) DefaultSpec(log logr.Logger) bool { @@ -41,6 +42,11 @@ func (in *DistributedRedisCluster) DefaultSpec(log logr.Logger) bool { mon := in.Spec.Monitor if mon != nil { + if mon.Image == "" { + mon.Image = defaultMonitorImage + update = true + } + if mon.Prometheus == nil { mon.Prometheus = &PrometheusSpec{} update = true diff --git a/pkg/apis/redis/v1alpha1/distributedrediscluster_types.go b/pkg/apis/redis/v1alpha1/distributedrediscluster_types.go index 4f8472fee..2a41fb433 100644 --- a/pkg/apis/redis/v1alpha1/distributedrediscluster_types.go +++ b/pkg/apis/redis/v1alpha1/distributedrediscluster_types.go @@ -24,16 +24,18 @@ type DistributedRedisClusterSpec struct { ClusterReplicas int32 `json:"clusterReplicas,omitempty"` ServiceName string `json:"serviceName,omitempty"` Config map[string]string `json:"config,omitempty"` - Affinity *corev1.Affinity `json:"affinity,omitempty"` - NodeSelector map[string]string `json:"nodeSelector,omitempty"` - ToleRations []corev1.Toleration `json:"toleRations,omitempty"` - SecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"` - Annotations map[string]string `json:"annotations,omitempty"` - Storage *RedisStorage `json:"storage,omitempty"` - Resources *corev1.ResourceRequirements `json:"resources,omitempty"` - PasswordSecret *corev1.LocalObjectReference `json:"passwordSecret,omitempty"` - Monitor *AgentSpec `json:"monitor,omitempty"` - Init *InitSpec `json:"init,omitempty"` + // Set RequiredAntiAffinity to force the master-slave node anti-affinity. + RequiredAntiAffinity bool `json:"requiredAntiAffinity,omitempty"` + Affinity *corev1.Affinity `json:"affinity,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + ToleRations []corev1.Toleration `json:"toleRations,omitempty"` + SecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + Storage *RedisStorage `json:"storage,omitempty"` + Resources *corev1.ResourceRequirements `json:"resources,omitempty"` + PasswordSecret *corev1.LocalObjectReference `json:"passwordSecret,omitempty"` + Monitor *AgentSpec `json:"monitor,omitempty"` + Init *InitSpec `json:"init,omitempty"` } type AgentSpec struct { diff --git a/pkg/apis/redis/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/redis/v1alpha1/zz_generated.deepcopy.go index effcdcdb5..16d8e247e 100644 --- a/pkg/apis/redis/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/redis/v1alpha1/zz_generated.deepcopy.go @@ -133,11 +133,23 @@ func (in *DistributedRedisClusterList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DistributedRedisClusterSpec) DeepCopyInto(out *DistributedRedisClusterSpec) { *out = *in + if in.ImagePullSecrets != nil { + in, out := &in.ImagePullSecrets, &out.ImagePullSecrets + *out = make([]v1.LocalObjectReference, len(*in)) + copy(*out, *in) + } if in.Command != nil { in, out := &in.Command, &out.Command *out = make([]string, len(*in)) copy(*out, *in) } + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]v1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.Config != nil { in, out := &in.Config, &out.Config *out = make(map[string]string, len(*in)) @@ -224,6 +236,7 @@ func (in *DistributedRedisClusterStatus) DeepCopyInto(out *DistributedRedisClust (*in)[i].DeepCopyInto(&(*out)[i]) } } + in.Restore.DeepCopyInto(&out.Restore) return } @@ -441,6 +454,11 @@ func (in *RedisClusterBackupSpec) DeepCopyInto(out *RedisClusterBackupSpec) { *out = new(PodSpec) (*in).DeepCopyInto(*out) } + if in.ActiveDeadlineSeconds != nil { + in, out := &in.ActiveDeadlineSeconds, &out.ActiveDeadlineSeconds + *out = new(int64) + **out = **in + } return } @@ -515,3 +533,24 @@ func (in *RedisStorage) DeepCopy() *RedisStorage { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Restore) DeepCopyInto(out *Restore) { + *out = *in + if in.Backup != nil { + in, out := &in.Backup, &out.Backup + *out = new(RedisClusterBackup) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Restore. +func (in *Restore) DeepCopy() *Restore { + if in == nil { + return nil + } + out := new(Restore) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/controller/manager/ensurer.go b/pkg/controller/manager/ensurer.go index 2070e6620..bf1912613 100644 --- a/pkg/controller/manager/ensurer.go +++ b/pkg/controller/manager/ensurer.go @@ -119,7 +119,25 @@ func shouldUpdateRedis(cluster *redisv1alpha1.DistributedRedisCluster, sts *apps if result := expectResource.Limits.Cpu().Cmp(*currentResource.Limits.Cpu()); result != 0 { return true } - return false + return monitorChanged(cluster, sts) +} + +func monitorChanged(cluster *redisv1alpha1.DistributedRedisCluster, sts *appsv1.StatefulSet) bool { + if cluster.Spec.Monitor != nil { + for _, container := range sts.Spec.Template.Spec.Containers { + if container.Name == statefulsets.ExporterContainerName { + return false + } + } + return true + } else { + for _, container := range sts.Spec.Template.Spec.Containers { + if container.Name == statefulsets.ExporterContainerName { + return true + } + } + return false + } } func (r *realEnsureResource) ensureRedisPDB(cluster *redisv1alpha1.DistributedRedisCluster, name string, labels map[string]string) error { diff --git a/pkg/controller/redisclusterbackup/sync_handler.go b/pkg/controller/redisclusterbackup/sync_handler.go index 6d97e70ef..ceb867279 100644 --- a/pkg/controller/redisclusterbackup/sync_handler.go +++ b/pkg/controller/redisclusterbackup/sync_handler.go @@ -267,15 +267,14 @@ func (r *ReconcileRedisClusterBackup) getBackupJob(reqLogger logr.Logger, backup }, } if backup.Spec.PodSpec != nil { - podSpec := job.Spec.Template.Spec - podSpec.NodeSelector = backup.Spec.PodSpec.NodeSelector - podSpec.Affinity = backup.Spec.PodSpec.Affinity - podSpec.SchedulerName = backup.Spec.PodSpec.SchedulerName - podSpec.Tolerations = backup.Spec.PodSpec.Tolerations - podSpec.PriorityClassName = backup.Spec.PodSpec.PriorityClassName - podSpec.Priority = backup.Spec.PodSpec.Priority - podSpec.SecurityContext = backup.Spec.PodSpec.SecurityContext - podSpec.ImagePullSecrets = backup.Spec.PodSpec.ImagePullSecrets + job.Spec.Template.Spec.NodeSelector = backup.Spec.PodSpec.NodeSelector + job.Spec.Template.Spec.Affinity = backup.Spec.PodSpec.Affinity + job.Spec.Template.Spec.SchedulerName = backup.Spec.PodSpec.SchedulerName + job.Spec.Template.Spec.Tolerations = backup.Spec.PodSpec.Tolerations + job.Spec.Template.Spec.PriorityClassName = backup.Spec.PodSpec.PriorityClassName + job.Spec.Template.Spec.Priority = backup.Spec.PodSpec.Priority + job.Spec.Template.Spec.SecurityContext = backup.Spec.PodSpec.SecurityContext + job.Spec.Template.Spec.ImagePullSecrets = backup.Spec.PodSpec.ImagePullSecrets } if backup.Spec.Backend.Local != nil { job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{ diff --git a/pkg/redisutil/admin.go b/pkg/redisutil/admin.go index 0756d1910..af5e1455a 100644 --- a/pkg/redisutil/admin.go +++ b/pkg/redisutil/admin.go @@ -8,6 +8,8 @@ import ( "time" "github.com/go-logr/logr" + + "github.com/ucloud/redis-cluster-operator/pkg/utils" ) const ( @@ -333,6 +335,24 @@ func (a *Admin) GetAllConfig(c IClient, addr string) (map[string]string, error) return raw, nil } +var parseConfigMap = map[string]int8{ + "maxmemory": 0, + "proto-max-bulk-len": 0, + "client-query-buffer-limit": 0, + "repl-backlog-size": 0, + "auto-aof-rewrite-min-size": 0, + "active-defrag-ignore-bytes": 0, + "hash-max-ziplist-entries": 0, + "hash-max-ziplist-value": 0, + "stream-node-max-bytes": 0, + "set-max-intset-entries": 0, + "zset-max-ziplist-entries": 0, + "zset-max-ziplist-value": 0, + "hll-sparse-max-bytes": 0, + // TODO parse client-output-buffer-limit + //"client-output-buffer-limit": 0, +} + // SetConfigIfNeed set redis config func (a *Admin) SetConfigIfNeed(newConfig map[string]string) error { for addr, c := range a.Connections().GetAll() { @@ -342,6 +362,14 @@ func (a *Admin) SetConfigIfNeed(newConfig map[string]string) error { } for key, value := range newConfig { + var err error + if _, ok := parseConfigMap[key]; ok { + value, err = utils.ParseRedisMemConf(value) + if err != nil { + a.log.Error(err, "redis config format err", "key", key, "value", value) + continue + } + } if value != oldConfig[key] { a.log.V(3).Info("CONFIG SET", key, value) resp := c.Cmd("CONFIG", "SET", key, value) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index 2591f1889..1f063162d 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -24,6 +24,7 @@ const ( redisRestoreLocalVolumeName = "redis-local" redisServerName = "redis" hostnameTopologyKey = "kubernetes.io/hostname" + ExporterContainerName = "exporter" graceTime = 30 @@ -61,7 +62,7 @@ func NewStatefulSetForCR(cluster *redisv1alpha1.DistributedRedisCluster, ssName, }, Spec: corev1.PodSpec{ ImagePullSecrets: cluster.Spec.ImagePullSecrets, - Affinity: getAffinity(spec.Affinity, labels), + Affinity: getAffinity(cluster, labels), Tolerations: spec.ToleRations, SecurityContext: spec.SecurityContext, NodeSelector: cluster.Spec.NodeSelector, @@ -96,11 +97,37 @@ func NewStatefulSetForCR(cluster *redisv1alpha1.DistributedRedisCluster, ssName, return ss, nil } -func getAffinity(affinity *corev1.Affinity, labels map[string]string) *corev1.Affinity { +func getAffinity(cluster *redisv1alpha1.DistributedRedisCluster, labels map[string]string) *corev1.Affinity { + affinity := cluster.Spec.Affinity if affinity != nil { return affinity } + if cluster.Spec.RequiredAntiAffinity { + return &corev1.Affinity{ + PodAntiAffinity: &corev1.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{ + { + Weight: 100, + PodAffinityTerm: corev1.PodAffinityTerm{ + TopologyKey: hostnameTopologyKey, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{redisv1alpha1.LabelClusterName: cluster.Name}, + }, + }, + }, + }, + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ + { + TopologyKey: hostnameTopologyKey, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + }, + }, + }, + } + } // return a SOFT anti-affinity by default return &corev1.Affinity{ PodAntiAffinity: &corev1.PodAntiAffinity{ @@ -110,7 +137,7 @@ func getAffinity(affinity *corev1.Affinity, labels map[string]string) *corev1.Af PodAffinityTerm: corev1.PodAffinityTerm{ TopologyKey: hostnameTopologyKey, LabelSelector: &metav1.LabelSelector{ - MatchLabels: labels, + MatchLabels: map[string]string{redisv1alpha1.LabelClusterName: cluster.Name}, }, }, }, @@ -282,7 +309,7 @@ func redisServerContainer(cluster *redisv1alpha1.DistributedRedisCluster, passwo func redisExporterContainer(cluster *redisv1alpha1.DistributedRedisCluster, password *corev1.EnvVar) corev1.Container { container := corev1.Container{ - Name: "exporter", + Name: ExporterContainerName, Args: append([]string{ fmt.Sprintf("--web.listen-address=:%v", cluster.Spec.Monitor.Prometheus.Port), fmt.Sprintf("--web.telemetry-path=%v", redisv1alpha1.PrometheusExporterTelemetryPath), diff --git a/pkg/utils/parse.go b/pkg/utils/parse.go new file mode 100644 index 000000000..352e920a7 --- /dev/null +++ b/pkg/utils/parse.go @@ -0,0 +1,42 @@ +package utils + +import ( + "strconv" + "strings" +) + +func ParseRedisMemConf(p string) (string, error) { + var mul int64 = 1 + u := strings.ToLower(p) + digits := u + + if strings.HasSuffix(u, "k") { + digits = u[:len(u)-len("k")] + mul = 1000 + } else if strings.HasSuffix(u, "kb") { + digits = u[:len(u)-len("kb")] + mul = 1024 + } else if strings.HasSuffix(u, "m") { + digits = u[:len(u)-len("m")] + mul = 1000 * 1000 + } else if strings.HasSuffix(u, "mb") { + digits = u[:len(u)-len("mb")] + mul = 1024 * 1024 + } else if strings.HasSuffix(u, "g") { + digits = u[:len(u)-len("g")] + mul = 1000 * 1000 * 1000 + } else if strings.HasSuffix(u, "gb") { + digits = u[:len(u)-len("gb")] + mul = 1024 * 1024 * 1024 + } else if strings.HasSuffix(u, "b") { + digits = u[:len(u)-len("b")] + mul = 1 + } + + val, err := strconv.ParseInt(digits, 10, 64) + if err != nil { + return "", err + } + + return strconv.FormatInt(val*mul, 10), nil +} diff --git a/pkg/utils/parse_test.go b/pkg/utils/parse_test.go new file mode 100644 index 000000000..368f637de --- /dev/null +++ b/pkg/utils/parse_test.go @@ -0,0 +1,116 @@ +package utils + +import "testing" + +func TestParseRedisMemConf(t *testing.T) { + type args struct { + p string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "b", + args: args{ + p: "12b", + }, + want: "12", + wantErr: false, + }, + { + name: "digit", + args: args{ + p: "1202", + }, + want: "1202", + wantErr: false, + }, + { + name: "B", + args: args{ + p: "12B", + }, + want: "12", + wantErr: false, + }, + { + name: "k", + args: args{ + p: "12k", + }, + want: "12000", + wantErr: false, + }, + { + name: "kk", + args: args{ + p: "12kk", + }, + want: "", + wantErr: true, + }, + { + name: "kb", + args: args{ + p: "12kb", + }, + want: "12288", + wantErr: false, + }, + { + name: "Kb", + args: args{ + p: "12Kb", + }, + want: "12288", + wantErr: false, + }, + { + name: "m", + args: args{ + p: "12m", + }, + want: "12000000", + wantErr: false, + }, + { + name: "mB", + args: args{ + p: "12mb", + }, + want: "12582912", + wantErr: false, + }, + { + name: "g", + args: args{ + p: "12g", + }, + want: "12000000000", + wantErr: false, + }, + { + name: "gb", + args: args{ + p: "12gb", + }, + want: "12884901888", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseRedisMemConf(tt.args.p) + if (err != nil) != tt.wantErr { + t.Errorf("ParseRedisMemConf() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("ParseRedisMemConf() got = %v, want %v", got, tt.want) + } + }) + } +}