Skip to content

Commit

Permalink
fix
Browse files Browse the repository at this point in the history
  • Loading branch information
ideascf committed Apr 2, 2024
1 parent 1ea0ffd commit ab80380
Show file tree
Hide file tree
Showing 8 changed files with 47 additions and 30 deletions.
2 changes: 1 addition & 1 deletion pkg/apis/label/label.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ const (
AnnTiflashMountCMInTiflashContainer = "tiflash.tidb.pingcap.com/mount-cm-in-tiflash-container"
// AnnoConfigMapNameForNewSTSPrefix indicates that the xxx_member_manager should use the annotation value as name of ConfigMap
// if the value is not empty when xxx_member_manager try to CREATE sts.
AnnoConfigMapNameForNewSTSPrefix = "tidb.pingcap.com/configmap-name-for-new-sts/"
AnnoConfigMapNameForNewSTSPrefix = "tidb.pingcap.com/configmap-name-for-new-sts-"

// AnnPVCScaleInTime is pvc scaled in time key used in PVC for e2e test only
AnnPVCScaleInTime = "tidb.pingcap.com/scale-in-time"
Expand Down
9 changes: 6 additions & 3 deletions pkg/manager/member/pd_member_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
package member

import (
"context"
"fmt"
"path"
"regexp"
Expand Down Expand Up @@ -214,9 +215,6 @@ func (m *pdMemberManager) syncPDStatefulSetForTidbCluster(tc *v1alpha1.TidbClust
return err
}
if setNotExist {
if _, err := mngerutils.KeepConfigMapNameUnchangedWhenCreateSTS(klog.V(4), m.deps.ConfigMapLister, tc, v1alpha1.PDMemberType, cm); err != nil {
return err
}
err = mngerutils.SetStatefulSetLastAppliedConfigAnnotation(newPDSet)
if err != nil {
return err
Expand Down Expand Up @@ -453,6 +451,11 @@ func (m *pdMemberManager) syncPDConfigMap(tc *v1alpha1.TidbCluster, set *apps.St
inUseName = mngerutils.FindConfigMapVolume(&set.Spec.Template.Spec, func(name string) bool {
return strings.HasPrefix(name, controller.PDMemberName(tc.Name))
})
} else {
inUseName, err = mngerutils.FindConfigMapNameFromTCAnno(context.Background(), m.deps.ConfigMapLister, tc, v1alpha1.PDMemberType, newCm)
if err != nil {
return nil, err
}
}

err = mngerutils.UpdateConfigMapIfNeed(m.deps.ConfigMapLister, tc.BasePDSpec().ConfigUpdateStrategy(), inUseName, newCm)
Expand Down
8 changes: 5 additions & 3 deletions pkg/manager/member/tidb_member_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -272,9 +272,6 @@ func (m *tidbMemberManager) syncTiDBStatefulSetForTidbCluster(tc *v1alpha1.TidbC
}

if setNotExist {
if _, err := mngerutils.KeepConfigMapNameUnchangedWhenCreateSTS(klog.V(4), m.deps.ConfigMapLister, tc, v1alpha1.TiDBMemberType, cm); err != nil {
return err
}
err = mngerutils.SetStatefulSetLastAppliedConfigAnnotation(newTiDBSet)
if err != nil {
return err
Expand Down Expand Up @@ -557,6 +554,11 @@ func (m *tidbMemberManager) syncTiDBConfigMap(tc *v1alpha1.TidbCluster, set *app
inUseName = mngerutils.FindConfigMapVolume(&set.Spec.Template.Spec, func(name string) bool {
return strings.HasPrefix(name, controller.TiDBMemberName(tc.Name))
})
} else {
inUseName, err = mngerutils.FindConfigMapNameFromTCAnno(context.Background(), m.deps.ConfigMapLister, tc, v1alpha1.TiDBMemberType, newCm)
if err != nil {
return nil, err
}
}

klog.V(3).Info("get tidb in use config map name: ", inUseName)
Expand Down
9 changes: 6 additions & 3 deletions pkg/manager/member/tiflash_member_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
package member

import (
"context"
"fmt"
"reflect"
"regexp"
Expand Down Expand Up @@ -251,9 +252,6 @@ func (m *tiflashMemberManager) syncStatefulSet(tc *v1alpha1.TidbCluster) error {
klog.Infof("TidbCluster: %s/%s, waiting for PD cluster running", ns, tcName)
return nil
}
if _, err := mngerutils.KeepConfigMapNameUnchangedWhenCreateSTS(klog.V(4), m.deps.ConfigMapLister, tc, v1alpha1.TiFlashMemberType, cm); err != nil {
return err
}
err = mngerutils.SetStatefulSetLastAppliedConfigAnnotation(newSet)
if err != nil {
return err
Expand Down Expand Up @@ -307,6 +305,11 @@ func (m *tiflashMemberManager) syncConfigMap(tc *v1alpha1.TidbCluster, set *apps
inUseName = mngerutils.FindConfigMapVolume(&set.Spec.Template.Spec, func(name string) bool {
return strings.HasPrefix(name, controller.TiFlashMemberName(tc.Name))
})
} else {
inUseName, err = mngerutils.FindConfigMapNameFromTCAnno(context.Background(), m.deps.ConfigMapLister, tc, v1alpha1.TiFlashMemberType, newCm)
if err != nil {
return nil, err
}
}

err = mngerutils.UpdateConfigMapIfNeed(m.deps.ConfigMapLister, tc.BaseTiFlashSpec().ConfigUpdateStrategy(), inUseName, newCm)
Expand Down
9 changes: 6 additions & 3 deletions pkg/manager/member/tikv_member_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
package member

import (
"context"
"fmt"
"path"
"reflect"
Expand Down Expand Up @@ -239,9 +240,6 @@ func (m *tikvMemberManager) syncStatefulSetForTidbCluster(tc *v1alpha1.TidbClust
return err
}
if setNotExist {
if _, err := mngerutils.KeepConfigMapNameUnchangedWhenCreateSTS(klog.V(4), m.deps.ConfigMapLister, tc, v1alpha1.TiKVMemberType, cm); err != nil {
return err
}
err = mngerutils.SetStatefulSetLastAppliedConfigAnnotation(newSet)
if err != nil {
return err
Expand Down Expand Up @@ -312,6 +310,11 @@ func (m *tikvMemberManager) syncTiKVConfigMap(tc *v1alpha1.TidbCluster, set *app
inUseName = mngerutils.FindConfigMapVolume(&set.Spec.Template.Spec, func(name string) bool {
return strings.HasPrefix(name, controller.TiKVMemberName(tc.Name))
})
} else {
inUseName, err = mngerutils.FindConfigMapNameFromTCAnno(context.Background(), m.deps.ConfigMapLister, tc, v1alpha1.TiKVMemberType, newCm)
if err != nil {
return nil, err
}
}

err = mngerutils.UpdateConfigMapIfNeed(m.deps.ConfigMapLister, tc.BaseTiKVSpec().ConfigUpdateStrategy(), inUseName, newCm)
Expand Down
9 changes: 6 additions & 3 deletions pkg/manager/member/tiproxy_member_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
package member

import (
"context"
"fmt"
"path"
"path/filepath"
Expand Down Expand Up @@ -194,6 +195,11 @@ func (m *tiproxyMemberManager) syncConfigMap(tc *v1alpha1.TidbCluster, set *apps
inUseName = mngerutils.FindConfigMapVolume(&set.Spec.Template.Spec, func(name string) bool {
return strings.HasPrefix(name, controller.TiProxyMemberName(tc.Name))
})
} else {
inUseName, err = mngerutils.FindConfigMapNameFromTCAnno(context.Background(), m.deps.ConfigMapLister, tc, v1alpha1.TiProxyMemberType, newCm)
if err != nil {
return nil, err
}
}

klog.V(4).Info("get tiproxy in use config map name: ", inUseName)
Expand Down Expand Up @@ -235,9 +241,6 @@ func (m *tiproxyMemberManager) syncStatefulSet(tc *v1alpha1.TidbCluster) error {
}

if stsNotExist {
if _, err := mngerutils.KeepConfigMapNameUnchangedWhenCreateSTS(klog.V(4), m.deps.ConfigMapLister, tc, v1alpha1.PDMemberType, cm); err != nil {
return err
}
err = mngerutils.SetStatefulSetLastAppliedConfigAnnotation(newSts)
if err != nil {
return err
Expand Down
23 changes: 12 additions & 11 deletions pkg/manager/utils/configmap.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
package utils

import (
"context"
"crypto/sha256"
"encoding/json"
"fmt"
Expand Down Expand Up @@ -55,26 +56,26 @@ func FindConfigMapVolume(podSpec *corev1.PodSpec, pred func(string) bool) string
return ""
}

// KeepConfigMapNameUnchangedWhenCreateSTS is used to overwrite ConfigMap name to keep ConfigMap name remains unchanged
// when create a new StatefulSet.
// FindConfigMapNameFromTCAnno is used to find ConfigMap name from tc.annotations to keep ConfigMap name remains unchanged
// when recreating a StatefulSet. And more, it will check ConfigMap data against the newCm to ensure no data change happen.
// In some cases, we may need to delete and recreate STS for updating some immutable fields and are
// expected to keep the name of ConfigMap unchanged to ensure no accidentally restart of pod.
// For example: Updating storage size, iops or throughput of PVC using by TiKV. Now,
// the annotation is set by pvc_resizer(not supported yet), pvc_modifier or pvc_replacer, See pkg/manager/utils/statefulset.go:DeleteStatefulSetWithOrphan.
func KeepConfigMapNameUnchangedWhenCreateSTS(logger klog.Verbose, cmLister corelisters.ConfigMapLister, tc *v1alpha1.TidbCluster, componentType v1alpha1.MemberType, cm *corev1.ConfigMap) (overwritten bool, _ error) {
func FindConfigMapNameFromTCAnno(ctx context.Context, cmLister corelisters.ConfigMapLister, tc *v1alpha1.TidbCluster, componentType v1alpha1.MemberType, newCm *corev1.ConfigMap) (cmName string, _ error) {
logger := klog.FromContext(ctx).WithValues("comp", componentType, "tc", fmt.Sprintf("%s/%s", tc.Namespace, tc.Name))
cmNameInAnno := tc.Annotations[label.AnnoKeyOfConfigMapNameForNewSTS(string(componentType))]
if cmNameInAnno == "" || cm.Name == cmNameInAnno {
return false, nil
if cmNameInAnno == "" || cmNameInAnno == newCm.Name {
return cmNameInAnno, nil
}

logger.Infof("another cm name found in AnnoConfigMapNameForNewSTSPrefix, use it. comp=%s, name=%s, nameInAnno=%s", componentType, cm.Name, cmNameInAnno)
logger.Info("another cm name found in AnnoConfigMapNameForNewSTSPrefix, use it.", "name", newCm.Name, "nameInAnno", cmNameInAnno)
cmInAnno, err := cmLister.ConfigMaps(tc.Namespace).Get(cmNameInAnno)
if err != nil {
return false, fmt.Errorf("failed to get configmap %s/%s: %w", tc.Namespace, cmNameInAnno, err)
return "", fmt.Errorf("failed to get configmap %s/%s: %w", tc.Namespace, cmNameInAnno, err)
}
if !equality.Semantic.DeepEqual(cmInAnno.Data, cm.Data) {
return false, fmt.Errorf("unexpected ConfigMap data change. comp=%s, name=%s, nameInAnno=%s", componentType, cm.Name, cmNameInAnno)
if !equality.Semantic.DeepEqual(cmInAnno.Data, newCm.Data) {
return "", fmt.Errorf("unexpected ConfigMap data change. comp=%s, name=%s, nameInAnno=%s", componentType, newCm.Name, cmNameInAnno)
}
cm.Name = cmNameInAnno
return true, nil
return cmNameInAnno, nil
}
8 changes: 5 additions & 3 deletions pkg/manager/utils/statefulset.go
Original file line number Diff line number Diff line change
Expand Up @@ -213,12 +213,14 @@ func DeleteStatefulSetWithOrphan(
tc *v1alpha1.TidbCluster, sts *apps.StatefulSet) error {

// Store the name of currently using configmap into TC to make sure xxx_member_manager can use the same ConfigMap name
// when creating(restore) new StatefulSet. See pkg/manager/utils/configmap.go:KeepConfigMapNameUnchangedWhenCreateSTS.
// when creating(restore) new StatefulSet. See pkg/manager/utils/configmap.go:FindConfigMapNameFromTCAnno.
memberType := v1alpha1.MemberType(label.Label(sts.Labels).ComponentType())
inUseCMName := FindConfigMapVolume(&sts.Spec.Template.Spec, func(name string) bool {
return strings.HasPrefix(name, controller.MemberName(name, memberType))
return strings.HasPrefix(name, controller.MemberName(tc.Name, memberType))
})
tc.Annotations[(label.AnnoKeyOfConfigMapNameForNewSTS(string(memberType)] = inUseCMName
tc.Annotations[label.AnnoKeyOfConfigMapNameForNewSTS(string(memberType))] = inUseCMName
logger := klog.FromContext(ctx).WithValues("comp", memberType, "tc", fmt.Sprintf("%s/%s", tc.Namespace, tc.Name))
logger.Info("store inuse configmap name in tc annotation", "name", inUseCMName)
if _, err := tcCtl.Update(tc); err != nil {
return fmt.Errorf("update tc to save name of currently using configmap: %w", err)
}
Expand Down

0 comments on commit ab80380

Please sign in to comment.