From 7d3e67d3ec5811f08cbb9ffc3601b7d6918ddc7f Mon Sep 17 00:00:00 2001 From: evanlixin Date: Thu, 19 Aug 2021 11:00:12 +0800 Subject: [PATCH 1/3] feature: bcs-gamedeployment-operator gamedeployment pod inject unique index; #974 --- .../pkg/apis/tkex/v1alpha1/types.go | 4 ++ .../gamedeployment/game_deployment.go | 4 +- .../pkg/core/api.go | 2 +- .../pkg/core/gamedeployment_core.go | 41 +++++++++++++++---- .../pkg/scale/gamedeployment_scale.go | 7 ++-- .../pkg/scale/gamedeployment_scale_util.go | 37 +++++++++++++++++ 6 files changed, 80 insertions(+), 15 deletions(-) diff --git a/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1/types.go b/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1/types.go index 0dfa6c1d27..9e16544ee9 100644 --- a/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1/types.go +++ b/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1/types.go @@ -26,6 +26,10 @@ const ( // GameDeploymentInstanceID is a unique id for Pods and PVCs. // Each pod and the pvcs it owns have the same instance-id. GameDeploymentInstanceID = "tkex.bkbcs.tencent.com/gamedeployment-instance-id" + // GameDeploymentIndexID is a unique index id + GameDeploymentIndexID = "tkex.bkbcs.tencent.com/gamedeployment-index-id" + // GameDeploymentIndexEnv for deployment pod index key + GameDeploymentIndexEnv = "POD_INDEX" // DefaultGameDeploymentMaxUnavailable is the default value of maxUnavailable for GameDeployment update strategy. DefaultGameDeploymentMaxUnavailable = "20%" diff --git a/bcs-k8s/bcs-gamedeployment-operator/pkg/controllers/gamedeployment/game_deployment.go b/bcs-k8s/bcs-gamedeployment-operator/pkg/controllers/gamedeployment/game_deployment.go index 1da0f05d31..ccbad641f2 100644 --- a/bcs-k8s/bcs-gamedeployment-operator/pkg/controllers/gamedeployment/game_deployment.go +++ b/bcs-k8s/bcs-gamedeployment-operator/pkg/controllers/gamedeployment/game_deployment.go @@ -482,8 +482,8 @@ func (gdc *GameDeploymentController) sync(key string) (retErr error) { return err } - // in some case, the GameStatefulSet get from the informer cache may not be the latest, so get from apiserver directly - //deploy, err := gdc.gdLister.GameDeployments(namespace).Get(name) + // in some case, the GameDeployment get from the informer cache may not be the latest, so get from apiserver directly + // deploy, err := gdc.gdLister.GameDeployments(namespace).Get(name) deploy, err := gdc.gdClient.TkexV1alpha1().GameDeployments(namespace).Get(name, metav1.GetOptions{}) if errors.IsNotFound(err) { // Object not found, return. Created objects are automatically garbage collected. diff --git a/bcs-k8s/bcs-gamedeployment-operator/pkg/core/api.go b/bcs-k8s/bcs-gamedeployment-operator/pkg/core/api.go index e34b1b9c32..fa68e24d52 100644 --- a/bcs-k8s/bcs-gamedeployment-operator/pkg/core/api.go +++ b/bcs-k8s/bcs-gamedeployment-operator/pkg/core/api.go @@ -31,7 +31,7 @@ type Control interface { NewVersionedPods(currentCS, updateCS *gdv1alpha1.GameDeployment, currentRevision, updateRevision string, expectedCreations, expectedCurrentCreations int, - availableIDs []string, + availableIDs []string, availableIndex []int, ) ([]*v1.Pod, error) // update diff --git a/bcs-k8s/bcs-gamedeployment-operator/pkg/core/gamedeployment_core.go b/bcs-k8s/bcs-gamedeployment-operator/pkg/core/gamedeployment_core.go index b9f7b53643..335b6ae16c 100644 --- a/bcs-k8s/bcs-gamedeployment-operator/pkg/core/gamedeployment_core.go +++ b/bcs-k8s/bcs-gamedeployment-operator/pkg/core/gamedeployment_core.go @@ -17,6 +17,7 @@ import ( "encoding/json" "fmt" "regexp" + "strconv" gdv1alpha1 "github.com/Tencent/bk-bcs/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1" gdutil "github.com/Tencent/bk-bcs/bcs-k8s/bcs-gamedeployment-operator/pkg/util" @@ -63,19 +64,21 @@ func (c *commonControl) IsReadyToScale() bool { func (c *commonControl) NewVersionedPods(currentGD, updateGD *gdv1alpha1.GameDeployment, currentRevision, updateRevision string, expectedCreations, expectedCurrentCreations int, - availableIDs []string, + availableIDs []string, availableIndex []int, ) ([]*v1.Pod, error) { var newPods []*v1.Pod if expectedCreations <= expectedCurrentCreations { - newPods = c.newVersionedPods(currentGD, currentRevision, expectedCreations, &availableIDs) + newPods = c.newVersionedPods(currentGD, currentRevision, expectedCreations, &availableIDs, availableIndex) } else { - newPods = c.newVersionedPods(currentGD, currentRevision, expectedCurrentCreations, &availableIDs) - newPods = append(newPods, c.newVersionedPods(updateGD, updateRevision, expectedCreations-expectedCurrentCreations, &availableIDs)...) + newPods = c.newVersionedPods(currentGD, currentRevision, expectedCurrentCreations, &availableIDs, availableIndex) + newPods = append(newPods, + c.newVersionedPods(updateGD, updateRevision, expectedCreations-expectedCurrentCreations, &availableIDs, availableIndex)...) } return newPods, nil } -func (c *commonControl) newVersionedPods(cs *gdv1alpha1.GameDeployment, revision string, replicas int, availableIDs *[]string) []*v1.Pod { +func (c *commonControl) newVersionedPods(cs *gdv1alpha1.GameDeployment, revision string, replicas int, + availableIDs *[]string, availableIndex []int) []*v1.Pod { var newPods []*v1.Pod for i := 0; i < replicas; i++ { if len(*availableIDs) == 0 { @@ -84,6 +87,9 @@ func (c *commonControl) newVersionedPods(cs *gdv1alpha1.GameDeployment, revision id := (*availableIDs)[0] *availableIDs = (*availableIDs)[1:] + index := availableIndex[0] + availableIndex =availableIndex[1:] + pod, _ := kubecontroller.GetPodFromTemplate(&cs.Spec.Template, cs, metav1.NewControllerRef(cs, gdutil.ControllerKind)) if pod.Labels == nil { pod.Labels = make(map[string]string) @@ -93,7 +99,9 @@ func (c *commonControl) newVersionedPods(cs *gdv1alpha1.GameDeployment, revision pod.Name = fmt.Sprintf("%s-%s", cs.Name, id) pod.Namespace = cs.Namespace pod.Labels[gdv1alpha1.GameDeploymentInstanceID] = id + pod.Annotations[gdv1alpha1.GameDeploymentIndexID] = strconv.Itoa(index) + injectDeploymentPodIndexToEnv(pod, strconv.Itoa(index)) inplaceupdate.InjectReadinessGate(pod) newPods = append(newPods, pod) @@ -133,13 +141,13 @@ func (c *commonControl) GetUpdateOptions() *inplaceupdate.UpdateOptions { return opts } -func (c *commonControl) ValidateGameDeploymentUpdate(oldCS, newCS *gdv1alpha1.GameDeployment) error { - if newCS.Spec.UpdateStrategy.Type != gdv1alpha1.InPlaceGameDeploymentUpdateStrategyType { +func (c *commonControl) ValidateGameDeploymentUpdate(oldGD, newGD *gdv1alpha1.GameDeployment) error { + if newGD.Spec.UpdateStrategy.Type != gdv1alpha1.InPlaceGameDeploymentUpdateStrategyType { return nil } - oldTempJSON, _ := json.Marshal(oldCS.Spec.Template.Spec) - newTempJSON, _ := json.Marshal(newCS.Spec.Template.Spec) + oldTempJSON, _ := json.Marshal(oldGD.Spec.Template.Spec) + newTempJSON, _ := json.Marshal(newGD.Spec.Template.Spec) patches, err := jsonpatch.CreatePatch(oldTempJSON, newTempJSON) if err != nil { return fmt.Errorf("failed calculate patches between old/new template spec") @@ -153,3 +161,18 @@ func (c *commonControl) ValidateGameDeploymentUpdate(oldCS, newCS *gdv1alpha1.Ga } return nil } + +func injectDeploymentPodIndexToEnv(pod *v1.Pod, index string) { + if pod == nil { + return + } + + for i := range pod.Spec.Containers { + pod.Spec.Containers[i].Env = append(pod.Spec.Containers[i].Env, + v1.EnvVar{ + Name: gdv1alpha1.GameDeploymentIndexEnv, + Value: index, + }) + } + return +} diff --git a/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale.go b/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale.go index a6c1fbc769..c6d96b462a 100644 --- a/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale.go +++ b/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale.go @@ -109,9 +109,10 @@ func (r *realControl) Manage( // generate available ids availableIDs := genAvailableIDs(expectedCreations, pods) + availableIndex := genAvailableIndex(expectedCreations, pods) return r.createPods(expectedCreations, expectedCurrentCreations, - currentDeploy, updateDeploy, currentRevision, updateRevision, availableIDs.List()) + currentDeploy, updateDeploy, currentRevision, updateRevision, availableIDs.List(), availableIndex) } else if diff > 0 { klog.V(3).Infof("GameDeployment %s begin to scale in %d pods including %d (current rev)", @@ -129,12 +130,12 @@ func (r *realControl) createPods( expectedCreations, expectedCurrentCreations int, currentGD, updateGD *gdv1alpha1.GameDeployment, currentRevision, updateRevision string, - availableIDs []string, + availableIDs []string, availableIndex []int, ) (bool, error) { // new all pods need to create coreControl := gdcore.New(updateGD) newPods, err := coreControl.NewVersionedPods(currentGD, updateGD, currentRevision, updateRevision, - expectedCreations, expectedCurrentCreations, availableIDs) + expectedCreations, expectedCurrentCreations, availableIDs, availableIndex) if err != nil { return false, err } diff --git a/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale_util.go b/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale_util.go index ceabf7df83..eab34fac28 100644 --- a/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale_util.go +++ b/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale_util.go @@ -15,6 +15,7 @@ package scale import ( "sort" + "strconv" gdv1alpha1 "github.com/Tencent/bk-bcs/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1" canaryutil "github.com/Tencent/bk-bcs/bcs-k8s/bcs-gamedeployment-operator/pkg/util/canary" @@ -126,3 +127,39 @@ func choosePodsToDelete(totalDiff int, currentRevDiff int, notUpdatedPods, updat return podsToDelete } + +// Generate available index IDs, keep it unique +func genAvailableIndex(num int, pods []*v1.Pod) []int { + allIndex := num + len(pods) + existIDs := getExistPodsIndex(pods) + + needIDs := make([]int, 0) + for i := 0; i < allIndex; i++ { + _, ok := existIDs[i] + if !ok { + needIDs = append(needIDs, i) + } + } + + sort.Ints(needIDs) + return needIDs +} + +func getExistPodsIndex(pods []*v1.Pod) map[int]struct{} { + idIndex := make([]string, 0) + for _, pod := range pods { + if id := pod.Annotations[gdv1alpha1.GameDeploymentIndexID]; len(id) > 0 { + idIndex = append(idIndex, id) + } + } + + existIDs := make(map[int]struct{}, 0) + for _, id := range idIndex { + n, err := strconv.Atoi(id) + if err == nil { + existIDs[n] = struct{}{} + } + } + + return existIDs +} From 7173afe0c2cdbe7411576a84ea17ee8de4fdb27e Mon Sep 17 00:00:00 2001 From: evanlixin Date: Thu, 19 Aug 2021 21:43:45 +0800 Subject: [PATCH 2/3] feature: bcs-gamedeployment-operator gamedeployment pod inject unique index; #974 --- .../pkg/apis/tkex/v1alpha1/types.go | 10 ++++ .../gamedeployment/game_deployment_control.go | 2 +- .../pkg/core/gamedeployment_core.go | 24 +++++----- .../pkg/scale/gamedeployment_scale.go | 9 +++- .../pkg/scale/gamedeployment_scale_util.go | 46 +++++++++++++++++-- 5 files changed, 73 insertions(+), 18 deletions(-) diff --git a/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1/types.go b/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1/types.go index 9e16544ee9..92596e552e 100644 --- a/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1/types.go +++ b/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1/types.go @@ -30,6 +30,8 @@ const ( GameDeploymentIndexID = "tkex.bkbcs.tencent.com/gamedeployment-index-id" // GameDeploymentIndexEnv for deployment pod index key GameDeploymentIndexEnv = "POD_INDEX" + // GameDeploymentIndexOn for deployment pod index switch + GameDeploymentIndexOn = "tkex.bkbcs.tencent.com/gamedeployment-index-on" // DefaultGameDeploymentMaxUnavailable is the default value of maxUnavailable for GameDeployment update strategy. DefaultGameDeploymentMaxUnavailable = "20%" @@ -79,6 +81,14 @@ type GameDeploymentSpec struct { // without any of its container crashing, for it to be considered available. // Defaults to 0 (pod will be considered available as soon as it is ready) MinReadySeconds int32 `json:"minReadySeconds,omitempty"` + + // PodIndexRange indicates that gamedeployment pod will set unique index by podIndex(startIndex-endIndex) + PodIndexRange GameDeploymentPodIndexRange `json:"podIndexRange, omitempty"` +} + +type GameDeploymentPodIndexRange struct { + PodStartIndex int `json:"podStartIndex,omitempty"` + PodEndIndex int `json:"podEndIndex,omitempty"` } type GameDeploymentPreDeleteUpdateStrategy struct { diff --git a/bcs-k8s/bcs-gamedeployment-operator/pkg/controllers/gamedeployment/game_deployment_control.go b/bcs-k8s/bcs-gamedeployment-operator/pkg/controllers/gamedeployment/game_deployment_control.go index c9875f17b3..ecf020711d 100644 --- a/bcs-k8s/bcs-gamedeployment-operator/pkg/controllers/gamedeployment/game_deployment_control.go +++ b/bcs-k8s/bcs-gamedeployment-operator/pkg/controllers/gamedeployment/game_deployment_control.go @@ -159,7 +159,7 @@ func (gdc *defaultGameDeploymentControl) UpdateGameDeployment(deploy *gdv1alpha1 // scale and update pods delayDuration, updateErr := gdc.updateGameDeployment(deploy, canaryCtx.newStatus, currentRevision, updateRevision, revisions, pods, hrList) if updateErr != nil { - return 0, canaryCtx.newStatus, err + return 0, canaryCtx.newStatus, updateErr } unPauseDuration := gdc.reconcilePause(deploy) diff --git a/bcs-k8s/bcs-gamedeployment-operator/pkg/core/gamedeployment_core.go b/bcs-k8s/bcs-gamedeployment-operator/pkg/core/gamedeployment_core.go index 335b6ae16c..1f4c5e54a8 100644 --- a/bcs-k8s/bcs-gamedeployment-operator/pkg/core/gamedeployment_core.go +++ b/bcs-k8s/bcs-gamedeployment-operator/pkg/core/gamedeployment_core.go @@ -68,17 +68,17 @@ func (c *commonControl) NewVersionedPods(currentGD, updateGD *gdv1alpha1.GameDep ) ([]*v1.Pod, error) { var newPods []*v1.Pod if expectedCreations <= expectedCurrentCreations { - newPods = c.newVersionedPods(currentGD, currentRevision, expectedCreations, &availableIDs, availableIndex) + newPods = c.newVersionedPods(currentGD, currentRevision, expectedCreations, &availableIDs, &availableIndex) } else { - newPods = c.newVersionedPods(currentGD, currentRevision, expectedCurrentCreations, &availableIDs, availableIndex) + newPods = c.newVersionedPods(currentGD, currentRevision, expectedCurrentCreations, &availableIDs, &availableIndex) newPods = append(newPods, - c.newVersionedPods(updateGD, updateRevision, expectedCreations-expectedCurrentCreations, &availableIDs, availableIndex)...) + c.newVersionedPods(updateGD, updateRevision, expectedCreations-expectedCurrentCreations, &availableIDs, &availableIndex)...) } return newPods, nil } func (c *commonControl) newVersionedPods(cs *gdv1alpha1.GameDeployment, revision string, replicas int, - availableIDs *[]string, availableIndex []int) []*v1.Pod { + availableIDs *[]string, availableIndex *[]int) []*v1.Pod { var newPods []*v1.Pod for i := 0; i < replicas; i++ { if len(*availableIDs) == 0 { @@ -87,9 +87,6 @@ func (c *commonControl) newVersionedPods(cs *gdv1alpha1.GameDeployment, revision id := (*availableIDs)[0] *availableIDs = (*availableIDs)[1:] - index := availableIndex[0] - availableIndex =availableIndex[1:] - pod, _ := kubecontroller.GetPodFromTemplate(&cs.Spec.Template, cs, metav1.NewControllerRef(cs, gdutil.ControllerKind)) if pod.Labels == nil { pod.Labels = make(map[string]string) @@ -99,9 +96,14 @@ func (c *commonControl) newVersionedPods(cs *gdv1alpha1.GameDeployment, revision pod.Name = fmt.Sprintf("%s-%s", cs.Name, id) pod.Namespace = cs.Namespace pod.Labels[gdv1alpha1.GameDeploymentInstanceID] = id - pod.Annotations[gdv1alpha1.GameDeploymentIndexID] = strconv.Itoa(index) - injectDeploymentPodIndexToEnv(pod, strconv.Itoa(index)) + if len(*availableIndex) > 0 { + index := (*availableIndex)[0] + *availableIndex = (*availableIndex)[1:] + pod.Annotations[gdv1alpha1.GameDeploymentIndexID] = strconv.Itoa(index) + injectDeploymentPodIndexToEnv(pod, strconv.Itoa(index)) + } + inplaceupdate.InjectReadinessGate(pod) newPods = append(newPods, pod) @@ -170,8 +172,8 @@ func injectDeploymentPodIndexToEnv(pod *v1.Pod, index string) { for i := range pod.Spec.Containers { pod.Spec.Containers[i].Env = append(pod.Spec.Containers[i].Env, v1.EnvVar{ - Name: gdv1alpha1.GameDeploymentIndexEnv, - Value: index, + Name: gdv1alpha1.GameDeploymentIndexEnv, + Value: index, }) } return diff --git a/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale.go b/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale.go index c6d96b462a..77f9495527 100644 --- a/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale.go +++ b/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale.go @@ -79,6 +79,12 @@ func (r *realControl) Manage( return false, fmt.Errorf("spec.Replicas is nil") } + inject, err := validateGameDeploymentPodIndex(deploy) + if err != nil { + r.recorder.Eventf(deploy, v1.EventTypeWarning, "FailedScale", "failed to scale: %v", err) + return false, err + } + controllerKey := util.GetControllerKey(updateDeploy) coreControl := gdcore.New(updateDeploy) if !coreControl.IsReadyToScale() { @@ -109,7 +115,8 @@ func (r *realControl) Manage( // generate available ids availableIDs := genAvailableIDs(expectedCreations, pods) - availableIndex := genAvailableIndex(expectedCreations, pods) + availableIndex := genAvailableIndex(inject, deploy.Spec.PodIndexRange.PodStartIndex, + deploy.Spec.PodIndexRange.PodEndIndex, pods) return r.createPods(expectedCreations, expectedCurrentCreations, currentDeploy, updateDeploy, currentRevision, updateRevision, availableIDs.List(), availableIndex) diff --git a/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale_util.go b/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale_util.go index eab34fac28..295174eb14 100644 --- a/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale_util.go +++ b/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale_util.go @@ -14,6 +14,7 @@ package scale import ( + "fmt" "sort" "strconv" @@ -128,13 +129,48 @@ func choosePodsToDelete(totalDiff int, currentRevDiff int, notUpdatedPods, updat return podsToDelete } -// Generate available index IDs, keep it unique -func genAvailableIndex(num int, pods []*v1.Pod) []int { - allIndex := num + len(pods) - existIDs := getExistPodsIndex(pods) +func validateGameDeploymentPodIndex(deploy *gdv1alpha1.GameDeployment) (bool, error) { + if deploy == nil { + return false, nil + } + + ok := gameDeploymentIndexFeature(deploy) + if !ok { + return false, nil + } + + startIndex := deploy.Spec.PodIndexRange.PodStartIndex + endIndex := deploy.Spec.PodIndexRange.PodEndIndex + + if startIndex < 0 || endIndex < 0 || startIndex >= endIndex { + return false, nil + } + + if *deploy.Spec.Replicas > int32(endIndex-startIndex) { + return false, fmt.Errorf("deploy %s scale replicas gt available indexs", deploy.GetName()) + } + return true, nil +} + +func gameDeploymentIndexFeature(deploy *gdv1alpha1.GameDeployment) bool { + value, ok := deploy.Annotations[gdv1alpha1.GameDeploymentIndexOn] + if ok && value == "true" { + return true + } + + return false +} + +// Generate available index IDs, keep it unique +func genAvailableIndex(inject bool, start, end int, pods []*v1.Pod) []int { needIDs := make([]int, 0) - for i := 0; i < allIndex; i++ { + if !inject { + return needIDs + } + + existIDs := getExistPodsIndex(pods) + for i := start; i < end; i++ { _, ok := existIDs[i] if !ok { needIDs = append(needIDs, i) From 32098b6c3971292994a2d4de73b17c21c6224a17 Mon Sep 17 00:00:00 2001 From: evanlixin Date: Fri, 20 Aug 2021 20:53:13 +0800 Subject: [PATCH 3/3] fix: bcs-gamestatefulset-operator write pod index-range to annotations; #976 --- .../pkg/apis/tkex/v1alpha1/types.go | 5 +-- .../pkg/scale/gamedeployment_scale.go | 6 +-- .../pkg/scale/gamedeployment_scale_util.go | 39 ++++++++++++++----- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1/types.go b/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1/types.go index 92596e552e..6a95e963cd 100644 --- a/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1/types.go +++ b/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1/types.go @@ -32,6 +32,8 @@ const ( GameDeploymentIndexEnv = "POD_INDEX" // GameDeploymentIndexOn for deployment pod index switch GameDeploymentIndexOn = "tkex.bkbcs.tencent.com/gamedeployment-index-on" + // GameDeploymentIndexRange for pod inject index range + GameDeploymentIndexRange = "tkex.bkbcs.tencent.com/gamedeployment-index-range" // DefaultGameDeploymentMaxUnavailable is the default value of maxUnavailable for GameDeployment update strategy. DefaultGameDeploymentMaxUnavailable = "20%" @@ -81,9 +83,6 @@ type GameDeploymentSpec struct { // without any of its container crashing, for it to be considered available. // Defaults to 0 (pod will be considered available as soon as it is ready) MinReadySeconds int32 `json:"minReadySeconds,omitempty"` - - // PodIndexRange indicates that gamedeployment pod will set unique index by podIndex(startIndex-endIndex) - PodIndexRange GameDeploymentPodIndexRange `json:"podIndexRange, omitempty"` } type GameDeploymentPodIndexRange struct { diff --git a/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale.go b/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale.go index 77f9495527..b60a0919e3 100644 --- a/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale.go +++ b/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale.go @@ -79,8 +79,9 @@ func (r *realControl) Manage( return false, fmt.Errorf("spec.Replicas is nil") } - inject, err := validateGameDeploymentPodIndex(deploy) + inject, start, end, err := validateGameDeploymentPodIndex(deploy) if err != nil { + klog.V(3).Infof("GameDeployment %s validateGameDeploymentPodIndex failed: %v", deploy.Name, err) r.recorder.Eventf(deploy, v1.EventTypeWarning, "FailedScale", "failed to scale: %v", err) return false, err } @@ -115,8 +116,7 @@ func (r *realControl) Manage( // generate available ids availableIDs := genAvailableIDs(expectedCreations, pods) - availableIndex := genAvailableIndex(inject, deploy.Spec.PodIndexRange.PodStartIndex, - deploy.Spec.PodIndexRange.PodEndIndex, pods) + availableIndex := genAvailableIndex(inject, start, end, pods) return r.createPods(expectedCreations, expectedCurrentCreations, currentDeploy, updateDeploy, currentRevision, updateRevision, availableIDs.List(), availableIndex) diff --git a/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale_util.go b/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale_util.go index 295174eb14..b62c13d267 100644 --- a/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale_util.go +++ b/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale_util.go @@ -14,6 +14,7 @@ package scale import ( + "encoding/json" "fmt" "sort" "strconv" @@ -129,28 +130,30 @@ func choosePodsToDelete(totalDiff int, currentRevDiff int, notUpdatedPods, updat return podsToDelete } -func validateGameDeploymentPodIndex(deploy *gdv1alpha1.GameDeployment) (bool, error) { +func validateGameDeploymentPodIndex(deploy *gdv1alpha1.GameDeployment) (bool, int, int, error) { if deploy == nil { - return false, nil + return false, 0, 0, nil } ok := gameDeploymentIndexFeature(deploy) if !ok { - return false, nil + return false, 0, 0, nil } - startIndex := deploy.Spec.PodIndexRange.PodStartIndex - endIndex := deploy.Spec.PodIndexRange.PodEndIndex + start, end, err := getDeploymentIndexRange(deploy) + if err != nil { + return false, 0, 0, err + } - if startIndex < 0 || endIndex < 0 || startIndex >= endIndex { - return false, nil + if start < 0 || end < 0 || start >= end { + return false, 0, 0, fmt.Errorf("gamedeployment %s invalid index range", deploy.Name) } - if *deploy.Spec.Replicas > int32(endIndex-startIndex) { - return false, fmt.Errorf("deploy %s scale replicas gt available indexs", deploy.GetName()) + if *deploy.Spec.Replicas > int32(end-start) { + return false, 0, 0, fmt.Errorf("deploy %s scale replicas gt available indexs", deploy.GetName()) } - return true, nil + return true, start, end, nil } func gameDeploymentIndexFeature(deploy *gdv1alpha1.GameDeployment) bool { @@ -162,6 +165,22 @@ func gameDeploymentIndexFeature(deploy *gdv1alpha1.GameDeployment) bool { return false } +func getDeploymentIndexRange(deploy *gdv1alpha1.GameDeployment) (int, int, error) { + indexRange:= &gdv1alpha1.GameDeploymentPodIndexRange{} + + value, ok := deploy.Annotations[gdv1alpha1.GameDeploymentIndexRange] + if ok { + err := json.Unmarshal([]byte(value), indexRange) + if err != nil { + return 0, 0, err + } + + return indexRange.PodStartIndex, indexRange.PodEndIndex, nil + } + + return 0, 0, fmt.Errorf("gamedeployment %s inject index on, get index-range failed", deploy.Name) +} + // Generate available index IDs, keep it unique func genAvailableIndex(inject bool, start, end int, pods []*v1.Pod) []int { needIDs := make([]int, 0)