diff --git a/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1/types.go b/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1/types.go index 0dfa6c1d27..6a95e963cd 100644 --- a/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1/types.go +++ b/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1/types.go @@ -26,6 +26,14 @@ const ( // GameDeploymentInstanceID is a unique id for Pods and PVCs. // Each pod and the pvcs it owns have the same instance-id. GameDeploymentInstanceID = "tkex.bkbcs.tencent.com/gamedeployment-instance-id" + // GameDeploymentIndexID is a unique index id + GameDeploymentIndexID = "tkex.bkbcs.tencent.com/gamedeployment-index-id" + // GameDeploymentIndexEnv for deployment pod index key + GameDeploymentIndexEnv = "POD_INDEX" + // GameDeploymentIndexOn for deployment pod index switch + GameDeploymentIndexOn = "tkex.bkbcs.tencent.com/gamedeployment-index-on" + // GameDeploymentIndexRange for pod inject index range + GameDeploymentIndexRange = "tkex.bkbcs.tencent.com/gamedeployment-index-range" // DefaultGameDeploymentMaxUnavailable is the default value of maxUnavailable for GameDeployment update strategy. DefaultGameDeploymentMaxUnavailable = "20%" @@ -77,6 +85,11 @@ type GameDeploymentSpec struct { MinReadySeconds int32 `json:"minReadySeconds,omitempty"` } +type GameDeploymentPodIndexRange struct { + PodStartIndex int `json:"podStartIndex,omitempty"` + PodEndIndex int `json:"podEndIndex,omitempty"` +} + type GameDeploymentPreDeleteUpdateStrategy struct { Hook *hookv1alpha1.HookStep `json:"hook,omitempty"` RetryUnexpectedHooks bool `json:"retry,omitempty"` diff --git a/bcs-k8s/bcs-gamedeployment-operator/pkg/controllers/gamedeployment/game_deployment.go b/bcs-k8s/bcs-gamedeployment-operator/pkg/controllers/gamedeployment/game_deployment.go index 1da0f05d31..ccbad641f2 100644 --- a/bcs-k8s/bcs-gamedeployment-operator/pkg/controllers/gamedeployment/game_deployment.go +++ b/bcs-k8s/bcs-gamedeployment-operator/pkg/controllers/gamedeployment/game_deployment.go @@ -482,8 +482,8 @@ func (gdc *GameDeploymentController) sync(key string) (retErr error) { return err } - // in some case, the GameStatefulSet get from the informer cache may not be the latest, so get from apiserver directly - //deploy, err := gdc.gdLister.GameDeployments(namespace).Get(name) + // in some case, the GameDeployment get from the informer cache may not be the latest, so get from apiserver directly + // deploy, err := gdc.gdLister.GameDeployments(namespace).Get(name) deploy, err := gdc.gdClient.TkexV1alpha1().GameDeployments(namespace).Get(name, metav1.GetOptions{}) if errors.IsNotFound(err) { // Object not found, return. Created objects are automatically garbage collected. diff --git a/bcs-k8s/bcs-gamedeployment-operator/pkg/controllers/gamedeployment/game_deployment_control.go b/bcs-k8s/bcs-gamedeployment-operator/pkg/controllers/gamedeployment/game_deployment_control.go index c9875f17b3..ecf020711d 100644 --- a/bcs-k8s/bcs-gamedeployment-operator/pkg/controllers/gamedeployment/game_deployment_control.go +++ b/bcs-k8s/bcs-gamedeployment-operator/pkg/controllers/gamedeployment/game_deployment_control.go @@ -159,7 +159,7 @@ func (gdc *defaultGameDeploymentControl) UpdateGameDeployment(deploy *gdv1alpha1 // scale and update pods delayDuration, updateErr := gdc.updateGameDeployment(deploy, canaryCtx.newStatus, currentRevision, updateRevision, revisions, pods, hrList) if updateErr != nil { - return 0, canaryCtx.newStatus, err + return 0, canaryCtx.newStatus, updateErr } unPauseDuration := gdc.reconcilePause(deploy) diff --git a/bcs-k8s/bcs-gamedeployment-operator/pkg/core/api.go b/bcs-k8s/bcs-gamedeployment-operator/pkg/core/api.go index e34b1b9c32..fa68e24d52 100644 --- a/bcs-k8s/bcs-gamedeployment-operator/pkg/core/api.go +++ b/bcs-k8s/bcs-gamedeployment-operator/pkg/core/api.go @@ -31,7 +31,7 @@ type Control interface { NewVersionedPods(currentCS, updateCS *gdv1alpha1.GameDeployment, currentRevision, updateRevision string, expectedCreations, expectedCurrentCreations int, - availableIDs []string, + availableIDs []string, availableIndex []int, ) ([]*v1.Pod, error) // update diff --git a/bcs-k8s/bcs-gamedeployment-operator/pkg/core/gamedeployment_core.go b/bcs-k8s/bcs-gamedeployment-operator/pkg/core/gamedeployment_core.go index b9f7b53643..1f4c5e54a8 100644 --- a/bcs-k8s/bcs-gamedeployment-operator/pkg/core/gamedeployment_core.go +++ b/bcs-k8s/bcs-gamedeployment-operator/pkg/core/gamedeployment_core.go @@ -17,6 +17,7 @@ import ( "encoding/json" "fmt" "regexp" + "strconv" gdv1alpha1 "github.com/Tencent/bk-bcs/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1" gdutil "github.com/Tencent/bk-bcs/bcs-k8s/bcs-gamedeployment-operator/pkg/util" @@ -63,19 +64,21 @@ func (c *commonControl) IsReadyToScale() bool { func (c *commonControl) NewVersionedPods(currentGD, updateGD *gdv1alpha1.GameDeployment, currentRevision, updateRevision string, expectedCreations, expectedCurrentCreations int, - availableIDs []string, + availableIDs []string, availableIndex []int, ) ([]*v1.Pod, error) { var newPods []*v1.Pod if expectedCreations <= expectedCurrentCreations { - newPods = c.newVersionedPods(currentGD, currentRevision, expectedCreations, &availableIDs) + newPods = c.newVersionedPods(currentGD, currentRevision, expectedCreations, &availableIDs, &availableIndex) } else { - newPods = c.newVersionedPods(currentGD, currentRevision, expectedCurrentCreations, &availableIDs) - newPods = append(newPods, c.newVersionedPods(updateGD, updateRevision, expectedCreations-expectedCurrentCreations, &availableIDs)...) + newPods = c.newVersionedPods(currentGD, currentRevision, expectedCurrentCreations, &availableIDs, &availableIndex) + newPods = append(newPods, + c.newVersionedPods(updateGD, updateRevision, expectedCreations-expectedCurrentCreations, &availableIDs, &availableIndex)...) } return newPods, nil } -func (c *commonControl) newVersionedPods(cs *gdv1alpha1.GameDeployment, revision string, replicas int, availableIDs *[]string) []*v1.Pod { +func (c *commonControl) newVersionedPods(cs *gdv1alpha1.GameDeployment, revision string, replicas int, + availableIDs *[]string, availableIndex *[]int) []*v1.Pod { var newPods []*v1.Pod for i := 0; i < replicas; i++ { if len(*availableIDs) == 0 { @@ -94,6 +97,13 @@ func (c *commonControl) newVersionedPods(cs *gdv1alpha1.GameDeployment, revision pod.Namespace = cs.Namespace pod.Labels[gdv1alpha1.GameDeploymentInstanceID] = id + if len(*availableIndex) > 0 { + index := (*availableIndex)[0] + *availableIndex = (*availableIndex)[1:] + pod.Annotations[gdv1alpha1.GameDeploymentIndexID] = strconv.Itoa(index) + injectDeploymentPodIndexToEnv(pod, strconv.Itoa(index)) + } + inplaceupdate.InjectReadinessGate(pod) newPods = append(newPods, pod) @@ -133,13 +143,13 @@ func (c *commonControl) GetUpdateOptions() *inplaceupdate.UpdateOptions { return opts } -func (c *commonControl) ValidateGameDeploymentUpdate(oldCS, newCS *gdv1alpha1.GameDeployment) error { - if newCS.Spec.UpdateStrategy.Type != gdv1alpha1.InPlaceGameDeploymentUpdateStrategyType { +func (c *commonControl) ValidateGameDeploymentUpdate(oldGD, newGD *gdv1alpha1.GameDeployment) error { + if newGD.Spec.UpdateStrategy.Type != gdv1alpha1.InPlaceGameDeploymentUpdateStrategyType { return nil } - oldTempJSON, _ := json.Marshal(oldCS.Spec.Template.Spec) - newTempJSON, _ := json.Marshal(newCS.Spec.Template.Spec) + oldTempJSON, _ := json.Marshal(oldGD.Spec.Template.Spec) + newTempJSON, _ := json.Marshal(newGD.Spec.Template.Spec) patches, err := jsonpatch.CreatePatch(oldTempJSON, newTempJSON) if err != nil { return fmt.Errorf("failed calculate patches between old/new template spec") @@ -153,3 +163,18 @@ func (c *commonControl) ValidateGameDeploymentUpdate(oldCS, newCS *gdv1alpha1.Ga } return nil } + +func injectDeploymentPodIndexToEnv(pod *v1.Pod, index string) { + if pod == nil { + return + } + + for i := range pod.Spec.Containers { + pod.Spec.Containers[i].Env = append(pod.Spec.Containers[i].Env, + v1.EnvVar{ + Name: gdv1alpha1.GameDeploymentIndexEnv, + Value: index, + }) + } + return +} diff --git a/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale.go b/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale.go index a6c1fbc769..b60a0919e3 100644 --- a/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale.go +++ b/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale.go @@ -79,6 +79,13 @@ func (r *realControl) Manage( return false, fmt.Errorf("spec.Replicas is nil") } + inject, start, end, err := validateGameDeploymentPodIndex(deploy) + if err != nil { + klog.V(3).Infof("GameDeployment %s validateGameDeploymentPodIndex failed: %v", deploy.Name, err) + r.recorder.Eventf(deploy, v1.EventTypeWarning, "FailedScale", "failed to scale: %v", err) + return false, err + } + controllerKey := util.GetControllerKey(updateDeploy) coreControl := gdcore.New(updateDeploy) if !coreControl.IsReadyToScale() { @@ -109,9 +116,10 @@ func (r *realControl) Manage( // generate available ids availableIDs := genAvailableIDs(expectedCreations, pods) + availableIndex := genAvailableIndex(inject, start, end, pods) return r.createPods(expectedCreations, expectedCurrentCreations, - currentDeploy, updateDeploy, currentRevision, updateRevision, availableIDs.List()) + currentDeploy, updateDeploy, currentRevision, updateRevision, availableIDs.List(), availableIndex) } else if diff > 0 { klog.V(3).Infof("GameDeployment %s begin to scale in %d pods including %d (current rev)", @@ -129,12 +137,12 @@ func (r *realControl) createPods( expectedCreations, expectedCurrentCreations int, currentGD, updateGD *gdv1alpha1.GameDeployment, currentRevision, updateRevision string, - availableIDs []string, + availableIDs []string, availableIndex []int, ) (bool, error) { // new all pods need to create coreControl := gdcore.New(updateGD) newPods, err := coreControl.NewVersionedPods(currentGD, updateGD, currentRevision, updateRevision, - expectedCreations, expectedCurrentCreations, availableIDs) + expectedCreations, expectedCurrentCreations, availableIDs, availableIndex) if err != nil { return false, err } diff --git a/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale_util.go b/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale_util.go index ceabf7df83..b62c13d267 100644 --- a/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale_util.go +++ b/bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale_util.go @@ -14,7 +14,10 @@ package scale import ( + "encoding/json" + "fmt" "sort" + "strconv" gdv1alpha1 "github.com/Tencent/bk-bcs/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1" canaryutil "github.com/Tencent/bk-bcs/bcs-k8s/bcs-gamedeployment-operator/pkg/util/canary" @@ -126,3 +129,92 @@ func choosePodsToDelete(totalDiff int, currentRevDiff int, notUpdatedPods, updat return podsToDelete } + +func validateGameDeploymentPodIndex(deploy *gdv1alpha1.GameDeployment) (bool, int, int, error) { + if deploy == nil { + return false, 0, 0, nil + } + + ok := gameDeploymentIndexFeature(deploy) + if !ok { + return false, 0, 0, nil + } + + start, end, err := getDeploymentIndexRange(deploy) + if err != nil { + return false, 0, 0, err + } + + if start < 0 || end < 0 || start >= end { + return false, 0, 0, fmt.Errorf("gamedeployment %s invalid index range", deploy.Name) + } + + if *deploy.Spec.Replicas > int32(end-start) { + return false, 0, 0, fmt.Errorf("deploy %s scale replicas gt available indexs", deploy.GetName()) + } + + return true, start, end, nil +} + +func gameDeploymentIndexFeature(deploy *gdv1alpha1.GameDeployment) bool { + value, ok := deploy.Annotations[gdv1alpha1.GameDeploymentIndexOn] + if ok && value == "true" { + return true + } + + return false +} + +func getDeploymentIndexRange(deploy *gdv1alpha1.GameDeployment) (int, int, error) { + indexRange:= &gdv1alpha1.GameDeploymentPodIndexRange{} + + value, ok := deploy.Annotations[gdv1alpha1.GameDeploymentIndexRange] + if ok { + err := json.Unmarshal([]byte(value), indexRange) + if err != nil { + return 0, 0, err + } + + return indexRange.PodStartIndex, indexRange.PodEndIndex, nil + } + + return 0, 0, fmt.Errorf("gamedeployment %s inject index on, get index-range failed", deploy.Name) +} + +// Generate available index IDs, keep it unique +func genAvailableIndex(inject bool, start, end int, pods []*v1.Pod) []int { + needIDs := make([]int, 0) + if !inject { + return needIDs + } + + existIDs := getExistPodsIndex(pods) + for i := start; i < end; i++ { + _, ok := existIDs[i] + if !ok { + needIDs = append(needIDs, i) + } + } + + sort.Ints(needIDs) + return needIDs +} + +func getExistPodsIndex(pods []*v1.Pod) map[int]struct{} { + idIndex := make([]string, 0) + for _, pod := range pods { + if id := pod.Annotations[gdv1alpha1.GameDeploymentIndexID]; len(id) > 0 { + idIndex = append(idIndex, id) + } + } + + existIDs := make(map[int]struct{}, 0) + for _, id := range idIndex { + n, err := strconv.Atoi(id) + if err == nil { + existIDs[n] = struct{}{} + } + } + + return existIDs +}