Merge pull request #975 from evanlixin/fix-gamedeployment-index

bcs-gamedeployment-operator aplly pod unique index by spec.PodIndexRange

bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1/types.go

@@ -26,6 +26,14 @@ const (
 	// GameDeploymentInstanceID is a unique id for Pods and PVCs.
 	// Each pod and the pvcs it owns have the same instance-id.
 	GameDeploymentInstanceID = "tkex.bkbcs.tencent.com/gamedeployment-instance-id"
+	// GameDeploymentIndexID is a unique index id
+	GameDeploymentIndexID    = "tkex.bkbcs.tencent.com/gamedeployment-index-id"
+	// GameDeploymentIndexEnv for deployment pod index key
+	GameDeploymentIndexEnv   = "POD_INDEX"
+	// GameDeploymentIndexOn for deployment pod index switch
+	GameDeploymentIndexOn = "tkex.bkbcs.tencent.com/gamedeployment-index-on"
+	// GameDeploymentIndexRange for pod inject index range
+	GameDeploymentIndexRange = "tkex.bkbcs.tencent.com/gamedeployment-index-range"
 
 	// DefaultGameDeploymentMaxUnavailable is the default value of maxUnavailable for GameDeployment update strategy.
 	DefaultGameDeploymentMaxUnavailable = "20%"
@@ -77,6 +85,11 @@ type GameDeploymentSpec struct {
 	MinReadySeconds int32 `json:"minReadySeconds,omitempty"`
 }
 
+type GameDeploymentPodIndexRange struct {
+	PodStartIndex int  `json:"podStartIndex,omitempty"`
+	PodEndIndex   int  `json:"podEndIndex,omitempty"`
+}
+
 type GameDeploymentPreDeleteUpdateStrategy struct {
 	Hook                 *hookv1alpha1.HookStep `json:"hook,omitempty"`
 	RetryUnexpectedHooks bool                   `json:"retry,omitempty"`

bcs-k8s/bcs-gamedeployment-operator/pkg/controllers/gamedeployment/game_deployment.go

@@ -482,8 +482,8 @@ func (gdc *GameDeploymentController) sync(key string) (retErr error) {
 		return err
 	}
 
-	// in some case, the GameStatefulSet get from the informer cache may not be the latest, so get from apiserver directly
-	//deploy, err := gdc.gdLister.GameDeployments(namespace).Get(name)
+	// in some case, the GameDeployment get from the informer cache may not be the latest, so get from apiserver directly
+	// deploy, err := gdc.gdLister.GameDeployments(namespace).Get(name)
 	deploy, err := gdc.gdClient.TkexV1alpha1().GameDeployments(namespace).Get(name, metav1.GetOptions{})
 	if errors.IsNotFound(err) {
 		// Object not found, return.  Created objects are automatically garbage collected.

bcs-k8s/bcs-gamedeployment-operator/pkg/controllers/gamedeployment/game_deployment_control.go

@@ -159,7 +159,7 @@ func (gdc *defaultGameDeploymentControl) UpdateGameDeployment(deploy *gdv1alpha1
 	// scale and update pods
 	delayDuration, updateErr := gdc.updateGameDeployment(deploy, canaryCtx.newStatus, currentRevision, updateRevision, revisions, pods, hrList)
 	if updateErr != nil {
-		return 0, canaryCtx.newStatus, err
+		return 0, canaryCtx.newStatus, updateErr
 	}
 
 	unPauseDuration := gdc.reconcilePause(deploy)

bcs-k8s/bcs-gamedeployment-operator/pkg/core/api.go

@@ -31,7 +31,7 @@ type Control interface {
 	NewVersionedPods(currentCS, updateCS *gdv1alpha1.GameDeployment,
 		currentRevision, updateRevision string,
 		expectedCreations, expectedCurrentCreations int,
-		availableIDs []string,
+		availableIDs []string, availableIndex []int,
 	) ([]*v1.Pod, error)
 
 	// update

bcs-k8s/bcs-gamedeployment-operator/pkg/core/gamedeployment_core.go

@@ -17,6 +17,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"regexp"
+	"strconv"
 
 	gdv1alpha1 "github.com/Tencent/bk-bcs/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1"
 	gdutil "github.com/Tencent/bk-bcs/bcs-k8s/bcs-gamedeployment-operator/pkg/util"
@@ -63,19 +64,21 @@ func (c *commonControl) IsReadyToScale() bool {
 func (c *commonControl) NewVersionedPods(currentGD, updateGD *gdv1alpha1.GameDeployment,
 	currentRevision, updateRevision string,
 	expectedCreations, expectedCurrentCreations int,
-	availableIDs []string,
+	availableIDs []string, availableIndex []int,
 ) ([]*v1.Pod, error) {
 	var newPods []*v1.Pod
 	if expectedCreations <= expectedCurrentCreations {
-		newPods = c.newVersionedPods(currentGD, currentRevision, expectedCreations, &availableIDs)
+		newPods = c.newVersionedPods(currentGD, currentRevision, expectedCreations, &availableIDs, &availableIndex)
 	} else {
-		newPods = c.newVersionedPods(currentGD, currentRevision, expectedCurrentCreations, &availableIDs)
-		newPods = append(newPods, c.newVersionedPods(updateGD, updateRevision, expectedCreations-expectedCurrentCreations, &availableIDs)...)
+		newPods = c.newVersionedPods(currentGD, currentRevision, expectedCurrentCreations, &availableIDs, &availableIndex)
+		newPods = append(newPods,
+			c.newVersionedPods(updateGD, updateRevision, expectedCreations-expectedCurrentCreations, &availableIDs, &availableIndex)...)
 	}
 	return newPods, nil
 }
 
-func (c *commonControl) newVersionedPods(cs *gdv1alpha1.GameDeployment, revision string, replicas int, availableIDs *[]string) []*v1.Pod {
+func (c *commonControl) newVersionedPods(cs *gdv1alpha1.GameDeployment, revision string, replicas int,
+	availableIDs *[]string, availableIndex *[]int) []*v1.Pod {
 	var newPods []*v1.Pod
 	for i := 0; i < replicas; i++ {
 		if len(*availableIDs) == 0 {
@@ -94,6 +97,13 @@ func (c *commonControl) newVersionedPods(cs *gdv1alpha1.GameDeployment, revision
 		pod.Namespace = cs.Namespace
 		pod.Labels[gdv1alpha1.GameDeploymentInstanceID] = id
 
+		if len(*availableIndex) > 0 {
+			index := (*availableIndex)[0]
+			*availableIndex = (*availableIndex)[1:]
+			pod.Annotations[gdv1alpha1.GameDeploymentIndexID] = strconv.Itoa(index)
+			injectDeploymentPodIndexToEnv(pod, strconv.Itoa(index))
+		}
+
 		inplaceupdate.InjectReadinessGate(pod)
 
 		newPods = append(newPods, pod)
@@ -133,13 +143,13 @@ func (c *commonControl) GetUpdateOptions() *inplaceupdate.UpdateOptions {
 	return opts
 }
 
-func (c *commonControl) ValidateGameDeploymentUpdate(oldCS, newCS *gdv1alpha1.GameDeployment) error {
-	if newCS.Spec.UpdateStrategy.Type != gdv1alpha1.InPlaceGameDeploymentUpdateStrategyType {
+func (c *commonControl) ValidateGameDeploymentUpdate(oldGD, newGD *gdv1alpha1.GameDeployment) error {
+	if newGD.Spec.UpdateStrategy.Type != gdv1alpha1.InPlaceGameDeploymentUpdateStrategyType {
 		return nil
 	}
 
-	oldTempJSON, _ := json.Marshal(oldCS.Spec.Template.Spec)
-	newTempJSON, _ := json.Marshal(newCS.Spec.Template.Spec)
+	oldTempJSON, _ := json.Marshal(oldGD.Spec.Template.Spec)
+	newTempJSON, _ := json.Marshal(newGD.Spec.Template.Spec)
 	patches, err := jsonpatch.CreatePatch(oldTempJSON, newTempJSON)
 	if err != nil {
 		return fmt.Errorf("failed calculate patches between old/new template spec")
@@ -153,3 +163,18 @@ func (c *commonControl) ValidateGameDeploymentUpdate(oldCS, newCS *gdv1alpha1.Ga
 	}
 	return nil
 }
+
+func injectDeploymentPodIndexToEnv(pod *v1.Pod, index string) {
+	if pod == nil {
+		return
+	}
+
+	for i := range pod.Spec.Containers {
+		pod.Spec.Containers[i].Env = append(pod.Spec.Containers[i].Env,
+			v1.EnvVar{
+				Name:  gdv1alpha1.GameDeploymentIndexEnv,
+				Value: index,
+			})
+	}
+	return
+}

bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale.go

@@ -79,6 +79,13 @@ func (r *realControl) Manage(
 		return false, fmt.Errorf("spec.Replicas is nil")
 	}
 
+	inject, start, end, err := validateGameDeploymentPodIndex(deploy)
+	if err != nil {
+		klog.V(3).Infof("GameDeployment %s validateGameDeploymentPodIndex failed: %v", deploy.Name, err)
+		r.recorder.Eventf(deploy, v1.EventTypeWarning, "FailedScale", "failed to scale: %v", err)
+		return false, err
+	}
+
 	controllerKey := util.GetControllerKey(updateDeploy)
 	coreControl := gdcore.New(updateDeploy)
 	if !coreControl.IsReadyToScale() {
@@ -109,9 +116,10 @@ func (r *realControl) Manage(
 
 		// generate available ids
 		availableIDs := genAvailableIDs(expectedCreations, pods)
+		availableIndex := genAvailableIndex(inject, start, end, pods)
 
 		return r.createPods(expectedCreations, expectedCurrentCreations,
-			currentDeploy, updateDeploy, currentRevision, updateRevision, availableIDs.List())
+			currentDeploy, updateDeploy, currentRevision, updateRevision, availableIDs.List(), availableIndex)
 
 	} else if diff > 0 {
 		klog.V(3).Infof("GameDeployment %s begin to scale in %d pods including %d (current rev)",
@@ -129,12 +137,12 @@ func (r *realControl) createPods(
 	expectedCreations, expectedCurrentCreations int,
 	currentGD, updateGD *gdv1alpha1.GameDeployment,
 	currentRevision, updateRevision string,
-	availableIDs []string,
+	availableIDs []string, availableIndex []int,
 ) (bool, error) {
 	// new all pods need to create
 	coreControl := gdcore.New(updateGD)
 	newPods, err := coreControl.NewVersionedPods(currentGD, updateGD, currentRevision, updateRevision,
-		expectedCreations, expectedCurrentCreations, availableIDs)
+		expectedCreations, expectedCurrentCreations, availableIDs, availableIndex)
 	if err != nil {
 		return false, err
 	}

bcs-k8s/bcs-gamedeployment-operator/pkg/scale/gamedeployment_scale_util.go

@@ -14,7 +14,10 @@
 package scale
 
 import (
+	"encoding/json"
+	"fmt"
 	"sort"
+	"strconv"
 
 	gdv1alpha1 "github.com/Tencent/bk-bcs/bcs-k8s/bcs-gamedeployment-operator/pkg/apis/tkex/v1alpha1"
 	canaryutil "github.com/Tencent/bk-bcs/bcs-k8s/bcs-gamedeployment-operator/pkg/util/canary"
@@ -126,3 +129,92 @@ func choosePodsToDelete(totalDiff int, currentRevDiff int, notUpdatedPods, updat
 
 	return podsToDelete
 }
+
+func validateGameDeploymentPodIndex(deploy *gdv1alpha1.GameDeployment) (bool, int, int, error) {
+	if deploy == nil {
+		return false, 0, 0, nil
+	}
+
+	ok := gameDeploymentIndexFeature(deploy)
+	if !ok {
+		return false, 0, 0, nil
+	}
+
+	start, end, err := getDeploymentIndexRange(deploy)
+	if err != nil {
+		return false, 0, 0, err
+	}
+
+	if start < 0 || end < 0 || start >= end {
+		return false, 0, 0, fmt.Errorf("gamedeployment %s invalid index range", deploy.Name)
+	}
+
+	if *deploy.Spec.Replicas > int32(end-start) {
+		return false, 0, 0, fmt.Errorf("deploy %s scale replicas gt available indexs", deploy.GetName())
+	}
+
+	return true, start, end, nil
+}
+
+func gameDeploymentIndexFeature(deploy *gdv1alpha1.GameDeployment) bool {
+	value, ok := deploy.Annotations[gdv1alpha1.GameDeploymentIndexOn]
+	if ok && value == "true" {
+		return true
+	}
+
+	return false
+}
+
+func getDeploymentIndexRange(deploy *gdv1alpha1.GameDeployment) (int, int, error) {
+	indexRange:= &gdv1alpha1.GameDeploymentPodIndexRange{}
+
+	value, ok := deploy.Annotations[gdv1alpha1.GameDeploymentIndexRange]
+	if ok  {
+		err := json.Unmarshal([]byte(value), indexRange)
+		if err != nil {
+			return 0, 0, err
+		}
+
+		return indexRange.PodStartIndex, indexRange.PodEndIndex, nil
+	}
+
+	return 0, 0, fmt.Errorf("gamedeployment %s inject index on, get index-range failed", deploy.Name)
+}
+
+// Generate available index IDs, keep it unique
+func genAvailableIndex(inject bool, start, end int, pods []*v1.Pod) []int {
+	needIDs := make([]int, 0)
+	if !inject {
+		return needIDs
+	}
+
+	existIDs := getExistPodsIndex(pods)
+	for i := start; i < end; i++ {
+		_, ok := existIDs[i]
+		if !ok {
+			needIDs = append(needIDs, i)
+		}
+	}
+
+	sort.Ints(needIDs)
+	return needIDs
+}
+
+func getExistPodsIndex(pods []*v1.Pod) map[int]struct{} {
+	idIndex := make([]string, 0)
+	for _, pod := range pods {
+		if id := pod.Annotations[gdv1alpha1.GameDeploymentIndexID]; len(id) > 0 {
+			idIndex = append(idIndex, id)
+		}
+	}
+
+	existIDs := make(map[int]struct{}, 0)
+	for _, id := range idIndex {
+		n, err := strconv.Atoi(id)
+		if err == nil {
+			existIDs[n] = struct{}{}
+		}
+	}
+
+	return existIDs
+}