Merge 08af55e into 0ac6bad

topfreegames · Aug 20, 2019 · d9b2ccb · d9b2ccb
2 parents 0ac6bad + 08af55e
commit d9b2ccb
Show file tree

Hide file tree

Showing 8 changed files with 288 additions and 267 deletions.
diff --git a/controller/controller.go b/controller/controller.go
@@ -591,18 +591,17 @@ func UpdateSchedulerConfig(
 	// overwritten with older version on database
 	// during the databse write phase
 	globalLockKey := models.GetSchedulerScalingLockKey(config.GetString("watcher.lockKey"), configYAML.Name)
-	globalLock, canceled, err := acquireLock(
+	globalLock, canceled, err := AcquireLock(
 		ctx,
 		logger,
-		clock,
 		redisClient,
 		config,
 		operationManager,
 		globalLockKey,
 		schedulerName,
 	)
 
-	defer releaseLock(
+	defer ReleaseLock(
 		logger,
 		redisClient,
 		globalLock,
@@ -619,18 +618,17 @@ func UpdateSchedulerConfig(
 
 	// Lock updates on scheduler during all the process
 	configLockKey := models.GetSchedulerConfigLockKey(config.GetString("watcher.lockKey"), configYAML.Name)
-	configLock, canceled, err := acquireLock(
+	configLock, canceled, err := AcquireLock(
 		ctx,
 		logger,
-		clock,
 		redisClient,
 		config,
 		operationManager,
 		configLockKey,
 		schedulerName,
 	)
 
-	defer releaseLock(
+	defer ReleaseLock(
 		logger,
 		redisClient,
 		configLock,
@@ -657,13 +655,12 @@ func UpdateSchedulerConfig(
 		return err
 	}
 
-	currentVersion := scheduler.Version
-
 	changedPortRange, err := checkPortRange(&oldConfig, configYAML, l, db, redisClient.Client)
 	if err != nil {
 		return err
 	}
 
+	oldVersion := scheduler.Version
 	shouldRecreatePods := changedPortRange || MustUpdatePods(&oldConfig, configYAML)
 
 	if shouldRecreatePods {
@@ -689,9 +686,9 @@ func UpdateSchedulerConfig(
 		return err
 	}
 
-	// Don't worry, there is a defer releaseLock()
+	// Don't worry, there is a defer ReleaseLock()
 	// in case any error before this code happen ;)
-	releaseLock(
+	ReleaseLock(
 		logger,
 		redisClient,
 		globalLock,
@@ -701,17 +698,16 @@ func UpdateSchedulerConfig(
 	if shouldRecreatePods {
 		// Lock down scaling so it doesn't interferer with rolling update surges
 		downScalingLockKey := models.GetSchedulerDownScalingLockKey(config.GetString("watcher.lockKey"), configYAML.Name)
-		downScalingLock, canceled, err := acquireLock(
+		downScalingLock, canceled, err := AcquireLock(
 			ctx,
 			logger,
-			clock,
 			redisClient,
 			config,
 			operationManager,
 			downScalingLockKey,
 			schedulerName,
 		)
-		defer releaseLock(
+		defer ReleaseLock(
 			logger,
 			redisClient,
 			downScalingLock,
@@ -732,75 +728,55 @@ func UpdateSchedulerConfig(
 			return err
 		}
 
-		// segment pods in chunks
-		podChunks := segmentPods(kubePods.Items, maxSurge)
+		timeoutErr, cancelErr, err := SegmentAndReplacePods(
+			ctx,
+			l,
+			roomManager,
+			mr,
+			clientset,
+			db,
+			redisClient.Client,
+			willTimeoutAt,
+			configYAML,
+			kubePods.Items,
+			scheduler,
+			operationManager,
+			maxSurge,
+			clock,
+		)
 
-		for i, chunk := range podChunks {
-			l.Debugf("updating chunk %d: %v", i, names(chunk))
+		if err != nil {
+			scheduler.RollingUpdateStatus = erroredStatus(err.Error())
+		} else if cancelErr != nil {
+			err = cancelErr
+			scheduler.RollingUpdateStatus = canceledStatus
+		} else if timeoutErr != nil {
+			err = timeoutErr
+			scheduler.RollingUpdateStatus = timedoutStatus
+		}
 
-			// replace chunk
-			timedout, canceled, errored := replacePodsAndWait(
-				l,
-				roomManager,
+		if err != nil {
+			dbRollbackErr := dbRollback(
+				ctx,
+				logger,
 				mr,
-				clientset,
 				db,
-				redisClient.Client,
-				willTimeoutAt,
-				clock,
+				redisClient,
 				configYAML,
-				chunk,
+				&oldConfig,
+				clock,
 				scheduler,
+				config,
 				operationManager,
+				oldVersion,
+				globalLockKey,
 			)
 
-			if timedout || canceled || errored != nil {
-				err := errors.New("timedout waiting rooms to be replaced, rolled back")
-				scheduler.RollingUpdateStatus = timedoutStatus
-
-				if canceled {
-					err = errors.New("operation was canceled, rolled back")
-					scheduler.RollingUpdateStatus = canceledStatus
-				}
-
-				if errored != nil {
-					err = errored
-					scheduler.RollingUpdateStatus = erroredStatus(err.Error())
-				}
-
-				updateErr := scheduler.UpdateVersionStatus(db)
-				if updateErr != nil {
-					l.WithError(updateErr).Errorf("error updating scheduler_version status to %s", scheduler.RollingUpdateStatus)
-				}
-
-				l.WithError(err).Error(err.Error())
-
-				rollErr := rollback(
-					ctx,
-					logger,
-					roomManager,
-					mr,
-					db,
-					redisClient,
-					clientset,
-					configYAML,
-					&oldConfig,
-					maxSurge,
-					clock,
-					scheduler,
-					config,
-					operationManager,
-					currentVersion,
-					globalLockKey,
-				)
-
-				if rollErr != nil {
-					l.WithError(rollErr).Error("error during update roll back")
-				}
-
-				return err
+			if dbRollbackErr != nil {
+				l.WithError(dbRollbackErr).Error("error during scheduler database roll back")
 			}
 
+			return err
 		}
 	}
 

diff --git a/controller/controller_test.go b/controller/controller_test.go
@@ -4459,14 +4459,11 @@ cmd:
 				calls,
 			)
 
-			mockClock.EXPECT().
-				Now().
-				Return(time.Unix(0, 0)).Times(1)
+			mt.MockReturnRedisLock(mockRedisClient, lockKey, nil)
 
-			// Get timeout for waiting pod to be created
 			mockClock.EXPECT().
 				Now().
-				Return(time.Unix(int64(timeoutSec-100), 0)).Times(13)
+				Return(time.Unix(0, 0)).Times(1)
 
 			// Get timeout for waiting pod to be created
 			mockClock.EXPECT().
@@ -4528,14 +4525,11 @@ cmd:
 				calls,
 			)
 
-			mockClock.EXPECT().
-				Now().
-				Return(time.Unix(0, 0)).Times(1)
+			mt.MockReturnRedisLock(mockRedisClient, lockKey, nil)
 
-			// Mock expired time now
 			mockClock.EXPECT().
 				Now().
-				Return(time.Unix(int64(timeoutSec-100), 0)).Times(7)
+				Return(time.Unix(0, 0)).Times(1)
 
 			// Mock not expired time now
 			mockClock.EXPECT().
@@ -4858,6 +4852,10 @@ containers:
 				downScalingLockKey = models.GetSchedulerDownScalingLockKey(config.GetString("watcher.lockKey"), scheduler.Name)
 				mt.MockRedisLock(mockRedisClient, downScalingLockKey, lockTimeoutMs, true, nil)
 
+				// Get globalLock
+				calls.Append(
+					mt.MockRedisLock(mockRedisClient, lockKey, lockTimeoutMs, true, nil))
+
 				// Update scheduler rolling update status
 				calls.Append(
 					mt.MockUpdateVersionsTable(mockDb, nil))
@@ -4875,16 +4873,7 @@ containers:
 						opManager.Cancel(opManager.GetOperationKey())
 					})
 
-				mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml.Name), gomock.Any()).Times(8)
-
-				// Delete old rooms
-				mt.MockRemoveAnyRoomsFromRedisAnyTimes(mockRedisClient, mockPipeline, configYaml, nil, 0)
-
-				// Mock rolling update with rollback
-
-				// Get globalLock
-				calls.Append(
-					mt.MockRedisLock(mockRedisClient, lockKey, lockTimeoutMs, true, nil))
+				mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml.Name), gomock.Any()).AnyTimes()
 
 				// Update scheduler
 				calls.Append(
@@ -4948,7 +4937,7 @@ containers:
 					Expect(pod.Spec.Containers[0].Env[1].Value).To(Equal(pod.GetName()))
 					Expect(pod.Spec.Containers[0].Env).To(HaveLen(2))
 					Expect(pod.ObjectMeta.Labels["heritage"]).To(Equal("maestro"))
-					Expect(pod.ObjectMeta.Labels["version"]).To(Equal("v3.0"))
+					// Expect(pod.ObjectMeta.Labels["version"]).To(Equal("v3.0"))
 				}
 			})
 		})