Merge 9044079 into 0ac6bad

topfreegames · Aug 20, 2019 · 97c41a9 · 97c41a9
2 parents 0ac6bad + 9044079
commit 97c41a9
Show file tree

Hide file tree

Showing 10 changed files with 283 additions and 560 deletions.
diff --git a/api/scheduler_handler_test.go b/api/scheduler_handler_test.go
diff --git a/api/scheduler_rollback_handler_test.go b/api/scheduler_rollback_handler_test.go
@@ -84,12 +84,8 @@ autoscaling:
 
 			calls := NewCalls()
 
-			lockKey := models.GetSchedulerScalingLockKey(config.GetString("watcher.lockKey"), scheduler1.Name)
 			configLockKey := models.GetSchedulerConfigLockKey(config.GetString("watcher.lockKey"), scheduler1.Name)
 
-			// Get global lock
-			MockRedisLock(mockRedisClient, lockKey, lockTimeoutMs, true, nil)
-
 			// Get config lock
 			MockRedisLock(mockRedisClient, configLockKey, lockTimeoutMs, true, nil)
 
@@ -109,12 +105,6 @@ autoscaling:
 			// Count to delete old versions if necessary
 			MockCountNumberOfVersions(scheduler1, numberOfVersions, mockDb, nil)
 
-			// Release globalLock
-			MockReturnRedisLock(mockRedisClient, lockKey, nil)
-
-			// Release globalLock (again defer)
-			MockReturnRedisLock(mockRedisClient, lockKey, nil)
-
 			// Release configLock
 			MockReturnRedisLock(mockRedisClient, configLockKey, nil)
 
@@ -222,12 +212,8 @@ autoscaling:
 
 			calls := NewCalls()
 
-			lockKey := models.GetSchedulerScalingLockKey(config.GetString("watcher.lockKey"), scheduler1.Name)
 			configLockKey := models.GetSchedulerConfigLockKey(config.GetString("watcher.lockKey"), scheduler1.Name)
 
-			// Get global lock
-			MockRedisLock(mockRedisClient, lockKey, lockTimeoutMs, true, nil)
-
 			// Get config lock
 			MockRedisLock(mockRedisClient, configLockKey, lockTimeoutMs, true, nil)
 
@@ -247,12 +233,6 @@ autoscaling:
 			// Count to delete old versions if necessary
 			MockCountNumberOfVersions(scheduler1, numberOfVersions, mockDb, nil)
 
-			// Release globalLock
-			MockReturnRedisLock(mockRedisClient, lockKey, nil)
-
-			// Release globalLock (again defer)
-			MockReturnRedisLock(mockRedisClient, lockKey, nil)
-
 			// Release configLock
 			MockReturnRedisLock(mockRedisClient, configLockKey, nil)
 

diff --git a/controller/controller.go b/controller/controller.go
@@ -587,50 +587,19 @@ func UpdateSchedulerConfig(
 		return err
 	}
 
-	// Lock watchers so they don't scale up or down and the scheduler is not
-	// overwritten with older version on database
-	// during the databse write phase
-	globalLockKey := models.GetSchedulerScalingLockKey(config.GetString("watcher.lockKey"), configYAML.Name)
-	globalLock, canceled, err := acquireLock(
-		ctx,
-		logger,
-		clock,
-		redisClient,
-		config,
-		operationManager,
-		globalLockKey,
-		schedulerName,
-	)
-
-	defer releaseLock(
-		logger,
-		redisClient,
-		globalLock,
-		schedulerName,
-	)
-
-	if err != nil {
-		return err
-	}
-
-	if canceled {
-		return nil
-	}
-
 	// Lock updates on scheduler during all the process
 	configLockKey := models.GetSchedulerConfigLockKey(config.GetString("watcher.lockKey"), configYAML.Name)
-	configLock, canceled, err := acquireLock(
+	configLock, canceled, err := AcquireLock(
 		ctx,
 		logger,
-		clock,
 		redisClient,
 		config,
 		operationManager,
 		configLockKey,
 		schedulerName,
 	)
 
-	defer releaseLock(
+	defer ReleaseLock(
 		logger,
 		redisClient,
 		configLock,
@@ -657,13 +626,12 @@ func UpdateSchedulerConfig(
 		return err
 	}
 
-	currentVersion := scheduler.Version
-
 	changedPortRange, err := checkPortRange(&oldConfig, configYAML, l, db, redisClient.Client)
 	if err != nil {
 		return err
 	}
 
+	oldVersion := scheduler.Version
 	shouldRecreatePods := changedPortRange || MustUpdatePods(&oldConfig, configYAML)
 
 	if shouldRecreatePods {
@@ -689,29 +657,19 @@ func UpdateSchedulerConfig(
 		return err
 	}
 
-	// Don't worry, there is a defer releaseLock()
-	// in case any error before this code happen ;)
-	releaseLock(
-		logger,
-		redisClient,
-		globalLock,
-		schedulerName,
-	)
-
 	if shouldRecreatePods {
 		// Lock down scaling so it doesn't interferer with rolling update surges
 		downScalingLockKey := models.GetSchedulerDownScalingLockKey(config.GetString("watcher.lockKey"), configYAML.Name)
-		downScalingLock, canceled, err := acquireLock(
+		downScalingLock, canceled, err := AcquireLock(
 			ctx,
 			logger,
-			clock,
 			redisClient,
 			config,
 			operationManager,
 			downScalingLockKey,
 			schedulerName,
 		)
-		defer releaseLock(
+		defer ReleaseLock(
 			logger,
 			redisClient,
 			downScalingLock,
@@ -732,75 +690,53 @@ func UpdateSchedulerConfig(
 			return err
 		}
 
-		// segment pods in chunks
-		podChunks := segmentPods(kubePods.Items, maxSurge)
+		timeoutErr, cancelErr, err := SegmentAndReplacePods(
+			ctx,
+			l,
+			roomManager,
+			mr,
+			clientset,
+			db,
+			redisClient.Client,
+			willTimeoutAt,
+			configYAML,
+			kubePods.Items,
+			scheduler,
+			operationManager,
+			maxSurge,
+			clock,
+		)
 
-		for i, chunk := range podChunks {
-			l.Debugf("updating chunk %d: %v", i, names(chunk))
+		if err != nil {
+			scheduler.RollingUpdateStatus = erroredStatus(err.Error())
+		} else if cancelErr != nil {
+			err = cancelErr
+			scheduler.RollingUpdateStatus = canceledStatus
+		} else if timeoutErr != nil {
+			err = timeoutErr
+			scheduler.RollingUpdateStatus = timedoutStatus
+		}
 
-			// replace chunk
-			timedout, canceled, errored := replacePodsAndWait(
-				l,
-				roomManager,
+		if err != nil {
+			dbRollbackErr := dbRollback(
+				ctx,
+				logger,
 				mr,
-				clientset,
 				db,
-				redisClient.Client,
-				willTimeoutAt,
-				clock,
+				redisClient,
 				configYAML,
-				chunk,
+				&oldConfig,
+				clock,
 				scheduler,
-				operationManager,
+				config,
+				oldVersion,
 			)
 
-			if timedout || canceled || errored != nil {
-				err := errors.New("timedout waiting rooms to be replaced, rolled back")
-				scheduler.RollingUpdateStatus = timedoutStatus
-
-				if canceled {
-					err = errors.New("operation was canceled, rolled back")
-					scheduler.RollingUpdateStatus = canceledStatus
-				}
-
-				if errored != nil {
-					err = errored
-					scheduler.RollingUpdateStatus = erroredStatus(err.Error())
-				}
-
-				updateErr := scheduler.UpdateVersionStatus(db)
-				if updateErr != nil {
-					l.WithError(updateErr).Errorf("error updating scheduler_version status to %s", scheduler.RollingUpdateStatus)
-				}
-
-				l.WithError(err).Error(err.Error())
-
-				rollErr := rollback(
-					ctx,
-					logger,
-					roomManager,
-					mr,
-					db,
-					redisClient,
-					clientset,
-					configYAML,
-					&oldConfig,
-					maxSurge,
-					clock,
-					scheduler,
-					config,
-					operationManager,
-					currentVersion,
-					globalLockKey,
-				)
-
-				if rollErr != nil {
-					l.WithError(rollErr).Error("error during update roll back")
-				}
-
-				return err
+			if dbRollbackErr != nil {
+				l.WithError(dbRollbackErr).Error("error during scheduler database roll back")
 			}
 
+			return err
 		}
 	}