Use goroutines to perform rolling update

controller/controller_test.go

@@ -4419,22 +4419,19 @@ cmd:
 			calls.Append(
 				mt.MockSelectScheduler(yaml1, mockDb, nil))
 
-			calls.Append(mt.MockGetPortsFromPool(&configYaml1, mockRedisClient, mockPortChooser,
-				workerPortRange, portStart, portEnd, 1))
-
 			// Create rooms
-			calls.Append(
-				mt.MockCreateRooms(mockRedisClient, mockPipeline, &configYaml1, 1))
+			mt.MockCreateRoomsAnyTimes(mockRedisClient, mockPipeline, &configYaml1, 0)
+			mt.MockGetPortsFromPoolAnyTimes(&configYaml1, mockRedisClient, mockPortChooser,
+				workerPortRange, portStart, portEnd)
 
 			// Get timeout for waiting pod to be created
-			calls.Add(
-				mockClock.EXPECT().
-					Now().
-					Return(time.Unix(int64(timeoutSec+100), 0)))
+			mockClock.EXPECT().
+				Now().
+				Return(time.Unix(int64(timeoutSec+100), 0)).Times(3).Do(func() {
+			})
 
 			// Delete newly created rooms
-			calls.Append(
-				mt.MockRemoveAnyRoomsFromRedis(mockRedisClient, mockPipeline, 1, &configYaml1))
+			mt.MockRemoveAnyRoomsFromRedisAnyTimes(mockRedisClient, mockPipeline, &configYaml1, nil)
 
 			calls.Append(
 				mt.MockReturnRedisLock(mockRedisClient, lockKey, nil))
@@ -4464,11 +4461,6 @@ cmd:
 			// Update scheduler
 			calls := mt.NewCalls()
 
-			// Get lock
-			calls.Add(
-				mockClock.EXPECT().
-					Now().
-					Return(time.Unix(0, 0)))
 			calls.Append(
 				mt.MockRedisLock(mockRedisClient, lockKey, lockTimeoutMs, true, nil))
 
@@ -4481,34 +4473,27 @@ cmd:
 				mt.MockSelectScheduler(yaml1, mockDb, nil))
 
 			// Create room
-			calls.Append(
-				mt.MockCreateRooms(mockRedisClient, mockPipeline, &configYaml1, 1))
-			calls.Append(mt.MockGetPortsFromPool(&configYaml1, mockRedisClient, mockPortChooser,
-				workerPortRange, portStart, portEnd, 1))
-			calls.Add(
-				mockClock.EXPECT().
-					Now().
-					Return(time.Unix(int64(timeoutSec-100), 0)))
+			mt.MockCreateRoomsAnyTimes(mockRedisClient, mockPipeline, &configYaml1, 0)
+			mt.MockGetPortsFromPoolAnyTimes(&configYaml1, mockRedisClient, mockPortChooser,
+				workerPortRange, portStart, portEnd)
 
-			// Delete old rooms
-			calls.Append(
-				mt.MockRemoveAnyRoomsFromRedis(mockRedisClient, mockPipeline, 1, &configYaml1))
+			// Get lock
+			mockClock.EXPECT().
+				Now().
+				Return(time.Unix(0, 0)).Times(1)
 
-			// Get timeout for waiting old pods to be deleted
-			calls.Add(
-				mockClock.EXPECT().
-					Now().
-					Return(time.Unix(int64(timeoutSec+100), 0)))
+			// Mock expired time now
+			mockClock.EXPECT().
+				Now().
+				Return(time.Unix(int64(timeoutSec+100), 0)).Times(3)
 
-			// Delete newly created rooms
-			calls.Append(
-				mt.MockRemoveAnyRoomsFromRedis(mockRedisClient, mockPipeline, 1, &configYaml1))
+			// Mock not expired time now
+			mockClock.EXPECT().
+				Now().
+				Return(time.Unix(int64(timeoutSec-100), 0)).Times(3)
 
-			// Recreate old rooms
-			calls.Append(
-				mt.MockCreateRooms(mockRedisClient, mockPipeline, &configYaml1, 1))
-			calls.Append(mt.MockGetPortsFromPool(&configYaml1, mockRedisClient, mockPortChooser,
-				workerPortRange, portStart, portEnd, 1))
+			// Delete old rooms
+			mt.MockRemoveAnyRoomsFromRedisAnyTimes(mockRedisClient, mockPipeline, &configYaml1, nil)
 
 			calls.Append(
 				mt.MockReturnRedisLock(mockRedisClient, lockKey, nil))
@@ -4557,50 +4542,18 @@ cmd:
 			// Get scheduler from DB
 			calls.Append(mt.MockSelectScheduler(yaml1, mockDb, nil))
 
-			// Create and Delete rooms
-			errRedis := errors.New("redis error")
-			for _, pod := range pods.Items {
+			// Create room
+			mt.MockCreateRoomsAnyTimes(mockRedisClient, mockPipeline, &configYaml1, 3)
+			mt.MockGetPortsFromPoolAnyTimes(&configYaml1, mockRedisClient, mockPortChooser,
+				workerPortRange, portStart, portEnd)
 
-				// Create new pod
-				calls.Append(
-					mt.MockCreateRooms(mockRedisClient, mockPipeline, &configYaml2, 1))
-				calls.Append(
-					mt.MockGetPortsFromPool(&configYaml2, mockRedisClient, mockPortChooser,
-						workerPortRange, portStart, portEnd, 1))
-				// Get time.Now()
-				calls.Add(
-					mockClock.EXPECT().
-						Now().
-						Return(time.Unix(int64(timeoutSec-100), 1)))
-
-				// Retrieve ports to pool
-				room := models.NewRoom(pod.GetName(), pod.GetNamespace())
-				calls.Add(
-					mockRedisClient.EXPECT().
-						TxPipeline().
-						Return(mockPipeline))
-				for _, status := range allStatus {
-					calls.Add(
-						mockPipeline.EXPECT().
-							SRem(models.GetRoomStatusSetRedisKey(room.SchedulerName, status), room.GetRoomRedisKey()))
-					calls.Add(
-						mockPipeline.EXPECT().
-							ZRem(models.GetLastStatusRedisKey(room.SchedulerName, status), room.ID))
-				}
-				calls.Add(
-					mockPipeline.EXPECT().
-						ZRem(models.GetRoomPingRedisKey(pod.GetNamespace()), room.ID))
-				for _, mt := range allMetrics {
-					mockPipeline.EXPECT().ZRem(models.GetRoomMetricsRedisKey(room.SchedulerName, mt), gomock.Any())
-				}
-				calls.Add(
-					mockPipeline.EXPECT().
-						Del(room.GetRoomRedisKey()))
-				calls.Add(
-					mockPipeline.EXPECT().
-						Exec().
-						Return(nil, errRedis))
-			}
+			// Delete old rooms
+			mt.MockRemoveAnyRoomsFromRedisAnyTimes(mockRedisClient, mockPipeline, &configYaml1, errors.New("redis error"))
+
+			// Mock not expired time now
+			mockClock.EXPECT().
+				Now().
+				Return(time.Unix(int64(timeoutSec-100), 0)).AnyTimes()
 
 			calls.Append(
 				mt.MockUpdateSchedulersTable(mockDb, nil))

controller/utils.go

@@ -110,14 +110,11 @@ func createNewRemoveOldPod(
 	deletedPods *[]v1.Pod,
 ) (timedout, canceled bool) {
 	logger.Debug("creating pod")
-	fmt.Printf("\nARNALDO - creating pod %s", pod.Name)
 
 	// create new pod
 	newPod, err := roomManager.Create(logger, mr, redisClient,
 		db, clientset, configYAML, scheduler)
 
-	fmt.Printf("\nARNALDO - pod created %s", pod.Name)
-
 	if err != nil {
 		logger.WithError(err).Debug("error creating pod")
 		return false, false

testing/common.go

@@ -317,6 +317,98 @@ func mockCreateRooms(
 	return calls
 }
 
+// MockRemoveAnyRoomsFromRedisAnyTimes removes any rooms from redis
+func MockRemoveAnyRoomsFromRedisAnyTimes(
+	mockRedisClient *redismocks.MockRedisClient,
+	mockPipeline *redismocks.MockPipeliner,
+	configYaml *models.ConfigYAML, redisErrOrNil error,
+) {
+
+	allStatus := []string{
+		models.StatusCreating,
+		models.StatusReady,
+		models.StatusOccupied,
+		models.StatusTerminating,
+		models.StatusTerminated,
+	}
+
+	allMetrics := []string{
+		string(models.CPUAutoScalingPolicyType),
+		string(models.MemAutoScalingPolicyType),
+	}
+
+	mockRedisClient.EXPECT().
+		TxPipeline().
+		Return(mockPipeline).AnyTimes()
+	for _, status := range allStatus {
+		mockPipeline.EXPECT().
+			SRem(models.GetRoomStatusSetRedisKey(configYaml.Name, status), gomock.Any()).AnyTimes()
+		mockPipeline.EXPECT().
+			ZRem(models.GetLastStatusRedisKey(configYaml.Name, status), gomock.Any()).AnyTimes()
+	}
+	mockPipeline.EXPECT().
+		ZRem(models.GetRoomPingRedisKey(configYaml.Name), gomock.Any()).AnyTimes()
+	for _, mt := range allMetrics {
+		mockPipeline.EXPECT().ZRem(models.GetRoomMetricsRedisKey(configYaml.Name, mt), gomock.Any()).AnyTimes()
+	}
+	mockPipeline.EXPECT().
+		Del(gomock.Any()).AnyTimes()
+
+	if redisErrOrNil == nil {
+		mockPipeline.EXPECT().
+			Exec().AnyTimes()
+	} else {
+		mockPipeline.EXPECT().
+			Exec().Return(nil, redisErrOrNil).AnyTimes()
+	}
+
+}
+
+// MockCreateRoomsAnyTimes mocks the creation of rooms on redis
+func MockCreateRoomsAnyTimes(
+	mockRedisClient *redismocks.MockRedisClient,
+	mockPipeline *redismocks.MockPipeliner,
+	configYaml *models.ConfigYAML,
+	times int,
+) {
+	tx := mockRedisClient.EXPECT().
+		TxPipeline().
+		Return(mockPipeline)
+
+	hmset := mockPipeline.EXPECT().
+		HMSet(gomock.Any(), gomock.Any()).
+		Do(func(schedulerName string, statusInfo map[string]interface{}) {
+			gomega.Expect(statusInfo["status"]).
+				To(gomega.Equal(models.StatusCreating))
+			gomega.Expect(statusInfo["lastPing"]).
+				To(gomega.BeNumerically("~", time.Now().Unix(), 1))
+		})
+
+	sadd := mockPipeline.EXPECT().
+		SAdd(models.GetRoomStatusSetRedisKey(configYaml.Name, "creating"),
+			gomock.Any())
+
+	zadd := mockPipeline.EXPECT().
+		ZAdd(models.GetRoomPingRedisKey(configYaml.Name), gomock.Any())
+
+	exec := mockPipeline.EXPECT().Exec()
+
+	if times > 0 {
+		tx.Times(times)
+		hmset.Times(times)
+		sadd.Times(times)
+		zadd.Times(times)
+		exec.Times(times)
+
+		return
+	}
+	tx.AnyTimes()
+	hmset.AnyTimes()
+	sadd.AnyTimes()
+	zadd.AnyTimes()
+	exec.AnyTimes()
+}
+
 // MockCreateRooms mocks the creation of rooms on redis
 func MockCreateRooms(
 	mockRedisClient *redismocks.MockRedisClient,
@@ -457,6 +549,50 @@ func MockGetPortsFromPool(
 	return calls
 }
 
+// MockGetPortsFromPoolAnyTimes mocks the function that chooses random ports
+// to be used as HostPort in the pods
+func MockGetPortsFromPoolAnyTimes(
+	configYaml *models.ConfigYAML,
+	mockRedisClient *redismocks.MockRedisClient,
+	mockPortChooser *mocks.MockPortChooser,
+	workerPortRange string,
+	portStart, portEnd int,
+) {
+	if !configYaml.HasPorts() {
+		return
+	}
+
+	if !configYaml.PortRange.IsSet() {
+		mockRedisClient.EXPECT().
+			Get(models.GlobalPortsPoolKey).
+			Return(goredis.NewStringResult(workerPortRange, nil)).
+			AnyTimes()
+	}
+
+	if mockPortChooser == nil {
+		return
+	}
+
+	givePorts := func(nPorts int) {
+		ports := make([]int, nPorts)
+		for i := 0; i < nPorts; i++ {
+			ports[i] = portStart + i
+		}
+		mockPortChooser.EXPECT().
+			Choose(portStart, portEnd, nPorts).
+			Return(ports).
+			AnyTimes()
+	}
+
+	if configYaml.Version() == "v1" {
+		givePorts(len(configYaml.Ports))
+	} else if configYaml.Version() == "v2" {
+		for _, container := range configYaml.Containers {
+			givePorts(len(container.Ports))
+		}
+	}
+}
+
 // MockInsertScheduler inserts a new scheduler into database
 func MockInsertScheduler(
 	mockDb *pgmocks.MockDB,