Always set status to RollingUpdate if operation exists (#93)

* Always set status to RollingUpdate if operation exists * Fix tests * Log state transition * Add test
topfreegames · Jul 10, 2020 · 1e7e89b · 1e7e89b
1 parent 82080ea
commit 1e7e89b
Show file tree

Hide file tree

Showing 5 changed files with 56 additions and 13 deletions.
diff --git a/helm/charts/maestro-postgres/templates/maestro-postgres.yaml b/helm/charts/maestro-postgres/templates/maestro-postgres.yaml
@@ -38,6 +38,8 @@ spec:
           value: maestro
         - name: POSTGRES_DB
           value: maestro
+        - name: POSTGRES_HOST_AUTH_METHOD
+          value: trust
         livenessProbe:
           exec:
             command:

diff --git a/helm/charts/maestro/Chart.yaml b/helm/charts/maestro/Chart.yaml
@@ -1,7 +1,7 @@
 name: maestro
 home: https://github.com/topfreegames/maestro
 description: Maestro api and worker
-version: 7.2.2
+version: 9.5.1
 maintainers:
   - name: TFGCo
     email: backend@tfgco.com
diff --git a/metadata/version.go b/metadata/version.go
@@ -8,7 +8,7 @@
 package metadata
 
 //Version of Maestro
-var Version = "9.4.1"
+var Version = "9.5.2"
 
 //KubeVersion is the desired Kubernetes version
 var KubeVersion = "v1.13.9"
diff --git a/watcher/watcher.go b/watcher/watcher.go
@@ -1262,13 +1262,6 @@ func (w *Watcher) EnsureCorrectRooms() error {
 
 	invalidPods := append(incorrectPods, unregisteredPods...)
 
-	if len(invalidPods) <= 0 {
-		// delete invalidRooms key for safety
-		models.RemoveInvalidRoomsKey(w.RedisClient.Trace(ctx), w.MetricsReporter, w.SchedulerName)
-		logger.Debug("no invalid pods to replace")
-		return nil
-	}
-
 	// get operation manager if it exists.
 	// It won't exist if not in a UpdateSchedulerConfig operation
 	operationManager, err := w.getOperation(ctx, logger)
@@ -1289,11 +1282,17 @@ func (w *Watcher) EnsureCorrectRooms() error {
 		invalidPods = incorrectPods
 
 		if status["description"] != models.OpManagerRollingUpdate {
-			err = models.SetInvalidRooms(w.RedisClient.Trace(ctx), w.MetricsReporter, w.SchedulerName, incorrectPodNames)
-			if err != nil {
-				logger.WithError(err).Error("error trying to save invalid rooms to track progress")
-				return err
+			if len(invalidPods) > 0 {
+				err = models.SetInvalidRooms(w.RedisClient.Trace(ctx), w.MetricsReporter, w.SchedulerName, incorrectPodNames)
+				if err != nil {
+					logger.WithError(err).Error("error trying to save invalid rooms to track progress")
+					return err
+				}
 			}
+
+			logger.WithFields(logrus.Fields{
+				"operation": operationManager.GetOperationKey(),
+			}).Infof(`changing state from "%s" to "%s"`, status["description"], models.OpManagerRollingUpdate)
 			err = operationManager.SetDescription(models.OpManagerRollingUpdate)
 			if err != nil {
 				logger.WithError(err).Error("error trying to set opmanager to rolling update status")
@@ -1302,6 +1301,13 @@ func (w *Watcher) EnsureCorrectRooms() error {
 		}
 	}
 
+	if len(invalidPods) <= 0 {
+		// delete invalidRooms key for safety
+		models.RemoveInvalidRoomsKey(w.RedisClient.Trace(ctx), w.MetricsReporter, w.SchedulerName)
+		logger.Debug("no invalid pods to replace")
+		return nil
+	}
+
 	// replace invalid pods using rolling strategy
 	timeoutSec := w.Config.GetInt("updateTimeoutSeconds")
 	timeoutDur := time.Duration(timeoutSec) * time.Second

diff --git a/watcher/watcher_test.go b/watcher/watcher_test.go
@@ -494,6 +494,9 @@ var _ = Describe("Watcher", func() {
 				w.Run = false
 			}, mockDb, nil, nil)
 
+			opManager := models.NewOperationManager(configYaml.Name, mockRedisClient, logger)
+			testing.MockGetCurrentOperationKey(opManager, mockRedisClient, nil)
+
 			mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline)
 			mockPipeline.EXPECT().Del(models.GetInvalidRoomsKey(configYaml.Name))
 			mockPipeline.EXPECT().Exec()
@@ -4427,6 +4430,38 @@ var _ = Describe("Watcher", func() {
 			Expect(hook.Entries).To(testing.ContainLogMessage(fmt.Sprintf("error deleting pod %s", podNames[1])))
 		})
 
+		It("should set state to rolling update when there is no invalid pods and a operation is running", func() {
+			testing.MockSelectScheduler(yaml1, mockDb, nil)
+			testing.MockListPods(mockPipeline, mockRedisClient, w.SchedulerName, []string{}, nil)
+			testing.MockGetRegisteredRooms(mockRedisClient, mockPipeline,
+				w.SchedulerName, [][]string{}, nil)
+
+			opKey := fmt.Sprintf("opmanager:%s:1234", configYaml.Name)
+			opManager := models.NewOperationManager(configYaml.Name, mockRedisClient, logger)
+			mockRedisClient.EXPECT().
+				Get(opManager.BuildCurrOpKey()).
+				Return(redis.NewStringResult(opKey, nil))
+
+			mockRedisClient.EXPECT().
+				HGetAll(opKey).
+				Return(redis.NewStringStringMapResult(map[string]string{
+					"description": models.OpManagerRunning,
+				}, nil))
+
+			mockRedisClient.EXPECT().
+				HMSet(opKey, map[string]interface{}{
+					"description": models.OpManagerRollingUpdate,
+				}).Return(redis.NewStatusResult("", nil))
+
+			mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline)
+			mockPipeline.EXPECT().Del(models.GetInvalidRoomsKey(configYaml.Name))
+			mockPipeline.EXPECT().Exec()
+
+			err := w.EnsureCorrectRooms()
+
+			Expect(err).ToNot(HaveOccurred())
+		})
+
 		It("should delete invalid pods", func() {
 			podNames := []string{"room-1", "room-2"}
 			room := models.NewRoom(podNames[0], w.SchedulerName)