List all 3 locks (#69)

topfreegames · Aug 14, 2019 · 6780438 · 6780438
1 parent ea72f15
commit 6780438
Show file tree

Hide file tree

Showing 4 changed files with 79 additions and 40 deletions.
diff --git a/api/scheduler_locks_handler.go b/api/scheduler_locks_handler.go
@@ -29,9 +29,9 @@ func (h *SchedulerLocksListHandler) ServeHTTP(w http.ResponseWriter, r *http.Req
 		"scheduler": params.SchedulerName,
 	})
 	logger.Debug("Getting scheduler locks")
-	locksPrefixes := []string{h.App.Config.GetString("watcher.lockKey")}
+	lockPrefix := h.App.Config.GetString("watcher.lockKey")
 	locks, err := models.ListSchedulerLocks(
-		h.App.RedisClient.Trace(r.Context()), params.SchedulerName, locksPrefixes,
+		h.App.RedisClient.Trace(r.Context()), lockPrefix, params.SchedulerName,
 	)
 	if err != nil {
 		logger.WithError(err).Error("error listing scheduler locks")
@@ -68,9 +68,9 @@ func (h *SchedulerLockDeleteHandler) ServeHTTP(w http.ResponseWriter, r *http.Re
 		"lock":      params.LockKey,
 	})
 	logger.Debug("Getting scheduler locks")
-	locksPrefixes := []string{h.App.Config.GetString("watcher.lockKey")}
+	prefix := h.App.Config.GetString("watcher.lockKey")
 	isLockValid := false
-	validLocksKeys := models.ListSchedulerLocksKeys(params.SchedulerName, locksPrefixes)
+	validLocksKeys := models.ListSchedulerLocksKeys(prefix, params.SchedulerName)
 	for _, possible := range validLocksKeys {
 		if possible == params.LockKey {
 			isLockValid = true

diff --git a/api/scheduler_locks_handler_test.go b/api/scheduler_locks_handler_test.go
@@ -59,40 +59,67 @@ var _ = Describe("SchedulerLocksHandler", func() {
 			request, _ = http.NewRequest("GET", url, nil)
 		})
 
-		It("should return unlocked watcher lock", func() {
-			mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient)
+		It("should return unlocked watcher locks", func() {
+			for _, lockKey := range []string{
+				"maestro-lock-key-scheduler-name",
+				"maestro-lock-key-scheduler-name-config",
+				"maestro-lock-key-scheduler-name-downscaling",
+			} {
+				mockPipeline.EXPECT().TTL(lockKey).
+					Return(redis.NewDurationResult(9*time.Second, nil))
+				mockPipeline.EXPECT().Exists(lockKey).
+					Return(redis.NewIntResult(1, nil))
+			}
 			mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline)
-			mockPipeline.EXPECT().TTL("maestro-lock-key-scheduler-name").
-				Return(redis.NewDurationResult(9*time.Second, nil))
-			mockPipeline.EXPECT().Exists("maestro-lock-key-scheduler-name").
-				Return(redis.NewIntResult(1, nil))
+			mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient)
 			mockPipeline.EXPECT().Exec()
+
 			app.Router.ServeHTTP(recorder, request)
 			Expect(recorder.Code).To(Equal(http.StatusOK))
 			var locks []models.SchedulerLock
 			json.Unmarshal(recorder.Body.Bytes(), &locks)
-			Expect(locks).To(HaveLen(1))
-			Expect(locks[0].Key).To(Equal("maestro-lock-key-scheduler-name"))
-			Expect(locks[0].TTLInSec).To(Equal(int64(9)))
-			Expect(locks[0].IsLocked).To(BeTrue())
+			Expect(locks).To(HaveLen(3))
+
+			for i, lockKey := range []string{
+				"maestro-lock-key-scheduler-name",
+				"maestro-lock-key-scheduler-name-config",
+				"maestro-lock-key-scheduler-name-downscaling",
+			} {
+				Expect(locks[i].Key).To(Equal(lockKey))
+				Expect(locks[i].TTLInSec).To(Equal(int64(9)))
+				Expect(locks[i].IsLocked).To(BeTrue())
+			}
 		})
 
 		It("should return locked watcher lock", func() {
-			mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient)
+			for _, lockKey := range []string{
+				"maestro-lock-key-scheduler-name",
+				"maestro-lock-key-scheduler-name-config",
+				"maestro-lock-key-scheduler-name-downscaling",
+			} {
+				mockPipeline.EXPECT().TTL(lockKey).
+					Return(redis.NewDurationResult(time.Duration(0), redis.Nil))
+				mockPipeline.EXPECT().Exists(lockKey).
+					Return(redis.NewIntResult(0, nil))
+			}
 			mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline)
-			mockPipeline.EXPECT().TTL("maestro-lock-key-scheduler-name").
-				Return(redis.NewDurationResult(time.Duration(0), redis.Nil))
-			mockPipeline.EXPECT().Exists("maestro-lock-key-scheduler-name").
-				Return(redis.NewIntResult(0, nil))
+			mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient)
 			mockPipeline.EXPECT().Exec()
+
 			app.Router.ServeHTTP(recorder, request)
 			Expect(recorder.Code).To(Equal(http.StatusOK))
 			var locks []models.SchedulerLock
 			json.Unmarshal(recorder.Body.Bytes(), &locks)
-			Expect(locks).To(HaveLen(1))
-			Expect(locks[0].Key).To(Equal("maestro-lock-key-scheduler-name"))
-			Expect(locks[0].TTLInSec).To(Equal(int64(0)))
-			Expect(locks[0].IsLocked).To(BeFalse())
+
+			for i, lockKey := range []string{
+				"maestro-lock-key-scheduler-name",
+				"maestro-lock-key-scheduler-name-config",
+				"maestro-lock-key-scheduler-name-downscaling",
+			} {
+				Expect(locks[i].Key).To(Equal(lockKey))
+				Expect(locks[i].TTLInSec).To(Equal(int64(0)))
+				Expect(locks[i].IsLocked).To(BeFalse())
+			}
 		})
 	})
 
@@ -103,15 +130,18 @@ var _ = Describe("SchedulerLocksHandler", func() {
 		})
 
 		It("should remove lockName key in redis", func() {
-			url = fmt.Sprintf(
-				"http://%s/scheduler/%s/locks/%s", app.Address, configYaml.Name,
+			for _, lockKey := range []string{
 				models.GetSchedulerScalingLockKey(app.Config.GetString("watcher.lockKey"), configYaml.Name),
-			)
-			request, _ = http.NewRequest("DELETE", url, nil)
-			mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient)
-			mockRedisClient.EXPECT().Del("maestro-lock-key-scheduler-name").Return(redis.NewIntResult(1, nil))
-			app.Router.ServeHTTP(recorder, request)
-			Expect(recorder.Code).To(Equal(http.StatusNoContent))
+				models.GetSchedulerConfigLockKey(app.Config.GetString("watcher.lockKey"), configYaml.Name),
+				models.GetSchedulerDownScalingLockKey(app.Config.GetString("watcher.lockKey"), configYaml.Name),
+			} {
+				url = fmt.Sprintf("http://%s/scheduler/%s/locks/%s", app.Address, configYaml.Name, lockKey)
+				request, _ = http.NewRequest("DELETE", url, nil)
+				mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient)
+				mockRedisClient.EXPECT().Del(lockKey).Return(redis.NewIntResult(1, nil))
+				app.Router.ServeHTTP(recorder, request)
+				Expect(recorder.Code).To(Equal(http.StatusNoContent))
+			}
 		})
 
 		It("should return 400 when trying to remove invalid lock key", func() {

diff --git a/controller/utils.go b/controller/utils.go
@@ -334,6 +334,15 @@ func waitTerminatingPods(
 					exit = false
 					break
 				}
+
+				if err != nil && !strings.Contains(err.Error(), "not found") {
+					logger.
+						WithError(err).
+						WithField("pod", pod.GetName()).
+						Info("error getting pod")
+					exit = false
+					break
+				}
 			}
 		case <-timeoutTimer.C:
 			logger.Error("timeout waiting for rooms to be removed")

diff --git a/models/scheduler.go b/models/scheduler.go
@@ -477,11 +477,11 @@ func LoadConfigWithVersion(db interfaces.DB, schedulerName, version string) (str
 
 // ListSchedulerLocks returns the list of locks of a scheduler
 func ListSchedulerLocks(
-	redisClient redisinterfaces.RedisClient, schedulerName string, prefixes []string,
+	redisClient redisinterfaces.RedisClient, prefix, schedulerName string,
 ) ([]SchedulerLock, error) {
-	locks := make([]SchedulerLock, 0, len(prefixes))
 	pipe := redisClient.TxPipeline()
-	keys := ListSchedulerLocksKeys(schedulerName, prefixes)
+	keys := ListSchedulerLocksKeys(prefix, schedulerName)
+	locks := make([]SchedulerLock, 0, len(keys))
 	for _, key := range keys {
 		duration, err := pipe.TTL(key).Result()
 		if err != nil && err != redis.Nil {
@@ -504,13 +504,13 @@ func ListSchedulerLocks(
 	return locks, nil
 }
 
-// ListSchedulerLocksKeys lists a slice of locks keys for schedulerName and prefixes
-func ListSchedulerLocksKeys(schedulerName string, prefixes []string) []string {
-	names := make([]string, 0, len(prefixes))
-	for _, prefix := range prefixes {
-		names = append(names, GetSchedulerScalingLockKey(prefix, schedulerName))
+// ListSchedulerLocksKeys lists a slice of locks keys for schedulerName and prefix
+func ListSchedulerLocksKeys(prefix, schedulerName string) []string {
+	return []string{
+		GetSchedulerScalingLockKey(prefix, schedulerName),
+		GetSchedulerConfigLockKey(prefix, schedulerName),
+		GetSchedulerDownScalingLockKey(prefix, schedulerName),
 	}
-	return names
 }
 
 // DeleteSchedulerLock deletes a scheduler lock