Watcher delete old occupied rooms

topfreegames · Jun 20, 2017 · 9630067 · 9630067
1 parent b94cd73
commit 9630067
Show file tree

Hide file tree

Showing 5 changed files with 122 additions and 7 deletions.
diff --git a/config/local.yaml b/config/local.yaml
@@ -23,6 +23,7 @@ scaleUpTimeoutSeconds: 300
 scaleDownTimeoutSeconds: 300
 deleteTimeoutSeconds: 600
 pingTimeout: 60
+occupiedTimeout: 180
 updateTimeoutSeconds: 300
 sentry:
   url: ""

diff --git a/config/test.yaml b/config/test.yaml
@@ -23,6 +23,7 @@ scaleUpTimeoutSeconds: 300
 scaleDownTimeoutSeconds: 300
 deleteTimeoutSeconds: 150
 pingTimeout: 300
+occupiedTimeout: 180
 updateTimeoutSeconds: 600
 newrelic:
   app: maestro

diff --git a/helm/charts/maestro/Chart.yaml b/helm/charts/maestro/Chart.yaml
@@ -1,7 +1,7 @@
 name: maestro
 home: https://github.com/topfreegames/maestro
 description: Maestro api and worker
-version: 2.12.1
+version: 2.13.0
 maintainers:
   - name: TFGCo
     email: backend@tfgco.com
diff --git a/watcher/watcher.go b/watcher/watcher.go
@@ -80,6 +80,7 @@ func (w *Watcher) loadConfigurationDefaults() {
 	w.Config.SetDefault("watcher.lockTimeoutMs", 180000)
 	w.Config.SetDefault("watcher.gracefulShutdownTimeout", 300)
 	w.Config.SetDefault("pingTimeout", 30)
+	w.Config.SetDefault("occupiedTimeout", 60*60)
 }
 
 func (w *Watcher) configure() {
@@ -157,6 +158,24 @@ func (w *Watcher) RemoveDeadRooms() {
 	if err != nil {
 		logger.WithError(err).Error("error removing dead rooms")
 	}
+
+	since = time.Now().Unix() - w.Config.GetInt64("occupiedTimeout")
+	logger = w.Logger.WithFields(logrus.Fields{
+		"executionID": uuid.NewV4().String(),
+		"operation":   "removeDeadOccupiedRooms",
+		"since":       since,
+	})
+	err = controller.DeleteRoomsOccupiedTimeout(
+		logger,
+		w.MetricsReporter,
+		w.RedisClient.Client,
+		w.KubernetesClient,
+		w.SchedulerName,
+		since,
+	)
+	if err != nil {
+		logger.WithError(err).Error("error removing old occupied rooms")
+	}
 }
 
 // AutoScale checks if the GRUs state is as expected and scale up or down if necessary

diff --git a/watcher/watcher_test.go b/watcher/watcher_test.go
@@ -126,6 +126,13 @@ var _ = Describe("Watcher", func() {
 				pKey, gomock.Any(),
 			).Return(redis.NewStringSliceResult([]string{}, nil))
 
+			//DeleteRoomsOccupiedTimeout
+			lKey := models.GetLastStatusRedisKey(configYaml1.Name, models.StatusOccupied)
+			mockRedisClient.EXPECT().ZRangeByScore(
+				lKey,
+				gomock.Any(),
+			).Return(redis.NewStringSliceResult([]string{}, nil))
+
 			// GetSchedulerScalingInfo
 			mockDb.EXPECT().Query(gomock.Any(), "SELECT * FROM schedulers WHERE name = ?", configYaml1.Name).Do(func(scheduler *models.Scheduler, query string, modifier string) {
 				scheduler.YAML = yaml1
@@ -685,8 +692,10 @@ var _ = Describe("Watcher", func() {
 			Expect(w).NotTo(BeNil())
 		})
 
-		It("should call controller DeleteRoomsNoPingSince", func() {
-			pKey := models.GetRoomPingRedisKey(configYaml1.Name)
+		It("should call controller DeleteRoomsNoPingSince and DeleteRoomsOccupiedTimeout", func() {
+			schedulerName := configYaml1.Name
+			pKey := models.GetRoomPingRedisKey(schedulerName)
+			lKey := models.GetLastStatusRedisKey(schedulerName, models.StatusOccupied)
 			ts := time.Now().Unix() - w.Config.GetInt64("pingTimeout")
 			// DeleteRoomsNoPingSince
 			mockRedisClient.EXPECT().ZRangeByScore(
@@ -696,14 +705,41 @@ var _ = Describe("Watcher", func() {
 				Expect(zrangeby.Min).To(Equal("-inf"))
 				max, err := strconv.Atoi(zrangeby.Max)
 				Expect(err).NotTo(HaveOccurred())
-				Expect(max).To(BeNumerically("~", ts, 1))
+				Expect(max).To(BeNumerically("~", ts, 1*time.Second))
 			}).Return(redis.NewStringSliceResult([]string{}, nil))
 
+			// DeleteRoomsOccupiedTimeout
+			ts = time.Now().Unix() - w.Config.GetInt64("occupiedTimeout")
+			expectedRooms := []string{"room1", "room2", "room3"}
+			mockRedisClient.EXPECT().ZRangeByScore(
+				lKey,
+				redis.ZRangeBy{Min: "-inf", Max: strconv.FormatInt(ts, 10)},
+			).Do(func(key string, zrangeby redis.ZRangeBy) {
+				Expect(zrangeby.Min).To(Equal("-inf"))
+				max, err := strconv.Atoi(zrangeby.Max)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(max).To(BeNumerically("~", ts, 1*time.Second))
+			}).Return(redis.NewStringSliceResult(expectedRooms, nil))
+
+			for _, roomName := range expectedRooms {
+				room := models.NewRoom(roomName, schedulerName)
+				mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline)
+				for _, status := range allStatus {
+					mockPipeline.EXPECT().SRem(models.GetRoomStatusSetRedisKey(schedulerName, status), room.GetRoomRedisKey())
+					mockPipeline.EXPECT().ZRem(models.GetLastStatusRedisKey(schedulerName, status), roomName)
+				}
+				mockPipeline.EXPECT().ZRem(models.GetRoomPingRedisKey(schedulerName), roomName)
+				mockPipeline.EXPECT().Del(room.GetRoomRedisKey())
+				mockPipeline.EXPECT().Exec()
+			}
+
 			Expect(func() { w.RemoveDeadRooms() }).ShouldNot(Panic())
 		})
 
-		It("should log and not panic in case of error", func() {
-			pKey := models.GetRoomPingRedisKey(configYaml1.Name)
+		It("should log and not panic in case of no ping since error", func() {
+			schedulerName := configYaml1.Name
+			pKey := models.GetRoomPingRedisKey(schedulerName)
+			lKey := models.GetLastStatusRedisKey(schedulerName, models.StatusOccupied)
 			ts := time.Now().Unix() - w.Config.GetInt64("pingTimeout")
 			// DeleteRoomsNoPingSince
 			mockRedisClient.EXPECT().ZRangeByScore(
@@ -713,11 +749,69 @@ var _ = Describe("Watcher", func() {
 				Expect(zrangeby.Min).To(Equal("-inf"))
 				max, err := strconv.Atoi(zrangeby.Max)
 				Expect(err).NotTo(HaveOccurred())
-				Expect(max).To(BeNumerically("~", ts, 1))
+				Expect(max).To(BeNumerically("~", ts, 1*time.Second))
 			}).Return(redis.NewStringSliceResult([]string{}, errors.New("some error")))
 
+			// DeleteRoomsOccupiedTimeout
+			ts = time.Now().Unix() - w.Config.GetInt64("occupiedTimeout")
+			expectedRooms := []string{"room1", "room2", "room3"}
+			mockRedisClient.EXPECT().ZRangeByScore(
+				lKey,
+				redis.ZRangeBy{Min: "-inf", Max: strconv.FormatInt(ts, 10)},
+			).Do(func(key string, zrangeby redis.ZRangeBy) {
+				Expect(zrangeby.Min).To(Equal("-inf"))
+				max, err := strconv.Atoi(zrangeby.Max)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(max).To(BeNumerically("~", ts, 1*time.Second))
+			}).Return(redis.NewStringSliceResult(expectedRooms, nil))
+
+			for _, roomName := range expectedRooms {
+				room := models.NewRoom(roomName, schedulerName)
+				mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline)
+				for _, status := range allStatus {
+					mockPipeline.EXPECT().SRem(models.GetRoomStatusSetRedisKey(schedulerName, status), room.GetRoomRedisKey())
+					mockPipeline.EXPECT().ZRem(models.GetLastStatusRedisKey(schedulerName, status), roomName)
+				}
+				mockPipeline.EXPECT().ZRem(models.GetRoomPingRedisKey(schedulerName), roomName)
+				mockPipeline.EXPECT().Del(room.GetRoomRedisKey())
+				mockPipeline.EXPECT().Exec()
+			}
+
 			Expect(func() { w.RemoveDeadRooms() }).ShouldNot(Panic())
 			Expect(hook.Entries).To(testing.ContainLogMessage("error removing dead rooms"))
 		})
+
+		It("should log and not panic in case of occupied error", func() {
+			schedulerName := configYaml1.Name
+			pKey := models.GetRoomPingRedisKey(schedulerName)
+			lKey := models.GetLastStatusRedisKey(schedulerName, models.StatusOccupied)
+			ts := time.Now().Unix() - w.Config.GetInt64("pingTimeout")
+			// DeleteRoomsNoPingSince
+			mockRedisClient.EXPECT().ZRangeByScore(
+				pKey,
+				gomock.Any(),
+			).Do(func(key string, zrangeby redis.ZRangeBy) {
+				Expect(zrangeby.Min).To(Equal("-inf"))
+				max, err := strconv.Atoi(zrangeby.Max)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(max).To(BeNumerically("~", ts, 1*time.Second))
+			}).Return(redis.NewStringSliceResult([]string{}, nil))
+
+			// DeleteRoomsOccupiedTimeout
+			ts = time.Now().Unix() - w.Config.GetInt64("occupiedTimeout")
+			expectedRooms := []string{"room1", "room2", "room3"}
+			mockRedisClient.EXPECT().ZRangeByScore(
+				lKey,
+				redis.ZRangeBy{Min: "-inf", Max: strconv.FormatInt(ts, 10)},
+			).Do(func(key string, zrangeby redis.ZRangeBy) {
+				Expect(zrangeby.Min).To(Equal("-inf"))
+				max, err := strconv.Atoi(zrangeby.Max)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(max).To(BeNumerically("~", ts, 1*time.Second))
+			}).Return(redis.NewStringSliceResult(expectedRooms, errors.New("redis error")))
+
+			Expect(func() { w.RemoveDeadRooms() }).ShouldNot(Panic())
+			Expect(hook.Entries).To(testing.ContainLogMessage("error removing old occupied rooms"))
+		})
 	})
 })