[manager/orchestrator/reaper] Clean out the task reaper dirty set at …

…the end of tick()

Signed-off-by: Anshul Pundir <anshul.pundir@docker.com>

manager/orchestrator/taskreaper/task_reaper.go

@@ -184,12 +184,23 @@ func (tr *TaskReaper) tick() {
 	}
 
 	// Check history of dirty tasks for cleanup.
+	// Note: Clean out the dirty set at the end of this tick iteration
+	// in all but one scenarios (documented below).
+	// When tick() finishes, the tasks in the slot were either cleaned up,
+	// or it was skipped because it didn't meet the criteria for cleaning.
+	// Either way, we can discard the dirty set because future events on
+	// that slot will cause the task to be readded to the dirty set
+	// at that point.
+	//
+	// The only case when we keep the slot dirty is when there are more
+	// than one running tasks present for a given slot.
+	// In that case, we need to keep the slot dirty to allow it to be
+	// cleaned when tick() is called next and one or more the tasks
+	// in that slot have stopped running.
 	tr.store.View(func(tx store.ReadTx) {
 		for dirty := range tr.dirty {
 			service := store.GetService(tx, dirty.ServiceID)
 			if service == nil {
-				// If the service can't be found, assume that it was deleted
-				// and remove the slot from the dirty list.
 				delete(tr.dirty, dirty)
 				continue
 			}
@@ -214,6 +225,7 @@ func (tr *TaskReaper) tick() {
 
 			// Negative value for TaskHistoryRetentionLimit is an indication to never clean up task history.
 			if taskHistory < 0 {
+				delete(tr.dirty, dirty)
 				continue
 			}
 
@@ -243,6 +255,7 @@ func (tr *TaskReaper) tick() {
 			}
 
 			if int64(len(historicTasks)) <= taskHistory {
+				delete(tr.dirty, dirty)
 				continue
 			}
 
@@ -271,10 +284,16 @@ func (tr *TaskReaper) tick() {
 				if int64(len(historicTasks)) <= taskHistory {
 					break
 				}
-			}
 
-			if runningTasks <= 1 {
-				delete(tr.dirty, dirty)
+				// The only case when we keep the slot dirty at the end of tick()
+				// is when there are more than one running tasks present
+				// for a given slot.
+				// In that case, we keep the slot dirty to allow it to be
+				// cleaned when tick() is called next and one or more of
+				// the tasks in that slot have stopped running.
+				if runningTasks <= 1 {
+					delete(tr.dirty, dirty)
+				}
 			}
 		}
 	})