use rate limiting queue with delaying for pod deletion events (#2774)

kubeovn · May 9, 2023 · 3b2b071 · 3b2b071
1 parent 04e4d25
commit 3b2b071
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 12 deletions.
diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go
@@ -377,10 +377,13 @@ func Run(ctx context.Context, config *Configuration) {
 		providerNetworksLister: providerNetworkInformer.Lister(),
 		providerNetworkSynced:  providerNetworkInformer.Informer().HasSynced,
 
-		podsLister:             podInformer.Lister(),
-		podsSynced:             podInformer.Informer().HasSynced,
-		addOrUpdatePodQueue:    workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "AddOrUpdatePod"),
-		deletePodQueue:         workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "DeletePod"),
+		podsLister:          podInformer.Lister(),
+		podsSynced:          podInformer.Informer().HasSynced,
+		addOrUpdatePodQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "AddOrUpdatePod"),
+		deletePodQueue: workqueue.NewRateLimitingQueueWithDelayingInterface(
+			workqueue.NewNamedDelayingQueue("DeletePod"),
+			workqueue.DefaultControllerRateLimiter(),
+		),
 		updatePodSecurityQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "UpdatePodSecurity"),
 		podKeyMutex:            keymutex.New(97),
 

diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go
@@ -289,30 +289,38 @@ func (c *Controller) enqueueUpdatePod(oldObj, newObj interface{}) {
 		return
 	}
 
+	// enqueue delay
+	var delay time.Duration
+	if newPod.Spec.TerminationGracePeriodSeconds != nil {
+		if newPod.DeletionTimestamp != nil {
+			delay = time.Until(newPod.DeletionTimestamp.Add(time.Duration(*newPod.Spec.TerminationGracePeriodSeconds) * time.Second))
+		} else {
+			delay = time.Duration(*newPod.Spec.TerminationGracePeriodSeconds) * time.Second
+		}
+	}
+
 	if newPod.DeletionTimestamp != nil && !isStateful && !isVmPod {
 		go func() {
 			// In case node get lost and pod can not be deleted,
 			// the ip address will not be recycled
-			time.Sleep(time.Duration(*newPod.Spec.TerminationGracePeriodSeconds) * time.Second)
-			c.deletePodQueue.Add(newObj)
+			klog.V(3).Infof("enqueue delete pod %s after %v", key, delay)
+			c.deletePodQueue.AddAfter(newObj, delay)
 		}()
 		return
 	}
 
 	// do not delete statefulset pod unless ownerReferences is deleted
 	if isStateful && isStatefulSetPodToDel(c.config.KubeClient, newPod, statefulSetName) {
 		go func() {
-			klog.V(3).Infof("enqueue delete pod %s", key)
-			time.Sleep(time.Duration(*newPod.Spec.TerminationGracePeriodSeconds) * time.Second)
-			c.deletePodQueue.Add(newObj)
+			klog.V(3).Infof("enqueue delete pod %s after %v", key, delay)
+			c.deletePodQueue.AddAfter(newObj, delay)
 		}()
 		return
 	}
 	if isVmPod && c.isVmPodToDel(newPod, vmName) {
 		go func() {
-			klog.V(3).Infof("enqueue delete pod %s", key)
-			time.Sleep(time.Duration(*newPod.Spec.TerminationGracePeriodSeconds) * time.Second)
-			c.deletePodQueue.Add(newObj)
+			klog.V(3).Infof("enqueue delete pod %s after %v", key, delay)
+			c.deletePodQueue.AddAfter(newObj, delay)
 		}()
 		return
 	}