feat(controller): Pod deletion grace period. Fixes #4719 (#4725)

Signed-off-by: Alex Collins <alex_collins@intuit.com>

workflow/controller/operator.go

@@ -903,21 +903,18 @@ func (woc *wfOperationCtx) podReconciliation() error {
 		}
 		if _, ok := seenPods[nodeID]; !ok {
 
+			// grace-period to allow informer sync
+			recentlyStarted := recentlyStarted(node)
+			woc.log.WithFields(log.Fields{"nodeName": node.Name, "nodePhase": node.Phase, "recentlyStarted": recentlyStarted}).Info("Workflow pod is missing")
+			metrics.PodMissingMetric.WithLabelValues(strconv.FormatBool(recentlyStarted), string(node.Phase)).Inc()
+
 			// If the node is pending and the pod does not exist, it could be the case that we want to try to submit it
 			// again instead of marking it as an error. Check if that's the case.
-			if node.Pending() {
+			if node.Pending() || recentlyStarted {
 				continue
 			}
 
-			node.Message = "pod deleted"
-			node.Phase = wfv1.NodeError
-			// FinishedAt must be set since retry strategy depends on it to determine the backoff duration.
-			// See processNodeRetries for more details.
-			node.FinishedAt = metav1.Time{Time: time.Now().UTC()}
-			woc.wf.Status.Nodes[nodeID] = node
-			woc.log.WithField("displayName", node.DisplayName).WithField("templateName", node.TemplateName).
-				WithField("node", node.Name).Error("Pod for node deleted")
-			woc.updated = true
+			woc.markNodePhase(node.Name, wfv1.NodeError, "pod deleted")
 		} else {
 			// At this point we are certain that the pod associated with our node is running or has been run;
 			// it is safe to extract the k8s-node information given this knowledge.
@@ -931,6 +928,10 @@ func (woc *wfOperationCtx) podReconciliation() error {
 	return nil
 }
 
+func recentlyStarted(node wfv1.NodeStatus) bool {
+	return time.Since(node.StartedAt.Time) <= envutil.LookupEnvDurationOr("RECENTLY_STARTED_POD_DURATION", 10*time.Second)
+}
+
 // shouldPrintPodSpec return eligible to print to the pod spec
 func (woc *wfOperationCtx) shouldPrintPodSpec(node wfv1.NodeStatus) bool {
 	return woc.controller.Config.PodSpecLogStrategy.AllPods ||

workflow/metrics/pod_missing_metric.go

@@ -0,0 +1,14 @@
+package metrics
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var (
+	PodMissingMetric = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: argoNamespace,
+			Name:      "pod_missing",
+			Help:      "Incidents of pod missing",
+		},
+		[]string{"recently_started", "node_phase"},
+	)
+)

workflow/metrics/server.go

@@ -70,13 +70,15 @@ func (m *Metrics) Describe(ch chan<- *prometheus.Desc) {
 		ch <- metric.Desc()
 	}
 	m.logMetric.Describe(ch)
+	PodMissingMetric.Describe(ch)
 }
 
 func (m *Metrics) Collect(ch chan<- prometheus.Metric) {
 	for _, metric := range m.allMetrics() {
 		ch <- metric
 	}
 	m.logMetric.Collect(ch)
+	PodMissingMetric.Collect(ch)
 }
 
 func (m *Metrics) garbageCollector(ctx context.Context) {