add pod graceful termination to pod lifecycle

e2echart/e2e-chart-template.html

@@ -81,7 +81,7 @@ <h5 class="modal-title">Resource</h5>
     }
 
     function isPodLifecycle(eventInterval) {
-        if (eventInterval.locator.includes("pod/") && (eventInterval.message.includes("reason/Created") || eventInterval.message.includes("reason/Scheduled"))) {
+        if (eventInterval.locator.includes("pod/") && (eventInterval.message.includes("reason/Created") || eventInterval.message.includes("reason/Scheduled") || eventInterval.message.includes("reason/GracefulDelete"))) {
             return true
         }
         return false
@@ -164,6 +164,9 @@ <h5 class="modal-title">Resource</h5>
             if (m[2] == "Scheduled") {
                 return [item.locator, ` (pod lifecycle)`, "PodScheduled"];
             }
+            if (m[2] == "GracefulDelete") {
+                return [item.locator, ` (pod lifecycle)`, "PodTerminating"];
+            }
         }
         if (m && isContainerLifecycle(item)){
             if (m[2] == "ContainerWait") {
@@ -369,14 +372,14 @@ <h5 class="modal-title">Resource</h5>
             'OperatorUnavailable', 'OperatorDegraded', 'OperatorProgressing', // operators
             'Update', 'Drain', 'Reboot', 'OperatingSystemUpdate', 'NodeNotReady', // nodes
             'Passed', 'Skipped', 'Flaked', 'Failed',  // tests
-            'PodCreated', 'PodScheduled', 'ContainerWait', 'ContainerStart', 'ContainerNotReady', 'ContainerReady',  // pods
+            'PodCreated', 'PodScheduled', 'PodTerminating','ContainerWait', 'ContainerStart', 'ContainerNotReady', 'ContainerReady',  // pods
             'Degraded', 'Upgradeable', 'False', 'Unknown'])
         .range([
             '#fada5e','#fada5e','#ffa500','#d0312d',  // alerts
             '#d0312d', '#ffa500', '#fada5e', // operators
             '#1e7bd9', '#4294e6', '#6aaef2', '#96cbff', '#fada5e', // nodes
             '#3cb043', '#ceba76', '#ffa500', '#d0312d', // tests
-            '#96cbff', '#1e7bd9', '#ca8dfd', '#9300ff', '#fada5e','#3cb043', // pods
+            '#96cbff', '#1e7bd9', '#ffa500', '#ca8dfd', '#9300ff', '#fada5e','#3cb043', // pods
             '#b65049', '#32b8b6', '#ffffff', '#bbbbbb']);
     myChart.
     data(timelineGroups).

pkg/monitor/intervalcreation/pod.go

@@ -151,17 +151,20 @@ func (t podLifecycleTimeBounder) getEndTime(inLocator string) time.Time {
 		return *runOnceContainerTermination
 	}
 
-	// for other pod types, use the deletion time if we have it
-	if objDelete := t.getPodDeletionTime(inLocator); objDelete != nil {
-		return *objDelete
-	}
+	// pods will logically be gone once the pod deletion + grace period is over. Or at least they should be
+	lastPossiblePodDelete := t.getPodDeletionPlusGraceTime(inLocator)
 
 	podEvents, ok := t.podToStateTransitions[locator]
 	if !ok {
 		return t.delegate.getEndTime(locator)
 	}
 	for _, event := range podEvents {
 		if monitorapi.ReasonFrom(event.Message) == monitorapi.PodReasonDeleted {
+			// if the last possible pod delete is before the delete from teh watch stream, it just means our watch was delayed.
+			// use the pod time instead.
+			if lastPossiblePodDelete != nil && lastPossiblePodDelete.Before(event.From) {
+				return *lastPossiblePodDelete
+			}
 			return event.From
 		}
 	}
@@ -206,7 +209,7 @@ func (t podLifecycleTimeBounder) getPodCreationTime(inLocator string) *time.Time
 	return &temp.Time
 }
 
-func (t podLifecycleTimeBounder) getPodDeletionTime(inLocator string) *time.Time {
+func (t podLifecycleTimeBounder) getPodDeletionPlusGraceTime(inLocator string) *time.Time {
 	podCoordinates := monitorapi.PodFrom(inLocator)
 	instanceKey := monitorapi.InstanceKey{
 		Namespace: podCoordinates.Namespace,
@@ -228,8 +231,11 @@ func (t podLifecycleTimeBounder) getPodDeletionTime(inLocator string) *time.Time
 		return nil
 	}
 
-	temp := pod.DeletionTimestamp
-	return &temp.Time
+	deletionTime := pod.DeletionTimestamp.Time
+	if pod.Spec.TerminationGracePeriodSeconds != nil {
+		deletionTime = deletionTime.Add(time.Duration(*pod.Spec.TerminationGracePeriodSeconds * int64(time.Second)))
+	}
+	return &deletionTime
 }
 
 func (t podLifecycleTimeBounder) getRunOnceContainerEnd(inLocator string) *time.Time {

pkg/monitor/monitorapi/identification_pod.go

@@ -98,9 +98,10 @@ const (
 	// PodIPReused means the same pod IP is in use by two pods at the same time.
 	PodIPReused = "ReusedPodIP"
 
-	PodReasonCreated   = "Created"
-	PodReasonDeleted   = "Deleted"
-	PodReasonScheduled = "Scheduled"
+	PodReasonCreated               = "Created"
+	PodReasonGracefulDeleteStarted = "GracefulDelete"
+	PodReasonDeleted               = "Deleted"
+	PodReasonScheduled             = "Scheduled"
 
 	ContainerReasonContainerExit  = "ContainerExit"
 	ContainerReasonContainerStart = "ContainerStart"
@@ -121,6 +122,7 @@ var (
 	PodLifecycleTransitionReasons = sets.NewString(
 		PodReasonCreated,
 		PodReasonScheduled,
+		PodReasonGracefulDeleteStarted,
 		PodReasonDeleted,
 	)
 

pkg/monitor/pod.go

@@ -434,7 +434,7 @@ func startPodMonitoring(ctx context.Context, m Recorder, client kubernetes.Inter
 						conditions = append(conditions, monitorapi.Condition{
 							Level:   monitorapi.Info,
 							Locator: monitorapi.LocatePod(pod),
-							Message: fmt.Sprintf("reason/GracefulDelete duration/%ds", *pod.DeletionGracePeriodSeconds),
+							Message: monitorapi.ReasonedMessagef(monitorapi.PodReasonGracefulDeleteStarted, "duration/%ds", *pod.DeletionGracePeriodSeconds),
 						})
 					}
 				}