diff --git a/pkg/controller/nodelifecycle/node_lifecycle_controller.go b/pkg/controller/nodelifecycle/node_lifecycle_controller.go index 4f135963c746..846420a1c038 100644 --- a/pkg/controller/nodelifecycle/node_lifecycle_controller.go +++ b/pkg/controller/nodelifecycle/node_lifecycle_controller.go @@ -858,7 +858,7 @@ func (nc *Controller) monitorNodeHealth() error { nodeutil.RecordNodeStatusChange(nc.recorder, node, "NodeNotReady") fallthrough case needsRetry && observedReadyCondition.Status != v1.ConditionTrue: - if err = nodeutil.MarkPodsNotReady(nc.kubeClient, pods, node.Name); err != nil { + if err = nodeutil.MarkPodsNotReady(nc.kubeClient, nc.recorder, pods, node.Name); err != nil { utilruntime.HandleError(fmt.Errorf("unable to mark all pods NotReady on node %v: %v; queuing for retry", node.Name, err)) nc.nodesToRetry.Store(node.Name, struct{}{}) continue @@ -1341,7 +1341,7 @@ func (nc *Controller) processPod(podItem podUpdateItem) { } if currentReadyCondition.Status != v1.ConditionTrue { - if err := nodeutil.MarkPodsNotReady(nc.kubeClient, pods, nodeName); err != nil { + if err := nodeutil.MarkPodsNotReady(nc.kubeClient, nc.recorder, pods, nodeName); err != nil { klog.Warningf("Unable to mark pod %+v NotReady on node %v: %v.", podItem, nodeName, err) nc.podUpdateQueue.AddRateLimited(podItem) } diff --git a/pkg/controller/util/node/controller_utils.go b/pkg/controller/util/node/controller_utils.go index ffcdd654dac9..9cf1de2cf6d7 100644 --- a/pkg/controller/util/node/controller_utils.go +++ b/pkg/controller/util/node/controller_utils.go @@ -118,7 +118,7 @@ func SetPodTerminationReason(kubeClient clientset.Interface, pod *v1.Pod, nodeNa // MarkPodsNotReady updates ready status of given pods running on // given node from master return true if success -func MarkPodsNotReady(kubeClient clientset.Interface, pods []*v1.Pod, nodeName string) error { +func MarkPodsNotReady(kubeClient clientset.Interface, recorder record.EventRecorder, pods []*v1.Pod, nodeName string) error { klog.V(2).Infof("Update ready status of pods on node [%v]", nodeName) errMsg := []string{} @@ -136,6 +136,7 @@ func MarkPodsNotReady(kubeClient clientset.Interface, pods []*v1.Pod, nodeName s if !utilpod.UpdatePodCondition(&pod.Status, &cond) { break } + klog.V(2).Infof("Updating ready status of pod %v to false", pod.Name) _, err := kubeClient.CoreV1().Pods(pod.Namespace).UpdateStatus(context.TODO(), pod, metav1.UpdateOptions{}) if err != nil { @@ -147,6 +148,8 @@ func MarkPodsNotReady(kubeClient clientset.Interface, pods []*v1.Pod, nodeName s klog.Warningf("Failed to update status for pod %q: %v", format.Pod(pod), err) errMsg = append(errMsg, fmt.Sprintf("%v", err)) } + // record NodeNotReady event after updateStatus to make sure pod still exists + recorder.Event(pod, v1.EventTypeWarning, "NodeNotReady", "Node is not ready") break } }