kubelet: Make probe to be on for time drift

The initialDelaySeconds holds liveness or readiness probes for specified
seconds after container has started. To achieve this, kubelet relies on
StartedAt time of container which never change during its lifecycle. But
in case of time drift scenario (example: During maintenance, NVRAM is
flashed, looks like it resets the BIOS settings and RTC is also affected),
The StartedAt time may contain future time which makes probe to be disabled
until system reaches StartedAt time. Hence this commit handles this scenario
and makes probe to be still initiated for those cases.

Signed-off-by: Periyasamy Palanisamy <pepalani@redhat.com>

pkg/kubelet/prober/worker.go

@@ -265,8 +265,11 @@ func (w *worker) doProbe(ctx context.Context) (keepGoing bool) {
 		return false
 	}
 
-	// Probe disabled for InitialDelaySeconds.
-	if int32(time.Since(c.State.Running.StartedAt.Time).Seconds()) < w.spec.InitialDelaySeconds {
+	// Probe disabled for InitialDelaySeconds only if container's StartedAt time is
+	// earlier than system current time. But probe must be on for time drift case,
+	// This may happen due to nvram flash which resets system BIOS settings and RTC.
+	startedAtInSecs := time.Since(c.State.Running.StartedAt.Time).Seconds()
+	if startedAtInSecs > 0 && int32(startedAtInSecs) < w.spec.InitialDelaySeconds {
 		return true
 	}
 

pkg/kubelet/prober/worker_test.go

@@ -198,6 +198,39 @@ func TestInitialDelay(t *testing.T) {
 	}
 }
 
+func TestInitialDelayForTimeDrift(t *testing.T) {
+	ctx := context.Background()
+	m := newTestManager()
+
+	for _, probeType := range [...]probeType{liveness, readiness, startup} {
+		w := newTestWorker(m, probeType, v1.Probe{
+			InitialDelaySeconds: 10,
+		})
+		m.statusManager.SetPodStatus(w.pod, getTestRunningStatusWithStarted(probeType != startup))
+
+		expectContinue(t, w, w.doProbe(ctx), "during initial delay")
+		// Default value depends on probe, Success for liveness, Failure for readiness, Unknown for startup
+		switch probeType {
+		case liveness:
+			expectResult(t, w, results.Success, "during initial delay")
+		case readiness:
+			expectResult(t, w, results.Failure, "during initial delay")
+		case startup:
+			expectResult(t, w, results.Unknown, "during initial delay")
+		}
+
+		laterStatus := getTestRunningStatusWithStarted(probeType != startup)
+		// Set StartedAt to a future time to test time drift.
+		laterStatus.ContainerStatuses[0].State.Running.StartedAt.Time =
+			time.Now().Add(600 * time.Second)
+		m.statusManager.SetPodStatus(w.pod, laterStatus)
+
+		// The doProbe call should succeed after time drift.
+		expectContinue(t, w, w.doProbe(ctx), "after time drift")
+		expectResult(t, w, results.Success, "after time drift")
+	}
+}
+
 func TestFailureThreshold(t *testing.T) {
 	ctx := context.Background()
 	m := newTestManager()