kubelet: Make probe to be on for time drift

pkg/kubelet/prober/worker.go

@@ -70,6 +70,13 @@ type worker struct {
 	// If set, skip probing.
 	onHold bool
 
+	// Set to true when time drift is detected for the pod, will be unset
+	// once container's elapsed start time reaches +ve value.
+	timeDrift bool
+	// timeDriftStartTimeInSec contains container's elapsed start time at
+	// the time of first occurance of time drift.
+	timeDriftStartTimeInSec float64
+
 	// proberResultsMetricLabels holds the labels attached to this worker
 	// for the ProberResults metric by result.
 	proberResultsSuccessfulMetricLabels metrics.Labels
@@ -265,8 +272,21 @@ func (w *worker) doProbe(ctx context.Context) (keepGoing bool) {
 		return false
 	}
 
-	// Probe disabled for InitialDelaySeconds.
-	if int32(time.Since(c.State.Running.StartedAt.Time).Seconds()) < w.spec.InitialDelaySeconds {
+	// Probe disabled for InitialDelaySeconds. This also considers
+	// time drift scenario (in that case container's started time is
+	// later than system current time).
+	startedAtInSecs := time.Since(c.State.Running.StartedAt.Time).Seconds()
+	if startedAtInSecs <= 0 && !w.timeDrift {
+		w.timeDrift = true
+		w.timeDriftStartTimeInSec = startedAtInSecs
+	} else if startedAtInSecs > 0 {
+		w.timeDrift = false
+	}
+	if startedAtInSecs <= 0 && w.timeDrift &&
+		int32(startedAtInSecs-w.timeDriftStartTimeInSec) < w.spec.InitialDelaySeconds {
+		return true
+	}
+	if startedAtInSecs > 0 && int32(startedAtInSecs) < w.spec.InitialDelaySeconds {
 		return true
 	}
 

pkg/kubelet/prober/worker_test.go

@@ -198,6 +198,65 @@ func TestInitialDelay(t *testing.T) {
 	}
 }
 
+func TestInitialDelayForTimeDrift(t *testing.T) {
+	ctx := context.Background()
+	m := newTestManager()
+
+	for _, probeType := range [...]probeType{liveness, readiness, startup} {
+		w := newTestWorker(m, probeType, v1.Probe{
+			InitialDelaySeconds: 10,
+		})
+		m.statusManager.SetPodStatus(w.pod, getTestRunningStatusWithStarted(probeType != startup))
+
+		expectContinue(t, w, w.doProbe(ctx), "during initial delay")
+		// Default value depends on probe, Success for liveness, Failure for readiness, Unknown for startup
+		switch probeType {
+		case liveness:
+			expectResult(t, w, results.Success, "during initial delay")
+		case readiness:
+			expectResult(t, w, results.Failure, "during initial delay")
+		case startup:
+			expectResult(t, w, results.Unknown, "during initial delay")
+		}
+
+		laterStatus := getTestRunningStatusWithStarted(probeType != startup)
+		// Set StartedAt to a future time to test time drift.
+		laterStatus.ContainerStatuses[0].State.Running.StartedAt.Time =
+			time.Now().Add(600 * time.Second)
+		m.statusManager.SetPodStatus(w.pod, laterStatus)
+
+		// The doProbe call should fail immediately after time drift.
+		expectContinue(t, w, w.doProbe(ctx), "during initial delay")
+		switch probeType {
+		case liveness:
+			expectResult(t, w, results.Success, "during initial delay")
+		case readiness:
+			expectResult(t, w, results.Failure, "during initial delay")
+		case startup:
+			expectResult(t, w, results.Unknown, "during initial delay")
+		}
+
+		// The doProbe call would still fail when it called during specified
+		// initial delay time.
+		time.Sleep(5 * time.Second)
+		expectContinue(t, w, w.doProbe(ctx), "during initial delay")
+		switch probeType {
+		case liveness:
+			expectResult(t, w, results.Success, "during initial delay")
+		case readiness:
+			expectResult(t, w, results.Failure, "during initial delay")
+		case startup:
+			expectResult(t, w, results.Unknown, "during initial delay")
+		}
+
+		// Sleep until initial delay time elapses and probe must
+		// succeed now.
+		time.Sleep(5 * time.Second)
+		expectContinue(t, w, w.doProbe(ctx), "during initial delay")
+		expectResult(t, w, results.Success, "during initial delay")
+	}
+}
+
 func TestFailureThreshold(t *testing.T) {
 	ctx := context.Background()
 	m := newTestManager()