Skip to content

Commit

Permalink
fix: pods not readying thus task not ready
Browse files Browse the repository at this point in the history
Sometimes final Ready event comes from low-level trackers too early,
before e.g. Pods are even added to StateStore. This forces to Task to go
to Ready state regardless of Pods.

Signed-off-by: Ilya Lesikov <ilya@lesikov.com>
  • Loading branch information
ilya-lesikov committed Feb 1, 2024
1 parent d545cba commit 95b81d0
Show file tree
Hide file tree
Showing 4 changed files with 39 additions and 0 deletions.
12 changes: 12 additions & 0 deletions pkg/trackers/dyntracker/dynamic_readiness_tracker.go
Original file line number Diff line number Diff line change
Expand Up @@ -976,6 +976,8 @@ func (t *DynamicReadinessTracker) handleDeploymentStatus(status *deployment.Depl
}

if status.IsReady {
taskState.SetStatus(statestore.ReadinessTaskStatusReady)

for _, state := range taskState.ResourceStates() {
state.RWTransaction(func(rs *statestore.ResourceState) {
rs.SetStatus(statestore.ResourceStatusReady)
Expand All @@ -1000,6 +1002,8 @@ func (t *DynamicReadinessTracker) handleStatefulSetStatus(status *statefulset.St
}

if status.IsReady {
taskState.SetStatus(statestore.ReadinessTaskStatusReady)

for _, state := range taskState.ResourceStates() {
state.RWTransaction(func(rs *statestore.ResourceState) {
rs.SetStatus(statestore.ResourceStatusReady)
Expand All @@ -1024,6 +1028,8 @@ func (t *DynamicReadinessTracker) handleDaemonSetStatus(status *daemonset.Daemon
}

if status.IsReady {
taskState.SetStatus(statestore.ReadinessTaskStatusReady)

for _, state := range taskState.ResourceStates() {
state.RWTransaction(func(rs *statestore.ResourceState) {
rs.SetStatus(statestore.ResourceStatusReady)
Expand All @@ -1046,6 +1052,8 @@ func (t *DynamicReadinessTracker) handleJobStatus(status *job.JobStatus, taskSta
}

if status.IsSucceeded {
taskState.SetStatus(statestore.ReadinessTaskStatusReady)

for _, state := range taskState.ResourceStates() {
state.RWTransaction(func(rs *statestore.ResourceState) {
rs.SetStatus(statestore.ResourceStatusReady)
Expand All @@ -1064,6 +1072,8 @@ func (t *DynamicReadinessTracker) handleCanaryStatus(status *canary.CanaryStatus
}

if status.IsSucceeded {
taskState.SetStatus(statestore.ReadinessTaskStatusReady)

for _, state := range taskState.ResourceStates() {
state.RWTransaction(func(rs *statestore.ResourceState) {
rs.SetStatus(statestore.ResourceStatusReady)
Expand All @@ -1088,6 +1098,8 @@ func (t *DynamicReadinessTracker) handleGenericResourceStatus(status *generic.Re
}

if status.IsReady() {
taskState.SetStatus(statestore.ReadinessTaskStatusReady)

for _, state := range taskState.ResourceStates() {
state.RWTransaction(func(rs *statestore.ResourceState) {
rs.SetStatus(statestore.ResourceStatusReady)
Expand Down
9 changes: 9 additions & 0 deletions pkg/trackers/dyntracker/statestore/absence_task_state.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ type AbsenceTaskState struct {
absentConditions []AbsenceTaskConditionFn
failureConditions []AbsenceTaskConditionFn

status AbsenceTaskStatus
uuid string
resourceState *util.Concurrent[*ResourceState]
}
Expand Down Expand Up @@ -56,7 +57,15 @@ func (s *AbsenceTaskState) ResourceState() *util.Concurrent[*ResourceState] {
return s.resourceState
}

func (s *AbsenceTaskState) SetStatus(status AbsenceTaskStatus) {
s.status = status
}

func (s *AbsenceTaskState) Status() AbsenceTaskStatus {
if s.status != "" {
return s.status
}

for _, failureCondition := range s.failureConditions {
if failureCondition(s) {
return AbsenceTaskStatusFailed
Expand Down
9 changes: 9 additions & 0 deletions pkg/trackers/dyntracker/statestore/presence_task_state.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ type PresenceTaskState struct {
presentConditions []PresenceTaskConditionFn
failureConditions []PresenceTaskConditionFn

status PresenceTaskStatus
uuid string
resourceState *util.Concurrent[*ResourceState]
}
Expand Down Expand Up @@ -56,7 +57,15 @@ func (s *PresenceTaskState) ResourceState() *util.Concurrent[*ResourceState] {
return s.resourceState
}

func (s *PresenceTaskState) SetStatus(status PresenceTaskStatus) {
s.status = status
}

func (s *PresenceTaskState) Status() PresenceTaskStatus {
if s.status != "" {
return s.status
}

for _, failureCondition := range s.failureConditions {
if failureCondition(s) {
return PresenceTaskStatusFailed
Expand Down
9 changes: 9 additions & 0 deletions pkg/trackers/dyntracker/statestore/readiness_task_state.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ type ReadinessTaskState struct {
readyConditions []ReadinessTaskConditionFn
failureConditions []ReadinessTaskConditionFn

status ReadinessTaskStatus
uuid string
resourceStatesTree domigraph.Graph[string, *util.Concurrent[*ResourceState]]
}
Expand Down Expand Up @@ -127,7 +128,15 @@ func (s *ReadinessTaskState) TraverseResourceStates(fromName, fromNamespace stri
return states
}

func (s *ReadinessTaskState) SetStatus(status ReadinessTaskStatus) {
s.status = status
}

func (s *ReadinessTaskState) Status() ReadinessTaskStatus {
if s.status != "" {
return s.status
}

for _, failureCondition := range s.failureConditions {
if failureCondition(s) {
return ReadinessTaskStatusFailed
Expand Down

0 comments on commit 95b81d0

Please sign in to comment.