Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Only declare job as finished after removing all finalizers #119159

Merged
merged 1 commit into from Jul 8, 2023
Merged

Only declare job as finished after removing all finalizers #119159

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
11 changes: 4 additions & 7 deletions pkg/controller/job/job_controller.go
View file
Open in desktop
Expand Up @@ -792,12 +792,7 @@ func (jm *Controller) syncJob(ctx context.Context, key string) (rErr error) {
var manageJobErr error
var finishedCondition *batch.JobCondition

jobHasNewFailure := failed > job.Status.Failed
// new failures happen when status does not reflect the failures and active
// is different than parallelism, otherwise the previous controller loop
// failed updating status so even if we pick up failure it is not a new one
exceedsBackoffLimit := jobHasNewFailure && (active != *job.Spec.Parallelism) &&
alculquicondor marked this conversation as resolved.
Show resolved Hide resolved
(failed > *job.Spec.BackoffLimit)
exceedsBackoffLimit := failed > *job.Spec.BackoffLimit

if feature.DefaultFeatureGate.Enabled(features.JobPodFailurePolicy) {
if failureTargetCondition := findConditionByType(job.Status.Conditions, batch.JobFailureTarget); failureTargetCondition != nil {
Expand Down Expand Up @@ -999,6 +994,7 @@ func (jm *Controller) trackJobStatusAndRemoveFinalizers(ctx context.Context, job
needsFlush = true
}
podFailureCountByPolicyAction := map[string]int{}
reachedMaxUncountedPods := false
for _, pod := range pods {
if !hasJobTrackingFinalizer(pod) || expectedRmFinalizers.Has(string(pod.UID)) {
// This pod was processed in a previous sync.
Expand Down Expand Up @@ -1049,6 +1045,7 @@ func (jm *Controller) trackJobStatusAndRemoveFinalizers(ctx context.Context, job
//
// The job will be synced again because the Job status and Pod updates
// will put the Job back to the work queue.
reachedMaxUncountedPods = true
alculquicondor marked this conversation as resolved.
Show resolved Hide resolved
break
}
}
Expand Down Expand Up @@ -1077,7 +1074,7 @@ func (jm *Controller) trackJobStatusAndRemoveFinalizers(ctx context.Context, job
if job, needsFlush, err = jm.flushUncountedAndRemoveFinalizers(ctx, job, podsToRemoveFinalizer, uidsWithFinalizer, &oldCounters, podFailureCountByPolicyAction, needsFlush, newBackoffRecord); err != nil {
return err
}
jobFinished := jm.enactJobFinished(job, finishedCond)
jobFinished := !reachedMaxUncountedPods && jm.enactJobFinished(job, finishedCond)
if jobFinished {
needsFlush = true
}
Expand Down
3 changes: 3 additions & 0 deletions test/integration/job/job_test.go
View file
Open in desktop
Expand Up @@ -1341,6 +1341,9 @@ func TestOrphanPodsFinalizersClearedWithGC(t *testing.T) {
}

func TestFinalizersClearedWhenBackoffLimitExceeded(t *testing.T) {
// Set a maximum number of uncounted pods below parallelism, to ensure it
// doesn't affect the termination of pods.
t.Cleanup(setDuringTest(&jobcontroller.MaxUncountedPods, 50))
closeFn, restConfig, clientSet, ns := setup(t, "simple")
defer closeFn()
ctx, cancel := startJobControllerAndWaitForCaches(restConfig)
Expand Down