diff --git a/integration/job_fail/Tiltfile b/integration/job_fail/Tiltfile new file mode 100644 index 0000000000..f02afc6811 --- /dev/null +++ b/integration/job_fail/Tiltfile @@ -0,0 +1,13 @@ +# -*- mode: Python -*- + +include('../Tiltfile') +docker_build('db', '.', dockerfile='db.dockerfile') +k8s_yaml('db.yaml') + +docker_build('db-init', '.', dockerfile='db-init.dockerfile') +k8s_yaml('db-init.yaml') +k8s_resource('job-fail-db-init', resource_deps=['job-fail-db']) + +docker_build('app', '.', dockerfile='app.dockerfile') +k8s_yaml('app.yaml') +k8s_resource('job-fail-app', resource_deps=['job-fail-db-init']) diff --git a/integration/job_fail/app.dockerfile b/integration/job_fail/app.dockerfile new file mode 100644 index 0000000000..8dc415c2f0 --- /dev/null +++ b/integration/job_fail/app.dockerfile @@ -0,0 +1,2 @@ +FROM busybox +ENTRYPOINT busybox httpd -f -p 8000 diff --git a/integration/job_fail/app.yaml b/integration/job_fail/app.yaml new file mode 100644 index 0000000000..bd800a19b6 --- /dev/null +++ b/integration/job_fail/app.yaml @@ -0,0 +1,19 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: job-fail-app + namespace: tilt-integration + labels: + app: job-fail-app +spec: + selector: + matchLabels: + app: job-fail-app + template: + metadata: + labels: + app: job-fail-app + spec: + containers: + - name: app + image: app diff --git a/integration/job_fail/db-init.dockerfile b/integration/job_fail/db-init.dockerfile new file mode 100644 index 0000000000..4b9d425ac6 --- /dev/null +++ b/integration/job_fail/db-init.dockerfile @@ -0,0 +1,3 @@ +FROM busybox +ENTRYPOINT ["sh", "-c", "sleep 1 && echo 'db-init job failed' && exit 1"] + diff --git a/integration/job_fail/db-init.yaml b/integration/job_fail/db-init.yaml new file mode 100644 index 0000000000..430f6d768d --- /dev/null +++ b/integration/job_fail/db-init.yaml @@ -0,0 +1,17 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: job-fail-db-init + namespace: tilt-integration + labels: + app: job-fail-db-init +spec: + template: + metadata: + labels: + app: job-fail-db-init + spec: + restartPolicy: Never + containers: + - name: db-init + image: db-init diff --git a/integration/job_fail/db.dockerfile b/integration/job_fail/db.dockerfile new file mode 100644 index 0000000000..118028c729 --- /dev/null +++ b/integration/job_fail/db.dockerfile @@ -0,0 +1,3 @@ +FROM busybox +ENTRYPOINT busybox httpd -f -p 8000 + diff --git a/integration/job_fail/db.yaml b/integration/job_fail/db.yaml new file mode 100644 index 0000000000..958b6a4a55 --- /dev/null +++ b/integration/job_fail/db.yaml @@ -0,0 +1,19 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: job-fail-db + namespace: tilt-integration + labels: + app: job-fail-db +spec: + selector: + matchLabels: + app: job-fail-db + template: + metadata: + labels: + app: job-fail-db + spec: + containers: + - name: db + image: db diff --git a/integration/job_fail_test.go b/integration/job_fail_test.go new file mode 100644 index 0000000000..2d694893c3 --- /dev/null +++ b/integration/job_fail_test.go @@ -0,0 +1,25 @@ +//go:build integration +// +build integration + +package integration + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" +) + +func TestJobFail(t *testing.T) { + f := newK8sFixture(t, "job_fail") + f.SetRestrictedCredentials() + + // Make sure 'ci' fails. + err := f.tilt.CI(f.ctx, f.LogWriter()) + require.Error(t, err) + assert.Contains(t, f.logs.String(), "db-init job failed") + + _, _, podNames := f.AllPodsInPhase(f.ctx, "app=job-fail-db-init", v1.PodFailed) + require.Equal(t, 1, len(podNames)) +} diff --git a/integration/job_reattach/Tiltfile b/integration/job_reattach/Tiltfile new file mode 100644 index 0000000000..6527e8f53a --- /dev/null +++ b/integration/job_reattach/Tiltfile @@ -0,0 +1,13 @@ +# -*- mode: Python -*- + +include('../Tiltfile') +docker_build('db', '.', dockerfile='db.dockerfile') +k8s_yaml('db.yaml') + +docker_build('db-init', '.', dockerfile='db-init.dockerfile') +k8s_yaml('db-init.yaml') +k8s_resource('job-reattach-db-init', resource_deps=['job-reattach-db']) + +docker_build('app', '.', dockerfile='app.dockerfile') +k8s_yaml('app.yaml') +k8s_resource('job-reattach-app', resource_deps=['job-reattach-db-init']) diff --git a/integration/job_reattach/app.dockerfile b/integration/job_reattach/app.dockerfile new file mode 100644 index 0000000000..8dc415c2f0 --- /dev/null +++ b/integration/job_reattach/app.dockerfile @@ -0,0 +1,2 @@ +FROM busybox +ENTRYPOINT busybox httpd -f -p 8000 diff --git a/integration/job_reattach/app.yaml b/integration/job_reattach/app.yaml new file mode 100644 index 0000000000..4e2f014ec8 --- /dev/null +++ b/integration/job_reattach/app.yaml @@ -0,0 +1,19 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: job-reattach-app + namespace: tilt-integration + labels: + app: job-reattach-app +spec: + selector: + matchLabels: + app: job-reattach-app + template: + metadata: + labels: + app: job-reattach-app + spec: + containers: + - name: app + image: app diff --git a/integration/job_reattach/db-init.dockerfile b/integration/job_reattach/db-init.dockerfile new file mode 100644 index 0000000000..641d51945a --- /dev/null +++ b/integration/job_reattach/db-init.dockerfile @@ -0,0 +1,3 @@ +FROM busybox +ENTRYPOINT echo INIT + diff --git a/integration/job_reattach/db-init.yaml b/integration/job_reattach/db-init.yaml new file mode 100644 index 0000000000..6f869d543b --- /dev/null +++ b/integration/job_reattach/db-init.yaml @@ -0,0 +1,17 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: job-reattach-db-init + namespace: tilt-integration + labels: + app: job-reattach-db-init +spec: + template: + metadata: + labels: + app: job-reattach-db-init + spec: + restartPolicy: Never + containers: + - name: db-init + image: db-init diff --git a/integration/job_reattach/db.dockerfile b/integration/job_reattach/db.dockerfile new file mode 100644 index 0000000000..118028c729 --- /dev/null +++ b/integration/job_reattach/db.dockerfile @@ -0,0 +1,3 @@ +FROM busybox +ENTRYPOINT busybox httpd -f -p 8000 + diff --git a/integration/job_reattach/db.yaml b/integration/job_reattach/db.yaml new file mode 100644 index 0000000000..0b7cf79876 --- /dev/null +++ b/integration/job_reattach/db.yaml @@ -0,0 +1,19 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: job-reattach-db + namespace: tilt-integration + labels: + app: job-reattach-db +spec: + selector: + matchLabels: + app: job-reattach-db + template: + metadata: + labels: + app: job-reattach-db + spec: + containers: + - name: db + image: db diff --git a/integration/job_reattach_test.go b/integration/job_reattach_test.go new file mode 100644 index 0000000000..2e2fa92728 --- /dev/null +++ b/integration/job_reattach_test.go @@ -0,0 +1,29 @@ +//go:build integration +// +build integration + +package integration + +import ( + "testing" + + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" +) + +func TestJobReattach(t *testing.T) { + f := newK8sFixture(t, "job_reattach") + f.SetRestrictedCredentials() + + f.TiltCI() + + _, _, podNames := f.AllPodsInPhase(f.ctx, "app=job-reattach-db-init", v1.PodSucceeded) + require.Equal(t, 1, len(podNames)) + + f.runCommandSilently("kubectl", "delete", "-n", "tilt-integration", "pod", podNames[0]) + + // Make sure 'ci' still succeeds, but we don't restart the Job pod. + f.TiltCI() + + _, _, podNames = f.AllPodsInPhase(f.ctx, "app=job-reattach-db-init", v1.PodSucceeded) + require.Equal(t, 0, len(podNames)) +} diff --git a/internal/controllers/core/session/reconciler_test.go b/internal/controllers/core/session/reconciler_test.go index 617b1d4962..e891534ea0 100644 --- a/internal/controllers/core/session/reconciler_test.go +++ b/internal/controllers/core/session/reconciler_test.go @@ -411,6 +411,42 @@ func TestExitControlCI_JobSuccess(t *testing.T) { f.requireDoneWithNoError() } +func TestExitControlCI_JobSuccessWithNoPods(t *testing.T) { + f := newFixture(t, store.EngineModeCI) + + m := manifestbuilder.New(f, "fe"). + WithK8sYAML(testyaml.JobYAML). + WithK8sPodReadiness(model.PodReadinessSucceeded). + Build() + f.upsertManifest(m) + f.Store.WithState(func(state *store.EngineState) { + state.ManifestTargets["fe"].State.AddCompletedBuild(model.BuildRecord{ + StartTime: time.Now(), + FinishTime: time.Now(), + }) + }) + + f.MustReconcile(sessionKey) + f.requireNotDone() + + f.Store.WithState(func(state *store.EngineState) { + mt := state.ManifestTargets["fe"] + krs := store.NewK8sRuntimeState(mt.Manifest) + krs.HasEverDeployedSuccessfully = true + // There are no pods but the job completed successfully. + krs.Conditions = []metav1.Condition{ + { + Type: v1alpha1.ApplyConditionJobComplete, + Status: metav1.ConditionTrue, + }, + } + mt.State.RuntimeState = krs + }) + + f.MustReconcile(sessionKey) + f.requireDoneWithNoError() +} + func TestExitControlCI_TriggerMode_Local(t *testing.T) { type tc struct { triggerMode model.TriggerMode diff --git a/internal/controllers/core/session/status.go b/internal/controllers/core/session/status.go index 420464e285..1b843dc0a9 100644 --- a/internal/controllers/core/session/status.go +++ b/internal/controllers/core/session/status.go @@ -98,7 +98,7 @@ func (r *Reconciler) processExitCondition(spec v1alpha1.SessionSpec, state *stor } if res.State.Waiting != nil { waiting = append(waiting, fmt.Sprintf("%v %v", res.Name, res.State.Waiting.WaitReason)) - } else if res.State.Active != nil && (!res.State.Active.Ready || res.Type == v1alpha1.TargetTypeJob) { + } else if res.State.Active != nil && !res.State.Active.Ready { // jobs must run to completion notReady = append(notReady, res.Name) }