Bugfix: Add image building job deletion delay

api/turing/config/config.go

@@ -58,7 +58,7 @@ func (qty *Quantity) MarshalJSON() ([]byte, error) {
 
 type EngineConfig map[string]interface{}
 
-// EnvironmentConfig is a abridged version of
+// EnvironmentConfig is an abridged version of
 // https://github.dev/caraml-dev/merlin/blob/98ada0d3aa8de30d73e441d3fd1000fe5d5ac266/api/config/environment.go#L26
 // Only requires Name and K8sConfig
 // This struct should be removed when Environments API is moved from Merlin to MLP

api/turing/imagebuilder/imagebuilder.go

@@ -43,7 +43,12 @@ const (
 type JobStatus int
 
 const (
-	tickDurationInSeconds = 5
+	// jobDeletionTimeoutInSeconds is the maximum time to wait for a job to be deleted from a cluster
+	jobDeletionTimeoutInSeconds = 30
+	// jobDeletionTickDurationInMilliseconds is the interval at which the API server checks if a job has been deleted
+	jobDeletionTickDurationInMilliseconds = 100
+	// jobCompletionTickDurationInSeconds is the interval at which the API server checks if a job has completed
+	jobCompletionTickDurationInSeconds = 5
 	// JobStatusActive is the status of the image building job is active
 	JobStatusActive = JobStatus(iota)
 	// JobStatusFailed is when the image building job has failed
@@ -164,6 +169,11 @@ func (ib *imageBuilder) BuildImage(request BuildImageRequest) (string, error) {
 				return "", ErrDeleteFailedJob
 			}
 
+			err = ib.waitForJobToBeDeleted(job)
+			if err != nil {
+				return "", ErrDeleteFailedJob
+			}
+
 			job, err = ib.createKanikoJob(kanikoJobName, imageRef, request.ArtifactURI, request.BuildLabels,
 				request.EnsemblerFolder, request.BaseImageRefTag)
 			if err != nil {
@@ -183,7 +193,7 @@ func (ib *imageBuilder) BuildImage(request BuildImageRequest) (string, error) {
 
 func (ib *imageBuilder) waitForJobToFinish(job *apibatchv1.Job) error {
 	timeout := time.After(ib.imageBuildingConfig.BuildTimeoutDuration)
-	ticker := time.NewTicker(time.Second * tickDurationInSeconds)
+	ticker := time.NewTicker(time.Second * jobCompletionTickDurationInSeconds)
 
 	for {
 		select {
@@ -207,6 +217,27 @@ func (ib *imageBuilder) waitForJobToFinish(job *apibatchv1.Job) error {
 	}
 }
 
+func (ib *imageBuilder) waitForJobToBeDeleted(job *apibatchv1.Job) error {
+	timeout := time.After(time.Second * jobDeletionTimeoutInSeconds)
+	ticker := time.NewTicker(time.Millisecond * jobDeletionTickDurationInMilliseconds)
+
+	for {
+		select {
+		case <-timeout:
+			return ErrDeleteFailedJob
+		case <-ticker.C:
+			_, err := ib.clusterController.GetJob(context.Background(), ib.imageBuildingConfig.BuildNamespace, job.Name)
+			if err != nil {
+				if kerrors.IsNotFound(err) {
+					return nil
+				}
+				log.Errorf("unable to get job status for job %s: %v", job.Name, err)
+				return ErrDeleteFailedJob
+			}
+		}
+	}
+}
+
 func (ib *imageBuilder) createKanikoJob(
 	kanikoJobName string,
 	imageRef string,

api/turing/imagebuilder/imagebuilder_test.go

@@ -229,7 +229,21 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) {
 					nil,
 				).Once()
 
-				// Second time it's called
+				// Second time GetJob is called
+				ctlr.On(
+					"GetJob",
+					mock.Anything,
+					mock.Anything,
+					mock.Anything,
+				).Return(
+					nil,
+					k8serrors.NewNotFound(
+						schema.GroupResource{},
+						fmt.Sprintf("batch-%s-%s-%d-%s", projectName, modelName, modelVersion, runID[:5]),
+					),
+				).Once()
+
+				// Third time it's called
 				ctlr.On(
 					"GetJob",
 					mock.Anything,
@@ -492,7 +506,21 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) {
 					nil,
 				).Once()
 
-				// Second time it's called
+				// Second time GetJob is called
+				ctlr.On(
+					"GetJob",
+					mock.Anything,
+					mock.Anything,
+					mock.Anything,
+				).Return(
+					nil,
+					k8serrors.NewNotFound(
+						schema.GroupResource{},
+						fmt.Sprintf("service-builder-%s-%s-%d-%s", projectName, modelName, modelVersion, runID[:5]),
+					),
+				).Once()
+
+				// Third time it's called
 				ctlr.On(
 					"GetJob",
 					mock.Anything,