fix: retry builds on depot (#4285)

Signed-off-by: Javier Lopez <javier@okteto.com>
okteto · May 8, 2024 · c7f0b9f · c7f0b9f
1 parent 89c6c7d
commit c7f0b9f
Show file tree

Hide file tree

Showing 4 changed files with 68 additions and 43 deletions.
diff --git a/pkg/cmd/build/build.go b/pkg/cmd/build/build.go
@@ -63,6 +63,7 @@ type OktetoBuilderInterface interface {
 type OktetoBuilder struct {
 	OktetoContext OktetoContextInterface
 	Fs            afero.Fs
+	isRetry       bool
 }
 
 // OktetoRegistryInterface checks if an image is at the registry
@@ -109,11 +110,11 @@ func (ob *OktetoBuilder) Run(ctx context.Context, buildOptions *types.BuildOptio
 	// use the internal cluster ip as if they were running their scripts on the k8s cluster
 	case IsDepotEnabled() && !isDeployOrDestroy:
 		depotManager := newDepotBuilder(depotProject, depotToken, ob.OktetoContext, ioCtrl)
-		return depotManager.Run(ctx, buildOptions, runAndHandleBuild)
+		return depotManager.Run(ctx, buildOptions, solveBuild)
 	case ob.OktetoContext.GetCurrentBuilder() == "":
 		return ob.buildWithDocker(ctx, buildOptions)
 	default:
-		return ob.buildWithOkteto(ctx, buildOptions, ioCtrl, runAndHandleBuild)
+		return ob.buildWithOkteto(ctx, buildOptions, ioCtrl, solveBuild)
 	}
 }
 
@@ -176,7 +177,30 @@ func (ob *OktetoBuilder) buildWithOkteto(ctx context.Context, buildOptions *type
 		return err
 	}
 
-	return run(ctx, buildkitClient, opt, buildOptions, ob.OktetoContext, ioCtrl)
+	err = run(ctx, buildkitClient, opt, buildOptions.OutputMode, ioCtrl)
+	if err != nil {
+		if shouldRetryBuild(err, buildOptions.Tag, ob.OktetoContext) {
+			ioCtrl.Logger().Infof("Failed to build image: %s", err.Error())
+			ioCtrl.Logger().Infof("isRetry: %t", ob.isRetry)
+			if !ob.isRetry {
+				retryBuilder := NewOktetoBuilder(ob.OktetoContext, ob.Fs)
+				retryBuilder.isRetry = true
+				err = retryBuilder.buildWithOkteto(ctx, buildOptions, ioCtrl, run)
+			}
+		}
+		err = getErrorMessage(err, buildOptions.Tag)
+		return err
+	}
+
+	var tag string
+	if buildOptions != nil {
+		tag = buildOptions.Tag
+		if buildOptions.Manifest != nil && buildOptions.Manifest.Deploy != nil {
+			tag = buildOptions.Manifest.Deploy.Image
+		}
+	}
+	err = getErrorMessage(err, tag)
+	return err
 }
 
 // https://github.com/docker/cli/blob/56e5910181d8ac038a634a203a4f3550bb64991f/cli/command/image/build.go#L209

diff --git a/pkg/cmd/build/buildkit.go b/pkg/cmd/build/buildkit.go
@@ -391,56 +391,29 @@ func solveBuild(ctx context.Context, c *client.Client, opt *client.SolveOpt, pro
 	return nil
 }
 
-func runAndHandleBuild(ctx context.Context, c *client.Client, opt *client.SolveOpt, buildOptions *types.BuildOptions, okCtx OktetoContextInterface, ioCtrl *io.Controller) error {
-	err := solveBuild(ctx, c, opt, buildOptions.OutputMode, ioCtrl)
-	if err != nil {
-		oktetoLog.Infof("Failed to build image: %s", err.Error())
-	}
+func shouldRetryBuild(err error, tag string, okCtx OktetoContextInterface) bool {
 	if isTransientError(err) {
 		oktetoLog.Yellow(`Failed to push '%s' to the registry:
   %s,
-  Retrying ...`, buildOptions.Tag, err.Error())
-		success := true
-		err := solveBuild(ctx, c, opt, buildOptions.OutputMode, ioCtrl)
-		if err != nil {
-			success = false
-			oktetoLog.Infof("Failed to build image: %s", err.Error())
-		}
-		err = getErrorMessage(err, buildOptions.Tag)
-		analytics.TrackBuildTransientError(success)
-		return err
+  Retrying...`, tag, err.Error())
+		analytics.TrackBuildTransientError(true)
+		return true
 	}
 
-	if err == nil && buildOptions.Tag != "" {
-		tags := strings.Split(buildOptions.Tag, ",")
+	if err == nil && tag != "" {
+		tags := strings.Split(tag, ",")
 		reg := registry.NewOktetoRegistry(GetRegistryConfigFromOktetoConfig(okCtx))
 		for _, tag := range tags {
 			if _, err := reg.GetImageTagWithDigest(tag); err != nil {
 				oktetoLog.Yellow(`Failed to push '%s' metadata to the registry:
 	  %s,
-	  Retrying ...`, buildOptions.Tag, err.Error())
-				success := true
-				err := solveBuild(ctx, c, opt, buildOptions.OutputMode, ioCtrl)
-				if err != nil {
-					success = false
-					oktetoLog.Infof("Failed to build image: %s", err.Error())
-				}
-				err = getErrorMessage(err, buildOptions.Tag)
-				analytics.TrackBuildPullError(success)
-				return err
+	  Retrying...`, tag, err.Error())
+				analytics.TrackBuildPullError(true)
+				return true
 			}
 		}
 	}
-
-	var tag string
-	if buildOptions != nil {
-		tag = buildOptions.Tag
-		if buildOptions.Manifest != nil && buildOptions.Manifest.Deploy != nil {
-			tag = buildOptions.Manifest.Deploy.Image
-		}
-	}
-	err = getErrorMessage(err, tag)
-	return err
+	return false
 }
 
 func (*buildWriter) Write(p []byte) (int, error) {

diff --git a/pkg/cmd/build/depot.go b/pkg/cmd/build/depot.go
@@ -52,6 +52,7 @@ type depotBuilder struct {
 	acquireMachine func(ctx context.Context, buildId, token, platform string) (depotMachineConnector, error)
 	token          string
 	project        string
+	isRetry        bool
 }
 
 func IsDepotEnabled() bool {
@@ -78,13 +79,16 @@ func newDepotBuilder(projectId, token string, okCtx OktetoContextInterface, ioCt
 func (db *depotBuilder) release(build build.Build) {
 	build.Finish(db.err)
 
+	if db.machine == nil {
+		return
+	}
 	err := db.machine.Release()
 	if err != nil {
 		db.ioCtrl.Logger().Infof("failed to release depot's machine: %s", err)
 	}
 }
 
-type runAndHandleBuildFn func(ctx context.Context, c *client.Client, opt *client.SolveOpt, buildOptions *types.BuildOptions, okCtx OktetoContextInterface, ioCtrl *io.Controller) error
+type runAndHandleBuildFn func(ctx context.Context, c *client.Client, opt *client.SolveOpt, progress string, ioCtrl *io.Controller) error
 
 func (db *depotBuilder) Run(ctx context.Context, buildOptions *types.BuildOptions, run runAndHandleBuildFn) error {
 	db.ioCtrl.Logger().Info("building your image on depot's machine")
@@ -161,7 +165,31 @@ func (db *depotBuilder) Run(ctx context.Context, buildOptions *types.BuildOption
 	}
 
 	db.ioCtrl.Logger().Infof("[depot] build URL: %s", build.BuildURL)
-	return run(ctx, client, opt, buildOptions, db.okCtx, db.ioCtrl)
+
+	err = run(ctx, client, opt, buildOptions.OutputMode, db.ioCtrl)
+	if err != nil {
+		if shouldRetryBuild(err, buildOptions.Tag, db.okCtx) {
+			db.ioCtrl.Logger().Infof("Failed to build image: %s", err.Error())
+			db.ioCtrl.Logger().Infof("isRetry: %t", db.isRetry)
+			if !db.isRetry {
+				retryBuilder := newDepotBuilder(db.project, db.token, db.okCtx, db.ioCtrl)
+				retryBuilder.isRetry = true
+				err = retryBuilder.Run(ctx, buildOptions, run)
+			}
+		}
+		err = getErrorMessage(err, buildOptions.Tag)
+		return err
+	}
+
+	var tag string
+	if buildOptions != nil {
+		tag = buildOptions.Tag
+		if buildOptions.Manifest != nil && buildOptions.Manifest.Deploy != nil {
+			tag = buildOptions.Manifest.Deploy.Image
+		}
+	}
+	err = getErrorMessage(err, tag)
+	return err
 }
 
 // getBuildkitClient returns a buildkit client connected to the depot's machine.

diff --git a/pkg/cmd/build/depot_test.go b/pkg/cmd/build/depot_test.go
@@ -165,7 +165,7 @@ func TestDepotRun(t *testing.T) {
 				Tag:       "okteto.dev/test:okteto",
 				DevTag:    "okteto.dev/test:okteto",
 			}
-			runAndHandle := func(ctx context.Context, c *client.Client, opt *client.SolveOpt, buildOptions *types.BuildOptions, okCtx OktetoContextInterface, ioCtrl *io.Controller) error {
+			runAndHandle := func(ctx context.Context, c *client.Client, opt *client.SolveOpt, progress string, ioCtrl *io.Controller) error {
 				return nil
 			}
 			err := db.Run(context.Background(), opts, runAndHandle)