Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: retry builds on depot #4285

Merged
merged 1 commit into from
May 8, 2024
Merged

fix: retry builds on depot #4285

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
30 changes: 27 additions & 3 deletions pkg/cmd/build/build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ type OktetoBuilderInterface interface {
type OktetoBuilder struct {
OktetoContext OktetoContextInterface
Fs afero.Fs
isRetry bool
}

// OktetoRegistryInterface checks if an image is at the registry
Expand Down Expand Up @@ -109,11 +110,11 @@ func (ob *OktetoBuilder) Run(ctx context.Context, buildOptions *types.BuildOptio
// use the internal cluster ip as if they were running their scripts on the k8s cluster
case IsDepotEnabled() && !isDeployOrDestroy:
depotManager := newDepotBuilder(depotProject, depotToken, ob.OktetoContext, ioCtrl)
return depotManager.Run(ctx, buildOptions, runAndHandleBuild)
return depotManager.Run(ctx, buildOptions, solveBuild)
case ob.OktetoContext.GetCurrentBuilder() == "":
return ob.buildWithDocker(ctx, buildOptions)
default:
return ob.buildWithOkteto(ctx, buildOptions, ioCtrl, runAndHandleBuild)
return ob.buildWithOkteto(ctx, buildOptions, ioCtrl, solveBuild)
}
}

Expand Down Expand Up @@ -176,7 +177,30 @@ func (ob *OktetoBuilder) buildWithOkteto(ctx context.Context, buildOptions *type
return err
}

return run(ctx, buildkitClient, opt, buildOptions, ob.OktetoContext, ioCtrl)
err = run(ctx, buildkitClient, opt, buildOptions.OutputMode, ioCtrl)
if err != nil {
if shouldRetryBuild(err, buildOptions.Tag, ob.OktetoContext) {
ioCtrl.Logger().Infof("Failed to build image: %s", err.Error())
ioCtrl.Logger().Infof("isRetry: %t", ob.isRetry)
if !ob.isRetry {
retryBuilder := NewOktetoBuilder(ob.OktetoContext, ob.Fs)
retryBuilder.isRetry = true
err = retryBuilder.buildWithOkteto(ctx, buildOptions, ioCtrl, run)
}
}
err = getErrorMessage(err, buildOptions.Tag)
return err
}

var tag string
if buildOptions != nil {
tag = buildOptions.Tag
if buildOptions.Manifest != nil && buildOptions.Manifest.Deploy != nil {
tag = buildOptions.Manifest.Deploy.Image
}
}
err = getErrorMessage(err, tag)
return err
}

// https://github.com/docker/cli/blob/56e5910181d8ac038a634a203a4f3550bb64991f/cli/command/image/build.go#L209
Expand Down
47 changes: 10 additions & 37 deletions pkg/cmd/build/buildkit.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -391,56 +391,29 @@ func solveBuild(ctx context.Context, c *client.Client, opt *client.SolveOpt, pro
return nil
}

func runAndHandleBuild(ctx context.Context, c *client.Client, opt *client.SolveOpt, buildOptions *types.BuildOptions, okCtx OktetoContextInterface, ioCtrl *io.Controller) error {
err := solveBuild(ctx, c, opt, buildOptions.OutputMode, ioCtrl)
if err != nil {
oktetoLog.Infof("Failed to build image: %s", err.Error())
}
func shouldRetryBuild(err error, tag string, okCtx OktetoContextInterface) bool {
if isTransientError(err) {
oktetoLog.Yellow(`Failed to push '%s' to the registry:
%s,
Retrying ...`, buildOptions.Tag, err.Error())
success := true
err := solveBuild(ctx, c, opt, buildOptions.OutputMode, ioCtrl)
if err != nil {
success = false
oktetoLog.Infof("Failed to build image: %s", err.Error())
}
err = getErrorMessage(err, buildOptions.Tag)
analytics.TrackBuildTransientError(success)
return err
Retrying...`, tag, err.Error())
analytics.TrackBuildTransientError(true)
return true
}

if err == nil && buildOptions.Tag != "" {
tags := strings.Split(buildOptions.Tag, ",")
if err == nil && tag != "" {
tags := strings.Split(tag, ",")
reg := registry.NewOktetoRegistry(GetRegistryConfigFromOktetoConfig(okCtx))
for _, tag := range tags {
if _, err := reg.GetImageTagWithDigest(tag); err != nil {
oktetoLog.Yellow(`Failed to push '%s' metadata to the registry:
%s,
Retrying ...`, buildOptions.Tag, err.Error())
success := true
err := solveBuild(ctx, c, opt, buildOptions.OutputMode, ioCtrl)
if err != nil {
success = false
oktetoLog.Infof("Failed to build image: %s", err.Error())
}
err = getErrorMessage(err, buildOptions.Tag)
analytics.TrackBuildPullError(success)
return err
Retrying...`, tag, err.Error())
analytics.TrackBuildPullError(true)
return true
}
}
}

var tag string
if buildOptions != nil {
tag = buildOptions.Tag
if buildOptions.Manifest != nil && buildOptions.Manifest.Deploy != nil {
tag = buildOptions.Manifest.Deploy.Image
}
}
err = getErrorMessage(err, tag)
return err
return false
}

func (*buildWriter) Write(p []byte) (int, error) {
Expand Down
32 changes: 30 additions & 2 deletions pkg/cmd/build/depot.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ type depotBuilder struct {
acquireMachine func(ctx context.Context, buildId, token, platform string) (depotMachineConnector, error)
token string
project string
isRetry bool
}

func IsDepotEnabled() bool {
Expand All @@ -78,13 +79,16 @@ func newDepotBuilder(projectId, token string, okCtx OktetoContextInterface, ioCt
func (db *depotBuilder) release(build build.Build) {
build.Finish(db.err)

if db.machine == nil {
return
}
err := db.machine.Release()
if err != nil {
db.ioCtrl.Logger().Infof("failed to release depot's machine: %s", err)
}
}

type runAndHandleBuildFn func(ctx context.Context, c *client.Client, opt *client.SolveOpt, buildOptions *types.BuildOptions, okCtx OktetoContextInterface, ioCtrl *io.Controller) error
type runAndHandleBuildFn func(ctx context.Context, c *client.Client, opt *client.SolveOpt, progress string, ioCtrl *io.Controller) error

func (db *depotBuilder) Run(ctx context.Context, buildOptions *types.BuildOptions, run runAndHandleBuildFn) error {
db.ioCtrl.Logger().Info("building your image on depot's machine")
Expand Down Expand Up @@ -161,7 +165,31 @@ func (db *depotBuilder) Run(ctx context.Context, buildOptions *types.BuildOption
}

db.ioCtrl.Logger().Infof("[depot] build URL: %s", build.BuildURL)
return run(ctx, client, opt, buildOptions, db.okCtx, db.ioCtrl)

err = run(ctx, client, opt, buildOptions.OutputMode, db.ioCtrl)
if err != nil {
if shouldRetryBuild(err, buildOptions.Tag, db.okCtx) {
db.ioCtrl.Logger().Infof("Failed to build image: %s", err.Error())
db.ioCtrl.Logger().Infof("isRetry: %t", db.isRetry)
if !db.isRetry {
retryBuilder := newDepotBuilder(db.project, db.token, db.okCtx, db.ioCtrl)
retryBuilder.isRetry = true
err = retryBuilder.Run(ctx, buildOptions, run)
}
}
err = getErrorMessage(err, buildOptions.Tag)
return err
}

var tag string
if buildOptions != nil {
tag = buildOptions.Tag
if buildOptions.Manifest != nil && buildOptions.Manifest.Deploy != nil {
tag = buildOptions.Manifest.Deploy.Image
}
}
err = getErrorMessage(err, tag)
return err
}

// getBuildkitClient returns a buildkit client connected to the depot's machine.
Expand Down
2 changes: 1 addition & 1 deletion pkg/cmd/build/depot_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ func TestDepotRun(t *testing.T) {
Tag: "okteto.dev/test:okteto",
DevTag: "okteto.dev/test:okteto",
}
runAndHandle := func(ctx context.Context, c *client.Client, opt *client.SolveOpt, buildOptions *types.BuildOptions, okCtx OktetoContextInterface, ioCtrl *io.Controller) error {
runAndHandle := func(ctx context.Context, c *client.Client, opt *client.SolveOpt, progress string, ioCtrl *io.Controller) error {
return nil
}
err := db.Run(context.Background(), opts, runAndHandle)
Expand Down