From 0f48f14ac3f495e45ac47ddc4e9e787c7dc86cf1 Mon Sep 17 00:00:00 2001 From: "Vlad A. Ionescu" <446771+vladaionescu@users.noreply.github.com> Date: Mon, 26 Oct 2020 19:00:18 -0700 Subject: [PATCH] Execute builds in a single BuildKit session (most pieces anyway), get rid of fakedep (#453) * Add gwClient param in converter. * Some prep work. * Working multi-export. Need to clean up though. * Minor cleanup. * Ensure that main state is built every time. * Consistent ordering of outputs. * Support for outputting dirs. * Cleanup - decouple meta and ref names. * Minor. * Output artifacts correctly. Correct printouts after success line. * Get rid of fake dep which was causing huge performance issues. * Bring --no-output back to life. * Fix --no-output. WIP * Working --image and --artifact modes. * Cleanup. * Point to earthly/buildkit:earthly-master again. * Get rid of some code that we no longer use. Co-authored-by: Vlad A. Ionescu --- buildcontext/git.go | 3 +- builder/builder.go | 385 ++++++++++++++++++---------- builder/solver.go | 144 +++++------ cmd/earth/main.go | 28 +- earth-buildkitd-wrapper.sh | 16 +- earthfile2llb/converter.go | 42 +-- earthfile2llb/earthfile2llb.go | 14 +- examples/integration-test/Earthfile | 1 - examples/ruby-on-rails/Earthfile | 2 - examples/terraform/Earthfile | 2 - go.mod | 2 +- go.sum | 11 + llbutil/fakedep.go | 27 -- states/states.go | 10 +- states/visited.go | 21 ++ 15 files changed, 396 insertions(+), 312 deletions(-) delete mode 100644 llbutil/fakedep.go create mode 100644 states/visited.go diff --git a/buildcontext/git.go b/buildcontext/git.go index 996f458e20..475a040157 100644 --- a/buildcontext/git.go +++ b/buildcontext/git.go @@ -124,7 +124,7 @@ func (gr *gitResolver) resolveGitProject(ctx context.Context, target domain.Targ llb.WithCustomNamef("[internal] COPY GIT CLONE %s Earthfile", target.ProjectCanonical()), } opImg := llb.Image( - defaultGitImage, llb.MarkImageInternal, + defaultGitImage, llb.MarkImageInternal, llb.ResolveModePreferLocal, llb.Platform(llbutil.TargetPlatform)) copyOp := opImg.Run(copyOpts...) earthfileState := copyOp.AddMount("/dest", llb.Scratch().Platform(llbutil.TargetPlatform)) @@ -160,6 +160,7 @@ func (gr *gitResolver) resolveGitProject(ctx context.Context, target domain.Targ gr.cleanCollection.Add(func() error { return os.RemoveAll(earthfileTmpDir) }) + // TODO: Use gwClient to download solved files instead of executing a full build like this. err = gr.artifactBuilderFun(ctx, mts, artifact, fmt.Sprintf("%s/", earthfileTmpDir)) if err != nil { return nil, "", "", errors.Wrap(err, "build git") diff --git a/builder/builder.go b/builder/builder.go index c0dbcdc41d..75d30d4789 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -4,11 +4,14 @@ import ( "context" "encoding/json" "fmt" + "io" "io/ioutil" "os" + "os/exec" "path" "path/filepath" "strings" + "sync" "github.com/earthly/earthly/buildcontext" "github.com/earthly/earthly/buildcontext/provider" @@ -26,6 +29,7 @@ import ( "github.com/moby/buildkit/util/entitlements" reccopy "github.com/otiai10/copy" "github.com/pkg/errors" + "golang.org/x/sync/errgroup" ) // Opt represent builder options. @@ -46,8 +50,12 @@ type Opt struct { // BuildOpt is a collection of build options. type BuildOpt struct { - PrintSuccess bool - Push bool + PrintSuccess bool + Push bool + NoOutput bool + OnlyFinalTargetImages bool + OnlyArtifact *domain.Artifact + OnlyArtifactDestPath string } // Builder executes Earthly builds. @@ -81,23 +89,9 @@ func (b *Builder) BuildTarget(ctx context.Context, target domain.Target, opt Bui if err != nil { return nil, err } - if opt.PrintSuccess { - b.opt.Console.PrintSuccess() - } return mts, nil } -// Output produces the output for all targets provided. -func (b *Builder) Output(ctx context.Context, mts *states.MultiTarget, opt BuildOpt) error { - for _, states := range mts.All() { - err := b.outputs(ctx, states, opt) - if err != nil { - return err - } - } - return nil -} - // OutputArtifact outputs a specific artifact found in an mts's final target. func (b *Builder) OutputArtifact(ctx context.Context, mts *states.MultiTarget, artifact domain.Artifact, destPath string, opt BuildOpt) error { // TODO: Should double check that the last state is the same as the one @@ -126,44 +120,25 @@ func (b *Builder) MakeImageAsTarBuilderFun() states.DockerBuilderFun { } } -// OutputImages outputs the images of a single target. -func (b *Builder) OutputImages(ctx context.Context, states *states.SingleTarget, opt BuildOpt) error { - for _, imageToSave := range states.SaveImages { - if imageToSave.DockerTag == "" { - // Not a docker export. Skip. - continue - } - err := b.outputImage(ctx, imageToSave, states, opt) - if err != nil { - return err - } - } - return nil -} - -// OutputArtifacts outputs the artifacts of a single target. -func (b *Builder) OutputArtifacts(ctx context.Context, states *states.SingleTarget, opt BuildOpt) error { +func (b *Builder) convertAndBuild(ctx context.Context, target domain.Target, opt BuildOpt) (*states.MultiTarget, error) { outDir, err := ioutil.TempDir(".", ".tmp-earth-out") if err != nil { - return errors.Wrap(err, "mk temp dir for artifacts") + return nil, errors.Wrap(err, "mk temp dir for artifacts") } defer os.RemoveAll(outDir) - solvedStates := make(map[int]bool) - for _, artifactToSaveLocally := range states.SaveLocals { - err = b.outputArtifact( - ctx, artifactToSaveLocally, outDir, solvedStates, states, opt) - if err != nil { - return err + var successOnce sync.Once + successFun := func() { + if opt.PrintSuccess { + b.opt.Console.PrintSuccess() } } - return nil -} - -func (b *Builder) convertAndBuild(ctx context.Context, target domain.Target, opt BuildOpt) (*states.MultiTarget, error) { + destPathWhitelist := make(map[string]bool) var mts *states.MultiTarget + finalTargetImageIndices := make(map[int]bool) bf := func(ctx context.Context, gwClient gwclient.Client) (*gwclient.Result, error) { var err error mts, err = earthfile2llb.Earthfile2LLB(ctx, target, earthfile2llb.ConvertOpt{ + GwClient: gwClient, Resolver: b.resolver, ImageResolveMode: b.opt.ImageResolveMode, DockerBuilderFun: b.MakeImageAsTarBuilderFun(), @@ -171,48 +146,225 @@ func (b *Builder) convertAndBuild(ctx context.Context, target domain.Target, opt CleanCollection: b.opt.CleanCollection, VarCollection: b.opt.VarCollection, BuildContextProvider: b.opt.BuildContextProvider, - MetaResolver: gwClient, }) if err != nil { return nil, err } - state := mts.Final.MainState - if b.opt.NoCache { - state = state.SetMarshalDefaults(llb.IgnoreCache) + ref, err := b.stateToRef(ctx, gwClient, mts.Final.MainState) + if err != nil { + return nil, err } - def, err := state.Marshal(ctx) + res := gwclient.NewResult() + res.AddRef("main", ref) + ref, err = b.stateToRef(ctx, gwClient, mts.Final.ArtifactsState) if err != nil { - return nil, errors.Wrap(err, "marshal main state") + return nil, err } - r, err := gwClient.Solve(ctx, gwclient.SolveRequest{ - Definition: def.ToPB(), + refKey := "final-artifact" + refPrefix := fmt.Sprintf("ref/%s", refKey) + res.AddRef(refKey, ref) + res.AddMeta(fmt.Sprintf("%s/export-dir", refPrefix), []byte("true")) + res.AddMeta(fmt.Sprintf("%s/final-artifact", refPrefix), []byte("true")) + + depIndex := 0 + imageIndex := 0 + dirIndex := 0 + for _, sts := range mts.All() { + for _, depRef := range sts.DepsRefs { + refKey := fmt.Sprintf("dep-%d", depIndex) + res.AddRef(refKey, depRef) + depIndex++ + } + for _, saveImage := range sts.SaveImages { + ref, err := b.stateToRef(ctx, gwClient, saveImage.State) + if err != nil { + return nil, err + } + config, err := json.Marshal(saveImage.Image) + if err != nil { + return nil, errors.Wrapf(err, "marshal save image config") + } + // TODO: Support multiple docker tags at the same time (improves export speed). + refKey := fmt.Sprintf("image-%d", imageIndex) + refPrefix := fmt.Sprintf("ref/%s", refKey) + res.AddMeta(fmt.Sprintf("%s/image.name", refPrefix), []byte(saveImage.DockerTag)) + res.AddMeta(fmt.Sprintf("%s/%s", refPrefix, exptypes.ExporterImageConfigKey), config) + res.AddMeta(fmt.Sprintf("%s/export-image", refPrefix), []byte("true")) + res.AddMeta(fmt.Sprintf("%s/image-index", refPrefix), []byte(fmt.Sprintf("%d", imageIndex))) + res.AddRef(refKey, ref) + if sts == mts.Final { + finalTargetImageIndices[imageIndex] = true + } + imageIndex++ + } + if sts.Target.IsRemote() { + // Don't do save local's for remote targets. + continue + } + for _, saveLocal := range sts.SaveLocals { + ref, err := b.stateToRef(ctx, gwClient, sts.SeparateArtifactsState[saveLocal.Index]) + if err != nil { + return nil, err + } + refKey := fmt.Sprintf("dir-%d", dirIndex) + refPrefix := fmt.Sprintf("ref/%s", refKey) + res.AddRef(refKey, ref) + artifact := domain.Artifact{ + Target: sts.Target, + Artifact: saveLocal.ArtifactPath, + } + res.AddMeta(fmt.Sprintf("%s/artifact", refPrefix), []byte(artifact.String())) + res.AddMeta(fmt.Sprintf("%s/src-path", refPrefix), []byte(saveLocal.ArtifactPath)) + res.AddMeta(fmt.Sprintf("%s/dest-path", refPrefix), []byte(saveLocal.DestPath)) + res.AddMeta(fmt.Sprintf("%s/export-dir", refPrefix), []byte("true")) + res.AddMeta(fmt.Sprintf("%s/dir-index", refPrefix), []byte(fmt.Sprintf("%d", dirIndex))) + destPathWhitelist[saveLocal.DestPath] = true + dirIndex++ + } + } + return res, nil + } + onImage := func(ctx context.Context, eg *errgroup.Group, index int, imageName string, digest string) (io.WriteCloser, error) { + successOnce.Do(successFun) + if opt.NoOutput || opt.OnlyArtifact != nil { + return nil, nil + } + if opt.OnlyFinalTargetImages && !finalTargetImageIndices[index] { + return nil, nil + } + pipeR, pipeW := io.Pipe() + eg.Go(func() error { + defer pipeR.Close() + err := loadDockerTar(ctx, pipeR) + if err != nil { + return errors.Wrapf(err, "load docker tar") + } + return nil }) + return pipeW, nil + } + onArtifact := func(ctx context.Context, index int, artifact domain.Artifact, artifactPath string, destPath string) (string, error) { + successOnce.Do(successFun) + if opt.NoOutput || opt.OnlyFinalTargetImages || opt.OnlyArtifact != nil { + return "", nil + } + if !destPathWhitelist[destPath] { + return "", errors.Errorf("dest path %s is not in the whitelist: %+v", destPath, destPathWhitelist) + } + artifactDir := filepath.Join(outDir, fmt.Sprintf("index-%d", index)) + err := os.MkdirAll(artifactDir, 0755) if err != nil { - return nil, errors.Wrap(err, "solve main state") + return "", errors.Wrapf(err, "create dir %s", artifactDir) + } + return artifactDir, nil + } + onFinalArtifact := func(ctx context.Context) (string, error) { + successOnce.Do(successFun) + if opt.NoOutput || opt.OnlyFinalTargetImages { + return "", nil } - ref, err := r.SingleRef() + if opt.OnlyArtifact == nil { + return "", nil + } + return outDir, nil + } + err = b.s.buildMainMulti(ctx, bf, onImage, onArtifact, onFinalArtifact) + if err != nil { + return nil, errors.Wrapf(err, "build main") + } + successOnce.Do(successFun) + if opt.NoOutput { + // Nothing. + } else if opt.OnlyArtifact != nil { + err := b.saveArtifactLocally(ctx, *opt.OnlyArtifact, outDir, opt.OnlyArtifactDestPath, mts.Final.Salt, opt) if err != nil { return nil, err } - config, err := json.Marshal(mts.Final.MainImage) - if err != nil { - return nil, errors.Wrapf(err, "marshal image config") + } else if opt.OnlyFinalTargetImages { + for _, saveImage := range mts.Final.SaveImages { + shouldPush := opt.Push && saveImage.Push + console := b.opt.Console.WithPrefixAndSalt(mts.Final.Target.String(), mts.Final.Salt) + if shouldPush { + err := pushDockerImage(ctx, saveImage.DockerTag) + if err != nil { + return nil, err + } + } + pushStr := "" + if shouldPush { + pushStr = " (pushed)" + } + console.Printf("Image %s as %s%s\n", mts.Final.Target.StringCanonical(), saveImage.DockerTag, pushStr) + if saveImage.Push && !opt.Push { + console.Printf("Did not push %s. Use earth --push to enable pushing\n", saveImage.DockerTag) + } } + } else { + // This needs to match with the same index used during output. + // TODO: This is a little brittle to future code changes. + dirIndex := 0 + for _, sts := range mts.All() { + for _, saveImage := range sts.SaveImages { + shouldPush := opt.Push && saveImage.Push && !sts.Target.IsRemote() + console := b.opt.Console.WithPrefixAndSalt(sts.Target.String(), sts.Salt) + if shouldPush { + err := pushDockerImage(ctx, saveImage.DockerTag) + if err != nil { + return nil, err + } + } + pushStr := "" + if shouldPush { + pushStr = " (pushed)" + } + console.Printf("Image %s as %s%s\n", sts.Target.StringCanonical(), saveImage.DockerTag, pushStr) + if saveImage.Push && !opt.Push && !sts.Target.IsRemote() { + console.Printf("Did not push %s. Use earth --push to enable pushing\n", saveImage.DockerTag) + } + } + for _, saveLocal := range sts.SaveLocals { + artifactDir := filepath.Join(outDir, fmt.Sprintf("index-%d", dirIndex)) + artifact := domain.Artifact{ + Target: sts.Target, + Artifact: saveLocal.ArtifactPath, + } + err := b.saveArtifactLocally(ctx, artifact, artifactDir, saveLocal.DestPath, sts.Salt, opt) + if err != nil { + return nil, err + } + dirIndex++ + } + if !sts.Target.IsRemote() { + err = b.executeRunPush(ctx, sts, opt) + if err != nil { + return nil, err + } + } + } + } - res := gwclient.NewResult() - res.AddMeta(exptypes.ExporterImageConfigKey, config) - res.SetRef(ref) - return res, nil + return mts, nil +} + +func (b *Builder) stateToRef(ctx context.Context, gwClient gwclient.Client, state llb.State) (gwclient.Reference, error) { + if b.opt.NoCache { + state = state.SetMarshalDefaults(llb.IgnoreCache) } - err := b.s.buildMain(ctx, bf) + def, err := state.Marshal(ctx) if err != nil { - return nil, errors.Wrapf(err, "build main") + return nil, errors.Wrap(err, "marshal main state") } - if opt.PrintSuccess { - targetConsole := b.opt.Console.WithPrefixAndSalt(target.String(), mts.Final.Salt) - targetConsole.Printf("Target %s built successfully\n", target.StringCanonical()) + r, err := gwClient.Solve(ctx, gwclient.SolveRequest{ + Definition: def.ToPB(), + }) + if err != nil { + return nil, errors.Wrap(err, "solve main state") } - return mts, nil + ref, err := r.SingleRef() + if err != nil { + return nil, errors.Wrap(err, "single ref") + } + return ref, nil } func (b *Builder) buildOnlyLastImageAsTar(ctx context.Context, mts *states.MultiTarget, dockerTag string, outFile string, opt BuildOpt) error { @@ -221,9 +373,6 @@ func (b *Builder) buildOnlyLastImageAsTar(ctx context.Context, mts *states.Multi if err != nil { return err } - if opt.PrintSuccess { - b.opt.Console.PrintSuccess() - } err = b.outputImageTar(ctx, saveImage, dockerTag, outFile) if err != nil { @@ -237,16 +386,11 @@ func (b *Builder) buildOnlyArtifact(ctx context.Context, mts *states.MultiTarget if err != nil { return err } - if opt.PrintSuccess { - b.opt.Console.PrintSuccess() - } return b.OutputArtifact(ctx, mts, artifact, destPath, opt) } func (b *Builder) buildMain(ctx context.Context, mts *states.MultiTarget, opt BuildOpt) error { - finalTarget := mts.Final.Target - finalTargetConsole := b.opt.Console.WithPrefixAndSalt(finalTarget.String(), mts.Final.Salt) state := mts.Final.MainState if b.opt.NoCache { state = state.SetMarshalDefaults(llb.IgnoreCache) @@ -255,28 +399,6 @@ func (b *Builder) buildMain(ctx context.Context, mts *states.MultiTarget, opt Bu if err != nil { return errors.Wrapf(err, "solve side effects") } - if opt.PrintSuccess { - finalTargetConsole.Printf("Target %s built successfully\n", finalTarget.StringCanonical()) - } - return nil -} - -func (b *Builder) outputs(ctx context.Context, states *states.SingleTarget, opt BuildOpt) error { - err := b.executeRunPush(ctx, states, opt) - if err != nil { - return err - } - err = b.OutputImages(ctx, states, opt) - if err != nil { - return err - } - if !states.Target.IsRemote() { - // Don't output artifacts for remote images. - err = b.OutputArtifacts(ctx, states, opt) - if err != nil { - return err - } - } return nil } @@ -299,24 +421,6 @@ func (b *Builder) executeRunPush(ctx context.Context, states *states.SingleTarge return nil } -func (b *Builder) outputImage(ctx context.Context, imageToSave states.SaveImage, states *states.SingleTarget, opt BuildOpt) error { - shouldPush := opt.Push && imageToSave.Push - console := b.opt.Console.WithPrefixAndSalt(states.Target.String(), states.Salt) - err := b.s.solveDocker(ctx, imageToSave.State, imageToSave.Image, imageToSave.DockerTag, shouldPush) - if err != nil { - return errors.Wrapf(err, "solve image %s", imageToSave.DockerTag) - } - pushStr := "" - if shouldPush { - pushStr = " (pushed)" - } - console.Printf("Image %s as %s%s\n", states.Target.StringCanonical(), imageToSave.DockerTag, pushStr) - if imageToSave.Push && !opt.Push { - console.Printf("Did not push %s. Use earth --push to enable pushing\n", imageToSave.DockerTag) - } - return nil -} - func (b *Builder) outputImageTar(ctx context.Context, saveImage states.SaveImage, dockerTag string, outFile string) error { err := b.s.solveDockerTar(ctx, saveImage.State, saveImage.Image, dockerTag, outFile) if err != nil { @@ -344,33 +448,6 @@ func (b *Builder) outputSpecifiedArtifact(ctx context.Context, artifact domain.A return nil } -func (b *Builder) outputArtifact(ctx context.Context, artifactToSaveLocally states.SaveLocal, outDir string, solvedStates map[int]bool, states *states.SingleTarget, opt BuildOpt) error { - index := artifactToSaveLocally.Index - indexOutDir := filepath.Join(outDir, fmt.Sprintf("index-%d", index)) - if !solvedStates[index] { - solvedStates[index] = true - artifactsState := states.SeparateArtifactsState[artifactToSaveLocally.Index] - err := os.Mkdir(indexOutDir, 0755) - if err != nil { - return errors.Wrap(err, "mk index dir") - } - err = b.s.solveArtifacts(ctx, artifactsState, indexOutDir) - if err != nil { - return errors.Wrap(err, "solve artifacts") - } - } - - artifact := domain.Artifact{ - Target: states.Target, - Artifact: artifactToSaveLocally.ArtifactPath, - } - err := b.saveArtifactLocally(ctx, artifact, indexOutDir, artifactToSaveLocally.DestPath, states.Salt, opt) - if err != nil { - return err - } - return nil -} - func (b *Builder) saveArtifactLocally(ctx context.Context, artifact domain.Artifact, indexOutDir string, destPath string, salt string, opt BuildOpt) error { console := b.opt.Console.WithPrefixAndSalt(artifact.Target.String(), salt) fromPattern := filepath.Join(indexOutDir, filepath.FromSlash(artifact.Artifact)) @@ -466,3 +543,27 @@ func (b *Builder) saveArtifactLocally(ctx context.Context, artifact domain.Artif } return nil } + +func loadDockerTar(ctx context.Context, r io.ReadCloser) error { + // TODO: This is a gross hack - should use proper docker client. + cmd := exec.CommandContext(ctx, "docker", "load") + cmd.Stdin = r + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + return errors.Wrap(err, "docker load") + } + return nil +} + +func pushDockerImage(ctx context.Context, imageName string) error { + cmd := exec.CommandContext(ctx, "docker", "push", imageName) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + return errors.Wrapf(err, "docker push %s", imageName) + } + return nil +} diff --git a/builder/solver.go b/builder/solver.go index 00cfec0a8a..542f14d980 100644 --- a/builder/solver.go +++ b/builder/solver.go @@ -6,8 +6,9 @@ import ( "encoding/json" "io" "os" - "os/exec" + "strconv" + "github.com/earthly/earthly/domain" "github.com/earthly/earthly/llbutil" "github.com/earthly/earthly/states/image" "github.com/moby/buildkit/client" @@ -19,6 +20,10 @@ import ( "golang.org/x/sync/errgroup" ) +type onImageFunc func(context.Context, *errgroup.Group, int, string, string) (io.WriteCloser, error) +type onArtifactFunc func(context.Context, int, domain.Artifact, string, string) (string, error) +type onFinalArtifactFunc func(context.Context) (string, error) + type solver struct { sm *solverMonitor bkClient *client.Client @@ -27,61 +32,6 @@ type solver struct { remoteCache string } -func (s *solver) solveDocker(ctx context.Context, state llb.State, img *image.Image, dockerTag string, push bool) error { - dt, err := state.Marshal(ctx, llb.Platform(llbutil.TargetPlatform)) - if err != nil { - return errors.Wrap(err, "state marshal") - } - pipeR, pipeW := io.Pipe() - solveOpt, err := s.newSolveOptDocker(img, dockerTag, pipeW) - if err != nil { - return errors.Wrap(err, "new solve opt") - } - ch := make(chan *client.SolveStatus) - ctx, cancel := context.WithCancel(ctx) - defer cancel() - eg, ctx := errgroup.WithContext(ctx) - eg.Go(func() error { - var err error - _, err = s.bkClient.Solve(ctx, dt, *solveOpt, ch) - if err != nil { - return errors.Wrap(err, "solve") - } - return nil - }) - eg.Go(func() error { - return s.sm.monitorProgress(ctx, ch) - }) - eg.Go(func() error { - defer pipeR.Close() - err := loadDockerTar(ctx, pipeR) - if err != nil { - return errors.Wrapf(err, "load docker tar for %s", dockerTag) - } - if push { - err := pushDockerImage(ctx, dockerTag) - if err != nil { - return err - } - } - return nil - }) - go func() { - for { - select { - case <-ctx.Done(): - // Close read pipe on cancels, otherwise the whole thing hangs. - pipeR.Close() - } - } - }() - err = eg.Wait() - if err != nil { - return err - } - return nil -} - func (s *solver) solveDockerTar(ctx context.Context, state llb.State, img *image.Image, dockerTag string, outFile string) error { dt, err := state.Marshal(ctx, llb.Platform(llbutil.TargetPlatform)) if err != nil { @@ -176,15 +126,15 @@ func (s *solver) solveArtifacts(ctx context.Context, state llb.State, outDir str return nil } -func (s *solver) buildMain(ctx context.Context, bf gwclient.BuildFunc) error { - solveOpt, err := s.newSolveOptMain() - if err != nil { - return errors.Wrap(err, "new solve opt") - } +func (s *solver) buildMainMulti(ctx context.Context, bf gwclient.BuildFunc, onImage onImageFunc, onArtifact onArtifactFunc, onFinalArtifact onFinalArtifactFunc) error { ch := make(chan *client.SolveStatus) ctx, cancel := context.WithCancel(ctx) defer cancel() eg, ctx := errgroup.WithContext(ctx) + solveOpt, err := s.newSolveOptMulti(ctx, eg, onImage, onArtifact, onFinalArtifact) + if err != nil { + return errors.Wrap(err, "new solve opt") + } eg.Go(func() error { var err error _, err = s.bkClient.Build(ctx, *solveOpt, "", bf, ch) @@ -270,6 +220,54 @@ func (s *solver) newSolveOptArtifacts(outDir string) (*client.SolveOpt, error) { }, nil } +func (s *solver) newSolveOptMulti(ctx context.Context, eg *errgroup.Group, onImage onImageFunc, onArtifact onArtifactFunc, onFinalArtifact onFinalArtifactFunc) (*client.SolveOpt, error) { + return &client.SolveOpt{ + Exports: []client.ExportEntry{ + { + Type: client.ExporterEarthly, + Attrs: map[string]string{}, + Output: func(md map[string]string) (io.WriteCloser, error) { + if md["export-image"] != "true" { + return nil, nil + } + indexStr := md["image-index"] + index, err := strconv.Atoi(indexStr) + if err != nil { + return nil, errors.Wrapf(err, "parse image-index %s", indexStr) + } + imageName := md["image.name"] + digest := md["containerimage.digest"] + return onImage(ctx, eg, index, imageName, digest) + }, + OutputDirFunc: func(md map[string]string) (string, error) { + if md["export-dir"] != "true" { + // Use the other fun for images. + return "", nil + } + if md["final-artifact"] == "true" { + return onFinalArtifact(ctx) + } + indexStr := md["dir-index"] + index, err := strconv.Atoi(indexStr) + if err != nil { + return "", errors.Wrapf(err, "parse dir-index %s", indexStr) + } + artifactStr := md["artifact"] + srcPath := md["src-path"] + destPath := md["dest-path"] + artifact, err := domain.ParseArtifact(artifactStr) + if err != nil { + return "", errors.Wrapf(err, "parse artifact %s", artifactStr) + } + return onArtifact(ctx, index, artifact, srcPath, destPath) + }, + }, + }, + Session: s.attachables, + AllowedEntitlements: s.enttlmnts, + }, nil +} + func (s *solver) newSolveOptMain() (*client.SolveOpt, error) { var cacheImportExport []client.CacheOptionsEntry if s.remoteCache != "" { @@ -292,27 +290,3 @@ func newRegistryCacheOpt(ref string) client.CacheOptionsEntry { Attrs: registryCacheOptAttrs, } } - -func loadDockerTar(ctx context.Context, r io.ReadCloser) error { - // TODO: This is a gross hack - should use proper docker client. - cmd := exec.CommandContext(ctx, "docker", "load") - cmd.Stdin = r - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - err := cmd.Run() - if err != nil { - return errors.Wrap(err, "docker load") - } - return nil -} - -func pushDockerImage(ctx context.Context, imageName string) error { - cmd := exec.CommandContext(ctx, "docker", "push", imageName) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - err := cmd.Run() - if err != nil { - return errors.Wrapf(err, "docker push %s", imageName) - } - return nil -} diff --git a/cmd/earth/main.go b/cmd/earth/main.go index e36af54f0e..676d4cd5c4 100644 --- a/cmd/earth/main.go +++ b/cmd/earth/main.go @@ -1423,29 +1423,19 @@ func (app *earthApp) actionBuild(c *cli.Context) error { } buildOpts := builder.BuildOpt{ - PrintSuccess: true, - Push: app.push, + PrintSuccess: true, + Push: app.push, + NoOutput: app.noOutput, + OnlyFinalTargetImages: app.imageMode, } - mts, err := b.BuildTarget(c.Context, target, buildOpts) + if app.artifactMode { + buildOpts.OnlyArtifact = &artifact + buildOpts.OnlyArtifactDestPath = destPath + } + _, err = b.BuildTarget(c.Context, target, buildOpts) if err != nil { return errors.Wrap(err, "build target") } - if app.imageMode { - err = b.OutputImages(c.Context, mts.Final, buildOpts) - if err != nil { - return errors.Wrap(err, "output images") - } - } else if app.artifactMode { - err = b.OutputArtifact(c.Context, mts, artifact, destPath, buildOpts) - if err != nil { - return errors.Wrap(err, "output artifact") - } - } else if !app.noOutput { - err = b.Output(c.Context, mts, buildOpts) - if err != nil { - return errors.Wrap(err, "output") - } - } return nil } diff --git a/earth-buildkitd-wrapper.sh b/earth-buildkitd-wrapper.sh index 8eed56f88c..bbec23d749 100755 --- a/earth-buildkitd-wrapper.sh +++ b/earth-buildkitd-wrapper.sh @@ -1,6 +1,9 @@ #!/bin/sh +set -eu + # Start buildkitd. +rm -f "/run/buildkit/buildkitd.sock" /usr/bin/entrypoint.sh \ buildkitd \ --allow-insecure-entitlement=security.insecure \ @@ -17,27 +20,32 @@ while [ ! -S "/run/buildkit/buildkitd.sock" ]; do sleep 1 i=$((i+1)) if [ "$i" -gt "$timeout" ]; then - kill -9 "$buildkitd_pid" >/dev/null 2>&1 + kill -9 "$buildkitd_pid" >/dev/null 2>&1 || true echo "Buildkitd did not start within $timeout seconds" + echo "Buildkitd log" + echo "==============" + cat /var/log/buildkitd.log + echo "==============" exit 1 fi done # Run earth with given args. +set +e earth "$@" exit_code="$?" +set -e # Shut down buildkitd. -kill "$buildkitd_pid" >/dev/null 2>&1 +kill "$buildkitd_pid" >/dev/null 2>&1 || true i=1 timeout=10 while kill -0 "$buildkitd_pid" >/dev/null 2>&1 ; do sleep 1 i=$((i+1)) if [ "$i" -gt "$timeout" ]; then - kill -9 "$buildkitd_pid" >/dev/null 2>&1 + kill -9 "$buildkitd_pid" >/dev/null 2>&1 || true fi done -rm -f "/run/buildkit/buildkitd.sock" exit "$exit_code" diff --git a/earthfile2llb/converter.go b/earthfile2llb/converter.go index 63a4dc376f..6ab1868d4a 100644 --- a/earthfile2llb/converter.go +++ b/earthfile2llb/converter.go @@ -29,6 +29,7 @@ import ( "github.com/earthly/earthly/states/image" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb" + gwclient "github.com/moby/buildkit/frontend/gateway/client" solverpb "github.com/moby/buildkit/solver/pb" "github.com/pkg/errors" ) @@ -36,6 +37,7 @@ import ( // Converter turns earth commands to buildkit LLB representation. type Converter struct { gitMeta *buildcontext.GitMetadata + gwClient gwclient.Client resolver *buildcontext.Resolver mts *states.MultiTarget directDeps []*states.SingleTarget @@ -76,9 +78,10 @@ func NewConverter(ctx context.Context, target domain.Target, bc *buildcontext.Da sts.TargetInput = sts.TargetInput.WithBuildArgInput(ovVar.BuildArgInput(key, "")) } targetStr := target.String() - opt.Visited[targetStr] = append(opt.Visited[targetStr], sts) + opt.Visited.Add(targetStr, sts) return &Converter{ gitMeta: bc.GitMetadata, + gwClient: opt.GwClient, resolver: opt.Resolver, imageResolveMode: opt.ImageResolveMode, mts: mts, @@ -415,6 +418,7 @@ func (c *Converter) Build(ctx context.Context, fullTargetName string, buildArgs // Recursion. mts, err := Earthfile2LLB( ctx, target, ConvertOpt{ + GwClient: c.gwClient, Resolver: c.resolver, ImageResolveMode: c.imageResolveMode, DockerBuilderFun: c.dockerBuilderFun, @@ -574,19 +578,29 @@ func (c *Converter) Healthcheck(ctx context.Context, isNone bool, cmdArgs []stri } // FinalizeStates returns the LLB states. -func (c *Converter) FinalizeStates() *states.MultiTarget { - // Create an artificial bond to depStates so that side-effects of deps are built automatically. +func (c *Converter) FinalizeStates(ctx context.Context) (*states.MultiTarget, error) { + // Store refs for all dep states. for _, depStates := range c.directDeps { - c.mts.Final.MainState = withDependency( - c.mts.Final.MainState, - c.mts.Final.Target, - depStates.MainState, - depStates.Target) + def, err := depStates.MainState.Marshal(ctx) + if err != nil { + return nil, errors.Wrapf(err, "marshal dep %s", depStates.Target.String()) + } + r, err := c.gwClient.Solve(ctx, gwclient.SolveRequest{ + Definition: def.ToPB(), + }) + if err != nil { + return nil, errors.Wrap(err, "gw solve") + } + ref, err := r.SingleRef() + if err != nil { + return nil, errors.Wrap(err, "single ref") + } + c.mts.Final.DepsRefs = append(c.mts.Final.DepsRefs, ref) } - c.buildContextProvider.AddDirs(c.mts.Final.LocalDirs) + c.mts.Final.Ongoing = false - return c.mts + return c.mts, nil } func (c *Converter) internalRun(ctx context.Context, args []string, secretKeyValues []string, isWithShell bool, shellWrap shellWrapFun, pushFlag bool, withSSH bool, commandStr string, opts ...llb.RunOption) error { @@ -803,14 +817,6 @@ func (c *Converter) vertexPrefixWithURL(url string) string { return fmt.Sprintf("[%s(%s) %s] ", c.mts.Final.Target.String(), url, url) } -func withDependency(state llb.State, target domain.Target, depState llb.State, depTarget domain.Target) llb.State { - return llbutil.WithDependency( - state, depState, - llb.WithCustomNamef( - "[internal] create artificial dependency: %s depends on %s", - target.String(), depTarget.String())) -} - func makeCacheContext(target domain.Target) llb.State { sessionID := cacheKey(target) opts := []llb.LocalOption{ diff --git a/earthfile2llb/earthfile2llb.go b/earthfile2llb/earthfile2llb.go index a2c7648db9..3029379b13 100644 --- a/earthfile2llb/earthfile2llb.go +++ b/earthfile2llb/earthfile2llb.go @@ -15,11 +15,14 @@ import ( "github.com/earthly/earthly/earthfile2llb/variables" "github.com/earthly/earthly/states" "github.com/moby/buildkit/client/llb" + gwclient "github.com/moby/buildkit/frontend/gateway/client" "github.com/pkg/errors" ) // ConvertOpt holds conversion parameters needed for conversion. type ConvertOpt struct { + // GwClient is the BuildKit gateway client. + GwClient gwclient.Client // Resolver is the build context resolver. Resolver *buildcontext.Resolver // The resolve mode for referenced images (force pull or prefer local). @@ -36,7 +39,7 @@ type ConvertOpt struct { CleanCollection *cleanup.Collection // Visited is a collection of target states which have been converted to LLB. // This is used for deduplication and infinite cycle detection. - Visited map[string][]*states.SingleTarget + Visited *states.VisitedCollection // VarCollection is a collection of build args used for overriding args in the build. VarCollection *variables.Collection // A cache for image solves. depTargetInputHash -> context containing image.tar. @@ -53,11 +56,14 @@ func Earthfile2LLB(ctx context.Context, target domain.Target, opt ConvertOpt) (m opt.SolveCache = make(map[string]llb.State) } if opt.Visited == nil { - opt.Visited = make(map[string][]*states.SingleTarget) + opt.Visited = states.NewVisitedCollection() + } + if opt.MetaResolver == nil { + opt.MetaResolver = opt.GwClient } // Check if we have previously converted this target, with the same build args. targetStr := target.String() - for _, sts := range opt.Visited[targetStr] { + for _, sts := range opt.Visited.Visited[targetStr] { same := true for _, bai := range sts.TargetInput.BuildArgs { if sts.Ongoing && !bai.IsConstant { @@ -128,7 +134,7 @@ func Earthfile2LLB(ctx context.Context, target domain.Target, opt ConvertOpt) (m if walkErr != nil { return nil, walkErr } - return converter.FinalizeStates(), nil + return converter.FinalizeStates(ctx) } func walkTree(l *listener, tree parser.IEarthFileContext) (err error) { diff --git a/examples/integration-test/Earthfile b/examples/integration-test/Earthfile index 3d218d5804..d188995b42 100644 --- a/examples/integration-test/Earthfile +++ b/examples/integration-test/Earthfile @@ -17,7 +17,6 @@ sbt: # This triggers a bunch of useful downloads. RUN sbt sbtVersion - SAVE IMAGE project-files: FROM +sbt diff --git a/examples/ruby-on-rails/Earthfile b/examples/ruby-on-rails/Earthfile index 969abc4394..e302af56c7 100644 --- a/examples/ruby-on-rails/Earthfile +++ b/examples/ruby-on-rails/Earthfile @@ -10,13 +10,11 @@ deps: RUN bundle config build.nokogiri --use-system-libraries RUN bundle check || bundle install SAVE ARTIFACT Gemfile.lock AS LOCAL ./Gemfile.lock - SAVE IMAGE build: FROM +deps COPY . . SAVE ARTIFACT . /. - SAVE IMAGE docker: FROM +build diff --git a/examples/terraform/Earthfile b/examples/terraform/Earthfile index bf04129263..7961b4c522 100644 --- a/examples/terraform/Earthfile +++ b/examples/terraform/Earthfile @@ -5,8 +5,6 @@ prep: COPY aws/ . RUN terraform init - SAVE IMAGE - localstack: FROM +prep diff --git a/go.mod b/go.mod index eea2b8cfe6..984c18dfd9 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,6 @@ replace ( github.com/docker/docker => github.com/docker/docker v17.12.0-ce-rc1.0.20200310163718-4634ce647cf2+incompatible github.com/hashicorp/go-immutable-radix => github.com/tonistiigi/go-immutable-radix v0.0.0-20170803185627-826af9ccf0fe github.com/jaguilar/vt100 => github.com/tonistiigi/vt100 v0.0.0-20190402012908-ad4c4a574305 - github.com/moby/buildkit => github.com/earthly/buildkit v0.7.1-0.20201022233159-4f9057a0a3a7 + github.com/moby/buildkit => github.com/earthly/buildkit v0.7.1-0.20201027004549-b3b1aa8fc1d5 github.com/urfave/cli/v2 => github.com/alexcb/cli/v2 v2.2.1-0.20200824212017-2ae03fa69ce7 ) diff --git a/go.sum b/go.sum index 6d5d6d0a7e..cf457442c0 100644 --- a/go.sum +++ b/go.sum @@ -208,6 +208,16 @@ github.com/earthly/buildkit v0.7.1-0.20201022230831-316425965ede h1:Qk7/DMHO+irJ github.com/earthly/buildkit v0.7.1-0.20201022230831-316425965ede/go.mod h1:1Whs/5ueNdTNZDg5rl8TXviT98lBjtQUHyR0JMtvpMU= github.com/earthly/buildkit v0.7.1-0.20201022233159-4f9057a0a3a7 h1:5I0QYrf79m2FPc00c0vJkSVDZN9ffesqMr8lRhVl7L4= github.com/earthly/buildkit v0.7.1-0.20201022233159-4f9057a0a3a7/go.mod h1:1Whs/5ueNdTNZDg5rl8TXviT98lBjtQUHyR0JMtvpMU= +github.com/earthly/buildkit v0.7.1-0.20201024183456-ffada3a1cbd3 h1:srvRQmMjlI/xO0pSd0aXpV6T+wqV0aLgLlx8SIvhy9k= +github.com/earthly/buildkit v0.7.1-0.20201024183456-ffada3a1cbd3/go.mod h1:1Whs/5ueNdTNZDg5rl8TXviT98lBjtQUHyR0JMtvpMU= +github.com/earthly/buildkit v0.7.1-0.20201024185454-67705c8589bd h1:LKTLyOBzz+BEGjf+nUUGVNbcFae1ymb4mem+s58ODqQ= +github.com/earthly/buildkit v0.7.1-0.20201024185454-67705c8589bd/go.mod h1:1Whs/5ueNdTNZDg5rl8TXviT98lBjtQUHyR0JMtvpMU= +github.com/earthly/buildkit v0.7.1-0.20201024195616-094f5b9dd40b h1:+fvcX8c0hg6SKQA6/1Y5BuJr8QbtYss9lQUjuSBLZ8s= +github.com/earthly/buildkit v0.7.1-0.20201024195616-094f5b9dd40b/go.mod h1:1Whs/5ueNdTNZDg5rl8TXviT98lBjtQUHyR0JMtvpMU= +github.com/earthly/buildkit v0.7.1-0.20201026225007-c9bab88a1c61 h1:u2hLEYnshUfPoa16szaep3mMpZNgF5X5v2uwiZpMcw4= +github.com/earthly/buildkit v0.7.1-0.20201026225007-c9bab88a1c61/go.mod h1:1Whs/5ueNdTNZDg5rl8TXviT98lBjtQUHyR0JMtvpMU= +github.com/earthly/buildkit v0.7.1-0.20201027004549-b3b1aa8fc1d5 h1:47ZAFcxMZyDrjkxuatTIP2bi5pTeM60g1ZStT0kXekI= +github.com/earthly/buildkit v0.7.1-0.20201027004549-b3b1aa8fc1d5/go.mod h1:1Whs/5ueNdTNZDg5rl8TXviT98lBjtQUHyR0JMtvpMU= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -708,6 +718,7 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/llbutil/fakedep.go b/llbutil/fakedep.go deleted file mode 100644 index 3c9e69606b..0000000000 --- a/llbutil/fakedep.go +++ /dev/null @@ -1,27 +0,0 @@ -package llbutil - -import ( - "github.com/moby/buildkit/client/llb" -) - -const fakeDepImg = "busybox:1.31.1" - -// WithDependency creates a fake dependency between two states. -func WithDependency(state llb.State, depState llb.State, opts ...llb.RunOption) llb.State { - // TODO: Is there a better way to mark two states as depending on each other? - if depState.Output() == nil { - // depState is Scratch. - return state - } - runOpts := []llb.RunOption{ - llb.Args([]string{"/bin/sh", "-c", "true"}), - llb.Dir("/"), - llb.ReadonlyRootFS(), - llb.AddMount("/fake-dep", depState, llb.Readonly), - } - runOpts = append(runOpts, opts...) - opImg := llb.Image( - fakeDepImg, llb.MarkImageInternal, llb.Platform(TargetPlatform), - llb.WithCustomNamef("[internal] helper image for fake dep operations")) - return opImg.Run(runOpts...).AddMount("/fake", state) -} diff --git a/states/states.go b/states/states.go index e55610c4bc..ba0e676639 100644 --- a/states/states.go +++ b/states/states.go @@ -5,6 +5,7 @@ import ( "github.com/earthly/earthly/states/dedup" "github.com/earthly/earthly/states/image" "github.com/moby/buildkit/client/llb" + gwclient "github.com/moby/buildkit/frontend/gateway/client" ) // MultiTarget holds LLB states representing multiple earth targets, @@ -13,7 +14,7 @@ type MultiTarget struct { // Visited represents the previously visited states, grouped by target // name. Duplicate targets are possible if same target is called with different // build args. - Visited map[string][]*SingleTarget + Visited *VisitedCollection // Final is the main target to be built. Final *SingleTarget } @@ -25,11 +26,7 @@ func (mts *MultiTarget) FinalTarget() domain.Target { // All returns all SingleTarget contained within. func (mts *MultiTarget) All() []*SingleTarget { - var ret []*SingleTarget - for _, stss := range mts.Visited { - ret = append(ret, stss...) - } - return ret + return mts.Visited.VisitedList } // SingleTarget holds LLB states representing a earth target. @@ -46,6 +43,7 @@ type SingleTarget struct { LocalDirs map[string]string Ongoing bool Salt string + DepsRefs []gwclient.Reference } // LastSaveImage returns the last save image available (if any). diff --git a/states/visited.go b/states/visited.go new file mode 100644 index 0000000000..4fe5c532af --- /dev/null +++ b/states/visited.go @@ -0,0 +1,21 @@ +package states + +// VisitedCollection is a collection of visited targets. +type VisitedCollection struct { + Visited map[string][]*SingleTarget + // Same collection as above, but as a list, to make the ordering consistent. + VisitedList []*SingleTarget +} + +// NewVisitedCollection returns a collection of visited targets. +func NewVisitedCollection() *VisitedCollection { + return &VisitedCollection{ + Visited: make(map[string][]*SingleTarget), + } +} + +// Add adds a target to the collection. +func (vc *VisitedCollection) Add(target string, sts *SingleTarget) { + vc.Visited[target] = append(vc.Visited[target], sts) + vc.VisitedList = append(vc.VisitedList, sts) +}