Skip to content

Commit

Permalink
[image-builder] Copy image rather than use buildkit
Browse files Browse the repository at this point in the history 
To work around:
containerd/containerd#5978
  • Loading branch information
@csweichel
csweichel committed Sep 24, 2021
1 parent 1337d47 commit 2cce40e
Showing 1 changed file with 199 additions and 45 deletions.
244 changes: 199 additions & 45 deletions components/image-builder-bob/pkg/builder/builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@ package builder

import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
Expand All @@ -17,14 +20,13 @@ import (
"github.com/gitpod-io/gitpod/common-go/log"

"github.com/containerd/console"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/util/contentutil"
"github.com/moby/buildkit/util/imageutil"
"github.com/moby/buildkit/util/progress/progressui"
"github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"golang.org/x/sync/errgroup"
"golang.org/x/xerrors"
Expand Down Expand Up @@ -193,20 +195,15 @@ func (b *Builder) buildBaseLayer(ctx context.Context, cl *client.Client) error {
}

func (b *Builder) buildWorkspaceImage(ctx context.Context, cl *client.Client) (err error) {
// Note: buildkit does not handle/export image config by default. That's why we need
// to download it ourselves and explicitely export it.
// See https://github.com/moby/buildkit/issues/2362 for details.

var (
sess []session.Attachable
resolver remotes.Resolver
)
// Workaround: buildkit/containerd currently does not support pushing multi-image builds
// with some registries, e.g. gcr.io. Until https://github.com/containerd/containerd/issues/5978
// is resolved, we'll manually copy the image.
var resolver remotes.Resolver
if gplayerAuth := b.Config.WorkspaceLayerAuth; gplayerAuth != "" {
auth, err := newAuthProviderFromEnvvar(gplayerAuth)
_, err := newAuthProviderFromEnvvar(gplayerAuth)
if err != nil {
return err
}
sess = append(sess, auth)

authorizer, err := newDockerAuthorizerFromEnvvar(gplayerAuth)
if err != nil {
Expand All @@ -218,55 +215,212 @@ func (b *Builder) buildWorkspaceImage(ctx context.Context, cl *client.Client) (e
} else {
resolver = docker.NewResolver(docker.ResolverOptions{})
}
return copyImage(ctx, resolver, b.Config.BaseRef, b.Config.TargetRef)

// // Note: buildkit does not handle/export image config by default. That's why we need
// // to download it ourselves and explicitely export it.
// // See https://github.com/moby/buildkit/issues/2362 for details.
// var sess []session.Attachable
// if gplayerAuth := b.Config.WorkspaceLayerAuth; gplayerAuth != "" {
// auth, err := newAuthProviderFromEnvvar(gplayerAuth)
// if err != nil {
// return err
// }
// sess = append(sess, auth)

// authorizer, err := newDockerAuthorizerFromEnvvar(gplayerAuth)
// if err != nil {
// return err
// }
// resolver = docker.NewResolver(docker.ResolverOptions{
// Authorizer: authorizer,
// })
// } else {
// resolver = docker.NewResolver(docker.ResolverOptions{})
// }

// platform := specs.Platform{OS: "linux", Architecture: "amd64"}
// _, cfg, err := imageutil.Config(ctx, b.Config.BaseRef, resolver, contentutil.NewBuffer(), nil, &platform)
// if err != nil {
// return err
// }
// state, err := llb.Image(b.Config.BaseRef).WithImageConfig(cfg)
// if err != nil {
// return err
// }

// def, err := state.Marshal(ctx, llb.Platform(platform))
// if err != nil {
// return err
// }

// // TODO(cw):
// // buildkit does not support setting raw annotations yet (https://github.com/moby/buildkit/issues/1220).
// // Once it does, we should set org.opencontainers.image.base.name as defined in https://github.com/opencontainers/image-spec/blob/main/annotations.md

// solveOpt := client.SolveOpt{
// Exports: []client.ExportEntry{
// {
// Type: "image",
// Attrs: map[string]string{
// "name": b.Config.TargetRef,
// "push": "true",
// "containerimage.config": string(cfg),
// },
// },
// },
// Session: sess,
// CacheImports: b.Config.LocalCacheImport(),
// }

// eg, ctx := errgroup.WithContext(ctx)
// ch := make(chan *client.SolveStatus)
// eg.Go(func() error {
// _, err := cl.Solve(ctx, def, solveOpt, ch)
// if err != nil {
// return xerrors.Errorf("cannot build Gitpod layer: %w", err)
// }
// return nil
// })
// eg.Go(func() error {
// var c console.Console
// return progressui.DisplaySolveStatus(ctx, "", c, os.Stdout, ch)
// })
// return eg.Wait()
}

platform := specs.Platform{OS: "linux", Architecture: "amd64"}
_, cfg, err := imageutil.Config(ctx, b.Config.BaseRef, resolver, contentutil.NewBuffer(), nil, &platform)
func copyImage(ctx context.Context, resolver remotes.Resolver, from, to string) error {
fromRef, fromDesc, err := resolver.Resolve(ctx, from)
if err != nil {
return err
}
state, err := llb.Image(b.Config.BaseRef).WithImageConfig(cfg)
fetcher, err := resolver.Fetcher(ctx, fromRef)
if err != nil {
return err
}

def, err := state.Marshal(ctx, llb.Platform(platform))
fetch := func(desc specs.Descriptor, out interface{}) error {
rc, err := fetcher.Fetch(ctx, fromDesc)
if err != nil {
return err
}
defer rc.Close()

return json.NewDecoder(rc).Decode(out)
}
if fromDesc.MediaType == specs.MediaTypeImageIndex {
var idx specs.Index
err := fetch(fromDesc, &idx)
if err != nil {
return err
}

var res *specs.Descriptor
for _, m := range idx.Manifests {
if m.Platform != nil && m.Platform.Architecture == "amd64" && m.Platform.OS == "linux" {
res = &m
break
}
}
if res == nil {
return fmt.Errorf("no manifest for amd64/linux found")
}
fromDesc = *res
}

var manifest specs.Manifest
err = fetch(fromDesc, &manifest)
if err != nil {
return err
}
manifestB, err := json.Marshal(manifest)
if err != nil {
return err
}

// TODO(cw):
// buildkit does not support setting raw annotations yet (https://github.com/moby/buildkit/issues/1220).
// Once it does, we should set org.opencontainers.image.base.name as defined in https://github.com/opencontainers/image-spec/blob/main/annotations.md
pusher, err := resolver.Pusher(ctx, to)
if err != nil {
return err
}

solveOpt := client.SolveOpt{
Exports: []client.ExportEntry{
{
Type: "image",
Attrs: map[string]string{
"name": b.Config.TargetRef,
"push": "true",
"containerimage.config": string(cfg),
},
},
},
Session: sess,
CacheImports: b.Config.LocalCacheImport(),
// TODO(cw): optimize layer copy - copy only when pushing to a different registry
eg, egctx := errgroup.WithContext(ctx)
for _, layer := range manifest.Layers {
layer := layer
eg.Go(func() error { return copyLayer(egctx, fetcher, pusher, layer) })
}
eg.Go(func() error { return copyLayer(egctx, fetcher, pusher, manifest.Config) })
err = eg.Wait()
if err != nil {
return err
}

eg, ctx := errgroup.WithContext(ctx)
ch := make(chan *client.SolveStatus)
eg.Go(func() error {
_, err := cl.Solve(ctx, def, solveOpt, ch)
mfspec := specs.Descriptor{
MediaType: specs.MediaTypeImageManifest,
Digest: digest.FromBytes(manifestB),
Size: int64(len(manifestB)),
Platform: &specs.Platform{OS: "linux", Architecture: "amd64"},
}
mfw, err := pusher.Push(ctx, mfspec)
if err != nil {
return err
}
defer mfw.Close()
mfwN, err := mfw.Write(manifestB)
if err != nil {
return err
}
if mfwN != len(manifestB) {
return fmt.Errorf("cannot write manifest: %w", io.ErrShortWrite)
}
err = mfw.Commit(ctx, mfspec.Size, mfspec.Digest)
if err != nil {
return err
}

return nil
}

func copyLayer(ctx context.Context, fetcher remotes.Fetcher, pusher remotes.Pusher, layer specs.Descriptor) (err error) {
log := log.WithField("blob", layer)
defer func() {
if errdefs.IsAlreadyExists(err) {
err = nil
}
if err != nil {
return xerrors.Errorf("cannot build Gitpod layer: %w", err)
log.WithError(err).Error("failed to copy blob")
} else {
log.Info("push complete")
}
return nil
})
eg.Go(func() error {
var c console.Console
return progressui.DisplaySolveStatus(ctx, "", c, os.Stdout, ch)
})
return eg.Wait()
}()
log.Info("pushing blob")

in, err := fetcher.Fetch(ctx, layer)
if err != nil {
return err
}
defer in.Close()

out, err := pusher.Push(ctx, layer)
if err != nil {
return err
}
defer out.Close()

n, err := io.Copy(out, in)
if err != nil {
return err
}
if n != layer.Size {
return fmt.Errorf("copied less than %d bytes from layer %s (expected %d bytes)", n, layer.Digest.String(), layer.Size)
}

err = out.Commit(ctx, n, layer.Digest)
if err != nil {
return err
}

return nil
}

func waitForBuildContext(ctx context.Context) error {
Expand Down

0 comments on commit 2cce40e

Please sign in to comment.