From e59b26087fffe0357d34b400740e2fb15c062cb5 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Wed, 14 Aug 2019 17:46:46 -0700 Subject: [PATCH] vendor: update buildkit to v0.6.1 Signed-off-by: Tonis Tiigi --- vendor.conf | 2 +- .../moby/buildkit/cache/remotecache/import.go | 21 ++++++++---- .../cache/remotecache/inline/inline.go | 7 ++++ .../buildkit/cache/remotecache/v1/utils.go | 8 +++-- .../frontend/dockerfile/builder/build.go | 19 +++++++++-- .../dockerfile/dockerfile2llb/convert.go | 4 +-- .../dockerfile2llb/convert_runmount.go | 3 +- .../moby/buildkit/solver/combinedcache.go | 7 ++-- .../github.com/moby/buildkit/solver/jobs.go | 2 +- .../buildkit/solver/llbsolver/ops/exec.go | 6 ++-- .../moby/buildkit/solver/pb/caps.go | 33 +++++++++++-------- .../moby/buildkit/source/git/gitsource.go | 26 ++++++++++++--- 12 files changed, 98 insertions(+), 40 deletions(-) diff --git a/vendor.conf b/vendor.conf index 93e9db553644e..a3e4bf0b4e2a6 100644 --- a/vendor.conf +++ b/vendor.conf @@ -27,7 +27,7 @@ github.com/imdario/mergo 7c29201646fa3de8506f70121347 golang.org/x/sync e225da77a7e68af35c70ccbf71af2b83e6acac3c # buildkit -github.com/moby/buildkit f5a55a9516d1c6e2ade9bec22b83259caeed3a84 +github.com/moby/buildkit be0d75f074e7a4b0f5b5877c719213a3f5057e60 # v0.6.1 github.com/tonistiigi/fsutil 3bbb99cdbd76619ab717299830c60f6f2a533a6b github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746 github.com/opentracing/opentracing-go 1361b9cd60be79c4c3a7fa9841b3c132e40066a7 diff --git a/vendor/github.com/moby/buildkit/cache/remotecache/import.go b/vendor/github.com/moby/buildkit/cache/remotecache/import.go index 229d45a07bb26..88223ff92018e 100644 --- a/vendor/github.com/moby/buildkit/cache/remotecache/import.go +++ b/vendor/github.com/moby/buildkit/cache/remotecache/import.go @@ -111,7 +111,7 @@ func (ci *contentCacheImporter) importInlineCache(ctx context.Context, dt []byte } var mu sync.Mutex - cc := v1.NewCacheChains() + var cMap = map[digest.Digest]*v1.CacheChains{} eg, ctx := errgroup.WithContext(ctx) for dgst, dt := range m { @@ -183,11 +183,12 @@ func (ci *contentCacheImporter) importInlineCache(ctx context.Context, dt []byte if err != nil { return errors.WithStack(err) } - - mu.Lock() + cc := v1.NewCacheChains() if err := v1.ParseConfig(config, layers, cc); err != nil { return err } + mu.Lock() + cMap[dgst] = cc mu.Unlock() return nil }) @@ -198,11 +199,17 @@ func (ci *contentCacheImporter) importInlineCache(ctx context.Context, dt []byte return nil, err } - keysStorage, resultStorage, err := v1.NewCacheKeyStorage(cc, w) - if err != nil { - return nil, err + cms := make([]solver.CacheManager, 0, len(cMap)) + + for _, cc := range cMap { + keysStorage, resultStorage, err := v1.NewCacheKeyStorage(cc, w) + if err != nil { + return nil, err + } + cms = append(cms, solver.NewCacheManager(id, keysStorage, resultStorage)) } - return solver.NewCacheManager(id, keysStorage, resultStorage), nil + + return solver.NewCombinedCacheManager(cms, nil), nil } func (ci *contentCacheImporter) allDistributionManifests(ctx context.Context, dt []byte, m map[digest.Digest][]byte) error { diff --git a/vendor/github.com/moby/buildkit/cache/remotecache/inline/inline.go b/vendor/github.com/moby/buildkit/cache/remotecache/inline/inline.go index 4ce7de876d24d..96b979bdd362a 100644 --- a/vendor/github.com/moby/buildkit/cache/remotecache/inline/inline.go +++ b/vendor/github.com/moby/buildkit/cache/remotecache/inline/inline.go @@ -31,6 +31,12 @@ func (ce *exporter) Finalize(ctx context.Context) (map[string]string, error) { return nil, nil } +func (ce *exporter) reset() { + cc := v1.NewCacheChains() + ce.CacheExporterTarget = cc + ce.chains = cc +} + func (ce *exporter) ExportForLayers(layers []digest.Digest) ([]byte, error) { config, descs, err := ce.chains.Marshal() if err != nil { @@ -82,6 +88,7 @@ func (ce *exporter) ExportForLayers(layers []digest.Digest) ([]byte, error) { if err != nil { return nil, err } + ce.reset() return dt, nil } diff --git a/vendor/github.com/moby/buildkit/cache/remotecache/v1/utils.go b/vendor/github.com/moby/buildkit/cache/remotecache/v1/utils.go index 0638b17aff363..dfc7fab36a174 100644 --- a/vendor/github.com/moby/buildkit/cache/remotecache/v1/utils.go +++ b/vendor/github.com/moby/buildkit/cache/remotecache/v1/utils.go @@ -78,10 +78,14 @@ func sortConfig(cc *CacheConfig) { if ri.Inputs[i][j].Selector != rj.Inputs[i][j].Selector { return ri.Inputs[i][j].Selector < rj.Inputs[i][j].Selector } - return cc.Records[ri.Inputs[i][j].LinkIndex].Digest < cc.Records[rj.Inputs[i][j].LinkIndex].Digest + inputDigesti := cc.Records[ri.Inputs[i][j].LinkIndex].Digest + inputDigestj := cc.Records[rj.Inputs[i][j].LinkIndex].Digest + if inputDigesti != inputDigestj { + return inputDigesti < inputDigestj + } } } - return ri.Digest < rj.Digest + return false }) for i, l := range sortedRecords { l.newIndex = i diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/builder/build.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/builder/build.go index fbd4c42b0a93f..84e45eff9e09c 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerfile/builder/build.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/builder/build.go @@ -49,6 +49,7 @@ const ( keyNameContext = "contextkey" keyNameDockerfile = "dockerfilekey" keyContextSubDir = "contextsubdir" + keyContextKeepGitDir = "build-arg:BUILDKIT_CONTEXT_KEEP_GIT_DIR" ) var httpPrefix = regexp.MustCompile(`^https?://`) @@ -129,7 +130,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { var buildContext *llb.State isScratchContext := false - if st, ok := detectGitContext(opts[localNameContext]); ok { + if st, ok := detectGitContext(opts[localNameContext], opts[keyContextKeepGitDir]); ok { if !forceLocalDockerfile { src = *st } @@ -451,12 +452,19 @@ func filter(opt map[string]string, key string) map[string]string { return m } -func detectGitContext(ref string) (*llb.State, bool) { +func detectGitContext(ref, gitContext string) (*llb.State, bool) { found := false if httpPrefix.MatchString(ref) && gitUrlPathWithFragmentSuffix.MatchString(ref) { found = true } + keepGit := false + if gitContext != "" { + if v, err := strconv.ParseBool(gitContext); err == nil { + keepGit = v + } + } + for _, prefix := range []string{"git://", "github.com/", "git@"} { if strings.HasPrefix(ref, prefix) { found = true @@ -472,7 +480,12 @@ func detectGitContext(ref string) (*llb.State, bool) { if len(parts) > 1 { branch = parts[1] } - st := llb.Git(parts[0], branch, dockerfile2llb.WithInternalName("load git source "+ref)) + gitOpts := []llb.GitOption{dockerfile2llb.WithInternalName("load git source " + ref)} + if keepGit { + gitOpts = append(gitOpts, llb.KeepGitDir()) + } + + st := llb.Git(parts[0], branch, gitOpts...) return &st, true } diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go index 126d8b6c4120f..8cec181ed5a12 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go @@ -746,9 +746,9 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS }}, copyOpt...) if a == nil { - a = llb.Copy(sourceState, src, dest, opts...) + a = llb.Copy(sourceState, filepath.Join("/", src), dest, opts...) } else { - a = a.Copy(sourceState, src, dest, opts...) + a = a.Copy(sourceState, filepath.Join("/", src), dest, opts...) } } } diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runmount.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runmount.go index 3d9a83c31dc05..33595abb7d74b 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runmount.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runmount.go @@ -12,6 +12,7 @@ import ( "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/frontend/dockerfile/instructions" + "github.com/moby/buildkit/solver/pb" "github.com/pkg/errors" ) @@ -113,7 +114,7 @@ func dispatchRunMounts(d *dispatchState, c *instructions.RunCommand, sources []* } if mount.ReadOnly { mountOpts = append(mountOpts, llb.Readonly) - } else if mount.Type == instructions.MountTypeBind { + } else if mount.Type == instructions.MountTypeBind && opt.llbCaps.Supports(pb.CapExecMountBindReadWriteNoOuput) == nil { mountOpts = append(mountOpts, llb.ForceNoOutput) } if mount.Type == instructions.MountTypeCache { diff --git a/vendor/github.com/moby/buildkit/solver/combinedcache.go b/vendor/github.com/moby/buildkit/solver/combinedcache.go index 21e83aeb8b3bb..07c494d1cb15c 100644 --- a/vendor/github.com/moby/buildkit/solver/combinedcache.go +++ b/vendor/github.com/moby/buildkit/solver/combinedcache.go @@ -11,7 +11,7 @@ import ( "golang.org/x/sync/errgroup" ) -func newCombinedCacheManager(cms []CacheManager, main CacheManager) CacheManager { +func NewCombinedCacheManager(cms []CacheManager, main CacheManager) CacheManager { return &combinedCacheManager{cms: cms, main: main} } @@ -80,7 +80,7 @@ func (cm *combinedCacheManager) Load(ctx context.Context, rec *CacheRecord) (res res.Result.Release(context.TODO()) } }() - if rec.cacheManager != cm.main { + if rec.cacheManager != cm.main && cm.main != nil { for _, res := range results { if _, err := cm.main.Save(res.CacheKey, res.Result, res.CacheResult.CreatedAt); err != nil { return nil, err @@ -91,6 +91,9 @@ func (cm *combinedCacheManager) Load(ctx context.Context, rec *CacheRecord) (res } func (cm *combinedCacheManager) Save(key *CacheKey, s Result, createdAt time.Time) (*ExportableCacheKey, error) { + if cm.main == nil { + return nil, nil + } return cm.main.Save(key, s, createdAt) } diff --git a/vendor/github.com/moby/buildkit/solver/jobs.go b/vendor/github.com/moby/buildkit/solver/jobs.go index 99c7c8b69fe71..925ae5c50a782 100644 --- a/vendor/github.com/moby/buildkit/solver/jobs.go +++ b/vendor/github.com/moby/buildkit/solver/jobs.go @@ -141,7 +141,7 @@ func (s *state) combinedCacheManager() CacheManager { return s.mainCache } - return newCombinedCacheManager(cms, s.mainCache) + return NewCombinedCacheManager(cms, s.mainCache) } func (s *state) Release() { diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/ops/exec.go b/vendor/github.com/moby/buildkit/solver/llbsolver/ops/exec.go index d0de3802114bd..d823261d37473 100644 --- a/vendor/github.com/moby/buildkit/solver/llbsolver/ops/exec.go +++ b/vendor/github.com/moby/buildkit/solver/llbsolver/ops/exec.go @@ -256,7 +256,7 @@ func (e *execOp) getRefCacheDir(ctx context.Context, ref cache.ImmutableRef, id } func (e *execOp) getRefCacheDirNoCache(ctx context.Context, key string, ref cache.ImmutableRef, id string, m *pb.Mount, block bool) (cache.MutableRef, error) { - makeMutable := func(cache.ImmutableRef) (cache.MutableRef, error) { + makeMutable := func(ref cache.ImmutableRef) (cache.MutableRef, error) { desc := fmt.Sprintf("cached mount %s from exec %s", m.Dest, strings.Join(e.op.Meta.Args, " ")) return e.cm.New(ctx, ref, cache.WithRecordType(client.UsageRecordTypeCacheMount), cache.WithDescription(desc), cache.CachePolicyRetain) } @@ -585,7 +585,7 @@ func (e *execOp) Exec(ctx context.Context, inputs []solver.Result) ([]solver.Res mountable = ref } - makeMutable := func(cache.ImmutableRef) (cache.MutableRef, error) { + makeMutable := func(ref cache.ImmutableRef) (cache.MutableRef, error) { desc := fmt.Sprintf("mount %s from exec %s", m.Dest, strings.Join(e.op.Meta.Args, " ")) return e.cm.New(ctx, ref, cache.WithDescription(desc)) } @@ -606,7 +606,7 @@ func (e *execOp) Exec(ctx context.Context, inputs []solver.Result) ([]solver.Res outputs = append(outputs, active) mountable = active } - } else if ref == nil { + } else if (!m.Readonly || ref == nil) && m.Dest != pb.RootMount { // this case is empty readonly scratch without output that is not really useful for anything but don't error active, err := makeMutable(ref) if err != nil { diff --git a/vendor/github.com/moby/buildkit/solver/pb/caps.go b/vendor/github.com/moby/buildkit/solver/pb/caps.go index 7ff0358e62e02..7649ce7d9db18 100644 --- a/vendor/github.com/moby/buildkit/solver/pb/caps.go +++ b/vendor/github.com/moby/buildkit/solver/pb/caps.go @@ -30,19 +30,20 @@ const ( CapBuildOpLLBFileName apicaps.CapID = "source.buildop.llbfilename" - CapExecMetaBase apicaps.CapID = "exec.meta.base" - CapExecMetaProxy apicaps.CapID = "exec.meta.proxyenv" - CapExecMetaNetwork apicaps.CapID = "exec.meta.network" - CapExecMetaSecurity apicaps.CapID = "exec.meta.security" - CapExecMetaSetsDefaultPath apicaps.CapID = "exec.meta.setsdefaultpath" - CapExecMountBind apicaps.CapID = "exec.mount.bind" - CapExecMountCache apicaps.CapID = "exec.mount.cache" - CapExecMountCacheSharing apicaps.CapID = "exec.mount.cache.sharing" - CapExecMountSelector apicaps.CapID = "exec.mount.selector" - CapExecMountTmpfs apicaps.CapID = "exec.mount.tmpfs" - CapExecMountSecret apicaps.CapID = "exec.mount.secret" - CapExecMountSSH apicaps.CapID = "exec.mount.ssh" - CapExecCgroupsMounted apicaps.CapID = "exec.cgroup" + CapExecMetaBase apicaps.CapID = "exec.meta.base" + CapExecMetaProxy apicaps.CapID = "exec.meta.proxyenv" + CapExecMetaNetwork apicaps.CapID = "exec.meta.network" + CapExecMetaSecurity apicaps.CapID = "exec.meta.security" + CapExecMetaSetsDefaultPath apicaps.CapID = "exec.meta.setsdefaultpath" + CapExecMountBind apicaps.CapID = "exec.mount.bind" + CapExecMountBindReadWriteNoOuput apicaps.CapID = "exec.mount.bind.readwrite-nooutput" + CapExecMountCache apicaps.CapID = "exec.mount.cache" + CapExecMountCacheSharing apicaps.CapID = "exec.mount.cache.sharing" + CapExecMountSelector apicaps.CapID = "exec.mount.selector" + CapExecMountTmpfs apicaps.CapID = "exec.mount.tmpfs" + CapExecMountSecret apicaps.CapID = "exec.mount.secret" + CapExecMountSSH apicaps.CapID = "exec.mount.ssh" + CapExecCgroupsMounted apicaps.CapID = "exec.cgroup" CapFileBase apicaps.CapID = "file.base" @@ -193,6 +194,12 @@ func init() { Status: apicaps.CapStatusExperimental, }) + Caps.Init(apicaps.Cap{ + ID: CapExecMountBindReadWriteNoOuput, + Enabled: true, + Status: apicaps.CapStatusExperimental, + }) + Caps.Init(apicaps.Cap{ ID: CapExecMountCache, Enabled: true, diff --git a/vendor/github.com/moby/buildkit/source/git/gitsource.go b/vendor/github.com/moby/buildkit/source/git/gitsource.go index b2a3eacfe4b7b..d2d6ff76179ea 100644 --- a/vendor/github.com/moby/buildkit/source/git/gitsource.go +++ b/vendor/github.com/moby/buildkit/source/git/gitsource.go @@ -153,6 +153,14 @@ type gitSourceHandler struct { cacheKey string } +func (gs *gitSourceHandler) shaToCacheKey(sha string) string { + key := sha + if gs.src.KeepGitDir { + key += ".git" + } + return key +} + func (gs *gitSource) Resolve(ctx context.Context, id source.Identifier, _ *session.Manager) (source.SourceInstance, error) { gitIdentifier, ok := id.(*source.GitIdentifier) if !ok { @@ -175,6 +183,7 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, index int) (string, bo defer gs.locker.Unlock(remote) if isCommitSHA(ref) { + ref = gs.shaToCacheKey(ref) gs.cacheKey = ref return ref, true, nil } @@ -201,6 +210,7 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, index int) (string, bo if !isCommitSHA(sha) { return "", false, errors.Errorf("invalid commit sha %q", sha) } + sha = gs.shaToCacheKey(sha) gs.cacheKey = sha return sha, true, nil } @@ -298,11 +308,15 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe }() if gs.src.KeepGitDir { - _, err = gitWithinDir(ctx, checkoutDir, "", "init") + checkoutDirGit := filepath.Join(checkoutDir, ".git") + if err := os.MkdirAll(checkoutDir, 0711); err != nil { + return nil, err + } + _, err = gitWithinDir(ctx, checkoutDirGit, "", "init") if err != nil { return nil, err } - _, err = gitWithinDir(ctx, checkoutDir, "", "remote", "add", "origin", gitDir) + _, err = gitWithinDir(ctx, checkoutDirGit, "", "remote", "add", "origin", gitDir) if err != nil { return nil, err } @@ -313,16 +327,18 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe if err != nil { return nil, err } + } else { + pullref += ":" + pullref } - _, err = gitWithinDir(ctx, checkoutDir, "", "fetch", "--depth=1", "origin", pullref) + _, err = gitWithinDir(ctx, checkoutDirGit, "", "fetch", "-u", "--depth=1", "origin", pullref) if err != nil { return nil, err } - _, err = gitWithinDir(ctx, checkoutDir, checkoutDir, "checkout", "FETCH_HEAD") + _, err = gitWithinDir(ctx, checkoutDirGit, checkoutDir, "checkout", "FETCH_HEAD") if err != nil { return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote) } - gitDir = checkoutDir + gitDir = checkoutDirGit } else { _, err = gitWithinDir(ctx, gitDir, checkoutDir, "checkout", ref, "--", ".") if err != nil {