Skip to content

Commit

Permalink
source: make sources pluggable
Browse files Browse the repository at this point in the history
Sources are a pretty neat extension point, except there are a few code
paths that hard-code against each type. This moves code around and
adjusts interfaces so that Source implementations are self-contained and
merely need to be registered with the source.Manager.

Signed-off-by: Alex Suraci <suraci.alex@gmail.com>
  • Loading branch information
vito authored and jedevc committed Aug 16, 2023
1 parent 53f503b commit 6b27487
Show file tree
Hide file tree
Showing 21 changed files with 741 additions and 613 deletions.
2 changes: 1 addition & 1 deletion solver/llbsolver/ops/source.go
Expand Up @@ -60,7 +60,7 @@ func (s *SourceOp) instance(ctx context.Context) (source.SourceInstance, error)
if s.src != nil {
return s.src, nil
}
id, err := source.FromLLB(s.op, s.platform)
id, err := s.sm.Identifier(s.op, s.platform)
if err != nil {
return nil, err
}
Expand Down
68 changes: 3 additions & 65 deletions solver/llbsolver/provenance.go
Expand Up @@ -20,7 +20,6 @@ import (
"github.com/moby/buildkit/solver/llbsolver/ops"
"github.com/moby/buildkit/solver/llbsolver/provenance"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/source"
"github.com/moby/buildkit/worker"
digest "github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
Expand Down Expand Up @@ -268,70 +267,9 @@ func captureProvenance(ctx context.Context, res solver.CachedResultWithProvenanc
switch op := pp.(type) {
case *ops.SourceOp:
id, pin := op.Pin()
switch s := id.(type) {
case *source.ImageIdentifier:
dgst, err := digest.Parse(pin)
if err != nil {
return errors.Wrapf(err, "failed to parse image digest %s", pin)
}
c.AddImage(provenance.ImageSource{
Ref: s.Reference.String(),
Platform: s.Platform,
Digest: dgst,
})
case *source.LocalIdentifier:
c.AddLocal(provenance.LocalSource{
Name: s.Name,
})
case *source.GitIdentifier:
url := s.Remote
if s.Ref != "" {
url += "#" + s.Ref
}
c.AddGit(provenance.GitSource{
URL: url,
Commit: pin,
})
if s.AuthTokenSecret != "" {
c.AddSecret(provenance.Secret{
ID: s.AuthTokenSecret,
Optional: true,
})
}
if s.AuthHeaderSecret != "" {
c.AddSecret(provenance.Secret{
ID: s.AuthHeaderSecret,
Optional: true,
})
}
if s.MountSSHSock != "" {
c.AddSSH(provenance.SSH{
ID: s.MountSSHSock,
Optional: true,
})
}
case *source.HTTPIdentifier:
dgst, err := digest.Parse(pin)
if err != nil {
return errors.Wrapf(err, "failed to parse HTTP digest %s", pin)
}
c.AddHTTP(provenance.HTTPSource{
URL: s.URL,
Digest: dgst,
})
case *source.OCIIdentifier:
dgst, err := digest.Parse(pin)
if err != nil {
return errors.Wrapf(err, "failed to parse OCI digest %s", pin)
}
c.AddImage(provenance.ImageSource{
Ref: s.Reference.String(),
Platform: s.Platform,
Digest: dgst,
Local: true,
})
default:
return errors.Errorf("unknown source identifier %T", id)
err := id.Capture(c, pin)
if err != nil {
return err
}
case *ops.ExecOp:
pr := op.Proto()
Expand Down
8 changes: 0 additions & 8 deletions solver/llbsolver/vertex.go
Expand Up @@ -9,7 +9,6 @@ import (
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/solver/llbsolver/ops/opsutils"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/source"
"github.com/moby/buildkit/util/entitlements"
digest "github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
Expand Down Expand Up @@ -322,13 +321,6 @@ func loadLLB(ctx context.Context, def *pb.Definition, polEngine SourcePolicyEval
func llbOpName(pbOp *pb.Op, load func(digest.Digest) (solver.Vertex, error)) (string, error) {
switch op := pbOp.Op.(type) {
case *pb.Op_Source:
if id, err := source.FromLLB(op, nil); err == nil {
if id, ok := id.(*source.LocalIdentifier); ok {
if len(id.IncludePatterns) == 1 {
return op.Source.Identifier + " (" + id.IncludePatterns[0] + ")", nil
}
}
}
return op.Source.Identifier, nil
case *pb.Op_Exec:
return strings.Join(op.Exec.Meta.Args, " "), nil
Expand Down
92 changes: 92 additions & 0 deletions source/containerimage/identifier.go
@@ -0,0 +1,92 @@
package containerimage

import (
"github.com/containerd/containerd/reference"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/solver/llbsolver/provenance"
"github.com/moby/buildkit/source"
srctypes "github.com/moby/buildkit/source/types"
"github.com/moby/buildkit/util/resolver"
digest "github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)

type ImageIdentifier struct {
Reference reference.Spec
Platform *ocispecs.Platform
ResolveMode resolver.ResolveMode
RecordType client.UsageRecordType
LayerLimit *int
}

func NewImageIdentifier(str string) (*ImageIdentifier, error) {
ref, err := reference.Parse(str)
if err != nil {
return nil, errors.WithStack(err)
}

if ref.Object == "" {
return nil, errors.WithStack(reference.ErrObjectRequired)
}
return &ImageIdentifier{Reference: ref}, nil
}

var _ source.Identifier = (*ImageIdentifier)(nil)

func (*ImageIdentifier) Scheme() string {
return srctypes.DockerImageScheme
}

func (id *ImageIdentifier) Capture(c *provenance.Capture, pin string) error {
dgst, err := digest.Parse(pin)
if err != nil {
return errors.Wrapf(err, "failed to parse image digest %s", pin)
}
c.AddImage(provenance.ImageSource{
Ref: id.Reference.String(),
Platform: id.Platform,
Digest: dgst,
})
return nil
}

type OCIIdentifier struct {
Reference reference.Spec
Platform *ocispecs.Platform
SessionID string
StoreID string
LayerLimit *int
}

func NewOCIIdentifier(str string) (*OCIIdentifier, error) {
ref, err := reference.Parse(str)
if err != nil {
return nil, errors.WithStack(err)
}

if ref.Object == "" {
return nil, errors.WithStack(reference.ErrObjectRequired)
}
return &OCIIdentifier{Reference: ref}, nil
}

var _ source.Identifier = (*OCIIdentifier)(nil)

func (*OCIIdentifier) Scheme() string {
return srctypes.OCIScheme
}

func (id *OCIIdentifier) Capture(c *provenance.Capture, pin string) error {
dgst, err := digest.Parse(pin)
if err != nil {
return errors.Wrapf(err, "failed to parse OCI digest %s", pin)
}
c.AddImage(provenance.ImageSource{
Ref: id.Reference.String(),
Platform: id.Platform,
Digest: dgst,
Local: true,
})
return nil
}
160 changes: 1 addition & 159 deletions source/containerimage/pull.go
Expand Up @@ -7,24 +7,18 @@ import (
"time"

"github.com/containerd/containerd/content"
"github.com/containerd/containerd/diff"
containerderrdefs "github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/leases"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/reference"
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
"github.com/containerd/containerd/snapshots"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/solver/errdefs"
"github.com/moby/buildkit/source"
srctypes "github.com/moby/buildkit/source/types"
"github.com/moby/buildkit/util/estargz"
"github.com/moby/buildkit/util/flightcontrol"
"github.com/moby/buildkit/util/imageutil"
Expand All @@ -39,164 +33,12 @@ import (
"github.com/pkg/errors"
)

// TODO: break apart containerd specifics like contentstore so the resolver
// code can be used with any implementation

type ResolverType int

const (
ResolverTypeRegistry ResolverType = iota
ResolverTypeOCILayout
)

type SourceOpt struct {
Snapshotter snapshot.Snapshotter
ContentStore content.Store
Applier diff.Applier
CacheAccessor cache.Accessor
ImageStore images.Store // optional
RegistryHosts docker.RegistryHosts
ResolverType
LeaseManager leases.Manager
}

type resolveImageResult struct {
ref string
dgst digest.Digest
dt []byte
}

type Source struct {
SourceOpt
g flightcontrol.Group[*resolveImageResult]
}

var _ source.Source = &Source{}

func NewSource(opt SourceOpt) (*Source, error) {
is := &Source{
SourceOpt: opt,
}

return is, nil
}

func (is *Source) ID() string {
if is.ResolverType == ResolverTypeOCILayout {
return srctypes.OCIScheme
}
return srctypes.DockerImageScheme
}

func (is *Source) ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt, sm *session.Manager, g session.Group) (string, digest.Digest, []byte, error) {
key := ref
if platform := opt.Platform; platform != nil {
key += platforms.Format(*platform)
}
var (
rm source.ResolveMode
rslvr remotes.Resolver
err error
)

switch is.ResolverType {
case ResolverTypeRegistry:
rm, err = source.ParseImageResolveMode(opt.ResolveMode)
if err != nil {
return "", "", nil, err
}
rslvr = resolver.DefaultPool.GetResolver(is.RegistryHosts, ref, "pull", sm, g).WithImageStore(is.ImageStore, rm)
case ResolverTypeOCILayout:
rm = source.ResolveModeForcePull
rslvr = getOCILayoutResolver(opt.Store, sm, g)
}
key += rm.String()
res, err := is.g.Do(ctx, key, func(ctx context.Context) (*resolveImageResult, error) {
newRef, dgst, dt, err := imageutil.Config(ctx, ref, rslvr, is.ContentStore, is.LeaseManager, opt.Platform, opt.SourcePolicies)
if err != nil {
return nil, err
}
return &resolveImageResult{dgst: dgst, dt: dt, ref: newRef}, nil
})
if err != nil {
return "", "", nil, err
}
return res.ref, res.dgst, res.dt, nil
}

func (is *Source) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager, vtx solver.Vertex) (source.SourceInstance, error) {
var (
p *puller
platform = platforms.DefaultSpec()
pullerUtil *pull.Puller
mode source.ResolveMode
recordType client.UsageRecordType
ref reference.Spec
store llb.ResolveImageConfigOptStore
layerLimit *int
)
switch is.ResolverType {
case ResolverTypeRegistry:
imageIdentifier, ok := id.(*source.ImageIdentifier)
if !ok {
return nil, errors.Errorf("invalid image identifier %v", id)
}

if imageIdentifier.Platform != nil {
platform = *imageIdentifier.Platform
}
mode = imageIdentifier.ResolveMode
recordType = imageIdentifier.RecordType
ref = imageIdentifier.Reference
layerLimit = imageIdentifier.LayerLimit
case ResolverTypeOCILayout:
ociIdentifier, ok := id.(*source.OCIIdentifier)
if !ok {
return nil, errors.Errorf("invalid OCI layout identifier %v", id)
}

if ociIdentifier.Platform != nil {
platform = *ociIdentifier.Platform
}
mode = source.ResolveModeForcePull // with OCI layout, we always just "pull"
store = llb.ResolveImageConfigOptStore{
SessionID: ociIdentifier.SessionID,
StoreID: ociIdentifier.StoreID,
}
ref = ociIdentifier.Reference
layerLimit = ociIdentifier.LayerLimit
default:
return nil, errors.Errorf("unknown resolver type: %v", is.ResolverType)
}
pullerUtil = &pull.Puller{
ContentStore: is.ContentStore,
Platform: platform,
Src: ref,
}
p = &puller{
CacheAccessor: is.CacheAccessor,
LeaseManager: is.LeaseManager,
Puller: pullerUtil,
RegistryHosts: is.RegistryHosts,
ResolverType: is.ResolverType,
ImageStore: is.ImageStore,
Mode: mode,
RecordType: recordType,
Ref: ref.String(),
SessionManager: sm,
vtx: vtx,
store: store,
layerLimit: layerLimit,
}
return p, nil
}

type puller struct {
CacheAccessor cache.Accessor
LeaseManager leases.Manager
RegistryHosts docker.RegistryHosts
ImageStore images.Store
Mode source.ResolveMode
Mode resolver.ResolveMode
RecordType client.UsageRecordType
Ref string
SessionManager *session.Manager
Expand Down

0 comments on commit 6b27487

Please sign in to comment.