Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

replace resolveimageconfig with generic sourcemetaresolver #4563

Merged
merged 2 commits into from
Feb 13, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 5 additions & 0 deletions client/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,11 @@ func (g *gatewayClientForBuild) ResolveImageConfig(ctx context.Context, in *gate
return g.gateway.ResolveImageConfig(ctx, in, opts...)
}

func (g *gatewayClientForBuild) ResolveSourceMeta(ctx context.Context, in *gatewayapi.ResolveSourceMetaRequest, opts ...grpc.CallOption) (*gatewayapi.ResolveSourceMetaResponse, error) {
ctx = buildid.AppendToOutgoingContext(ctx, g.buildID)
return g.gateway.ResolveSourceMeta(ctx, in, opts...)
}

func (g *gatewayClientForBuild) Solve(ctx context.Context, in *gatewayapi.SolveRequest, opts ...grpc.CallOption) (*gatewayapi.SolveResponse, error) {
ctx = buildid.AppendToOutgoingContext(ctx, g.buildID)
return g.gateway.Solve(ctx, in, opts...)
Expand Down
5 changes: 3 additions & 2 deletions client/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ import (
intoto "github.com/in-toto/in-toto-golang/in_toto"
controlapi "github.com/moby/buildkit/api/services/control"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/exporter/containerimage/exptypes"
gateway "github.com/moby/buildkit/frontend/gateway/client"
gatewaypb "github.com/moby/buildkit/frontend/gateway/pb"
Expand Down Expand Up @@ -3061,7 +3062,7 @@ func testSourceDateEpochClamp(t *testing.T, sb integration.Sandbox) {

var bboxConfig []byte
_, err = c.Build(sb.Context(), SolveOpt{}, "", func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
_, _, bboxConfig, err = c.ResolveImageConfig(ctx, "docker.io/library/busybox:latest", llb.ResolveImageConfigOpt{})
_, _, bboxConfig, err = c.ResolveImageConfig(ctx, "docker.io/library/busybox:latest", sourceresolver.Opt{})
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -10059,7 +10060,7 @@ func testSourcePolicy(t *testing.T, sb integration.Sandbox) {
},
}

ref, dgst, _, err := c.ResolveImageConfig(ctx, origRef, llb.ResolveImageConfigOpt{
ref, dgst, _, err := c.ResolveImageConfig(ctx, origRef, sourceresolver.Opt{
SourcePolicies: pol,
})
if err != nil {
Expand Down
16 changes: 8 additions & 8 deletions client/llb/imagemetaresolver/resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/util/contentutil"
"github.com/moby/buildkit/util/imageutil"
"github.com/moby/buildkit/version"
Expand Down Expand Up @@ -70,32 +71,31 @@ type imageMetaResolver struct {
}

type resolveResult struct {
ref string
config []byte
dgst digest.Digest
}

func (imr *imageMetaResolver) ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt) (string, digest.Digest, []byte, error) {
func (imr *imageMetaResolver) ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (string, digest.Digest, []byte, error) {
imr.locker.Lock(ref)
defer imr.locker.Unlock(ref)

platform := opt.Platform
if platform == nil {
platform = imr.platform
platform := imr.platform
if opt.Platform != nil {
platform = opt.Platform
}

k := imr.key(ref, platform)

if res, ok := imr.cache[k]; ok {
return res.ref, res.dgst, res.config, nil
return ref, res.dgst, res.config, nil
}

ref, dgst, config, err := imageutil.Config(ctx, ref, imr.resolver, imr.buffer, nil, platform, opt.SourcePolicies)
dgst, config, err := imageutil.Config(ctx, ref, imr.resolver, imr.buffer, nil, platform)
if err != nil {
return "", "", nil, err
}

imr.cache[k] = resolveResult{dgst: dgst, config: config, ref: ref}
imr.cache[k] = resolveResult{dgst: dgst, config: config}
return ref, dgst, config, nil
}

Expand Down
34 changes: 2 additions & 32 deletions client/llb/resolver.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,7 @@
package llb

import (
"context"

spb "github.com/moby/buildkit/sourcepolicy/pb"
digest "github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/moby/buildkit/client/llb/sourceresolver"
)

// WithMetaResolver adds a metadata resolver to an image
Expand All @@ -31,30 +27,4 @@ func WithLayerLimit(l int) ImageOption {
}

// ImageMetaResolver can resolve image config metadata from a reference
type ImageMetaResolver interface {
ResolveImageConfig(ctx context.Context, ref string, opt ResolveImageConfigOpt) (string, digest.Digest, []byte, error)
}

type ResolverType int

const (
ResolverTypeRegistry ResolverType = iota
ResolverTypeOCILayout
)

type ResolveImageConfigOpt struct {
ResolverType

Platform *ocispecs.Platform
ResolveMode string
LogName string

Store ResolveImageConfigOptStore

SourcePolicies []*spb.Policy
}

type ResolveImageConfigOptStore struct {
SessionID string
StoreID string
}
type ImageMetaResolver = sourceresolver.ImageMetaResolver
3 changes: 2 additions & 1 deletion client/llb/resolver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"testing"

"github.com/containerd/containerd/platforms"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/solver/pb"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
Expand Down Expand Up @@ -74,7 +75,7 @@ type testResolver struct {
platform string
}

func (r *testResolver) ResolveImageConfig(ctx context.Context, ref string, opt ResolveImageConfigOpt) (string, digest.Digest, []byte, error) {
func (r *testResolver) ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (string, digest.Digest, []byte, error) {
var img struct {
Config struct {
Env []string `json:"Env,omitempty"`
Expand Down
19 changes: 11 additions & 8 deletions client/llb/source.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"strings"

"github.com/distribution/reference"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/apicaps"
"github.com/moby/buildkit/util/gitutil"
Expand Down Expand Up @@ -136,10 +137,11 @@ func Image(ref string, opts ...ImageOption) State {
if p == nil {
p = c.Platform
}
_, _, dt, err := info.metaResolver.ResolveImageConfig(ctx, ref, ResolveImageConfigOpt{
Platform: p,
ResolveMode: info.resolveMode.String(),
ResolverType: ResolverTypeRegistry,
_, _, dt, err := info.metaResolver.ResolveImageConfig(ctx, ref, sourceresolver.Opt{
Platform: p,
ImageOpt: &sourceresolver.ResolveImageOpt{
ResolveMode: info.resolveMode.String(),
},
})
if err != nil {
return State{}, err
Expand All @@ -152,10 +154,11 @@ func Image(ref string, opts ...ImageOption) State {
if p == nil {
p = c.Platform
}
ref, dgst, dt, err := info.metaResolver.ResolveImageConfig(context.TODO(), ref, ResolveImageConfigOpt{
Platform: p,
ResolveMode: info.resolveMode.String(),
ResolverType: ResolverTypeRegistry,
ref, dgst, dt, err := info.metaResolver.ResolveImageConfig(context.TODO(), ref, sourceresolver.Opt{
Platform: p,
ImageOpt: &sourceresolver.ResolveImageOpt{
ResolveMode: info.resolveMode.String(),
},
})
if err != nil {
return State{}, err
Expand Down
59 changes: 59 additions & 0 deletions client/llb/sourceresolver/imageresolver.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package sourceresolver

import (
"context"
"strings"

"github.com/distribution/reference"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/imageutil"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
)

type ImageMetaResolver interface {
ResolveImageConfig(ctx context.Context, ref string, opt Opt) (string, digest.Digest, []byte, error)
}

type imageMetaResolver struct {
mr MetaResolver
}

var _ ImageMetaResolver = &imageMetaResolver{}

func NewImageMetaResolver(mr MetaResolver) ImageMetaResolver {
return &imageMetaResolver{
mr: mr,
}
}

func (imr *imageMetaResolver) ResolveImageConfig(ctx context.Context, ref string, opt Opt) (string, digest.Digest, []byte, error) {
parsed, err := reference.ParseNormalizedNamed(ref)
if err != nil {
return "", "", nil, errors.Wrapf(err, "could not parse reference %q", ref)
}
ref = parsed.String()
op := &pb.SourceOp{
Identifier: "docker-image://" + ref,
}
if opt := opt.OCILayoutOpt; opt != nil {
op.Identifier = "oci-layout://" + ref
op.Attrs = map[string]string{}
if opt.Store.SessionID != "" {
op.Attrs[pb.AttrOCILayoutSessionID] = opt.Store.SessionID
}
if opt.Store.StoreID != "" {
op.Attrs[pb.AttrOCILayoutStoreID] = opt.Store.StoreID
}
}
res, err := imr.mr.ResolveSourceMetadata(ctx, op, opt)
if err != nil {
return "", "", nil, errors.Wrapf(err, "failed to resolve source metadata for %s", ref)
}
if res.Image == nil {
return "", "", nil, &imageutil.ResolveToNonImageError{Ref: ref, Updated: res.Op.Identifier}
}
ref = strings.TrimPrefix(res.Op.Identifier, "docker-image://")
ref = strings.TrimPrefix(ref, "oci-layout://")
return ref, res.Image.Digest, res.Image.Config, nil
}
54 changes: 54 additions & 0 deletions client/llb/sourceresolver/types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package sourceresolver

import (
"context"

"github.com/moby/buildkit/solver/pb"
spb "github.com/moby/buildkit/sourcepolicy/pb"
digest "github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
)

type ResolverType int

const (
ResolverTypeRegistry ResolverType = iota
ResolverTypeOCILayout
)

type MetaResolver interface {
ResolveSourceMetadata(ctx context.Context, op *pb.SourceOp, opt Opt) (*MetaResponse, error)
}

type Opt struct {
LogName string
SourcePolicies []*spb.Policy
Platform *ocispecs.Platform

ImageOpt *ResolveImageOpt
OCILayoutOpt *ResolveOCILayoutOpt
}

type MetaResponse struct {
Op *pb.SourceOp

Image *ResolveImageResponse
}

type ResolveImageOpt struct {
ResolveMode string
}

type ResolveImageResponse struct {
Digest digest.Digest
Config []byte
}

type ResolveOCILayoutOpt struct {
Store ResolveImageConfigOptStore
}

type ResolveImageConfigOptStore struct {
SessionID string
StoreID string
}
9 changes: 9 additions & 0 deletions control/gateway/gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,15 @@ func (gwf *GatewayForwarder) ResolveImageConfig(ctx context.Context, req *gwapi.
return fwd.ResolveImageConfig(ctx, req)
}

func (gwf *GatewayForwarder) ResolveSourceMeta(ctx context.Context, req *gwapi.ResolveSourceMetaRequest) (*gwapi.ResolveSourceMetaResponse, error) {
fwd, err := gwf.lookupForwarder(ctx)
if err != nil {
return nil, errors.Wrap(err, "forwarding ResolveSourceMeta")
}

return fwd.ResolveSourceMeta(ctx, req)
}

func (gwf *GatewayForwarder) Solve(ctx context.Context, req *gwapi.SolveRequest) (*gwapi.SolveResponse, error) {
fwd, err := gwf.lookupForwarder(ctx)
if err != nil {
Expand Down
6 changes: 4 additions & 2 deletions frontend/attestations/sbom/sbom.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (

intoto "github.com/in-toto/in-toto-golang/in_toto"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
gatewaypb "github.com/moby/buildkit/frontend/gateway/pb"
"github.com/moby/buildkit/solver/result"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
Expand All @@ -33,12 +34,13 @@ const (
// attestation.
type Scanner func(ctx context.Context, name string, ref llb.State, extras map[string]llb.State, opts ...llb.ConstraintsOpt) (result.Attestation[*llb.State], error)

func CreateSBOMScanner(ctx context.Context, resolver llb.ImageMetaResolver, scanner string, resolveOpt llb.ResolveImageConfigOpt) (Scanner, error) {
func CreateSBOMScanner(ctx context.Context, resolver sourceresolver.MetaResolver, scanner string, resolveOpt sourceresolver.Opt) (Scanner, error) {
if scanner == "" {
return nil, nil
}

scanner, _, dt, err := resolver.ResolveImageConfig(ctx, scanner, resolveOpt)
imr := sourceresolver.NewImageMetaResolver(resolver)
scanner, _, dt, err := imr.ResolveImageConfig(ctx, scanner, resolveOpt)
if err != nil {
return nil, err
}
Expand Down
8 changes: 6 additions & 2 deletions frontend/dockerfile/builder/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (

"github.com/containerd/containerd/platforms"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/exporter/containerimage/image"
"github.com/moby/buildkit/frontend"
"github.com/moby/buildkit/frontend/attestations/sbom"
Expand Down Expand Up @@ -101,8 +102,11 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) {

var scanner sbom.Scanner
if bc.SBOM != nil {
scanner, err = sbom.CreateSBOMScanner(ctx, c, bc.SBOM.Generator, llb.ResolveImageConfigOpt{
ResolveMode: opts["image-resolve-mode"],
// TODO: scanner should pass policy
scanner, err = sbom.CreateSBOMScanner(ctx, c, bc.SBOM.Generator, sourceresolver.Opt{
ImageOpt: &sourceresolver.ResolveImageOpt{
ResolveMode: opts["image-resolve-mode"],
},
})
if err != nil {
return nil, err
Expand Down
13 changes: 7 additions & 6 deletions frontend/dockerfile/dockerfile2llb/convert.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/docker/go-connections/nat"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/imagemetaresolver"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/exporter/containerimage/image"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
"github.com/moby/buildkit/frontend/dockerfile/parser"
Expand Down Expand Up @@ -423,12 +424,12 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
prefix += platforms.Format(*platform) + " "
}
prefix += "internal]"
mutRef, dgst, dt, err := metaResolver.ResolveImageConfig(ctx, d.stage.BaseName, llb.ResolveImageConfigOpt{
Platform: platform,
ResolveMode: opt.ImageResolveMode.String(),
LogName: fmt.Sprintf("%s load metadata for %s", prefix, d.stage.BaseName),
ResolverType: llb.ResolverTypeRegistry,
SourcePolicies: nil,
mutRef, dgst, dt, err := metaResolver.ResolveImageConfig(ctx, d.stage.BaseName, sourceresolver.Opt{
LogName: fmt.Sprintf("%s load metadata for %s", prefix, d.stage.BaseName),
Platform: platform,
ImageOpt: &sourceresolver.ResolveImageOpt{
ResolveMode: opt.ImageResolveMode.String(),
},
})
if err != nil {
return suggest.WrapError(errors.Wrap(err, origName), origName, append(allStageNames, commonImageNames()...), true)
Expand Down