Skip to content

Commit

Permalink
Merge pull request #1494 from tonistiigi/errdefs2
Browse files Browse the repository at this point in the history 
llb: add source tracking support
  • Loading branch information
@AkihiroSuda
AkihiroSuda committed May 25, 2020
2 parents 7f4214f + fbee6cc commit d6f5e97
Show file tree
Hide file tree
Showing 28 changed files with 2,506 additions and 506 deletions.
124 changes: 124 additions & 0 deletions client/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ import (
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/secrets/secretsprovider"
"github.com/moby/buildkit/session/sshforward/sshprovider"
"github.com/moby/buildkit/solver/errdefs"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/contentutil"
"github.com/moby/buildkit/util/entitlements"
"github.com/moby/buildkit/util/testutil"
Expand Down Expand Up @@ -113,6 +115,8 @@ func TestIntegration(t *testing.T) {
testTarExporterWithSocket,
testTarExporterSymlink,
testMultipleRegistryCacheImportExport,
testSourceMap,
testSourceMapFromRef,
}, mirrors)

integration.Run(t, []integration.Test{
Expand Down Expand Up @@ -2681,6 +2685,126 @@ func testReadonlyRootFS(t *testing.T, sb integration.Sandbox) {
checkAllReleasable(t, c, sb, true)
}

func testSourceMap(t *testing.T, sb integration.Sandbox) {
c, err := New(context.TODO(), sb.Address())
require.NoError(t, err)
defer c.Close()

sm1 := llb.NewSourceMap(nil, "foo", []byte("data1"))
sm2 := llb.NewSourceMap(nil, "bar", []byte("data2"))

st := llb.Scratch().Run(
llb.Shlex("not-exist"),
sm1.Location([]*pb.Range{{Start: pb.Position{Line: 7}}}),
sm2.Location([]*pb.Range{{Start: pb.Position{Line: 8}}}),
sm1.Location([]*pb.Range{{Start: pb.Position{Line: 9}}}),
)

def, err := st.Marshal(context.TODO())
require.NoError(t, err)

_, err = c.Solve(context.TODO(), def, SolveOpt{}, nil)
require.Error(t, err)

srcs := errdefs.Sources(err)
require.Equal(t, 3, len(srcs))

// Source errors are wrapped in the order provided as llb.ConstraintOpts, so
// when they are unwrapped, the first unwrapped error is the last location
// provided.
require.Equal(t, "foo", srcs[0].Info.Filename)
require.Equal(t, []byte("data1"), srcs[0].Info.Data)
require.Nil(t, srcs[0].Info.Definition)

require.Equal(t, 1, len(srcs[0].Ranges))
require.Equal(t, int32(9), srcs[0].Ranges[0].Start.Line)
require.Equal(t, int32(0), srcs[0].Ranges[0].Start.Character)

require.Equal(t, "bar", srcs[1].Info.Filename)
require.Equal(t, []byte("data2"), srcs[1].Info.Data)
require.Nil(t, srcs[1].Info.Definition)

require.Equal(t, 1, len(srcs[1].Ranges))
require.Equal(t, int32(8), srcs[1].Ranges[0].Start.Line)
require.Equal(t, int32(0), srcs[1].Ranges[0].Start.Character)

require.Equal(t, "foo", srcs[2].Info.Filename)
require.Equal(t, []byte("data1"), srcs[2].Info.Data)
require.Nil(t, srcs[2].Info.Definition)

require.Equal(t, 1, len(srcs[2].Ranges))
require.Equal(t, int32(7), srcs[2].Ranges[0].Start.Line)
require.Equal(t, int32(0), srcs[2].Ranges[0].Start.Character)

}

func testSourceMapFromRef(t *testing.T, sb integration.Sandbox) {
requiresLinux(t)
c, err := New(context.TODO(), sb.Address())
require.NoError(t, err)
defer c.Close()

srcState := llb.Scratch().File(
llb.Mkfile("foo", 0600, []byte("data")))
sm := llb.NewSourceMap(&srcState, "bar", []byte("bardata"))

frontend := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
st := llb.Scratch().File(
llb.Mkdir("foo/bar", 0600), //fails because /foo doesn't exist
sm.Location([]*pb.Range{{Start: pb.Position{Line: 3, Character: 1}}}),
)

def, err := st.Marshal(context.TODO())
if err != nil {
return nil, err
}

res, err := c.Solve(ctx, gateway.SolveRequest{
Definition: def.ToPB(),
})
if err != nil {
return nil, err
}

ref, err := res.SingleRef()
if err != nil {
return nil, err
}

st2, err := ref.ToState()
if err != nil {
return nil, err
}

st = llb.Scratch().File(
llb.Copy(st2, "foo", "foo2"),
)

def, err = st.Marshal(context.TODO())
if err != nil {
return nil, err
}

return c.Solve(ctx, gateway.SolveRequest{
Definition: def.ToPB(),
})
}

_, err = c.Build(context.TODO(), SolveOpt{}, "", frontend, nil)
require.Error(t, err)

srcs := errdefs.Sources(err)
require.Equal(t, 1, len(srcs))

require.Equal(t, "bar", srcs[0].Info.Filename)
require.Equal(t, []byte("bardata"), srcs[0].Info.Data)
require.NotNil(t, srcs[0].Info.Definition)

require.Equal(t, 1, len(srcs[0].Ranges))
require.Equal(t, int32(3), srcs[0].Ranges[0].Start.Line)
require.Equal(t, int32(1), srcs[0].Ranges[0].Start.Character)
}

func testProxyEnv(t *testing.T, sb integration.Sandbox) {
c, err := New(context.TODO(), sb.Address())
require.NoError(t, err)
Expand Down
4 changes: 2 additions & 2 deletions client/llb/async.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,8 @@ type errVertex struct {
func (v *errVertex) Validate(context.Context) error {
return v.err
}
func (v *errVertex) Marshal(context.Context, *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
return "", nil, nil, v.err
func (v *errVertex) Marshal(context.Context, *Constraints) (digest.Digest, []byte, *pb.OpMetadata, []*SourceLocation, error) {
return "", nil, nil, nil, v.err
}
func (v *errVertex) Output() Output {
return nil
Expand Down
42 changes: 38 additions & 4 deletions client/llb/definition.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ type DefinitionOp struct {
ops map[digest.Digest]*pb.Op
defs map[digest.Digest][]byte
metas map[digest.Digest]pb.OpMetadata
sources map[digest.Digest][]*SourceLocation
platforms map[digest.Digest]*specs.Platform
dgst digest.Digest
index pb.OutputIndex
Expand Down Expand Up @@ -49,6 +50,38 @@ func NewDefinitionOp(def *pb.Definition) (*DefinitionOp, error) {
platforms[dgst] = platform
}

srcs := map[digest.Digest][]*SourceLocation{}

if def.Source != nil {
sourceMaps := make([]*SourceMap, len(def.Source.Infos))
for i, info := range def.Source.Infos {
var st *State
sdef := info.Definition
if sdef != nil {
op, err := NewDefinitionOp(sdef)
if err != nil {
return nil, err
}
state := NewState(op)
st = &state
}
sourceMaps[i] = NewSourceMap(st, info.Filename, info.Data)
}

for dgst, locs := range def.Source.Locations {
for _, loc := range locs.Locations {
if loc.SourceIndex < 0 || int(loc.SourceIndex) >= len(sourceMaps) {
return nil, errors.Errorf("failed to find source map with index %d", loc.SourceIndex)
}

srcs[digest.Digest(dgst)] = append(srcs[digest.Digest(dgst)], &SourceLocation{
SourceMap: sourceMaps[int(loc.SourceIndex)],
Ranges: loc.Ranges,
})
}
}
}

var index pb.OutputIndex
if dgst != "" {
index = ops[dgst].Inputs[0].Index
Expand All @@ -59,6 +92,7 @@ func NewDefinitionOp(def *pb.Definition) (*DefinitionOp, error) {
ops: ops,
defs: defs,
metas: def.Metadata,
sources: srcs,
platforms: platforms,
dgst: dgst,
index: index,
Expand Down Expand Up @@ -110,20 +144,20 @@ func (d *DefinitionOp) Validate(context.Context) error {
return nil
}

func (d *DefinitionOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
func (d *DefinitionOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, []*SourceLocation, error) {
if d.dgst == "" {
return "", nil, nil, errors.Errorf("cannot marshal empty definition op")
return "", nil, nil, nil, errors.Errorf("cannot marshal empty definition op")
}

if err := d.Validate(ctx); err != nil {
return "", nil, nil, err
return "", nil, nil, nil, err
}

d.mu.Lock()
defer d.mu.Unlock()

meta := d.metas[d.dgst]
return d.dgst, d.defs[d.dgst], &meta, nil
return d.dgst, d.defs[d.dgst], &meta, d.sources[d.dgst], nil

}

Expand Down
30 changes: 15 additions & 15 deletions client/llb/exec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ func (e *ExecOp) AddMount(target string, source Output, opt ...MountOption) Outp
}
m.output = o
}
e.Store(nil, nil, nil)
e.Store(nil, nil, nil, nil)
e.isValidated = false
return m.output
}
Expand Down Expand Up @@ -124,12 +124,12 @@ func (e *ExecOp) Validate(ctx context.Context) error {
return nil
}

func (e *ExecOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
func (e *ExecOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, []*SourceLocation, error) {
if e.Cached(c) {
return e.Load()
}
if err := e.Validate(ctx); err != nil {
return "", nil, nil, err
return "", nil, nil, nil, err
}
// make sure mounts are sorted
sort.Slice(e.mounts, func(i, j int) bool {
Expand All @@ -138,7 +138,7 @@ func (e *ExecOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []

env, err := getEnv(e.base)(ctx)
if err != nil {
return "", nil, nil, err
return "", nil, nil, nil, err
}

if len(e.ssh) > 0 {
Expand All @@ -161,17 +161,17 @@ func (e *ExecOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []

args, err := getArgs(e.base)(ctx)
if err != nil {
return "", nil, nil, err
return "", nil, nil, nil, err
}

cwd, err := getDir(e.base)(ctx)
if err != nil {
return "", nil, nil, err
return "", nil, nil, nil, err
}

user, err := getUser(e.base)(ctx)
if err != nil {
return "", nil, nil, err
return "", nil, nil, nil, err
}

meta := &pb.Meta{
Expand All @@ -182,7 +182,7 @@ func (e *ExecOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []
}
extraHosts, err := getExtraHosts(e.base)(ctx)
if err != nil {
return "", nil, nil, err
return "", nil, nil, nil, err
}
if len(extraHosts) > 0 {
hosts := make([]*pb.HostIP, len(extraHosts))
Expand All @@ -194,12 +194,12 @@ func (e *ExecOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []

network, err := getNetwork(e.base)(ctx)
if err != nil {
return "", nil, nil, err
return "", nil, nil, nil, err
}

security, err := getSecurity(e.base)(ctx)
if err != nil {
return "", nil, nil, err
return "", nil, nil, nil, err
}

peo := &pb.ExecOp{
Expand Down Expand Up @@ -252,7 +252,7 @@ func (e *ExecOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []
if e.constraints.Platform == nil {
p, err := getPlatform(e.base)(ctx)
if err != nil {
return "", nil, nil, err
return "", nil, nil, nil, err
}
e.constraints.Platform = p
}
Expand All @@ -267,11 +267,11 @@ func (e *ExecOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []
inputIndex := pb.InputIndex(len(pop.Inputs))
if m.source != nil {
if m.tmpfs {
return "", nil, nil, errors.Errorf("tmpfs mounts must use scratch")
return "", nil, nil, nil, errors.Errorf("tmpfs mounts must use scratch")
}
inp, err := m.source.ToInput(ctx, c)
if err != nil {
return "", nil, nil, err
return "", nil, nil, nil, err
}

newInput := true
Expand Down Expand Up @@ -356,9 +356,9 @@ func (e *ExecOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []

dt, err := pop.Marshal()
if err != nil {
return "", nil, nil, err
return "", nil, nil, nil, err
}
e.Store(dt, md, c)
e.Store(dt, md, e.constraints.SourceLocations, c)
return e.Load()
}

Expand Down
Loading

0 comments on commit d6f5e97

Please sign in to comment.