Merge pull request #442 from tonistiigi/runmount

dockerfile: add run --mount support

client/client_test.go

@@ -949,8 +949,7 @@ func testMountWithNoSource(t *testing.T, sb integration.Sandbox) {
 	require.NoError(t, err)
 
 	_, err = c.Solve(context.TODO(), def, SolveOpt{}, nil)
-	require.Error(t, err)
-	require.Contains(t, err.Error(), "has no input")
+	require.NoError(t, err)
 
 	checkAllReleasable(t, c, sb, true)
 }

frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile

@@ -1,6 +1,7 @@
 FROM golang:1.10-alpine AS builder
+ARG BUILDTAGS=""
 COPY . /go/src/github.com/moby/buildkit
-RUN CGO_ENABLED=0 go build -o /dockerfile-frontend --ldflags '-extldflags "-static"' github.com/moby/buildkit/frontend/dockerfile/cmd/dockerfile-frontend
+RUN CGO_ENABLED=0 go build -o /dockerfile-frontend -tags "$BUILDTAGS" --ldflags '-extldflags "-static"' github.com/moby/buildkit/frontend/dockerfile/cmd/dockerfile-frontend
 
 FROM scratch
 COPY --from=builder /dockerfile-frontend /bin/dockerfile-frontend

frontend/dockerfile/dockerfile2llb/convert.go

@@ -44,6 +44,8 @@ type ConvertOpt struct {
 	// IgnoreCache contains names of the stages that should not use build cache.
 	// Empty slice means ignore cache for all stages. Nil doesn't disable cache.
 	IgnoreCache []string
+	// CacheIDNamespace scopes the IDs for different cache mounts
+	CacheIDNamespace string
 }
 
 func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, *Image, error) {
@@ -125,15 +127,17 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 	for _, d := range allDispatchStates {
 		d.commands = make([]command, len(d.stage.Commands))
 		for i, cmd := range d.stage.Commands {
-			newCmd, created, err := toCommand(cmd, dispatchStatesByName, allDispatchStates)
+			newCmd, err := toCommand(cmd, dispatchStatesByName, allDispatchStates)
 			if err != nil {
 				return nil, nil, err
 			}
 			d.commands[i] = newCmd
-			if newCmd.copySource != nil {
-				d.deps[newCmd.copySource] = struct{}{}
-				if created {
-					allDispatchStates = append(allDispatchStates, newCmd.copySource)
+			for _, src := range newCmd.sources {
+				if src != nil {
+					d.deps[src] = struct{}{}
+					if src.unregistered {
+						allDispatchStates = append(allDispatchStates, src)
+					}
 				}
 			}
 		}
@@ -237,6 +241,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 			sessionID:            opt.SessionID,
 			buildContext:         llb.NewState(buildContext),
 			proxyEnv:             proxyEnv,
+			cacheIDNamespace:     opt.CacheIDNamespace,
 		}
 
 		if err = dispatchOnBuild(d, d.image.Config.OnBuild, opt); err != nil {
@@ -278,9 +283,8 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 	return &target.state, &target.image, nil
 }
 
-func toCommand(ic instructions.Command, dispatchStatesByName map[string]*dispatchState, allDispatchStates []*dispatchState) (command, bool, error) {
+func toCommand(ic instructions.Command, dispatchStatesByName map[string]*dispatchState, allDispatchStates []*dispatchState) (command, error) {
 	cmd := command{Command: ic}
-	created := false
 	if c, ok := ic.(*instructions.CopyCommand); ok {
 		if c.From != "" {
 			var stn *dispatchState
@@ -289,21 +293,26 @@ func toCommand(ic instructions.Command, dispatchStatesByName map[string]*dispatc
 				stn, ok = dispatchStatesByName[strings.ToLower(c.From)]
 				if !ok {
 					stn = &dispatchState{
-						stage: instructions.Stage{BaseName: c.From},
-						deps:  make(map[*dispatchState]struct{}),
+						stage:        instructions.Stage{BaseName: c.From},
+						deps:         make(map[*dispatchState]struct{}),
+						unregistered: true,
 					}
-					created = true
 				}
 			} else {
 				if index < 0 || index >= len(allDispatchStates) {
-					return command{}, false, errors.Errorf("invalid stage index %d", index)
+					return command{}, errors.Errorf("invalid stage index %d", index)
 				}
 				stn = allDispatchStates[index]
 			}
-			cmd.copySource = stn
+			cmd.sources = []*dispatchState{stn}
 		}
 	}
-	return cmd, created, nil
+
+	if ok := detectRunMount(&cmd, dispatchStatesByName, allDispatchStates); ok {
+		return cmd, nil
+	}
+
+	return cmd, nil
 }
 
 type dispatchOpt struct {
@@ -315,6 +324,7 @@ type dispatchOpt struct {
 	sessionID            string
 	buildContext         llb.State
 	proxyEnv             *llb.ProxyEnv
+	cacheIDNamespace     string
 }
 
 func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
@@ -334,7 +344,7 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
 	case *instructions.EnvCommand:
 		err = dispatchEnv(d, c, true)
 	case *instructions.RunCommand:
-		err = dispatchRun(d, c, opt.proxyEnv)
+		err = dispatchRun(d, c, opt.proxyEnv, cmd.sources, opt)
 	case *instructions.WorkdirCommand:
 		err = dispatchWorkdir(d, c, true)
 	case *instructions.AddCommand:
@@ -368,11 +378,11 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
 		err = dispatchArg(d, c, opt.metaArgs, opt.buildArgValues)
 	case *instructions.CopyCommand:
 		l := opt.buildContext
-		if cmd.copySource != nil {
-			l = cmd.copySource.state
+		if len(cmd.sources) != 0 {
+			l = cmd.sources[0].state
 		}
 		err = dispatchCopy(d, c.SourcesAndDest, l, false, c, c.Chown)
-		if err == nil && cmd.copySource == nil {
+		if err == nil && len(cmd.sources) == 0 {
 			for _, src := range c.Sources() {
 				d.ctxPaths[path.Join("/", filepath.ToSlash(src))] = struct{}{}
 			}
@@ -383,21 +393,22 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
 }
 
 type dispatchState struct {
-	state       llb.State
-	image       Image
-	stage       instructions.Stage
-	base        *dispatchState
-	deps        map[*dispatchState]struct{}
-	buildArgs   []instructions.ArgCommand
-	commands    []command
-	ctxPaths    map[string]struct{}
-	ignoreCache bool
-	cmdSet      bool
+	state        llb.State
+	image        Image
+	stage        instructions.Stage
+	base         *dispatchState
+	deps         map[*dispatchState]struct{}
+	buildArgs    []instructions.ArgCommand
+	commands     []command
+	ctxPaths     map[string]struct{}
+	ignoreCache  bool
+	cmdSet       bool
+	unregistered bool
 }
 
 type command struct {
 	instructions.Command
-	copySource *dispatchState
+	sources []*dispatchState
 }
 
 func dispatchOnBuild(d *dispatchState, triggers []string, opt dispatchOpt) error {
@@ -413,7 +424,7 @@ func dispatchOnBuild(d *dispatchState, triggers []string, opt dispatchOpt) error
 		if err != nil {
 			return err
 		}
-		cmd, _, err := toCommand(ic, opt.dispatchStatesByName, opt.allDispatchStates)
+		cmd, err := toCommand(ic, opt.dispatchStatesByName, opt.allDispatchStates)
 		if err != nil {
 			return err
 		}
@@ -437,7 +448,7 @@ func dispatchEnv(d *dispatchState, c *instructions.EnvCommand, commit bool) erro
 	return nil
 }
 
-func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyEnv) error {
+func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyEnv, sources []*dispatchState, dopt dispatchOpt) error {
 	var args []string = c.CmdLine
 	if c.PrependShell {
 		args = withShell(d.image, args)
@@ -455,6 +466,9 @@ func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyE
 	if proxy != nil {
 		opt = append(opt, llb.WithProxy(*proxy))
 	}
+
+	opt = append(opt, dispatchRunMounts(d, c, sources, dopt)...)
+
 	d.state = d.state.Run(opt...).Root()
 	return commitToHistory(&d.image, "RUN "+runCommandString(args, d.buildArgs), true, &d.state)
 }

frontend/dockerfile/dockerfile2llb/convert_norunmount.go

@@ -0,0 +1,16 @@
+// +build !dfrunmount,!dfextall
+
+package dockerfile2llb
+
+import (
+	"github.com/moby/buildkit/client/llb"
+	"github.com/moby/buildkit/frontend/dockerfile/instructions"
+)
+
+func detectRunMount(cmd *command, dispatchStatesByName map[string]*dispatchState, allDispatchStates []*dispatchState) bool {
+	return false
+}
+
+func dispatchRunMounts(d *dispatchState, c *instructions.RunCommand, sources []*dispatchState, opt dispatchOpt) []llb.RunOption {
+	return nil
+}

frontend/dockerfile/dockerfile2llb/convert_runmount.go

@@ -0,0 +1,70 @@
+// +build dfrunmount dfextall
+
+package dockerfile2llb
+
+import (
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/moby/buildkit/client/llb"
+	"github.com/moby/buildkit/frontend/dockerfile/instructions"
+)
+
+func detectRunMount(cmd *command, dispatchStatesByName map[string]*dispatchState, allDispatchStates []*dispatchState) bool {
+	if c, ok := cmd.Command.(*instructions.RunCommand); ok {
+		mounts := instructions.GetMounts(c)
+		sources := make([]*dispatchState, len(mounts))
+		for i, mount := range mounts {
+			if mount.From == "" && mount.Type == "cache" {
+				mount.From = emptyImageName
+			}
+			from := mount.From
+			if from == "" {
+				continue
+			}
+			stn, ok := dispatchStatesByName[strings.ToLower(from)]
+			if !ok {
+				stn = &dispatchState{
+					stage:        instructions.Stage{BaseName: from},
+					deps:         make(map[*dispatchState]struct{}),
+					unregistered: true,
+				}
+			}
+			sources[i] = stn
+		}
+		cmd.sources = sources
+		return true
+	}
+
+	return false
+}
+
+func dispatchRunMounts(d *dispatchState, c *instructions.RunCommand, sources []*dispatchState, opt dispatchOpt) []llb.RunOption {
+	var out []llb.RunOption
+	mounts := instructions.GetMounts(c)
+
+	for i, mount := range mounts {
+		if mount.From == "" && mount.Type == "cache" {
+			mount.From = emptyImageName
+		}
+		st := opt.buildContext
+		if mount.From != "" {
+			st = sources[i].state
+		}
+		var mountOpts []llb.MountOption
+		if mount.ReadOnly {
+			mountOpts = append(mountOpts, llb.Readonly)
+		}
+		if mount.Type == "cache" {
+			mountOpts = append(mountOpts, llb.AsPersistentCacheDir(opt.cacheIDNamespace+"/"+mount.CacheID))
+		}
+		if src := path.Join("/", mount.Source); src != "/" {
+			mountOpts = append(mountOpts, llb.SourcePath(src))
+		}
+		out = append(out, llb.AddMount(path.Join("/", mount.Target), st, mountOpts...))
+
+		d.ctxPaths[path.Join("/", filepath.ToSlash(mount.Source))] = struct{}{}
+	}
+	return out
+}

frontend/dockerfile/instructions/bflag.go

@@ -11,6 +11,7 @@ type FlagType int
 const (
 	boolType FlagType = iota
 	stringType
+	stringsType
 )
 
 // BFlags contains all flags information for the builder
@@ -23,10 +24,11 @@ type BFlags struct {
 
 // Flag contains all information for a flag
 type Flag struct {
-	bf       *BFlags
-	name     string
-	flagType FlagType
-	Value    string
+	bf           *BFlags
+	name         string
+	flagType     FlagType
+	Value        string
+	StringValues []string
 }
 
 // NewBFlags returns the new BFlags struct
@@ -70,6 +72,15 @@ func (bf *BFlags) AddString(name string, def string) *Flag {
 	return flag
 }
 
+// AddString adds a string flag to BFlags that can match multiple values
+func (bf *BFlags) AddStrings(name string) *Flag {
+	flag := bf.addFlag(name, stringsType)
+	if flag == nil {
+		return nil
+	}
+	return flag
+}
+
 // addFlag is a generic func used by the other AddXXX() func
 // to add a new flag to the BFlags struct.
 // Note, any error will be generated when Parse() is called (see Parse).
@@ -145,7 +156,7 @@ func (bf *BFlags) Parse() error {
 			return fmt.Errorf("Unknown flag: %s", arg)
 		}
 
-		if _, ok = bf.used[arg]; ok {
+		if _, ok = bf.used[arg]; ok && flag.flagType != stringsType {
 			return fmt.Errorf("Duplicate flag specified: %s", arg)
 		}
 
@@ -173,6 +184,12 @@ func (bf *BFlags) Parse() error {
 			}
 			flag.Value = value
 
+		case stringsType:
+			if index < 0 {
+				return fmt.Errorf("Missing a value on flag: %s", arg)
+			}
+			flag.StringValues = append(flag.StringValues, value)
+
 		default:
 			panic("No idea what kind of flag we have! Should never get here!")
 		}

frontend/dockerfile/instructions/commands.go

@@ -233,6 +233,7 @@ type ShellDependantCmdLine struct {
 //
 type RunCommand struct {
 	withNameAndCode
+	withExternalData
 	ShellDependantCmdLine
 }
 
@@ -416,3 +417,18 @@ func HasStage(s []Stage, name string) (int, bool) {
 	}
 	return -1, false
 }
+
+type withExternalData struct {
+	m map[interface{}]interface{}
+}
+
+func (c *withExternalData) getExternalValue(k interface{}) interface{} {
+	return c.m[k]
+}
+
+func (c *withExternalData) setExternalValue(k, v interface{}) {
+	if c.m == nil {
+		c.m = map[interface{}]interface{}{}
+	}
+	c.m[k] = v
+}