build: Add support for buildkit like --mount=type=bind

Following commit adds support for using buildkit like
`--mount=type=bind` with `RUN` statements. Mounts created by `--mount`
are transient in nature and only scoped to current RUN statements.

Signed-off-by: Aditya Rajan <arajan@redhat.com>

cmd/buildah/run.go

@@ -17,6 +17,7 @@ type runInputOptions struct {
 	addHistory  bool
 	capAdd      []string
 	capDrop     []string
+	contextDir  string
 	env         []string
 	hostname    string
 	isolation   string
@@ -58,6 +59,7 @@ func init() {
 	flags.BoolVar(&opts.addHistory, "add-history", false, "add an entry for this operation to the image's history.  Use BUILDAH_HISTORY environment variable to override. (default false)")
 	flags.StringSliceVar(&opts.capAdd, "cap-add", []string{}, "add the specified capability (default [])")
 	flags.StringSliceVar(&opts.capDrop, "cap-drop", []string{}, "drop the specified capability (default [])")
+	flags.StringVar(&opts.contextDir, "contextdir", "", "context directory path")
 	flags.StringArrayVarP(&opts.env, "env", "e", []string{}, "add environment variable to be set temporarily when running command (default [])")
 	flags.StringVar(&opts.hostname, "hostname", "", "set the hostname inside of the container")
 	flags.StringVar(&opts.isolation, "isolation", "", "`type` of process isolation to use. Use BUILDAH_ISOLATION environment variable to override.")
@@ -130,6 +132,7 @@ func runCmd(c *cobra.Command, args []string, iopts runInputOptions) error {
 		Isolation:        isolation,
 		NamespaceOptions: namespaceOptions,
 		ConfigureNetwork: networkPolicy,
+		ContextDir:       iopts.contextDir,
 		CNIPluginPath:    iopts.CNIPlugInPath,
 		CNIConfigDir:     iopts.CNIConfigDir,
 		AddCapabilities:  iopts.capAdd,
@@ -146,7 +149,7 @@ func runCmd(c *cobra.Command, args []string, iopts runInputOptions) error {
 		}
 	}
 
-	mounts, err := parse.GetVolumes(iopts.volumes, iopts.mounts)
+	mounts, err := parse.GetVolumes(iopts.volumes, iopts.mounts, iopts.contextDir)
 	if err != nil {
 		return err
 	}

docs/buildah-run.1.md

@@ -58,6 +58,12 @@ not disabled.
 List of directories in which the CNI plugins which will be used for configuring
 network namespaces can be found.
 
+**--contextdir** *directory*
+
+Allows setting context directory for current RUN invocation. Specifying a context
+directory causes RUN context to consider context directory as root directory for
+specified source in `--mount` of type 'bind'.
+
 **--env**, **-e** *env=value*
 
 Temporarily add a value (e.g. env=*value*) to the environment for the running

imagebuildah/stage_executor.go

@@ -439,6 +439,7 @@ func (s *StageExecutor) Run(run imagebuilder.Run, config docker.Config) error {
 		User:             config.User,
 		WorkingDir:       config.WorkingDir,
 		Entrypoint:       config.Entrypoint,
+		ContextDir:       s.executor.contextDir,
 		Cmd:              config.Cmd,
 		Stdin:            stdin,
 		Stdout:           s.executor.out,

pkg/parse/parse.go

@@ -257,8 +257,8 @@ func getVolumeMounts(volumes []string) (map[string]specs.Mount, error) {
 }
 
 // GetVolumes gets the volumes from --volume and --mount
-func GetVolumes(volumes []string, mounts []string) ([]specs.Mount, error) {
-	unifiedMounts, err := getMounts(mounts)
+func GetVolumes(volumes []string, mounts []string, contextDir string) ([]specs.Mount, error) {
+	unifiedMounts, err := getMounts(mounts, contextDir)
 	if err != nil {
 		return nil, err
 	}
@@ -284,7 +284,7 @@ func GetVolumes(volumes []string, mounts []string) ([]specs.Mount, error) {
 // spec mounts.
 // buildah run --mount type=bind,src=/etc/resolv.conf,target=/etc/resolv.conf ...
 // buildah run --mount type=tmpfs,target=/dev/shm ...
-func getMounts(mounts []string) (map[string]specs.Mount, error) {
+func getMounts(mounts []string, contextDir string) (map[string]specs.Mount, error) {
 	finalMounts := make(map[string]specs.Mount)
 
 	errInvalidSyntax := errors.Errorf("incorrect mount format: should be --mount type=<bind|tmpfs>,[src=<host-dir>,]target=<ctr-dir>[,options]")
@@ -307,7 +307,7 @@ func getMounts(mounts []string) (map[string]specs.Mount, error) {
 		tokens := strings.Split(arr[1], ",")
 		switch kv[1] {
 		case TypeBind:
-			mount, err := GetBindMount(tokens)
+			mount, err := GetBindMount(tokens, contextDir)
 			if err != nil {
 				return nil, err
 			}
@@ -333,27 +333,30 @@ func getMounts(mounts []string) (map[string]specs.Mount, error) {
 }
 
 // GetBindMount parses a single bind mount entry from the --mount flag.
-func GetBindMount(args []string) (specs.Mount, error) {
+func GetBindMount(args []string, contextDir string) (specs.Mount, error) {
 	newMount := specs.Mount{
 		Type: TypeBind,
 	}
 
-	setSource := false
 	setDest := false
+	bindNonRecursive := false
 
 	for _, val := range args {
 		kv := strings.SplitN(val, "=", 2)
 		switch kv[0] {
 		case "bind-nonrecursive":
 			newMount.Options = append(newMount.Options, "bind")
+			bindNonRecursive = true
 		case "ro", "nosuid", "nodev", "noexec":
 			// TODO: detect duplication of these options.
 			// (Is this necessary?)
 			newMount.Options = append(newMount.Options, kv[0])
+		case "rw", "readwrite":
+			newMount.Options = append(newMount.Options, "rw")
 		case "readonly":
 			// Alias for "ro"
 			newMount.Options = append(newMount.Options, "ro")
-		case "shared", "rshared", "private", "rprivate", "slave", "rslave", "Z", "z":
+		case "shared", "rshared", "private", "rprivate", "slave", "rslave", "Z", "z", "U":
 			newMount.Options = append(newMount.Options, kv[0])
 		case "bind-propagation":
 			if len(kv) == 1 {
@@ -368,7 +371,6 @@ func GetBindMount(args []string) (specs.Mount, error) {
 				return newMount, err
 			}
 			newMount.Source = kv[1]
-			setSource = true
 		case "target", "dst", "destination":
 			if len(kv) == 1 {
 				return newMount, errors.Wrapf(optionArgError, kv[0])
@@ -387,12 +389,20 @@ func GetBindMount(args []string) (specs.Mount, error) {
 		}
 	}
 
+	// buildkit parity: default bind option must be `rbind`
+	// unless specified
+	if !bindNonRecursive {
+		newMount.Options = append(newMount.Options, "rbind")
+	}
+
 	if !setDest {
 		return newMount, noDestError
 	}
 
-	if !setSource {
-		newMount.Source = newMount.Destination
+	// buildkit parity: support absolute path for sources from current build context
+	if strings.HasPrefix(newMount.Source, ".") || newMount.Source == "" || !filepath.IsAbs(newMount.Source) {
+		// path should be /contextDir/specified path
+		newMount.Source = filepath.Join(contextDir, filepath.Clean(string(filepath.Separator)+newMount.Source))
 	}
 
 	opts, err := parse.ValidateVolumeOpts(newMount.Options)

run.go

@@ -93,6 +93,8 @@ type RunOptions struct {
 	User string
 	// WorkingDir is an override for the working directory.
 	WorkingDir string
+	// ContextDir is used as the root directory for the source location for mounts that are of type "bind".
+	ContextDir string
 	// Shell is default shell to run in a container.
 	Shell string
 	// Cmd is an override for the configured default command.

run_linux.go

@@ -27,6 +27,7 @@ import (
 	"github.com/containers/buildah/copier"
 	"github.com/containers/buildah/define"
 	"github.com/containers/buildah/pkg/overlay"
+	"github.com/containers/buildah/pkg/parse"
 	"github.com/containers/buildah/pkg/sshagent"
 	"github.com/containers/buildah/util"
 	"github.com/containers/common/pkg/capabilities"
@@ -248,7 +249,7 @@ rootless=%d
 
 		bindFiles["/run/.containerenv"] = containerenvPath
 	}
-	runArtifacts, err := b.setupMounts(mountPoint, spec, path, options.Mounts, bindFiles, volumes, b.CommonBuildOpts.Volumes, b.CommonBuildOpts.ShmSize, namespaceOptions, options.Secrets, options.SSHSources, options.RunMounts)
+	runArtifacts, err := b.setupMounts(mountPoint, spec, path, options.Mounts, bindFiles, volumes, b.CommonBuildOpts.Volumes, b.CommonBuildOpts.ShmSize, namespaceOptions, options.Secrets, options.SSHSources, options.RunMounts, options.ContextDir)
 	if err != nil {
 		return errors.Wrapf(err, "error resolving mountpoints for container %q", b.ContainerID)
 	}
@@ -414,7 +415,7 @@ func runSetupBuiltinVolumes(mountLabel, mountPoint, containerDir string, builtin
 	return mounts, nil
 }
 
-func (b *Builder) setupMounts(mountPoint string, spec *specs.Spec, bundlePath string, optionMounts []specs.Mount, bindFiles map[string]string, builtinVolumes, volumeMounts []string, shmSize string, namespaceOptions define.NamespaceOptions, secrets map[string]string, sshSources map[string]*sshagent.Source, runFileMounts []string) (*runMountArtifacts, error) {
+func (b *Builder) setupMounts(mountPoint string, spec *specs.Spec, bundlePath string, optionMounts []specs.Mount, bindFiles map[string]string, builtinVolumes, volumeMounts []string, shmSize string, namespaceOptions define.NamespaceOptions, secrets map[string]string, sshSources map[string]*sshagent.Source, runFileMounts []string, contextDir string) (*runMountArtifacts, error) {
 	// Start building a new list of mounts.
 	var mounts []specs.Mount
 	haveMount := func(destination string) bool {
@@ -518,12 +519,18 @@ func (b *Builder) setupMounts(mountPoint string, spec *specs.Spec, bundlePath st
 		return nil, err
 	}
 
+	// Get host UID and GID of the container process.
+	processUID, processGID, err := util.GetHostIDs(spec.Linux.UIDMappings, spec.Linux.GIDMappings, spec.Process.User.UID, spec.Process.User.GID)
+	if err != nil {
+		return nil, err
+	}
+
 	// Get the list of subscriptions mounts.
 	subscriptionMounts := subscriptions.MountsWithUIDGID(b.MountLabel, cdir, b.DefaultMountsFilePath, mountPoint, int(rootUID), int(rootGID), unshare.IsRootless(), false)
 
 	// Get the list of mounts that are just for this Run() call.
 	// TODO: acui: de-spaghettify run mounts
-	runMounts, mountArtifacts, err := b.runSetupRunMounts(runFileMounts, secrets, sshSources, b.MountLabel, cdir, spec.Linux.UIDMappings, spec.Linux.GIDMappings, b.ProcessLabel)
+	runMounts, mountArtifacts, err := b.runSetupRunMounts(runFileMounts, secrets, sshSources, cdir, contextDir, spec.Linux.UIDMappings, spec.Linux.GIDMappings, int(rootUID), int(rootGID), int(processUID), int(processGID))
 	if err != nil {
 		return nil, err
 	}
@@ -533,11 +540,6 @@ func (b *Builder) setupMounts(mountPoint string, spec *specs.Spec, bundlePath st
 	if err != nil {
 		return nil, err
 	}
-	// Get host UID and GID of the container process.
-	processUID, processGID, err := util.GetHostIDs(spec.Linux.UIDMappings, spec.Linux.GIDMappings, spec.Process.User.UID, spec.Process.User.GID)
-	if err != nil {
-		return nil, err
-	}
 
 	// Get the list of explicitly-specified volume mounts.
 	volumes, err := b.runSetupVolumeMounts(spec.Linux.MountLabel, volumeMounts, optionMounts, int(rootUID), int(rootGID), int(processUID), int(processGID))
@@ -2347,7 +2349,7 @@ func init() {
 }
 
 // runSetupRunMounts sets up mounts that exist only in this RUN, not in subsequent runs
-func (b *Builder) runSetupRunMounts(mounts []string, secrets map[string]string, sshSources map[string]*sshagent.Source, mountlabel string, containerWorkingDir string, uidmap []spec.LinuxIDMapping, gidmap []spec.LinuxIDMapping, processLabel string) ([]spec.Mount, *runMountArtifacts, error) {
+func (b *Builder) runSetupRunMounts(mounts []string, secrets map[string]string, sshSources map[string]*sshagent.Source, containerWorkingDir string, contextDir string, uidmap []spec.LinuxIDMapping, gidmap []spec.LinuxIDMapping, rootUID int, rootGID int, processUID int, processGID int) ([]spec.Mount, *runMountArtifacts, error) {
 	mountTargets := make([]string, 0, 10)
 	finalMounts := make([]specs.Mount, 0, len(mounts))
 	agents := make([]*sshagent.AgentServer, 0, len(mounts))
@@ -2367,7 +2369,7 @@ func (b *Builder) runSetupRunMounts(mounts []string, secrets map[string]string,
 		// For now, we only support type secret.
 		switch kv[1] {
 		case "secret":
-			mount, err := getSecretMount(tokens, secrets, mountlabel, containerWorkingDir, uidmap, gidmap)
+			mount, err := getSecretMount(tokens, secrets, b.MountLabel, containerWorkingDir, uidmap, gidmap)
 			if err != nil {
 				return nil, nil, err
 			}
@@ -2377,7 +2379,7 @@ func (b *Builder) runSetupRunMounts(mounts []string, secrets map[string]string,
 
 			}
 		case "ssh":
-			mount, agent, err := b.getSSHMount(tokens, sshCount, sshSources, mountlabel, uidmap, gidmap, processLabel)
+			mount, agent, err := b.getSSHMount(tokens, sshCount, sshSources, b.MountLabel, uidmap, gidmap, b.ProcessLabel)
 			if err != nil {
 				return nil, nil, err
 			}
@@ -2391,6 +2393,13 @@ func (b *Builder) runSetupRunMounts(mounts []string, secrets map[string]string,
 				// Count is needed as the default destination of the ssh sock inside the container is  /run/buildkit/ssh_agent.{i}
 				sshCount++
 			}
+		case "bind":
+			mount, err := b.getBindMount(tokens, contextDir, rootUID, rootGID, processUID, processGID)
+			if err != nil {
+				return nil, nil, err
+			}
+			finalMounts = append(finalMounts, *mount)
+			mountTargets = append(mountTargets, mount.Destination)
 		default:
 			return nil, nil, errors.Errorf("invalid mount type %q", kv[1])
 		}
@@ -2403,6 +2412,23 @@ func (b *Builder) runSetupRunMounts(mounts []string, secrets map[string]string,
 	return finalMounts, artifacts, nil
 }
 
+func (b *Builder) getBindMount(tokens []string, contextDir string, rootUID, rootGID, processUID, processGID int) (*spec.Mount, error) {
+	if contextDir == "" {
+		return nil, errors.New("Context Directory for current run invocation is not configured")
+	}
+	var optionMounts []specs.Mount
+	mount, err := parse.GetBindMount(tokens, contextDir)
+	if err != nil {
+		return nil, err
+	}
+	optionMounts = append(optionMounts, mount)
+	volumes, err := b.runSetupVolumeMounts(b.MountLabel, nil, optionMounts, rootUID, rootGID, processUID, processGID)
+	if err != nil {
+		return nil, err
+	}
+	return &volumes[0], nil
+}
+
 func getSecretMount(tokens []string, secrets map[string]string, mountlabel string, containerWorkingDir string, uidmap []spec.LinuxIDMapping, gidmap []spec.LinuxIDMapping) (*spec.Mount, error) {
 	errInvalidSyntax := errors.New("secret should have syntax id=id[,target=path,required=bool,mode=uint,uid=uint,gid=uint")
 	if len(tokens) == 0 {

tests/bud.bats

@@ -3510,3 +3510,43 @@ _EOF
   ## exported /run should not be empty
   assert "$count" == "1"
 }
+
+@test "bud-with-mount-like-buildkit" {
+  skip_if_no_runtime
+  skip_if_in_container
+  run_buildah build -t testbud --signature-policy ${TESTSDIR}/policy.json -f ${TESTSDIR}/bud/buildkit-mount/Dockerfile ${TESTSDIR}/bud/buildkit-mount/
+  expect_output --substring "hello"
+  run_buildah rmi -f testbud
+}
+
+@test "bud-with-mount-no-source-like-buildkit" {
+  skip_if_no_runtime
+  skip_if_in_container
+  run_buildah build -t testbud --signature-policy ${TESTSDIR}/policy.json -f ${TESTSDIR}/bud/buildkit-mount/Dockerfile2 ${TESTSDIR}/bud/buildkit-mount/
+  expect_output --substring "hello"
+  run_buildah rmi -f testbud
+}
+
+@test "bud-with-mount-no-subdir-like-buildkit" {
+  skip_if_no_runtime
+  skip_if_in_container
+  run_buildah build -t testbud --signature-policy ${TESTSDIR}/policy.json -f ${TESTSDIR}/bud/buildkit-mount/Dockerfile ${TESTSDIR}/bud/buildkit-mount/subdir/
+  expect_output --substring "hello"
+  run_buildah rmi -f testbud
+}
+
+@test "bud-with-mount-with-rw-like-buildkit" {
+  skip_if_no_runtime
+  skip_if_in_container
+  run_buildah build --isolation chroot -t testbud --signature-policy ${TESTSDIR}/policy.json -f ${TESTSDIR}/bud/buildkit-mount/Dockerfile3 ${TESTSDIR}/bud/buildkit-mount/subdir/
+  expect_output --substring "world"
+  run_buildah rmi -f testbud
+}
+
+@test "bud-with-mount-relative-path-like-buildkit" {
+  skip_if_no_runtime
+  skip_if_in_container
+  run_buildah build -t testbud --signature-policy ${TESTSDIR}/policy.json -f ${TESTSDIR}/bud/buildkit-mount/Dockerfile4 ${TESTSDIR}/bud/buildkit-mount/
+  expect_output --substring "hello"
+  run_buildah rmi -f testbud
+}

tests/bud/buildkit-mount/Dockerfile

@@ -0,0 +1,4 @@
+FROM alpine
+RUN mkdir /test
+# use option z if selinux is enabled
+RUN --mount=type=bind,source=.,target=/test,z cat /test/input_file

tests/bud/buildkit-mount/Dockerfile2

@@ -0,0 +1,4 @@
+FROM alpine
+RUN mkdir /test
+# use option z if selinux is enabled
+RUN --mount=type=bind,target=/test,z cat /test/input_file

tests/bud/buildkit-mount/Dockerfile3

@@ -0,0 +1,4 @@
+FROM alpine
+RUN mkdir /test
+# use option z if selinux is enabled
+RUN --mount=type=bind,source=.,target=/test,z,rw echo world > /test/input_file && cat /test/input_file

tests/bud/buildkit-mount/Dockerfile4

@@ -0,0 +1,4 @@
+FROM alpine
+RUN mkdir /test
+# use option z if selinux is enabled
+RUN --mount=type=bind,source=subdir/,target=/test,z cat /test/input_file

tests/bud/buildkit-mount/input_file

@@ -0,0 +1 @@
+hello

tests/bud/buildkit-mount/subdir/input_file

@@ -0,0 +1 @@
+hello