feat(import): add "copy" support for import

Currently, "import:" directive only allows for temporarily mounting
previously layers, and actual copying would have to be done via "run:"
directive. But in cases where there is no shell in the current layer,
there was no way to achieve this, greatly limiting usability for
packaging golang binaries.

Adding support using "dest:" option under "import:" directive.

Thanks @smoser for the review and fix.

Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com>

base.go

@@ -31,14 +31,16 @@ type BaseLayerOpts struct {
 func GetBase(o BaseLayerOpts) error {
 	switch o.Layer.From.Type {
 	case types.BuiltLayer:
+		fallthrough
+	case types.ScratchLayer:
 		return nil
 	case types.TarLayer:
 		cacheDir := path.Join(o.Config.StackerDir, "layer-bases")
 		if err := os.MkdirAll(cacheDir, 0755); err != nil {
 			return err
 		}
 
-		_, err := acquireUrl(o.Config, o.Storage, o.Layer.From.Url, cacheDir, o.Progress, "")
+		_, err := acquireUrl(o.Config, o.Storage, o.Layer.From.Url, cacheDir, o.Progress, "", nil, -1, -1)
 		return err
 	/* now we can do all the containers/image types */
 	case types.OCILayer:
@@ -74,6 +76,8 @@ func SetupRootfs(o BaseLayerOpts) error {
 	}
 
 	switch o.Layer.From.Type {
+	case types.ScratchLayer:
+		return o.Storage.SetupEmptyRootfs(o.Name)
 	case types.TarLayer:
 		err := o.Storage.SetupEmptyRootfs(o.Name)
 		if err != nil {

build.go

@@ -352,10 +352,12 @@ func (b *Builder) build(s types.Storage, file string) error {
 			return err
 		}
 
-		if err := Import(opts.Config, s, name, l.Imports, opts.Progress); err != nil {
+		if err := Import(opts.Config, s, name, l.Imports, &l.OverlayDirs, opts.Progress); err != nil {
 			return err
 		}
 
+		log.Debugf("Overlaydirs after Import: %v", l.OverlayDirs)
+
 		// Need to check if the image has bind mounts, if the image has bind mounts,
 		// it needs to be rebuilt regardless of the build cache
 		// The reason is that tracking build cache for bind mounted folders

cache.go

@@ -207,6 +207,11 @@ func (c *BuildCache) Lookup(name string) (*CacheEntry, bool, error) {
 	}
 
 	for _, imp := range l.Imports {
+		if imp.Dest != "" {
+			// ignore imports which are copied
+			continue
+		}
+
 		cachedImport, ok := result.Imports[imp.Path]
 		if !ok {
 			log.Infof("cache miss because of new import: %s", imp.Path)
@@ -401,6 +406,9 @@ func (c *BuildCache) getBaseHash(name string) (string, error) {
 		}
 
 		return fmt.Sprintf("%d", baseHash), nil
+	case types.ScratchLayer:
+		// no base hash to use
+		return "", nil
 	case types.TarLayer:
 		// use the hash of the input tarball
 		cacheDir := path.Join(c.config.StackerDir, "layer-bases")
@@ -457,6 +465,11 @@ func (c *BuildCache) Put(name string, manifests map[types.LayerType]ispec.Descri
 	}
 
 	for _, imp := range l.Imports {
+		if imp.Dest != "" {
+			// ignore imports which are copied
+			continue
+		}
+
 		fname := path.Base(imp.Path)
 		importsDir := path.Join(c.config.StackerDir, "imports")
 		diskPath := path.Join(importsDir, name, fname)

cmd/grab.go

@@ -43,5 +43,5 @@ func doGrab(ctx *cli.Context) error {
 		return err
 	}
 
-	return stacker.Grab(config, s, name, parts[1], cwd)
+	return stacker.Grab(config, s, name, parts[1], cwd, nil, -1, -1)
 }

cmd/internal_go.go

@@ -24,6 +24,14 @@ var internalGoCmd = cli.Command{
 			Name:   "cp",
 			Action: doCP,
 		},
+		cli.Command{
+			Name:   "chmod",
+			Action: doChmod,
+		},
+		cli.Command{
+			Name:   "chown",
+			Action: doChown,
+		},
 		cli.Command{
 			Name:   "check-aa-profile",
 			Action: doCheckAAProfile,
@@ -109,6 +117,28 @@ func doCP(ctx *cli.Context) error {
 	)
 }
 
+func doChmod(ctx *cli.Context) error {
+	if len(ctx.Args()) != 2 {
+		return errors.Errorf("wrong number of args")
+	}
+
+	return lib.Chmod(
+		ctx.Args()[0],
+		ctx.Args()[1],
+	)
+}
+
+func doChown(ctx *cli.Context) error {
+	if len(ctx.Args()) != 2 {
+		return errors.Errorf("wrong number of args")
+	}
+
+	return lib.Chown(
+		ctx.Args()[0],
+		ctx.Args()[1],
+	)
+}
+
 func doCheckAAProfile(ctx *cli.Context) error {
 	toCheck := ctx.Args()[0]
 	command := fmt.Sprintf("changeprofile %s", toCheck)

cmd/main.go

@@ -72,7 +72,7 @@ func shouldSkipInternalUserns(ctx *cli.Context) bool {
 	}
 
 	if len(args) >= 2 && args[0] == "internal-go" {
-		if args[1] == "atomfs" || args[1] == "cp" {
+		if args[1] == "atomfs" || args[1] == "cp" || args[1] == "chown" || args[1] == "chmod" {
 			return true
 		}
 	}

grab.go

@@ -2,6 +2,7 @@ package stacker
 
 import (
 	"fmt"
+	"io/fs"
 	"os"
 	"path"
 
@@ -10,7 +11,9 @@ import (
 	"stackerbuild.io/stacker/types"
 )
 
-func Grab(sc types.StackerConfig, storage types.Storage, name string, source string, targetDir string) error {
+func Grab(sc types.StackerConfig, storage types.Storage, name string, source string, targetDir string,
+	mode *fs.FileMode, uid, gid int,
+) error {
 	c, err := container.New(sc, name)
 	if err != nil {
 		return err
@@ -38,5 +41,29 @@ func Grab(sc types.StackerConfig, storage types.Storage, name string, source str
 		return err
 	}
 
-	return c.Execute(fmt.Sprintf("/static-stacker internal-go cp %s /stacker/%s", source, path.Base(source)), nil)
+	err = c.Execute(fmt.Sprintf("/static-stacker internal-go cp %s /stacker/%s", source, path.Base(source)), nil)
+	if err != nil {
+		return err
+	}
+
+	if mode != nil {
+		err = c.Execute(fmt.Sprintf("/static-stacker internal-go chmod %s /stacker/%s", fmt.Sprintf("%o", *mode), path.Base(source)), nil)
+		if err != nil {
+			return err
+		}
+	}
+
+	if uid > 0 {
+		owns := fmt.Sprintf("%d", uid)
+		if gid > 0 {
+			owns += fmt.Sprintf(":%d", gid)
+		}
+
+		err = c.Execute(fmt.Sprintf("/static-stacker internal-go chown %s /stacker/%s", owns, path.Base(source)), nil)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
 }

import.go

@@ -1,6 +1,7 @@
 package stacker
 
 import (
+	"io/fs"
 	"os"
 	"path"
 	"strings"
@@ -84,7 +85,7 @@ func verifyImportFileHash(imp string, hash string) error {
 	return nil
 }
 
-func importFile(imp string, cacheDir string, hash string) (string, error) {
+func importFile(imp string, cacheDir string, hash string, mode *fs.FileMode, uid, gid int) (string, error) {
 	e1, err := os.Lstat(imp)
 	if err != nil {
 		return "", errors.Wrapf(err, "couldn't stat import %s", imp)
@@ -110,7 +111,7 @@ func importFile(imp string, cacheDir string, hash string) (string, error) {
 
 		if needsCopy {
 			log.Infof("copying %s", imp)
-			if err := lib.FileCopy(dest, imp); err != nil {
+			if err := lib.FileCopy(dest, imp, mode, uid, gid); err != nil {
 				return "", errors.Wrapf(err, "couldn't copy import %s", imp)
 			}
 		} else {
@@ -187,7 +188,7 @@ func importFile(imp string, cacheDir string, hash string) (string, error) {
 				if err != nil {
 					return "", err
 				}
-				err = lib.FileCopy(destpath, srcpath)
+				err = lib.FileCopy(destpath, srcpath, mode, uid, gid)
 			}
 			if err != nil {
 				return "", err
@@ -213,7 +214,9 @@ func validateHash(hash string) error {
 	return nil
 }
 
-func acquireUrl(c types.StackerConfig, storage types.Storage, i string, cache string, progress bool, expectedHash string) (string, error) {
+func acquireUrl(c types.StackerConfig, storage types.Storage, i string, cache string, progress bool, expectedHash string,
+	mode *fs.FileMode, uid, gid int,
+) (string, error) {
 	url, err := types.NewDockerishUrl(i)
 	if err != nil {
 		return "", err
@@ -226,7 +229,7 @@ func acquireUrl(c types.StackerConfig, storage types.Storage, i string, cache st
 
 	// It's just a path, let's copy it to .stacker.
 	if url.Scheme == "" {
-		return importFile(i, cache, expectedHash)
+		return importFile(i, cache, expectedHash, mode, uid, gid)
 	} else if url.Scheme == "http" || url.Scheme == "https" {
 		// otherwise, we need to download it
 		// first verify the hashes
@@ -242,7 +245,7 @@ func acquireUrl(c types.StackerConfig, storage types.Storage, i string, cache st
 			return "", errors.Errorf("The requested hash of %s import is different than the actual hash: %s != %s",
 				i, expectedHash, remoteHash)
 		}
-		return Download(cache, i, progress, expectedHash, remoteHash, remoteSize)
+		return Download(cache, i, progress, expectedHash, remoteHash, remoteSize, mode, uid, gid)
 	} else if url.Scheme == "stacker" {
 		// we always Grab() things from stacker://, because we need to
 		// mount the container's rootfs to get them and don't
@@ -254,7 +257,7 @@ func acquireUrl(c types.StackerConfig, storage types.Storage, i string, cache st
 			return "", err
 		}
 		defer cleanup()
-		err = Grab(c, storage, snap, url.Path, cache)
+		err = Grab(c, storage, snap, url.Path, cache, mode, uid, gid)
 		if err != nil {
 			return "", err
 		}
@@ -271,7 +274,11 @@ func acquireUrl(c types.StackerConfig, storage types.Storage, i string, cache st
 }
 
 func CleanImportsDir(c types.StackerConfig, name string, imports types.Imports, cache *BuildCache) error {
-	dir := path.Join(c.StackerDir, "imports", name)
+	// remove all copied imports
+	dir := path.Join(c.StackerDir, "imports-copy", name)
+	_ = os.RemoveAll(dir)
+
+	dir = path.Join(c.StackerDir, "imports", name)
 
 	cacheEntry, cacheHit := cache.Cache[name]
 	if !cacheHit {
@@ -298,20 +305,40 @@ func CleanImportsDir(c types.StackerConfig, name string, imports types.Imports,
 	return nil
 }
 
-func Import(c types.StackerConfig, storage types.Storage, name string, imports types.Imports, progress bool) error {
+// Import files from different sources to an ephemeral or permanent destination.
+func Import(c types.StackerConfig, storage types.Storage, name string, imports types.Imports, overlayDirs *types.OverlayDirs, progress bool) error {
 	dir := path.Join(c.StackerDir, "imports", name)
 
 	if err := os.MkdirAll(dir, 0755); err != nil {
 		return err
 	}
 
+	cpdir := path.Join(c.StackerDir, "imports-copy", name)
+
+	if err := os.MkdirAll(cpdir, 0755); err != nil {
+		return err
+	}
+
 	existing, err := os.ReadDir(dir)
 	if err != nil {
 		return errors.Wrapf(err, "couldn't read existing directory")
 	}
 
 	for _, i := range imports {
-		name, err := acquireUrl(c, storage, i.Path, dir, progress, i.Hash)
+		cache := dir
+		if i.Dest != "" {
+			tmpdir, err := os.MkdirTemp(cpdir, "")
+			if err != nil {
+				return errors.Wrapf(err, "couldn't create temp import copy directory")
+			}
+
+			ovl := types.OverlayDir{Source: tmpdir, Dest: "/"}
+			*overlayDirs = append(*overlayDirs, ovl)
+
+			cache = tmpdir
+		}
+
+		name, err := acquireUrl(c, storage, i.Path, cache, progress, i.Hash, i.Mode, i.Uid, i.Gid)
 		if err != nil {
 			return err
 		}

lib/dir.go

@@ -1,8 +1,11 @@
 package lib
 
 import (
+	"io/fs"
 	"os"
 	"path"
+	"strconv"
+	"strings"
 
 	"github.com/pkg/errors"
 )
@@ -62,7 +65,7 @@ func DirCopy(dest string, source string) error {
 				return err
 			}
 		} else {
-			if err = FileCopy(dstfp, srcfp); err != nil {
+			if err = FileCopy(dstfp, srcfp, nil, -1, -1); err != nil {
 				return err
 			}
 		}
@@ -80,6 +83,62 @@ func CopyThing(srcpath, destpath string) error {
 	if srcInfo.IsDir() {
 		return DirCopy(destpath, srcpath)
 	} else {
-		return FileCopy(destpath, srcpath)
+		return FileCopy(destpath, srcpath, nil, -1, -1)
 	}
 }
+
+// Chmod changes file permissions
+func Chmod(mode, destpath string) error {
+	destInfo, err := os.Lstat(destpath)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	if destInfo.IsDir() {
+		return errors.WithStack(os.ErrInvalid)
+	}
+
+	if destInfo.Mode()&os.ModeSymlink != 0 {
+		return errors.WithStack(os.ErrInvalid)
+	}
+
+	// read as an octal value
+	iperms, err := strconv.ParseUint(mode, 8, 32)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	return os.Chmod(destpath, fs.FileMode(iperms))
+}
+
+// Chown changes file ownership
+func Chown(owner, destpath string) error {
+	destInfo, err := os.Lstat(destpath)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	if destInfo.IsDir() {
+		return errors.WithStack(os.ErrInvalid)
+	}
+
+	owns := strings.Split(owner, ":")
+	if len(owns) > 2 {
+		return errors.WithStack(os.ErrInvalid)
+	}
+
+	uid, err := strconv.ParseInt(owns[0], 10, 32)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	gid := int64(-1)
+	if len(owns) > 1 {
+		gid, err = strconv.ParseInt(owns[1], 10, 32)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+	}
+
+	return os.Lchown(destpath, int(uid), int(gid))
+}