Move containerd fs package in tree

.travis.yml

@@ -1,4 +1,5 @@
 language: go
+sudo: required
 
 go:
   - 1.8.x
@@ -12,7 +13,6 @@ env:
   - TRAVIS_GOOS=windows
   - TRAVIS_GOOS=linux
   - TRAVIS_GOOS=darwin
-  - TRAVIS_GOOS=solaris
 
 script:
   - export GOOS=${TRAVIS_GOOS}

fs/copy.go

@@ -0,0 +1,119 @@
+package fs
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/pkg/errors"
+)
+
+var bufferPool = &sync.Pool{
+	New: func() interface{} {
+		buffer := make([]byte, 32*1024)
+		return &buffer
+	},
+}
+
+// CopyDir copies the directory from src to dst.
+// Most efficient copy of files is attempted.
+func CopyDir(dst, src string) error {
+	inodes := map[uint64]string{}
+	return copyDirectory(dst, src, inodes)
+}
+
+func copyDirectory(dst, src string, inodes map[uint64]string) error {
+	stat, err := os.Stat(src)
+	if err != nil {
+		return errors.Wrapf(err, "failed to stat %s", src)
+	}
+	if !stat.IsDir() {
+		return errors.Errorf("source is not directory")
+	}
+
+	if st, err := os.Stat(dst); err != nil {
+		if err := os.Mkdir(dst, stat.Mode()); err != nil {
+			return errors.Wrapf(err, "failed to mkdir %s", dst)
+		}
+	} else if !st.IsDir() {
+		return errors.Errorf("cannot copy to non-directory: %s", dst)
+	} else {
+		if err := os.Chmod(dst, stat.Mode()); err != nil {
+			return errors.Wrapf(err, "failed to chmod on %s", dst)
+		}
+	}
+
+	fis, err := ioutil.ReadDir(src)
+	if err != nil {
+		return errors.Wrapf(err, "failed to read %s", src)
+	}
+
+	if err := copyFileInfo(stat, dst); err != nil {
+		return errors.Wrapf(err, "failed to copy file info for %s", dst)
+	}
+
+	for _, fi := range fis {
+		source := filepath.Join(src, fi.Name())
+		target := filepath.Join(dst, fi.Name())
+
+		switch {
+		case fi.IsDir():
+			if err := copyDirectory(target, source, inodes); err != nil {
+				return err
+			}
+			continue
+		case (fi.Mode() & os.ModeType) == 0:
+			link, err := getLinkSource(target, fi, inodes)
+			if err != nil {
+				return errors.Wrap(err, "failed to get hardlink")
+			}
+			if link != "" {
+				if err := os.Link(link, target); err != nil {
+					return errors.Wrap(err, "failed to create hard link")
+				}
+			} else if err := copyFile(source, target); err != nil {
+				return errors.Wrap(err, "failed to copy files")
+			}
+		case (fi.Mode() & os.ModeSymlink) == os.ModeSymlink:
+			link, err := os.Readlink(source)
+			if err != nil {
+				return errors.Wrapf(err, "failed to read link: %s", source)
+			}
+			if err := os.Symlink(link, target); err != nil {
+				return errors.Wrapf(err, "failed to create symlink: %s", target)
+			}
+		case (fi.Mode() & os.ModeDevice) == os.ModeDevice:
+			if err := copyDevice(target, fi); err != nil {
+				return errors.Wrapf(err, "failed to create device")
+			}
+		default:
+			// TODO: Support pipes and sockets
+			return errors.Wrapf(err, "unsupported mode %s", fi.Mode())
+		}
+		if err := copyFileInfo(fi, target); err != nil {
+			return errors.Wrap(err, "failed to copy file info")
+		}
+
+		if err := copyXAttrs(target, source); err != nil {
+			return errors.Wrap(err, "failed to copy xattrs")
+		}
+	}
+
+	return nil
+}
+
+func copyFile(source, target string) error {
+	src, err := os.Open(source)
+	if err != nil {
+		return errors.Wrapf(err, "failed to open source %s", source)
+	}
+	defer src.Close()
+	tgt, err := os.Create(target)
+	if err != nil {
+		return errors.Wrapf(err, "failed to open target %s", target)
+	}
+	defer tgt.Close()
+
+	return copyFileContent(tgt, src)
+}

fs/copy_linux.go

@@ -0,0 +1,95 @@
+package fs
+
+import (
+	"io"
+	"os"
+	"syscall"
+
+	"github.com/containerd/continuity/sysx"
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+)
+
+func copyFileInfo(fi os.FileInfo, name string) error {
+	st := fi.Sys().(*syscall.Stat_t)
+	if err := os.Lchown(name, int(st.Uid), int(st.Gid)); err != nil {
+		if os.IsPermission(err) {
+			// Normally if uid/gid are the same this would be a no-op, but some
+			// filesystems may still return EPERM... for instance NFS does this.
+			// In such a case, this is not an error.
+			if dstStat, err2 := os.Lstat(name); err2 == nil {
+				st2 := dstStat.Sys().(*syscall.Stat_t)
+				if st.Uid == st2.Uid && st.Gid == st2.Gid {
+					err = nil
+				}
+			}
+		}
+		if err != nil {
+			return errors.Wrapf(err, "failed to chown %s", name)
+		}
+	}
+
+	if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink {
+		if err := os.Chmod(name, fi.Mode()); err != nil {
+			return errors.Wrapf(err, "failed to chmod %s", name)
+		}
+	}
+
+	timespec := []unix.Timespec{unix.Timespec(StatAtime(st)), unix.Timespec(StatMtime(st))}
+	if err := unix.UtimesNanoAt(unix.AT_FDCWD, name, timespec, unix.AT_SYMLINK_NOFOLLOW); err != nil {
+		return errors.Wrapf(err, "failed to utime %s", name)
+	}
+
+	return nil
+}
+
+func copyFileContent(dst, src *os.File) error {
+	st, err := src.Stat()
+	if err != nil {
+		return errors.Wrap(err, "unable to stat source")
+	}
+
+	n, err := unix.CopyFileRange(int(src.Fd()), nil, int(dst.Fd()), nil, int(st.Size()), 0)
+	if err != nil {
+		if err != unix.ENOSYS && err != unix.EXDEV {
+			return errors.Wrap(err, "copy file range failed")
+		}
+
+		buf := bufferPool.Get().(*[]byte)
+		_, err = io.CopyBuffer(dst, src, *buf)
+		bufferPool.Put(buf)
+		return err
+	}
+
+	if int64(n) != st.Size() {
+		return errors.Wrapf(err, "short copy: %d of %d", int64(n), st.Size())
+	}
+
+	return nil
+}
+
+func copyXAttrs(dst, src string) error {
+	xattrKeys, err := sysx.LListxattr(src)
+	if err != nil {
+		return errors.Wrapf(err, "failed to list xattrs on %s", src)
+	}
+	for _, xattr := range xattrKeys {
+		data, err := sysx.LGetxattr(src, xattr)
+		if err != nil {
+			return errors.Wrapf(err, "failed to get xattr %q on %s", xattr, src)
+		}
+		if err := sysx.LSetxattr(dst, xattr, data, 0); err != nil {
+			return errors.Wrapf(err, "failed to set xattr %q on %s", xattr, dst)
+		}
+	}
+
+	return nil
+}
+
+func copyDevice(dst string, fi os.FileInfo) error {
+	st, ok := fi.Sys().(*syscall.Stat_t)
+	if !ok {
+		return errors.New("unsupported stat type")
+	}
+	return unix.Mknod(dst, uint32(fi.Mode()), int(st.Rdev))
+}

fs/copy_test.go

@@ -0,0 +1,71 @@
+package fs
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	_ "crypto/sha256"
+
+	"github.com/containerd/continuity/fs/fstest"
+	"github.com/pkg/errors"
+)
+
+// TODO: Create copy directory which requires privilege
+//  chown
+//  mknod
+//  setxattr fstest.SetXAttr("/home", "trusted.overlay.opaque", "y"),
+
+func TestCopyDirectory(t *testing.T) {
+	apply := fstest.Apply(
+		fstest.CreateDir("/etc/", 0755),
+		fstest.CreateFile("/etc/hosts", []byte("localhost 127.0.0.1"), 0644),
+		fstest.Link("/etc/hosts", "/etc/hosts.allow"),
+		fstest.CreateDir("/usr/local/lib", 0755),
+		fstest.CreateFile("/usr/local/lib/libnothing.so", []byte{0x00, 0x00}, 0755),
+		fstest.Symlink("libnothing.so", "/usr/local/lib/libnothing.so.2"),
+		fstest.CreateDir("/home", 0755),
+	)
+
+	if err := testCopy(apply); err != nil {
+		t.Fatalf("Copy test failed: %+v", err)
+	}
+}
+
+// This test used to fail because link-no-nothing.txt would be copied first,
+// then file operations in dst during the CopyDir would follow the symlink and
+// fail.
+func TestCopyDirectoryWithLocalSymlink(t *testing.T) {
+	apply := fstest.Apply(
+		fstest.CreateFile("nothing.txt", []byte{0x00, 0x00}, 0755),
+		fstest.Symlink("nothing.txt", "link-no-nothing.txt"),
+	)
+
+	if err := testCopy(apply); err != nil {
+		t.Fatalf("Copy test failed: %+v", err)
+	}
+}
+
+func testCopy(apply fstest.Applier) error {
+	t1, err := ioutil.TempDir("", "test-copy-src-")
+	if err != nil {
+		return errors.Wrap(err, "failed to create temporary directory")
+	}
+	defer os.RemoveAll(t1)
+
+	t2, err := ioutil.TempDir("", "test-copy-dst-")
+	if err != nil {
+		return errors.Wrap(err, "failed to create temporary directory")
+	}
+	defer os.RemoveAll(t2)
+
+	if err := apply.Apply(t1); err != nil {
+		return errors.Wrap(err, "failed to apply changes")
+	}
+
+	if err := CopyDir(t2, t1); err != nil {
+		return errors.Wrap(err, "failed to copy")
+	}
+
+	return fstest.CheckDirectoryEqual(t1, t2)
+}

fs/copy_unix.go

@@ -0,0 +1,80 @@
+// +build solaris darwin freebsd
+
+package fs
+
+import (
+	"io"
+	"os"
+	"syscall"
+
+	"github.com/containerd/continuity/sysx"
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+)
+
+func copyFileInfo(fi os.FileInfo, name string) error {
+	st := fi.Sys().(*syscall.Stat_t)
+	if err := os.Lchown(name, int(st.Uid), int(st.Gid)); err != nil {
+		if os.IsPermission(err) {
+			// Normally if uid/gid are the same this would be a no-op, but some
+			// filesystems may still return EPERM... for instance NFS does this.
+			// In such a case, this is not an error.
+			if dstStat, err2 := os.Lstat(name); err2 == nil {
+				st2 := dstStat.Sys().(*syscall.Stat_t)
+				if st.Uid == st2.Uid && st.Gid == st2.Gid {
+					err = nil
+				}
+			}
+		}
+		if err != nil {
+			return errors.Wrapf(err, "failed to chown %s", name)
+		}
+	}
+
+	if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink {
+		if err := os.Chmod(name, fi.Mode()); err != nil {
+			return errors.Wrapf(err, "failed to chmod %s", name)
+		}
+	}
+
+	timespec := []syscall.Timespec{StatAtime(st), StatMtime(st)}
+	if err := syscall.UtimesNano(name, timespec); err != nil {
+		return errors.Wrapf(err, "failed to utime %s", name)
+	}
+
+	return nil
+}
+
+func copyFileContent(dst, src *os.File) error {
+	buf := bufferPool.Get().(*[]byte)
+	_, err := io.CopyBuffer(dst, src, *buf)
+	bufferPool.Put(buf)
+
+	return err
+}
+
+func copyXAttrs(dst, src string) error {
+	xattrKeys, err := sysx.LListxattr(src)
+	if err != nil {
+		return errors.Wrapf(err, "failed to list xattrs on %s", src)
+	}
+	for _, xattr := range xattrKeys {
+		data, err := sysx.LGetxattr(src, xattr)
+		if err != nil {
+			return errors.Wrapf(err, "failed to get xattr %q on %s", xattr, src)
+		}
+		if err := sysx.LSetxattr(dst, xattr, data, 0); err != nil {
+			return errors.Wrapf(err, "failed to set xattr %q on %s", xattr, dst)
+		}
+	}
+
+	return nil
+}
+
+func copyDevice(dst string, fi os.FileInfo) error {
+	st, ok := fi.Sys().(*syscall.Stat_t)
+	if !ok {
+		return errors.New("unsupported stat type")
+	}
+	return unix.Mknod(dst, uint32(fi.Mode()), int(st.Rdev))
+}