Merge pull request #849 from giuseppe/prepare-zstd-chunked

support patches to prepare #775

drivers/copy/copy_linux.go

@@ -38,20 +38,14 @@ const (
 	Hardlink
 )
 
-func copyRegular(srcPath, dstPath string, fileinfo os.FileInfo, copyWithFileRange, copyWithFileClone *bool) error {
+// CopyRegularToFile copies the content of a file to another
+func CopyRegularToFile(srcPath string, dstFile *os.File, fileinfo os.FileInfo, copyWithFileRange, copyWithFileClone *bool) error { // nolint: golint
 	srcFile, err := os.Open(srcPath)
 	if err != nil {
 		return err
 	}
 	defer srcFile.Close()
 
-	// If the destination file already exists, we shouldn't blow it away
-	dstFile, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, fileinfo.Mode())
-	if err != nil {
-		return err
-	}
-	defer dstFile.Close()
-
 	if *copyWithFileClone {
 		_, _, err = unix.Syscall(unix.SYS_IOCTL, dstFile.Fd(), C.FICLONE, srcFile.Fd())
 		if err == nil {
@@ -76,6 +70,18 @@ func copyRegular(srcPath, dstPath string, fileinfo os.FileInfo, copyWithFileRang
 	return legacyCopy(srcFile, dstFile)
 }
 
+// CopyRegular copies the content of a file to another
+func CopyRegular(srcPath, dstPath string, fileinfo os.FileInfo, copyWithFileRange, copyWithFileClone *bool) error { // nolint: golint
+	// If the destination file already exists, we shouldn't blow it away
+	dstFile, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, fileinfo.Mode())
+	if err != nil {
+		return err
+	}
+	defer dstFile.Close()
+
+	return CopyRegularToFile(srcPath, dstFile, fileinfo, copyWithFileRange, copyWithFileClone)
+}
+
 func doCopyWithFileRange(srcFile, dstFile *os.File, fileinfo os.FileInfo) error {
 	amountLeftToCopy := fileinfo.Size()
 
@@ -164,7 +170,7 @@ func DirCopy(srcDir, dstDir string, copyMode Mode, copyXattrs bool) error {
 					return err2
 				}
 			} else {
-				if err2 := copyRegular(srcPath, dstPath, f, &copyWithFileRange, &copyWithFileClone); err2 != nil {
+				if err2 := CopyRegular(srcPath, dstPath, f, &copyWithFileRange, &copyWithFileClone); err2 != nil {
 					return err2
 				}
 				copiedFiles[id] = dstPath

drivers/copy/copy_test.go

@@ -123,7 +123,7 @@ func doCopyTest(t *testing.T, copyWithFileRange, copyWithFileClone *bool) {
 	fileinfo, err := os.Stat(srcFilename)
 	assert.NilError(t, err)
 
-	assert.NilError(t, copyRegular(srcFilename, dstFilename, fileinfo, copyWithFileRange, copyWithFileClone))
+	assert.NilError(t, CopyRegular(srcFilename, dstFilename, fileinfo, copyWithFileRange, copyWithFileClone))
 	readBuf, err := ioutil.ReadFile(dstFilename)
 	assert.NilError(t, err)
 	assert.Check(t, is.DeepEqual(buf, readBuf))

drivers/copy/copy_unsupported.go

@@ -2,7 +2,12 @@
 
 package copy
 
-import "github.com/containers/storage/pkg/chrootarchive"
+import (
+	"io"
+	"os"
+
+	"github.com/containers/storage/pkg/chrootarchive"
+)
 
 // Mode indicates whether to use hardlink or copy content
 type Mode int
@@ -17,3 +22,19 @@ const (
 func DirCopy(srcDir, dstDir string, _ Mode, _ bool) error {
 	return chrootarchive.NewArchiver(nil).CopyWithTar(srcDir, dstDir)
 }
+
+// CopyRegularToFile copies the content of a file to another
+func CopyRegularToFile(srcPath string, dstFile *os.File, fileinfo os.FileInfo, copyWithFileRange, copyWithFileClone *bool) error {
+	f, err := os.Open(srcPath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	_, err = io.Copy(dstFile, f)
+	return err
+}
+
+// CopyRegular copies the content of a file to another
+func CopyRegular(srcPath, dstPath string, fileinfo os.FileInfo, copyWithFileRange, copyWithFileClone *bool) error {
+	return chrootarchive.NewArchiver(nil).CopyWithTar(srcPath, dstPath)
+}

pkg/archive/archive.go

@@ -443,9 +443,16 @@ func ReadUserXattrToTarHeader(path string, hdr *tar.Header) error {
 	return nil
 }
 
-type tarWhiteoutConverter interface {
+type TarWhiteoutHandler interface {
+	Setxattr(path, name string, value []byte) error
+	Mknod(path string, mode uint32, dev int) error
+	Chown(path string, uid, gid int) error
+}
+
+type TarWhiteoutConverter interface {
 	ConvertWrite(*tar.Header, string, os.FileInfo) (*tar.Header, error)
 	ConvertRead(*tar.Header, string) (bool, error)
+	ConvertReadWithHandler(*tar.Header, string, TarWhiteoutHandler) (bool, error)
 }
 
 type tarAppender struct {
@@ -461,7 +468,7 @@ type tarAppender struct {
 	// non standard format. The whiteout files defined
 	// by the AUFS standard are used as the tar whiteout
 	// standard.
-	WhiteoutConverter tarWhiteoutConverter
+	WhiteoutConverter TarWhiteoutConverter
 	// CopyPass indicates that the contents of any archive we're creating
 	// will instantly be extracted and written to disk, so we can deviate
 	// from the traditional behavior/format to get features like subsecond
@@ -798,7 +805,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error)
 			compressWriter,
 			options.ChownOpts,
 		)
-		ta.WhiteoutConverter = getWhiteoutConverter(options.WhiteoutFormat, options.WhiteoutData)
+		ta.WhiteoutConverter = GetWhiteoutConverter(options.WhiteoutFormat, options.WhiteoutData)
 		ta.CopyPass = options.CopyPass
 
 		defer func() {
@@ -958,11 +965,11 @@ func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) err
 	var dirs []*tar.Header
 	idMappings := idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps)
 	rootIDs := idMappings.RootPair()
-	whiteoutConverter := getWhiteoutConverter(options.WhiteoutFormat, options.WhiteoutData)
+	whiteoutConverter := GetWhiteoutConverter(options.WhiteoutFormat, options.WhiteoutData)
 	buffer := make([]byte, 1<<20)
 
 	if options.ForceMask != nil {
-		uid, gid, mode, err := getFileOwner(dest)
+		uid, gid, mode, err := GetFileOwner(dest)
 		if err == nil {
 			value := fmt.Sprintf("%d:%d:0%o", uid, gid, mode)
 			if err := system.Lsetxattr(dest, containersOverrideXattr, []byte(value), 0); err != nil {

pkg/archive/archive_linux.go

@@ -12,7 +12,7 @@ import (
 	"golang.org/x/sys/unix"
 )
 
-func getWhiteoutConverter(format WhiteoutFormat, data interface{}) tarWhiteoutConverter {
+func GetWhiteoutConverter(format WhiteoutFormat, data interface{}) TarWhiteoutConverter {
 	if format == OverlayWhiteoutFormat {
 		if rolayers, ok := data.([]string); ok && len(rolayers) > 0 {
 			return overlayWhiteoutConverter{rolayers: rolayers}
@@ -108,13 +108,13 @@ func (o overlayWhiteoutConverter) ConvertWrite(hdr *tar.Header, path string, fi
 	return
 }
 
-func (overlayWhiteoutConverter) ConvertRead(hdr *tar.Header, path string) (bool, error) {
+func (overlayWhiteoutConverter) ConvertReadWithHandler(hdr *tar.Header, path string, handler TarWhiteoutHandler) (bool, error) {
 	base := filepath.Base(path)
 	dir := filepath.Dir(path)
 
 	// if a directory is marked as opaque by the AUFS special file, we need to translate that to overlay
 	if base == WhiteoutOpaqueDir {
-		err := unix.Setxattr(dir, "trusted.overlay.opaque", []byte{'y'}, 0)
+		err := handler.Setxattr(dir, "trusted.overlay.opaque", []byte{'y'})
 		// don't write the file itself
 		return false, err
 	}
@@ -124,10 +124,10 @@ func (overlayWhiteoutConverter) ConvertRead(hdr *tar.Header, path string) (bool,
 		originalBase := base[len(WhiteoutPrefix):]
 		originalPath := filepath.Join(dir, originalBase)
 
-		if err := unix.Mknod(originalPath, unix.S_IFCHR, 0); err != nil {
+		if err := handler.Mknod(originalPath, unix.S_IFCHR, 0); err != nil {
 			return false, err
 		}
-		if err := idtools.SafeChown(originalPath, hdr.Uid, hdr.Gid); err != nil {
+		if err := handler.Chown(originalPath, hdr.Uid, hdr.Gid); err != nil {
 			return false, err
 		}
 
@@ -138,12 +138,32 @@ func (overlayWhiteoutConverter) ConvertRead(hdr *tar.Header, path string) (bool,
 	return true, nil
 }
 
+type directHandler struct {
+}
+
+func (d directHandler) Setxattr(path, name string, value []byte) error {
+	return unix.Setxattr(path, name, value, 0)
+}
+
+func (d directHandler) Mknod(path string, mode uint32, dev int) error {
+	return unix.Mknod(path, mode, dev)
+}
+
+func (d directHandler) Chown(path string, uid, gid int) error {
+	return idtools.SafeChown(path, uid, gid)
+}
+
+func (o overlayWhiteoutConverter) ConvertRead(hdr *tar.Header, path string) (bool, error) {
+	var handler directHandler
+	return o.ConvertReadWithHandler(hdr, path, handler)
+}
+
 func isWhiteOut(stat os.FileInfo) bool {
 	s := stat.Sys().(*syscall.Stat_t)
 	return major(uint64(s.Rdev)) == 0 && minor(uint64(s.Rdev)) == 0
 }
 
-func getFileOwner(path string) (uint32, uint32, uint32, error) {
+func GetFileOwner(path string) (uint32, uint32, uint32, error) {
 	f, err := os.Stat(path)
 	if err != nil {
 		return 0, 0, 0, err

pkg/archive/archive_other.go

@@ -2,10 +2,10 @@
 
 package archive
 
-func getWhiteoutConverter(format WhiteoutFormat, data interface{}) tarWhiteoutConverter {
+func GetWhiteoutConverter(format WhiteoutFormat, data interface{}) TarWhiteoutConverter {
 	return nil
 }
 
-func getFileOwner(path string) (uint32, uint32, uint32, error) {
+func GetFileOwner(path string) (uint32, uint32, uint32, error) {
 	return 0, 0, 0, nil
 }

pkg/ioutils/fswriters.go

@@ -156,6 +156,9 @@ type syncFileCloser struct {
 }
 
 func (w syncFileCloser) Close() error {
+	if !defaultWriterOptions.NoSync {
+		return w.File.Close()
+	}
 	err := fdatasync(w.File)
 	if err1 := w.File.Close(); err == nil {
 		err = err1