Skip to content

Commit

Permalink
Merge pull request #4395 from TBBle/use_gowinio_for_reading_tarballs
Browse files Browse the repository at this point in the history 
Use go-winio for applying tarballs
  • Loading branch information
@dmcgowan
dmcgowan committed Dec 6, 2020
2 parents e98d7f8 + 78f31af commit 9c398e1
Show file tree
Hide file tree
Showing 9 changed files with 507 additions and 230 deletions.
6 changes: 4 additions & 2 deletions archive/tar.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,16 +114,18 @@ func Apply(ctx context.Context, root string, r io.Reader, opts ...ApplyOpt) (int
options.applyFunc = applyNaive
}

return options.applyFunc(ctx, root, tar.NewReader(r), options)
return options.applyFunc(ctx, root, r, options)
}

// applyNaive applies a tar stream of an OCI style diff tar to a directory
// applying each file as either a whole file or whiteout.
// See https://github.com/opencontainers/image-spec/blob/master/layer.md#applying-changesets
func applyNaive(ctx context.Context, root string, tr *tar.Reader, options ApplyOptions) (size int64, err error) {
func applyNaive(ctx context.Context, root string, r io.Reader, options ApplyOptions) (size int64, err error) {
var (
dirs []*tar.Header

tr = tar.NewReader(r)

// Used for handling opaque directory markers which
// may occur out of order
unpackedPaths = make(map[string]struct{})
Expand Down
3 changes: 2 additions & 1 deletion archive/tar_opts.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package archive
import (
"archive/tar"
"context"
"io"
)

// ApplyOptions provides additional options for an Apply operation
Expand All @@ -27,7 +28,7 @@ type ApplyOptions struct {
ConvertWhiteout ConvertWhiteout // Convert whiteout files
Parents []string // Parent directories to handle inherited attributes without CoW

applyFunc func(context.Context, string, *tar.Reader, ApplyOptions) (int64, error)
applyFunc func(context.Context, string, io.Reader, ApplyOptions) (int64, error)
}

// ApplyOpt allows setting mutable archive apply properties on creation
Expand Down
215 changes: 16 additions & 199 deletions archive/tar_windows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,38 +22,20 @@ import (
"archive/tar"
"bufio"
"context"
"encoding/base64"
"fmt"
"io"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"syscall"

"github.com/Microsoft/go-winio"
"github.com/Microsoft/go-winio/backuptar"
"github.com/Microsoft/hcsshim"
"github.com/containerd/containerd/sys"
"github.com/pkg/errors"
)

const (
// MSWINDOWS pax vendor extensions
hdrMSWindowsPrefix = "MSWINDOWS."

hdrFileAttributes = hdrMSWindowsPrefix + "fileattr"
hdrSecurityDescriptor = hdrMSWindowsPrefix + "sd"
hdrRawSecurityDescriptor = hdrMSWindowsPrefix + "rawsd"
hdrMountPoint = hdrMSWindowsPrefix + "mountpoint"
hdrEaPrefix = hdrMSWindowsPrefix + "xattr."

// LIBARCHIVE pax vendor extensions
hdrLibArchivePrefix = "LIBARCHIVE."

hdrCreateTime = hdrLibArchivePrefix + "creationtime"
)

var (
// mutatedFiles is a list of files that are mutated by the import process
// and must be backed up and restored.
Expand Down Expand Up @@ -149,10 +131,21 @@ func setxattr(path, key, value string) error {
return errors.New("xattrs not supported on Windows")
}

func copyDirInfo(fi os.FileInfo, path string) error {
if err := os.Chmod(path, fi.Mode()); err != nil {
return errors.Wrapf(err, "failed to chmod %s", path)
}
return nil
}

func copyUpXAttrs(dst, src string) error {
return nil
}

// applyWindowsLayer applies a tar stream of an OCI style diff tar of a Windows
// layer using the hcsshim layer writer and backup streams.
// See https://github.com/opencontainers/image-spec/blob/master/layer.md#applying-changesets
func applyWindowsLayer(ctx context.Context, root string, tr *tar.Reader, options ApplyOptions) (size int64, err error) {
func applyWindowsLayer(ctx context.Context, root string, r io.Reader, options ApplyOptions) (size int64, err error) {
home, id := filepath.Split(root)
info := hcsshim.DriverInfo{
HomeDir: home,
Expand All @@ -172,6 +165,7 @@ func applyWindowsLayer(ctx context.Context, root string, tr *tar.Reader, options
}
}()

tr := tar.NewReader(r)
buf := bufio.NewWriter(nil)
hdr, nextErr := tr.Next()
// Iterate through the files in the archive.
Expand Down Expand Up @@ -208,7 +202,7 @@ func applyWindowsLayer(ctx context.Context, root string, tr *tar.Reader, options
}
hdr, nextErr = tr.Next()
} else {
name, fileSize, fileInfo, err := fileInfoFromHeader(hdr)
name, fileSize, fileInfo, err := backuptar.FileInfoFromHeader(hdr)
if err != nil {
return 0, err
}
Expand All @@ -223,42 +217,6 @@ func applyWindowsLayer(ctx context.Context, root string, tr *tar.Reader, options
return
}

// fileInfoFromHeader retrieves basic Win32 file information from a tar header, using the additional metadata written by
// WriteTarFileFromBackupStream.
func fileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *winio.FileBasicInfo, err error) {
name = hdr.Name
if hdr.Typeflag == tar.TypeReg || hdr.Typeflag == tar.TypeRegA {
size = hdr.Size
}
fileInfo = &winio.FileBasicInfo{
LastAccessTime: syscall.NsecToFiletime(hdr.AccessTime.UnixNano()),
LastWriteTime: syscall.NsecToFiletime(hdr.ModTime.UnixNano()),
ChangeTime: syscall.NsecToFiletime(hdr.ChangeTime.UnixNano()),

// Default CreationTime to ModTime, updated below if MSWINDOWS.createtime exists
CreationTime: syscall.NsecToFiletime(hdr.ModTime.UnixNano()),
}
if attrStr, ok := hdr.PAXRecords[hdrFileAttributes]; ok {
attr, err := strconv.ParseUint(attrStr, 10, 32)
if err != nil {
return "", 0, nil, err
}
fileInfo.FileAttributes = uint32(attr)
} else {
if hdr.Typeflag == tar.TypeDir {
fileInfo.FileAttributes |= syscall.FILE_ATTRIBUTE_DIRECTORY
}
}
if createStr, ok := hdr.PAXRecords[hdrCreateTime]; ok {
createTime, err := parsePAXTime(createStr)
if err != nil {
return "", 0, nil, err
}
fileInfo.CreationTime = syscall.NsecToFiletime(createTime.UnixNano())
}
return
}

// tarToBackupStreamWithMutatedFiles reads data from a tar stream and
// writes it to a backup stream, and also saves any files that will be mutated
// by the import layer process to a backup location.
Expand Down Expand Up @@ -299,146 +257,5 @@ func tarToBackupStreamWithMutatedFiles(buf *bufio.Writer, w io.Writer, t *tar.Re
}
}()

return writeBackupStreamFromTarFile(buf, t, hdr)
}

// writeBackupStreamFromTarFile writes a Win32 backup stream from the current tar file. Since this function may process multiple
// tar file entries in order to collect all the alternate data streams for the file, it returns the next
// tar file that was not processed, or io.EOF is there are no more.
func writeBackupStreamFromTarFile(w io.Writer, t *tar.Reader, hdr *tar.Header) (*tar.Header, error) {
bw := winio.NewBackupStreamWriter(w)
var sd []byte
var err error
// Maintaining old SDDL-based behavior for backward compatibility. All new tar headers written
// by this library will have raw binary for the security descriptor.
if sddl, ok := hdr.PAXRecords[hdrSecurityDescriptor]; ok {
sd, err = winio.SddlToSecurityDescriptor(sddl)
if err != nil {
return nil, err
}
}
if sdraw, ok := hdr.PAXRecords[hdrRawSecurityDescriptor]; ok {
sd, err = base64.StdEncoding.DecodeString(sdraw)
if err != nil {
return nil, err
}
}
if len(sd) != 0 {
bhdr := winio.BackupHeader{
Id: winio.BackupSecurity,
Size: int64(len(sd)),
}
err := bw.WriteHeader(&bhdr)
if err != nil {
return nil, err
}
_, err = bw.Write(sd)
if err != nil {
return nil, err
}
}
var eas []winio.ExtendedAttribute
for k, v := range hdr.PAXRecords {
if !strings.HasPrefix(k, hdrEaPrefix) {
continue
}
data, err := base64.StdEncoding.DecodeString(v)
if err != nil {
return nil, err
}
eas = append(eas, winio.ExtendedAttribute{
Name: k[len(hdrEaPrefix):],
Value: data,
})
}
if len(eas) != 0 {
eadata, err := winio.EncodeExtendedAttributes(eas)
if err != nil {
return nil, err
}
bhdr := winio.BackupHeader{
Id: winio.BackupEaData,
Size: int64(len(eadata)),
}
err = bw.WriteHeader(&bhdr)
if err != nil {
return nil, err
}
_, err = bw.Write(eadata)
if err != nil {
return nil, err
}
}
if hdr.Typeflag == tar.TypeSymlink {
_, isMountPoint := hdr.PAXRecords[hdrMountPoint]
rp := winio.ReparsePoint{
Target: filepath.FromSlash(hdr.Linkname),
IsMountPoint: isMountPoint,
}
reparse := winio.EncodeReparsePoint(&rp)
bhdr := winio.BackupHeader{
Id: winio.BackupReparseData,
Size: int64(len(reparse)),
}
err := bw.WriteHeader(&bhdr)
if err != nil {
return nil, err
}
_, err = bw.Write(reparse)
if err != nil {
return nil, err
}
}

buf := bufPool.Get().(*[]byte)
defer bufPool.Put(buf)

if hdr.Typeflag == tar.TypeReg || hdr.Typeflag == tar.TypeRegA {
bhdr := winio.BackupHeader{
Id: winio.BackupData,
Size: hdr.Size,
}
err := bw.WriteHeader(&bhdr)
if err != nil {
return nil, err
}
_, err = io.CopyBuffer(bw, t, *buf)
if err != nil {
return nil, err
}
}
// Copy all the alternate data streams and return the next non-ADS header.
for {
ahdr, err := t.Next()
if err != nil {
return nil, err
}
if ahdr.Typeflag != tar.TypeReg || !strings.HasPrefix(ahdr.Name, hdr.Name+":") {
return ahdr, nil
}
bhdr := winio.BackupHeader{
Id: winio.BackupAlternateData,
Size: ahdr.Size,
Name: ahdr.Name[len(hdr.Name):] + ":$DATA",
}
err = bw.WriteHeader(&bhdr)
if err != nil {
return nil, err
}
_, err = io.CopyBuffer(bw, t, *buf)
if err != nil {
return nil, err
}
}
}

func copyDirInfo(fi os.FileInfo, path string) error {
if err := os.Chmod(path, fi.Mode()); err != nil {
return errors.Wrapf(err, "failed to chmod %s", path)
}
return nil
}

func copyUpXAttrs(dst, src string) error {
return nil
return backuptar.WriteBackupStreamFromTarFile(buf, t, hdr)
}
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.15

require (
github.com/BurntSushi/toml v0.3.1
github.com/Microsoft/go-winio v0.4.15-0.20200908182639-5b44b70ab3ab
github.com/Microsoft/go-winio v0.4.15
github.com/Microsoft/hcsshim v0.8.10
github.com/Microsoft/hcsshim/test v0.0.0-20201119174602-966bebae11b4
github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4
Expand Down
3 changes: 2 additions & 1 deletion go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,9 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
github.com/Microsoft/go-winio v0.4.15-0.20200908182639-5b44b70ab3ab h1:9pygWVFqbY9lPxM0peffumuVDyMuIMzNLyO9uFjJuQo=
github.com/Microsoft/go-winio v0.4.15-0.20200908182639-5b44b70ab3ab/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
github.com/Microsoft/go-winio v0.4.15 h1:qkLXKzb1QoVatRyd/YlXZ/Kg0m5K3SPuoD82jjSOaBc=
github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ=
Expand Down
4 changes: 4 additions & 0 deletions vendor/github.com/Microsoft/go-winio/backuptar/noop.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 9c398e1

Please sign in to comment.