Skip to content

Commit

Permalink
Merge pull request #11 from Ahuge/master
Browse files Browse the repository at this point in the history
Prepare for Release v2.3.0
  • Loading branch information
Ahuge committed Mar 7, 2024
2 parents a0fc23a + 46af49d commit 572c3fd
Show file tree
Hide file tree
Showing 112 changed files with 22,192 additions and 12 deletions.
79 changes: 78 additions & 1 deletion command/cp.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,20 @@ Examples:
> s5cmd {{.HelpName}} --version-id VERSION_ID s3://bucket/prefix/object .
24. Pass arbitrary metadata to the object during upload or copy
> s5cmd {{.HelpName}} --metadata "camera=Nixon D750" --metadata "imageSize=6032x4032" flowers.png s3://bucket/prefix/flowers.png
> s5cmd {{.HelpName}} --metadata "camera=Nixon D750" --metadata "imageSize=6032x4032" flowers.png s3://bucket/prefix/flowers.png
25. Upload a file to S3 preserving the timestamp on disk
> s5cmd --preserve-timestamp myfile.css.br s3://bucket/
26. Download a file from S3 preserving the timestamp it was originally uploaded with
> s5cmd --preserve-timestamp s3://bucket/myfile.css.br myfile.css.br
27. Upload a file to S3 preserving the ownership of files
> s5cmd --preserve-ownership myfile.css.br s3://bucket/
28. Download a file from S3 preserving the ownership it was originally uploaded with
> s5cmd --preserve-ownership s3://bucket/myfile.css.br myfile.css.br
`

func NewSharedFlags() []cli.Flag {
Expand Down Expand Up @@ -207,6 +220,14 @@ func NewSharedFlags() []cli.Flag {
DefaultText: "0",
Hidden: true,
},
&cli.BoolFlag{
Name: "preserve-timestamp",
Usage: "preserve the timestamp on disk while uploading and set the timestamp from s3 while downloading.",
},
&cli.BoolFlag{
Name: "preserve-ownership",
Usage: "preserve the ownership (owner/group) on disk while uploading and set the ownership from s3 while downloading.",
},
}
}

Expand Down Expand Up @@ -306,6 +327,8 @@ type Copy struct {
contentDisposition string
metadata map[string]string
showProgress bool
preserveTimestamp bool
preserveOwnership bool
progressbar progressbar.ProgressBar

// patterns
Expand Down Expand Up @@ -384,6 +407,8 @@ func NewCopy(c *cli.Context, deleteSource bool) (*Copy, error) {
metadata: metadata,
showProgress: c.Bool("show-progress"),
progressbar: commandProgressBar,
preserveTimestamp: c.Bool("preserve-timestamp"),
preserveOwnership: c.Bool("preserve-ownership"),

// region settings
srcRegion: c.String("source-region"),
Expand Down Expand Up @@ -648,6 +673,42 @@ func (c Copy) doDownload(ctx context.Context, srcurl *url.URL, dsturl *url.URL)
return err
}

if c.preserveOwnership {
obj, err := srcClient.Stat(ctx, srcurl)
if err != nil {
return err
}
// SetFileUserGroup may return an InvalidOwnershipFormatError which signifies that it cannot
// understand the UserID or GroupID format.
// This is most common when a file is being ported across windows/linux.
// We aren't implementing a fix for it here, just a note that it cannot be resolved.
err = storage.SetFileUserGroup(dsturl.Absolute(), obj.UserID, obj.GroupID)
if err != nil {
invalidOwnershipFormat := &storage.InvalidOwnershipFormatError{}
if errors.As(err, &invalidOwnershipFormat) {
msg := log.ErrorMessage{
Operation: c.op,
Command: c.fullCommand,
Err: fmt.Sprintf("UserID: %s or GroupID: %s are not valid on this operating system.", obj.UserID, obj.GroupID),
}
log.Debug(msg)
}

return err
}
}

if c.preserveTimestamp {
obj, err := srcClient.Stat(ctx, srcurl)
if err != nil {
return err
}
err = storage.SetFileTime(dsturl.Absolute(), *obj.AccessTime, *obj.ModTime, *obj.CreateTime)
if err != nil {
return err
}
}

if !c.showProgress {
msg := log.InfoMessage{
Operation: c.op,
Expand Down Expand Up @@ -702,6 +763,22 @@ func (c Copy) doUpload(ctx context.Context, srcurl *url.URL, dsturl *url.URL, ex
EncryptionKeyID: c.encryptionKeyID,
}

if c.preserveTimestamp {
aTime, mTime, cTime, err := storage.GetFileTime(srcurl.Absolute())
if err != nil {
return err
}
storage.SetMetadataTimestamp(metadata, aTime, mTime, cTime)
}

if c.preserveOwnership {
userID, groupID, err := storage.GetFileUserGroup(srcurl.Absolute())
if err != nil {
return err
}
storage.SetMetadataOwnership(metadata, userID, groupID)
}

if c.contentType != "" {
metadata.ContentType = c.contentType
} else {
Expand Down
24 changes: 18 additions & 6 deletions command/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,9 +128,11 @@ type Sync struct {
fullCommand string

// flags
delete bool
sizeOnly bool
exitOnError bool
delete bool
sizeOnly bool
exitOnError bool
preserveTimestamp bool
preserveOwnership bool

// s3 options
storageOpts storage.Options
Expand All @@ -152,9 +154,11 @@ func NewSync(c *cli.Context) Sync {
fullCommand: commandFromContext(c),

// flags
delete: c.Bool("delete"),
sizeOnly: c.Bool("size-only"),
exitOnError: c.Bool("exit-on-error"),
delete: c.Bool("delete"),
sizeOnly: c.Bool("size-only"),
exitOnError: c.Bool("exit-on-error"),
preserveTimestamp: c.Bool("preserve-timestamp"),
preserveOwnership: c.Bool("preserve-ownership"),

// flags
followSymlinks: !c.Bool("no-follow-symlinks"),
Expand Down Expand Up @@ -451,6 +455,14 @@ func (s Sync) planRun(
"raw": true,
}

if s.preserveOwnership {
defaultFlags["preserve-ownership"] = s.preserveOwnership
}

if s.preserveTimestamp {
defaultFlags["preserve-timestamp"] = s.preserveTimestamp
}

// it should wait until both of the child goroutines for onlySource and common channels
// are completed before closing the WriteCloser w to ensure that all URLs are processed.
var wg sync.WaitGroup
Expand Down
4 changes: 3 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/peak/s5cmd/v2
go 1.19

require (
github.com/Microsoft/go-winio v0.6.1
github.com/aws/aws-sdk-go v1.44.256
github.com/cheggaaa/pb/v3 v3.1.4
github.com/golang/mock v1.6.0
Expand All @@ -15,6 +16,7 @@ require (
github.com/lanrat/extsort v1.0.0
github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae
github.com/urfave/cli/v2 v2.11.2
golang.org/x/sys v0.7.0
gotest.tools/v3 v3.0.3
)

Expand All @@ -36,8 +38,8 @@ require (
github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
golang.org/x/mod v0.10.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/sys v0.7.0 // indirect
golang.org/x/tools v0.8.0 // indirect
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect
Expand Down
3 changes: 3 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
github.com/aws/aws-sdk-go v1.44.256 h1:O8VH+bJqgLDguqkH/xQBFz5o/YheeZqgcOYIgsTVWY4=
Expand Down Expand Up @@ -83,6 +85,7 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
Expand Down
8 changes: 8 additions & 0 deletions storage/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,14 @@ import (
"github.com/peak/s5cmd/v2/storage/url"
)

type InvalidOwnershipFormatError struct {
Err error
}

func (e *InvalidOwnershipFormatError) Error() string {
return fmt.Sprintf("InvalidOwnershipFormatError: %v\n", e.Err)
}

// Filesystem is the Storage implementation of a local filesystem.
type Filesystem struct {
dryRun bool
Expand Down
87 changes: 87 additions & 0 deletions storage/fs_darwin.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
//go:build darwin

package storage

import (
"os"
"strconv"
"strings"
"syscall"
"time"
)

func GetFileTime(filename string) (time.Time, time.Time, time.Time, error) {
fi, err := os.Stat(filename)
if err != nil {
return time.Time{}, time.Time{}, time.Time{}, err
}

stat := fi.Sys().(*syscall.Stat_t)
cTime := time.Unix(int64(stat.Ctimespec.Sec), int64(stat.Ctimespec.Nsec))
aTime := time.Unix(int64(stat.Atimespec.Sec), int64(stat.Atimespec.Nsec))

mTime := fi.ModTime()

return aTime, mTime, cTime, nil
}

func SetFileTime(filename string, accessTime, modificationTime, creationTime time.Time) error {
var err error
if accessTime.IsZero() && modificationTime.IsZero() {
// Nothing recorded in s3. Return fast.
return nil
} else if accessTime.IsZero() {
accessTime, _, _, err = GetFileTime(filename)
if err != nil {
return err
}
} else if modificationTime.IsZero() {
_, modificationTime, _, err = GetFileTime(filename)
if err != nil {
return err
}
}
err = os.Chtimes(filename, accessTime, modificationTime)
if err != nil {
return err
}
return nil
}

// GetFileUserGroup will take a filename and return the userId and groupId associated with it.
//
// On windows this is in the format of a SID, on linux/darwin this is in the format of a UID/GID.
func GetFileUserGroup(filename string) (username, group string, err error) {
info, err := os.Stat(filename)
if err != nil {
return "", "", err
}

stat := info.Sys().(*syscall.Stat_t)

username = strconv.Itoa(int(stat.Uid))
group = strconv.Itoa(int(stat.Gid))
return username, group, nil
}

// SetFileUserGroup will set the UserId and GroupId on a filename.
//
// If the UserId/GroupId format does not match the platform, it will return an InvalidOwnershipFormatError.
//
// Windows expects the UserId/GroupId to be in SID format, Linux and Darwin expect it in UID/GID format.
func SetFileUserGroup(filename, uid, gid string) error {
uidI, err := strconv.Atoi(uid)
if err != nil && strings.Contains(err.Error(), "") {
return err
}
gidI, err := strconv.Atoi(gid)
if err != nil {
return err
}

err = os.Lchown(filename, uidI, gidI)
if err != nil {
return err
}
return nil
}
88 changes: 88 additions & 0 deletions storage/fs_linux.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
//go:build linux

package storage

import (
"os"
"strconv"
"syscall"
"time"
)

func GetFileTime(filename string) (time.Time, time.Time, time.Time, error) {
fi, err := os.Stat(filename)
if err != nil {
return time.Time{}, time.Time{}, time.Time{}, err
}

stat := fi.Sys().(*syscall.Stat_t)
cTime := time.Unix(int64(stat.Ctim.Sec), int64(stat.Ctim.Nsec))
aTime := time.Unix(int64(stat.Atim.Sec), int64(stat.Atim.Nsec))

mTime := fi.ModTime()

return aTime, mTime, cTime, nil
}

func SetFileTime(filename string, accessTime, modificationTime, creationTime time.Time) error {
if accessTime.IsZero() && modificationTime.IsZero() {
// Nothing recorded in s3. Return fast.
return nil
}
var err error
if accessTime.IsZero() {
accessTime, _, _, err = GetFileTime(filename)
if err != nil {
return err
}
}
if modificationTime.IsZero() {
_, modificationTime, _, err = GetFileTime(filename)
if err != nil {
return err
}
}
err = os.Chtimes(filename, accessTime, modificationTime)
if err != nil {
return err
}
return nil
}

// GetFileUserGroup will take a filename and return the userId and groupId associated with it.
//
// On windows this is in the format of a SID, on linux/darwin this is in the format of a UID/GID.
func GetFileUserGroup(filename string) (username, group string, err error) {
info, err := os.Stat(filename)
if err != nil {
return "", "", err
}

stat := info.Sys().(*syscall.Stat_t)

username = strconv.Itoa(int(stat.Uid))
group = strconv.Itoa(int(stat.Gid))
return username, group, nil
}

// SetFileUserGroup will set the UserId and GroupId on a filename.
//
// If the UserId/GroupId format does not match the platform, it will return an InvalidOwnershipFormatError.
//
// Windows expects the UserId/GroupId to be in SID format, Linux and Darwin expect it in UID/GID format.
func SetFileUserGroup(filename, uid, gid string) error {
uidI, err := strconv.Atoi(uid)
if err != nil {
return &InvalidOwnershipFormatError{Err: err}
}
gidI, err := strconv.Atoi(gid)
if err != nil {
return &InvalidOwnershipFormatError{Err: err}
}

err = os.Lchown(filename, uidI, gidI)
if err != nil {
return err
}
return nil
}
Loading

0 comments on commit 572c3fd

Please sign in to comment.