Skip to content

Commit

Permalink
re-implement metadata handling to preserve nanosecs as well
Browse files Browse the repository at this point in the history
  • Loading branch information
@harshavardhana
harshavardhana committed Sep 10, 2020
1 parent 8f2ffbf commit 6d6a5af
Show file tree
Hide file tree
Showing 9 changed files with 77 additions and 85 deletions.
76 changes: 37 additions & 39 deletions cmd/client-fs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package cmd
import (
"context"
"errors"
"fmt"
"io"
"os"
"path"
Expand All @@ -41,6 +42,7 @@ import (
"github.com/minio/minio-go/v7/pkg/encrypt"
"github.com/minio/minio-go/v7/pkg/lifecycle"
"github.com/minio/minio-go/v7/pkg/replication"
"github.com/minio/minio/pkg/console"
)

// filesystem client
Expand Down Expand Up @@ -225,49 +227,33 @@ func (f *fsClient) Watch(ctx context.Context, options WatchOptions) (*WatchObjec
func preserveAttributes(fd *os.File, attr map[string]string) *probe.Error {
if val, ok := attr["mode"]; ok {
mode, e := strconv.ParseUint(val, 0, 32)
if e != nil {
return probe.NewError(e)
}

// Attempt to change the file mode.
if e := fd.Chmod(os.FileMode(mode)); e != nil {
return probe.NewError(e)
if e == nil {
// Attempt to change the file mode.
if e = fd.Chmod(os.FileMode(mode)); e != nil {
return probe.NewError(e)
}
}
}

var uid, gid int
var gidExists, uidExists bool
var e error
if val, ok := attr["uid"]; ok {
uid, e = strconv.Atoi(val)
if e != nil {
return probe.NewError(e)
uid = -1
}
uidExists = true
}

if val, ok := attr["gid"]; ok {
gid, e = strconv.Atoi(val)
if e != nil {
return probe.NewError(e)
gid = -1
}
gidExists = true
}

// Attempt to change the owner.
if gidExists && uidExists {
if e := fd.Chown(uid, gid); e != nil {
return probe.NewError(e)
}
} else if uidExists {
if e := fd.Chown(uid, -1); e != nil {
return probe.NewError(e)
}

} else {
if e := fd.Chown(-1, gid); e != nil {
return probe.NewError(e)
}
if e = fd.Chown(uid, gid); e != nil {
return probe.NewError(e)
}

return nil
Expand Down Expand Up @@ -317,9 +303,9 @@ func (f *fsClient) put(ctx context.Context, reader io.Reader, size int64, meta m
tmpFile.Close()
return 0, probe.NewError(e)
}
if err := preserveAttributes(tmpFile, attr); err != nil {
tmpFile.Close()
return 0, err.Trace(objectPath)
err := preserveAttributes(tmpFile, attr)
if err != nil {
console.Println(console.Colorize("Error", fmt.Sprintf("unable to preserve attributes, continuing to copy the content %s\n", err.ToGoError())))
}
}

Expand Down Expand Up @@ -370,28 +356,42 @@ func (f *fsClient) put(ctx context.Context, reader io.Reader, size int64, meta m
}

if len(attr) != 0 && preserve {
var atime, mtime int64
var e error
var atime, mtime syscall.Timespec
var atimeChanged, mtimeChanged bool
if val, ok := attr["atime"]; ok {
atime, e = strconv.ParseInt(val, 10, 64)
vals := strings.SplitN(val, "#", 2)
atim, e := strconv.ParseInt(vals[0], 10, 64)
if e != nil {
return totalWritten, probe.NewError(e)
}
atimeChanged = true
atime.Sec = atim
if len(vals) == 2 {
atimnsec, e := strconv.ParseInt(vals[1], 10, 64)
if e != nil {
return totalWritten, probe.NewError(e)
}
atime.Nsec = atimnsec
}
}

if val, ok := attr["mtime"]; ok {
mtime, e = strconv.ParseInt(val, 10, 64)
vals := strings.SplitN(val, "#", 2)
mtim, e := strconv.ParseInt(vals[0], 10, 64)
if e != nil {
return totalWritten, probe.NewError(e)
}
mtimeChanged = true
mtime.Sec = mtim
if len(vals) == 2 {
mtimnsec, e := strconv.ParseInt(vals[1], 10, 64)
if e != nil {
return totalWritten, probe.NewError(e)
}
mtime.Nsec = mtimnsec
}
}

// Attempt to change the access and modify time
if atimeChanged && mtimeChanged {
if e := os.Chtimes(objectPath, time.Unix(atime, 0), time.Unix(mtime, 0)); e != nil {
if e := os.Chtimes(objectPath, time.Unix(atime.Unix()), time.Unix(mtime.Unix())); e != nil {
return totalWritten, probe.NewError(e)
}
}
Expand Down Expand Up @@ -600,9 +600,7 @@ func readDir(dirname string) ([]os.FileInfo, error) {
if e != nil {
return nil, e
}
if e = f.Close(); e != nil {
return nil, e
}
defer f.Close()
sort.Sort(byDirName(list))
return list, nil
}
Expand Down
5 changes: 1 addition & 4 deletions cmd/client-s3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1341,10 +1341,7 @@ func (c *S3Client) listObjectWrapper(ctx context.Context, bucket, object string,
// https://github.com/minio/mc/issues/3073
return c.api.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: object, Recursive: isRecursive, UseV1: true})
}
if metadata {
return c.api.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: object, Recursive: isRecursive})
}
return c.api.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: object, Recursive: isRecursive})
return c.api.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: object, Recursive: isRecursive, WithMetadata: metadata})
}

func (c *S3Client) statIncompleteUpload(ctx context.Context, bucket, object string) (*ClientContent, *probe.Error) {
Expand Down
12 changes: 7 additions & 5 deletions cmd/difference.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,11 +94,15 @@ func activeActiveModTimeUpdated(src, dst *ClientContent) bool {
return false
}

srcActualModTime := src.Time
dstActualModTime := dst.Time

srcModTime := getSourceModTimeKey(src.UserMetadata)
dstModTime := getSourceModTimeKey(dst.UserMetadata)
if srcModTime == "" && dstModTime == "" {
// No active-active mirror context found, consider src & dst as similar
return false
// No active-active mirror context found, fallback to modTimes presented
// by the client content
return srcActualModTime.After(dstActualModTime)
}

var srcOriginLastModified, dstOriginLastModified time.Time
Expand All @@ -118,12 +122,10 @@ func activeActiveModTimeUpdated(src, dst *ClientContent) bool {
}
}

srcActualModTime := src.Time
if !srcOriginLastModified.IsZero() && srcOriginLastModified.After(src.Time) {
srcActualModTime = srcOriginLastModified
}

dstActualModTime := dst.Time
if !dstOriginLastModified.IsZero() && dstOriginLastModified.After(dst.Time) {
dstActualModTime = dstOriginLastModified
}
Expand Down Expand Up @@ -265,7 +267,7 @@ func differenceInternal(ctx context.Context, sourceClnt, targetClnt Client, sour
}
continue
}
if (srcType.IsRegular() && tgtType.IsRegular()) && srcSize != tgtSize {
if srcSize != tgtSize {
// Regular files differing in size.
diffCh <- diffMessage{
FirstURL: srcCtnt.URL.String(),
Expand Down
38 changes: 23 additions & 15 deletions cmd/mirror-main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,8 +180,8 @@ EXAMPLES:
{{.Prompt}} {{.HelpName}} -a backup/ s3/archive
15. Cross mirror between sites in a active-active deployment.
Site-A: {{.Prompt}} {{.HelpName}} --watch --active-active siteA siteB
Site-B: {{.Prompt}} {{.HelpName}} --watch --active-active siteB siteA
Site-A: {{.Prompt}} {{.HelpName}} --active-active siteA siteB
Site-B: {{.Prompt}} {{.HelpName}} --active-active siteB siteA
`,
}

Expand Down Expand Up @@ -719,13 +719,6 @@ func getEventPathURLWin(srcURL, eventPath string) string {

// runMirror - mirrors all buckets to another S3 server
func runMirror(ctx context.Context, cancelMirror context.CancelFunc, srcURL, dstURL string, cli *cli.Context, encKeyDB map[string][]prefixSSEPair) bool {
// This is kept for backward compatibility, `--force` means
// --overwrite.
isOverwrite := cli.Bool("force")
if !isOverwrite {
isOverwrite = cli.Bool("overwrite")
}

// Parse metadata.
userMetadata := make(map[string]string)
if cli.String("attr") != "" {
Expand All @@ -745,13 +738,25 @@ func runMirror(ctx context.Context, cancelMirror context.CancelFunc, srcURL, dst
(srcClt.GetURL().Type == objectStorage &&
srcClt.GetURL().Path == string(srcClt.GetURL().Separator))

// Create a new mirror job and execute it
mj := newMirrorJob(srcURL, dstURL, mirrorOptions{
// This is kept for backward compatibility, `--force` means
// --overwrite.
isOverwrite := cli.Bool("force")
if !isOverwrite {
isOverwrite = cli.Bool("overwrite")
}

isWatch := cli.Bool("watch") || cli.Bool("multi-master") || cli.Bool("active-active")

// preserve is also expected to be overwritten if necessary
isMetadata := cli.Bool("a") || isWatch || len(userMetadata) > 0
isOverwrite = isOverwrite || isMetadata

mopts := mirrorOptions{
isFake: cli.Bool("fake"),
isRemove: cli.Bool("remove"),
isOverwrite: isOverwrite,
isWatch: cli.Bool("watch") || cli.Bool("multi-master") || cli.Bool("active-active"),
isMetadata: cli.Bool("a") || cli.Bool("multi-master") || cli.Bool("active-active") || len(userMetadata) > 0,
isWatch: isWatch,
isMetadata: isMetadata,
md5: cli.Bool("md5"),
disableMultipart: cli.Bool("disable-multipart"),
excludeOptions: cli.StringSlice("exclude"),
Expand All @@ -760,8 +765,11 @@ func runMirror(ctx context.Context, cancelMirror context.CancelFunc, srcURL, dst
storageClass: cli.String("storage-class"),
userMetadata: userMetadata,
encKeyDB: encKeyDB,
activeActive: cli.Bool("multi-master") || cli.Bool("active-active"),
})
activeActive: isWatch,
}

// Create a new mirror job and execute it
mj := newMirrorJob(srcURL, dstURL, mopts)

if mirrorAllBuckets {
// Synchronize buckets using dirDifference function
Expand Down
2 changes: 1 addition & 1 deletion cmd/utils_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ func TestParseAttribute(t *testing.T) {
// When "atime:" is passed.
{"atime:", map[string]string{"atime": ""}, nil, true},
// Passing a valid value
{"atime:1/ctime:1/gid:1/gname:a/md:/mode:3/mtime:1/uid:1/uname:a", map[string]string{"atime": "1", "ctime": "1", "gid": "1", "gname": "a", "md": "", "mode": "3", "mtime": "1", "uid": "1", "uname": "a"}, nil, true},
{"atime:1/gid:1/gname:a/md:/mode:3/mtime:1/uid:1/uname:a", map[string]string{"atime": "1", "gid": "1", "gname": "a", "md": "", "mode": "3", "mtime": "1", "uid": "1", "uname": "a"}, nil, true},
}

for idx, testCase := range metaDataCases {
Expand Down
11 changes: 2 additions & 9 deletions pkg/disk/stat_darwin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (
"strconv"
"strings"
"syscall"
"time"
)

// GetFileSystemAttrs return the file system attribute as string; containing mode,
Expand All @@ -37,9 +36,7 @@ func GetFileSystemAttrs(file string) (string, error) {

var fileAttr strings.Builder
fileAttr.WriteString("atime:")
fileAttr.WriteString(timespecToTime(st.Atimespec).String())
fileAttr.WriteString("/ctime:")
fileAttr.WriteString(timespecToTime(st.Ctimespec).String())
fileAttr.WriteString(strconv.FormatInt(st.Atimespec.Sec, 10) + "#" + strconv.FormatInt(st.Atimespec.Nsec, 10))
fileAttr.WriteString("/gid:")
fileAttr.WriteString(strconv.Itoa(int(st.Gid)))

Expand All @@ -52,7 +49,7 @@ func GetFileSystemAttrs(file string) (string, error) {
fileAttr.WriteString("/mode:")
fileAttr.WriteString(strconv.Itoa(int(st.Mode)))
fileAttr.WriteString("/mtime:")
fileAttr.WriteString(timespecToTime(st.Mtimespec).String())
fileAttr.WriteString(strconv.FormatInt(st.Mtimespec.Sec, 10) + "#" + strconv.FormatInt(st.Mtimespec.Nsec, 10))
fileAttr.WriteString("/uid:")
fileAttr.WriteString(strconv.Itoa(int(st.Uid)))

Expand All @@ -64,7 +61,3 @@ func GetFileSystemAttrs(file string) (string, error) {

return fileAttr.String(), nil
}

func timespecToTime(ts syscall.Timespec) time.Time {
return time.Unix(int64(ts.Sec), int64(ts.Nsec))
}
6 changes: 2 additions & 4 deletions pkg/disk/stat_freebsd_netbsd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,7 @@ func GetFileSystemAttrs(file string) (string, error) {

var fileAttr strings.Builder
fileAttr.WriteString("atime:")
fileAttr.WriteString(strconv.Itoa(int(st.Atimespec.Sec)))
fileAttr.WriteString("/ctime:")
fileAttr.WriteString(strconv.Itoa(int(st.Ctimespec.Sec)))
fileAttr.WriteString(strconv.FormatInt(st.Atimespec.Sec, 10) + "#" + strconv.FormatInt(st.Atimespec.Nsec, 10))
fileAttr.WriteString("/gid:")
fileAttr.WriteString(strconv.Itoa(int(st.Gid)))

Expand All @@ -51,7 +49,7 @@ func GetFileSystemAttrs(file string) (string, error) {
fileAttr.WriteString("/mode:")
fileAttr.WriteString(strconv.Itoa(int(st.Mode)))
fileAttr.WriteString("/mtime:")
fileAttr.WriteString(strconv.Itoa(int(st.Mtimespec.Sec)))
fileAttr.WriteString(strconv.FormatInt(st.Mtimespec.Sec, 10) + "#" + strconv.FormatInt(st.Mtimespec.Nsec, 10))
fileAttr.WriteString("/uid:")
fileAttr.WriteString(strconv.Itoa(int(st.Uid)))

Expand Down
6 changes: 2 additions & 4 deletions pkg/disk/stat_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,7 @@ func GetFileSystemAttrs(file string) (string, error) {

var fileAttr strings.Builder
fileAttr.WriteString("atime:")
fileAttr.WriteString(strconv.Itoa(int(st.Atim.Sec)))
fileAttr.WriteString("/ctime:")
fileAttr.WriteString(strconv.Itoa(int(st.Ctim.Sec)))
fileAttr.WriteString(strconv.FormatInt(st.Atim.Sec, 10) + "#" + strconv.FormatInt(st.Atim.Nsec, 10))
fileAttr.WriteString("/gid:")
fileAttr.WriteString(strconv.Itoa(int(st.Gid)))

Expand All @@ -53,7 +51,7 @@ func GetFileSystemAttrs(file string) (string, error) {
fileAttr.WriteString("/mode:")
fileAttr.WriteString(strconv.Itoa(int(st.Mode)))
fileAttr.WriteString("/mtime:")
fileAttr.WriteString(strconv.Itoa(int(st.Mtim.Sec)))
fileAttr.WriteString(strconv.FormatInt(st.Mtim.Sec, 10) + "#" + strconv.FormatInt(st.Mtim.Nsec, 10))
fileAttr.WriteString("/uid:")
fileAttr.WriteString(strconv.Itoa(int(st.Uid)))

Expand Down
6 changes: 2 additions & 4 deletions pkg/disk/stat_other.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,7 @@ func GetFileSystemAttrs(file string) (string, error) {

var fileAttr strings.Builder
fileAttr.WriteString("atime:")
fileAttr.WriteString(strconv.Itoa(int(st.Atim.Sec)))
fileAttr.WriteString("/ctime:")
fileAttr.WriteString(strconv.Itoa(int(st.Ctim.Sec)))
fileAttr.WriteString(strconv.FormatInt(st.Atim.Sec, 10) + "#" + strconv.FormatInt(st.Atim.Nsec, 10))
fileAttr.WriteString("/gid:")
fileAttr.WriteString(strconv.Itoa(int(st.Gid)))

Expand All @@ -52,7 +50,7 @@ func GetFileSystemAttrs(file string) (string, error) {
fileAttr.WriteString("/mode:")
fileAttr.WriteString(strconv.Itoa(int(st.Mode)))
fileAttr.WriteString("/mtime:")
fileAttr.WriteString(strconv.Itoa(int(st.Mtim.Sec)))
fileAttr.WriteString(strconv.FormatInt(st.Mtim.Sec, 10) + "#" + strconv.FormatInt(st.Mtim.Nsec, 10))
fileAttr.WriteString("/uid:")
fileAttr.WriteString(strconv.Itoa(int(st.Uid)))

Expand Down

0 comments on commit 6d6a5af

Please sign in to comment.