Merge pull request #13 from Ahuge/master

Prepare for Release v2.3.1

command/cp.go

@@ -491,7 +491,7 @@ func (c Copy) Run(ctx context.Context) error {
 	}
 
 	for object := range objch {
-		if errorpkg.IsCancelation(object.Err) || object.Type.IsDir() {
+		if errorpkg.IsCancelation(object.Err) {
 			continue
 		}
 
@@ -541,7 +541,7 @@ func (c Copy) Run(ctx context.Context) error {
 		case srcurl.Type == c.dst.Type: // local->local or remote->remote
 			task = c.prepareCopyTask(ctx, srcurl, c.dst, isBatch, c.metadata)
 		case srcurl.IsRemote(): // remote->local
-			task = c.prepareDownloadTask(ctx, srcurl, c.dst, isBatch)
+			task = c.prepareDownloadTask(ctx, srcurl, c.dst, isBatch, object.Type.IsDir())
 		case c.dst.IsRemote(): // local->remote
 			task = c.prepareUploadTask(ctx, srcurl, c.dst, isBatch, c.metadata)
 		default:
@@ -583,9 +583,10 @@ func (c Copy) prepareDownloadTask(
 	srcurl *url.URL,
 	dsturl *url.URL,
 	isBatch bool,
+	srcIsDir bool,
 ) func() error {
 	return func() error {
-		dsturl, err := prepareLocalDestination(ctx, srcurl, dsturl, c.flatten, isBatch, c.storageOpts)
+		dsturl, err := prepareLocalDestination(ctx, srcurl, dsturl, c.flatten, isBatch, c.storageOpts, srcIsDir)
 		if err != nil {
 			return err
 		}
@@ -644,35 +645,53 @@ func (c Copy) doDownload(ctx context.Context, srcurl *url.URL, dsturl *url.URL)
 		}
 		return err
 	}
+	// Check to see if the source is a directory for locally creation a directory too
+	srcObj, err := srcClient.Stat(ctx, srcurl)
+	if err != nil {
+		var objNotFound *storage.ErrGivenObjectNotFound
+		if !errors.As(err, &objNotFound) {
+			return err
+		}
+
+	}
 
 	dstPath := filepath.Dir(dsturl.Absolute())
 	dstFile := filepath.Base(dsturl.Absolute())
-	file, err := dstClient.CreateTemp(dstPath, dstFile)
-	if err != nil {
-		return err
-	}
 
-	writer := newCountingReaderWriter(file, c.progressbar)
-	size, err := srcClient.Get(ctx, srcurl, writer, c.concurrency, c.partSize)
-	file.Close()
+	isDir := srcObj.Type.IsDir()
+	var size int64 = 0
+	if isDir {
+		err = dstClient.CreateDir(ctx, dsturl.Absolute(), storage.Metadata{})
+		if err != nil {
+			return err
+		}
+	} else {
+		file, err := dstClient.CreateTemp(dstPath, dstFile)
+		if err != nil {
+			return err
+		}
 
-	if err != nil {
-		dErr := dstClient.Delete(ctx, &url.URL{Path: file.Name(), Type: dsturl.Type})
-		if dErr != nil {
-			printDebug(c.op, dErr, srcurl, dsturl)
+		writer := newCountingReaderWriter(file, c.progressbar)
+		size, err = srcClient.Get(ctx, srcurl, writer, c.concurrency, c.partSize)
+
+		file.Close()
+		if err != nil {
+			dErr := dstClient.Delete(ctx, &url.URL{Path: file.Name(), Type: dsturl.Type})
+			if dErr != nil {
+				printDebug(c.op, dErr, srcurl, dsturl)
+			}
+			return err
+		}
+		err = dstClient.Rename(file, dsturl.Absolute())
+		if err != nil {
+			return err
 		}
-		return err
 	}
 
 	if c.deleteSource {
 		_ = srcClient.Delete(ctx, srcurl)
 	}
 
-	err = dstClient.Rename(file, dsturl.Absolute())
-	if err != nil {
-		return err
-	}
-
 	if c.preserveOwnership {
 		obj, err := srcClient.Stat(ctx, srcurl)
 		if err != nil {
@@ -786,7 +805,15 @@ func (c Copy) doUpload(ctx context.Context, srcurl *url.URL, dsturl *url.URL, ex
 	}
 
 	reader := newCountingReaderWriter(file, c.progressbar)
-	err = dstClient.Put(ctx, reader, dsturl, metadata, c.concurrency, c.partSize)
+	fi, err := file.Stat()
+	if err != nil {
+		return err
+	}
+	if fi.IsDir() {
+		err = dstClient.CreateDir(ctx, dsturl, metadata)
+	} else {
+		err = dstClient.Put(ctx, reader, dsturl, metadata, c.concurrency, c.partSize)
+	}
 
 	if err != nil {
 		return err
@@ -957,6 +984,10 @@ func prepareRemoteDestination(
 		objname = srcurl.Relative()
 	}
 
+	if objname == "." {
+		return dsturl
+	}
+
 	if dsturl.IsPrefix() || dsturl.IsBucket() {
 		dsturl = dsturl.Join(objname)
 	}
@@ -972,6 +1003,7 @@ func prepareLocalDestination(
 	flatten bool,
 	isBatch bool,
 	storageOpts storage.Options,
+	srcIsDir bool,
 ) (*url.URL, error) {
 	objname := srcurl.Base()
 	if isBatch && !flatten {
@@ -1008,11 +1040,11 @@ func prepareLocalDestination(
 		if err != nil {
 			return nil, err
 		}
-		if strings.HasSuffix(dsturl.Absolute(), "/") {
+		if strings.HasSuffix(dsturl.Absolute(), "/") && !srcIsDir {
 			dsturl = dsturl.Join(objname)
 		}
 	} else {
-		if obj.Type.IsDir() {
+		if obj.Type.IsDir() && !srcIsDir {
 			dsturl = obj.URL.Join(objname)
 		}
 	}
@@ -1058,7 +1090,7 @@ func validateCopyCommand(c *cli.Context) error {
 	}
 
 	// we don't operate on S3 prefixes for copy and delete operations.
-	if srcurl.IsBucket() || srcurl.IsPrefix() {
+	if srcurl.IsBucket() {
 		return fmt.Errorf("source argument must contain wildcard character")
 	}
 
@@ -1107,6 +1139,10 @@ func validateUpload(ctx context.Context, srcurl, dsturl *url.URL, storageOpts st
 		return err
 	}
 
+	if obj.Type.IsDir() {
+		return nil
+	}
+
 	// 'cp dir/ s3://bucket/prefix-without-slash': expect a trailing slash to
 	// avoid any surprises.
 	if obj.Type.IsDir() && !dsturl.IsBucket() && !dsturl.IsPrefix() {

command/expand.go

@@ -23,7 +23,7 @@ func expandSource(
 	// if the source is local, we send a Stat call to know if  we have
 	// directory or file to walk. For remote storage, we don't want to send
 	// Stat since it doesn't have any folder semantics.
-	if !srcurl.IsWildcard() && !srcurl.IsRemote() {
+	if !srcurl.IsWildcard() {
 		obj, err := client.Stat(ctx, srcurl)
 		if err != nil {
 			return nil, err

command/rm.go

@@ -275,7 +275,7 @@ func validateRMCommand(c *cli.Context) error {
 	)
 	for i, srcurl := range srcurls {
 		// we don't operate on S3 prefixes for copy and delete operations.
-		if srcurl.IsBucket() || srcurl.IsPrefix() {
+		if srcurl.IsBucket() {
 			return fmt.Errorf("s3 bucket/prefix cannot be used for delete operations (forgot wildcard character?)")
 		}
 

command/sync.go

@@ -547,6 +547,10 @@ func generateDestinationURL(srcurl, dsturl *url.URL, isBatch bool) *url.URL {
 		objname = srcurl.Relative()
 	}
 
+	if strings.HasSuffix(srcurl.Absolute(), "/") && !strings.HasSuffix(objname, "/") {
+		objname += "/"
+	}
+
 	if dsturl.IsRemote() {
 		if dsturl.IsPrefix() || dsturl.IsBucket() {
 			return dsturl.Join(objname)
@@ -560,7 +564,7 @@ func generateDestinationURL(srcurl, dsturl *url.URL, isBatch bool) *url.URL {
 
 // shouldSkipObject checks is object should be skipped.
 func (s Sync) shouldSkipObject(object *storage.Object, verbose bool) bool {
-	if object.Type.IsDir() || errorpkg.IsCancelation(object.Err) {
+	if errorpkg.IsCancelation(object.Err) {
 		return true
 	}
 

storage/fs.go

@@ -130,7 +130,7 @@ func walkDir(ctx context.Context, fs *Filesystem, src *url.URL, followSymlinks b
 		Callback: func(pathname string, dirent *godirwalk.Dirent) error {
 			// we're interested in files
 			if dirent.IsDir() {
-				return nil
+				pathname += string(os.PathSeparator)
 			}
 
 			fileurl, err := url.New(pathname)
@@ -230,6 +230,24 @@ func (f *Filesystem) Create(path string) (*os.File, error) {
 	return os.Create(path)
 }
 
+func (f *Filesystem) CreateDir(
+	ctx context.Context,
+	path string,
+	metadata Metadata,
+) error {
+	if f.dryRun {
+		return nil
+	}
+	if _, err := os.Stat(path); err != nil {
+		if os.IsNotExist(err) {
+			return os.Mkdir(path, os.ModePerm)
+		} else {
+			return err
+		}
+	}
+	return nil
+}
+
 // Open opens the given source.
 func (f *Filesystem) Open(path string) (*os.File, error) {
 	file, err := os.OpenFile(path, os.O_RDONLY, 0644)

storage/fs_windows.go

@@ -59,7 +59,13 @@ func SetFileTime(filename string, accessTime, modificationTime, creationTime tim
 	mft := syscall.NsecToFiletime(modificationTime.UnixNano())
 	cft := syscall.NsecToFiletime(creationTime.UnixNano())
 
-	fd, err := syscall.Open(filename, os.O_RDWR, 0775)
+	var fd syscall.Handle
+	fi, err := os.Stat(filename)
+	if fi.IsDir() {
+		fd, err = getDirectoryHandle(filename)
+	} else {
+		fd, err = syscall.Open(filename, os.O_RDWR, 0775)
+	}
 	if err != nil {
 		return err
 	}
@@ -73,6 +79,21 @@ func SetFileTime(filename string, accessTime, modificationTime, creationTime tim
 	return nil
 }
 
+func getDirectoryHandle(filename string) (syscall.Handle, error) {
+	pathp, err := syscall.UTF16PtrFromString(filename)
+	if err != nil {
+		return syscall.InvalidHandle, err
+	}
+
+	h, e := syscall.CreateFile(pathp,
+		syscall.FILE_WRITE_ATTRIBUTES, syscall.FILE_SHARE_WRITE, nil,
+		syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS, 0)
+	if e != nil {
+		return syscall.InvalidHandle, e
+	}
+	return h, nil
+}
+
 // GetFileUserGroup will take a filename and return the userId and groupId associated with it.
 //
 //	On windows this is in the format of a SID, on linux/darwin this is in the format of a UID/GID.