operations: Implement DirMove for moving a directory rclone#2539

This does the equivalent of sync.Move but is specialised for moving
files in one backend.

fs/operations/operations.go

@@ -26,6 +26,7 @@ import (
 	"github.com/ncw/rclone/fs/walk"
 	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
 )
 
 // CheckHashes checks the two files to see if they have common
@@ -1585,3 +1586,81 @@ func (l *ListFormat) Format(entry fs.DirEntry) (result string) {
 	}
 	return result
 }
+
+// DirMove renames srcRemote to dstRemote
+//
+// It does this by loading the directory tree into memory (using ListR
+// if available) and doing renames in parallel.
+func DirMove(f fs.Fs, srcRemote, dstRemote string) (err error) {
+	// Use DirMove if possible
+	if doDirMove := f.Features().DirMove; doDirMove != nil {
+		return doDirMove(f, srcRemote, dstRemote)
+	}
+
+	// Load the directory tree into memory
+	tree, err := walk.NewDirTree(f, srcRemote, true, -1)
+	if err != nil {
+		return errors.Wrap(err, "RenameDir tree walk")
+	}
+
+	// Get the directories in sorted order
+	dirs := tree.Dirs()
+
+	// Make the destination directories - must be done in order not in parallel
+	for _, dir := range dirs {
+		dstPath := dstRemote + dir[len(srcRemote):]
+		err := f.Mkdir(dstPath)
+		if err != nil {
+			return errors.Wrap(err, "RenameDir mkdir")
+		}
+	}
+
+	// Rename the files in parallel
+	type rename struct {
+		o       fs.Object
+		newPath string
+	}
+	renames := make(chan rename, fs.Config.Transfers)
+	g, ctx := errgroup.WithContext(context.Background())
+	for i := 0; i < fs.Config.Transfers; i++ {
+		g.Go(func() error {
+			for job := range renames {
+				dstOverwritten, _ := f.NewObject(job.newPath)
+				_, err := Move(f, dstOverwritten, job.newPath, job.o)
+				if err != nil {
+					return err
+				}
+				select {
+				case <-ctx.Done():
+					return ctx.Err()
+				default:
+				}
+
+			}
+			return nil
+		})
+	}
+	for dir, entries := range tree {
+		dstPath := dstRemote + dir[len(srcRemote):]
+		for _, entry := range entries {
+			if o, ok := entry.(fs.Object); ok {
+				renames <- rename{o, path.Join(dstPath, path.Base(o.Remote()))}
+			}
+		}
+	}
+	close(renames)
+	err = g.Wait()
+	if err != nil {
+		return errors.Wrap(err, "RenameDir renames")
+	}
+
+	// Remove the source directories in reverse order
+	for i := len(dirs) - 1; i >= 0; i-- {
+		err := f.Rmdir(dirs[i])
+		if err != nil {
+			return errors.Wrap(err, "RenameDir rmdir")
+		}
+	}
+
+	return nil
+}

fs/operations/operations_test.go

@@ -956,3 +956,90 @@ func TestListFormat(t *testing.T) {
 	assert.Equal(t, fmt.Sprintf("%d", items[1].Size())+"|subdir/|"+items[1].ModTime().Local().Format("2006-01-02 15:04:05"), list.Format(items[1]))
 
 }
+
+func TestDirMove(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	r.Mkdir(r.Fremote)
+
+	// Make some files and dirs
+	r.ForceMkdir(r.Fremote)
+	files := []fstest.Item{
+		r.WriteObject("A1/one", "one", t1),
+		r.WriteObject("A1/two", "two", t2),
+		r.WriteObject("A1/B1/three", "three", t3),
+		r.WriteObject("A1/B1/C1/four", "four", t1),
+		r.WriteObject("A1/B1/C2/five", "five", t2),
+	}
+	require.NoError(t, operations.Mkdir(r.Fremote, "A1/B2"))
+	require.NoError(t, operations.Mkdir(r.Fremote, "A1/B1/C3"))
+
+	fstest.CheckListingWithPrecision(
+		t,
+		r.Fremote,
+		files,
+		[]string{
+			"A1",
+			"A1/B1",
+			"A1/B2",
+			"A1/B1/C1",
+			"A1/B1/C2",
+			"A1/B1/C3",
+		},
+		fs.GetModifyWindow(r.Fremote),
+	)
+
+	require.NoError(t, operations.DirMove(r.Fremote, "A1", "A2"))
+
+	for i := range files {
+		files[i].Path = strings.Replace(files[i].Path, "A1/", "A2/", -1)
+		files[i].WinPath = ""
+	}
+
+	fstest.CheckListingWithPrecision(
+		t,
+		r.Fremote,
+		files,
+		[]string{
+			"A2",
+			"A2/B1",
+			"A2/B2",
+			"A2/B1/C1",
+			"A2/B1/C2",
+			"A2/B1/C3",
+		},
+		fs.GetModifyWindow(r.Fremote),
+	)
+
+	// Disable DirMove
+	features := r.Fremote.Features()
+	oldDirMove := features.DirMove
+	features.DirMove = nil
+	defer func() {
+		features.DirMove = oldDirMove
+	}()
+
+	require.NoError(t, operations.DirMove(r.Fremote, "A2", "A3"))
+
+	for i := range files {
+		files[i].Path = strings.Replace(files[i].Path, "A2/", "A3/", -1)
+		files[i].WinPath = ""
+	}
+
+	fstest.CheckListingWithPrecision(
+		t,
+		r.Fremote,
+		files,
+		[]string{
+			"A3",
+			"A3/B1",
+			"A3/B2",
+			"A3/B1/C1",
+			"A3/B1/C2",
+			"A3/B1/C3",
+		},
+		fs.GetModifyWindow(r.Fremote),
+	)
+
+}