bisync: rc fixes (WIP)

cmd/bisync/cmd.go

@@ -151,6 +151,7 @@ func init() {
 	flags.StringVarP(cmdFlags, &Opt.ConflictSuffixFlag, "conflict-suffix", "", Opt.ConflictSuffixFlag, "Suffix to use when renaming a --conflict-loser. Can be either one string or two comma-separated strings to assign different suffixes to Path1/Path2. (default: 'conflict')", "")
 	_ = cmdFlags.MarkHidden("debugname")
 	_ = cmdFlags.MarkHidden("localtime")
+	addRC()
 }
 
 // bisync command definition

cmd/bisync/help.go

@@ -1,13 +1,18 @@
 package bisync
 
 import (
+	"fmt"
 	"strconv"
 	"strings"
+
+	"github.com/spf13/pflag"
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
 )
 
 func makeHelp(help string) string {
 	replacer := strings.NewReplacer(
-		"|", "`",
+		"||", "`",
 		"{MAXDELETE}", strconv.Itoa(DefaultMaxDelete),
 		"{CHECKFILE}", DefaultCheckFilename,
 		// "{WORKDIR}", DefaultWorkdir,
@@ -17,34 +22,20 @@ func makeHelp(help string) string {
 
 var shortHelp = `Perform bidirectional synchronization between two paths.`
 
-var rcHelp = makeHelp(`This takes the following parameters
+// RcHelp returns the rc help
+func RcHelp() string {
+	return makeHelp(`This takes the following parameters:
+
+- path1 (required) - (string) a remote directory string e.g. ||drive:path1||
+- path2 (required) - (string) a remote directory string e.g. ||drive:path2||
+- dryRun - (bool) dry-run mode
+` + GenerateParams() + `
 
-- path1 - a remote directory string e.g. |drive:path1|
-- path2 - a remote directory string e.g. |drive:path2|
-- dryRun - dry-run mode
-- resync - performs the resync run
-- checkAccess - abort if {CHECKFILE} files are not found on both filesystems
-- checkFilename - file name for checkAccess (default: {CHECKFILE})
-- maxDelete - abort sync if percentage of deleted files is above
-  this threshold (default: {MAXDELETE})
-- force - Bypass maxDelete safety check and run the sync
-- checkSync - |true| by default, |false| disables comparison of final listings,
-              |only| will skip sync, only compare listings from the last run
-- createEmptySrcDirs - Sync creation and deletion of empty directories. 
-			  (Not compatible with --remove-empty-dirs)
-- removeEmptyDirs - remove empty directories at the final cleanup step
-- filtersFile - read filtering patterns from a file
-- ignoreListingChecksum - Do not use checksums for listings
-- resilient - Allow future runs to retry after certain less-serious errors, instead of requiring resync. 
-            Use at your own risk!
-- workdir - server directory for history files (default: |~/.cache/rclone/bisync|)
-- backupdir1 - --backup-dir for Path1. Must be a non-overlapping path on the same remote.
-- backupdir2 - --backup-dir for Path2. Must be a non-overlapping path on the same remote.
-- noCleanup - retain working files
 
 See [bisync command help](https://rclone.org/commands/rclone_bisync/)
 and [full bisync description](https://rclone.org/bisync/)
 for more information.`)
+}
 
 var longHelp = shortHelp + makeHelp(`
 
@@ -53,7 +44,7 @@ bidirectional cloud sync solution in rclone.
 It retains the Path1 and Path2 filesystem listings from the prior run.
 On each successive run it will:
 - list files on Path1 and Path2, and check for changes on each side.
-  Changes include |New|, |Newer|, |Older|, and |Deleted| files.
+  Changes include ||New||, ||Newer||, ||Older||, and ||Deleted|| files.
 - Propagate changes on Path1 to Path2, and vice-versa.
 
 Bisync is **in beta** and is considered an **advanced command**, so use with care.
@@ -63,3 +54,30 @@ or data loss can result. Questions can be asked in the [Rclone Forum](https://fo
 
 See [full bisync description](https://rclone.org/bisync/) for details.
 `)
+
+// example: "create-empty-src-dirs" -> "createEmptySrcDirs"
+func toCamel(s string) string {
+	split := strings.Split(s, "-")
+	builder := strings.Builder{}
+	for i, word := range split {
+		if i == 0 { // first word always all lowercase
+			builder.WriteString(strings.ToLower(word))
+			continue
+		}
+		builder.WriteString(cases.Title(language.AmericanEnglish).String(word))
+	}
+	return builder.String()
+}
+
+// GenerateParams automatically generates the param list from commandDefinition.Flags
+func GenerateParams() string {
+	builder := strings.Builder{}
+	fn := func(flag *pflag.Flag) {
+		if flag.Hidden {
+			return
+		}
+		builder.WriteString(fmt.Sprintf("- %s - (%s) %s  \n", toCamel(flag.Name), flag.Value.Type(), flag.Usage))
+	}
+	commandDefinition.Flags().VisitAll(fn)
+	return builder.String()
+}

cmd/bisync/rc.go

@@ -4,19 +4,22 @@ import (
 	"context"
 	"errors"
 	"log"
+	"path/filepath"
 
+	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/cmd/bisync/bilib"
 	"github.com/rclone/rclone/fs"
+	fslog "github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/fs/rc"
 )
 
-func init() {
+func addRC() {
 	rc.Add(rc.Call{
 		Path:         "sync/bisync",
 		AuthRequired: true,
 		Fn:           rcBisync,
 		Title:        shortHelp,
-		Help:         rcHelp,
+		Help:         RcHelp(),
 	})
 }
 
@@ -41,56 +44,80 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {
 	}
 
 	if opt.Resync, err = in.GetBool("resync"); rc.NotErrParamNotFound(err) {
-		return
+		fs.Debugf("resync", "optional parameter is missing. using default value: %v", opt.Resync)
 	}
 	if opt.CheckAccess, err = in.GetBool("checkAccess"); rc.NotErrParamNotFound(err) {
-		return
+		fs.Debugf("checkAccess", "optional parameter is missing. using default value: %v", opt.CheckAccess)
 	}
 	if opt.Force, err = in.GetBool("force"); rc.NotErrParamNotFound(err) {
-		return
+		fs.Debugf("force", "optional parameter is missing. using default value: %v", opt.Force)
 	}
 	if opt.CreateEmptySrcDirs, err = in.GetBool("createEmptySrcDirs"); rc.NotErrParamNotFound(err) {
-		return
+		fs.Debugf("createEmptySrcDirs", "optional parameter is missing. using default value: %v", opt.CreateEmptySrcDirs)
 	}
 	if opt.RemoveEmptyDirs, err = in.GetBool("removeEmptyDirs"); rc.NotErrParamNotFound(err) {
-		return
+		fs.Debugf("removeEmptyDirs", "optional parameter is missing. using default value: %v", opt.RemoveEmptyDirs)
 	}
 	if opt.NoCleanup, err = in.GetBool("noCleanup"); rc.NotErrParamNotFound(err) {
-		return
+		fs.Debugf("noCleanup", "optional parameter is missing. using default value: %v", opt.NoCleanup)
 	}
 	if opt.IgnoreListingChecksum, err = in.GetBool("ignoreListingChecksum"); rc.NotErrParamNotFound(err) {
-		return
+		fs.Debugf("ignoreListingChecksum", "optional parameter is missing. using default value: %v", opt.IgnoreListingChecksum)
 	}
 	if opt.Resilient, err = in.GetBool("resilient"); rc.NotErrParamNotFound(err) {
-		return
+		fs.Debugf("resilient", "optional parameter is missing. using default value: %v", opt.Resilient)
 	}
-
 	if opt.CheckFilename, err = in.GetString("checkFilename"); rc.NotErrParamNotFound(err) {
-		return
+		opt.CheckFilename = DefaultCheckFilename
+		fs.Debugf("checkFilename", "optional parameter is missing. using default value: %v", opt.CheckFilename)
 	}
 	if opt.FiltersFile, err = in.GetString("filtersFile"); rc.NotErrParamNotFound(err) {
-		return
+		fs.Debugf("filtersFile", "optional parameter is missing. using default value: %v", opt.FiltersFile)
 	}
 	if opt.Workdir, err = in.GetString("workdir"); rc.NotErrParamNotFound(err) {
-		return
+		// "" sets correct default later
+		fs.Debugf("workdir", "optional parameter is missing. using default value: %v", opt.Workdir)
 	}
 	if opt.BackupDir1, err = in.GetString("backupdir1"); rc.NotErrParamNotFound(err) {
-		return
+		fs.Debugf("backupdir1", "optional parameter is missing. using default value: %v", opt.BackupDir1)
 	}
 	if opt.BackupDir2, err = in.GetString("backupdir2"); rc.NotErrParamNotFound(err) {
-		return
+		fs.Debugf("backupdir2", "optional parameter is missing. using default value: %v", opt.BackupDir2)
 	}
-
-	checkSync, err := in.GetString("checkSync")
-	if rc.NotErrParamNotFound(err) {
+	if err = setEnum(in, "checkSync", "true", opt.CheckSync.Set); err != nil {
+		return nil, err
+	}
+	if err = setEnum(in, "resyncMode", opt.ResyncMode.String(), opt.ResyncMode.Set); err != nil {
 		return nil, err
 	}
-	if checkSync == "" {
-		checkSync = "true"
+	if err = setEnum(in, "conflictResolve", opt.ConflictResolve.String(), opt.ConflictResolve.Set); err != nil {
+		return nil, err
 	}
-	if err := opt.CheckSync.Set(checkSync); err != nil {
+	if err = setEnum(in, "conflictLoser", opt.ConflictLoser.String(), opt.ConflictLoser.Set); err != nil {
 		return nil, err
 	}
+	if opt.ConflictSuffixFlag, err = in.GetString("conflictSuffix"); rc.NotErrParamNotFound(err) {
+		fs.Debugf("conflictSuffix", "optional parameter is missing. using default value: %v", opt.ConflictSuffixFlag)
+	}
+	if opt.Recover, err = in.GetBool("recover"); rc.NotErrParamNotFound(err) {
+		fs.Debugf("recover", "optional parameter is missing. using default value: %v", opt.Recover)
+	}
+	if opt.CompareFlag, err = in.GetString("compare"); rc.NotErrParamNotFound(err) {
+		fs.Debugf("compare", "optional parameter is missing. using default value: %v", opt.CompareFlag)
+	}
+	if opt.Compare.NoSlowHash, err = in.GetBool("noSlowHash"); rc.NotErrParamNotFound(err) {
+		fs.Debugf("noSlowHash", "optional parameter is missing. using default value: %v", opt.Compare.NoSlowHash)
+	}
+	if opt.Compare.SlowHashSyncOnly, err = in.GetBool("slowHashSyncOnly"); rc.NotErrParamNotFound(err) {
+		fs.Debugf("slowHashSyncOnly", "optional parameter is missing. using default value: %v", opt.Compare.SlowHashSyncOnly)
+	}
+	if opt.Compare.DownloadHash, err = in.GetBool("downloadHash"); rc.NotErrParamNotFound(err) {
+		fs.Debugf("downloadHash", "optional parameter is missing. using default value: %v", opt.Compare.DownloadHash)
+	}
+	if opt.MaxLock, err = in.GetDuration("maxLock"); rc.NotErrParamNotFound(err) {
+		opt.MaxLock = 0
+		fs.Debugf("maxLock", "optional parameter is missing. using default value: %v", opt.MaxLock)
+	}
 
 	fs1, err := rc.GetFsNamed(octx, in, "path1")
 	if err != nil {
@@ -102,9 +129,46 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {
 		return nil, err
 	}
 
-	output := bilib.CaptureOutput(func() {
+	fn := func() {
 		err = Bisync(octx, fs1, fs2, opt)
-	})
+	}
+
+	if ci.Progress {
+		fn = func() {
+			cmd.Run(false, true, commandDefinition, func() error {
+				err = Bisync(octx, fs1, fs2, opt)
+				return err
+			})
+		}
+	}
+
+	workDir, _ := filepath.Abs(DefaultWorkdir)
+	if opt.Workdir != "" {
+		workDir, _ = filepath.Abs(opt.Workdir)
+	}
+	basePath := bilib.BasePath(ctx, workDir, fs1, fs2)
+
+	output := bilib.CaptureOutput(fn)
 	_, _ = log.Writer().Write(output)
-	return rc.Params{"output": string(output)}, err
+	return rc.Params{
+		"output":   string(output),
+		"session":  bilib.SessionName(fs1, fs2),
+		"workDir":  workDir,
+		"basePath": basePath,
+		"listing1": basePath + ".path1.lst",
+		"listing2": basePath + ".path2.lst",
+		"logFile":  fslog.Opt.File,
+	}, err
+}
+
+func setEnum(in rc.Params, name string, defaultVal string, set func(s string) error) error {
+	v, err := in.GetString(name)
+	if rc.NotErrParamNotFound(err) || v == "" {
+		v = defaultVal
+		fs.Debugf(name, "optional parameter is missing. using default value: %v", v)
+	}
+	if err := set(v); err != nil {
+		return err
+	}
+	return nil
 }

librclone/librclone/librclone.go

@@ -37,6 +37,7 @@ func Initialize() {
 
 	// Load the config - this may need to be configurable
 	configfile.Install()
+	rc.AddOption("main", fs.GetConfig(ctx))
 
 	// Start accounting
 	accounting.Start(ctx)
@@ -105,11 +106,11 @@ func RPC(method string, input string) (output string, status int) {
 	if call.NeedsRequest {
 		return writeError(method, in, fmt.Errorf("method %q needs request, not supported", method), http.StatusNotFound)
 		// Add the request to RC
-		//in["_request"] = r
+		// in["_request"] = r
 	}
 	if call.NeedsResponse {
 		return writeError(method, in, fmt.Errorf("method %q need response, not supported", method), http.StatusNotFound)
-		//in["_response"] = w
+		// in["_response"] = w
 	}
 
 	fs.Debugf(nil, "rc: %q: with parameters %+v", method, in)