Skip to content

Commit

Permalink
operations: disable multi thread copy for local to local copies
Browse files Browse the repository at this point in the history
...unless --multi-thread-streams has been set explicitly
  • Loading branch information
ncw committed Aug 12, 2019
1 parent 43a6774 commit 784e544
Show file tree
Hide file tree
Showing 4 changed files with 14 additions and 1 deletion.
4 changes: 4 additions & 0 deletions docs/content/docs.md
Expand Up @@ -743,6 +743,10 @@ above.
**NB** that this **only** works for a local destination but will work
with any source.

**NB** that multi thread copies are disabled for local to local copies
as they are faster without unless `--multi-thread-streams` is set
explicitly.

### --multi-thread-streams=N ###

When using multi thread downloads (see above `--multi-thread-cutoff`)
Expand Down
1 change: 1 addition & 0 deletions fs/config.go
Expand Up @@ -100,6 +100,7 @@ type ConfigInfo struct {
ClientKey string // Client Side Key
MultiThreadCutoff SizeSuffix
MultiThreadStreams int
MultiThreadSet bool // whether MultiThreadStreams was set (set in fs/config/configflags)
}

// NewConfig creates a new config with everything set to the default
Expand Down
5 changes: 5 additions & 0 deletions fs/config/configflags/configflags.go
Expand Up @@ -213,4 +213,9 @@ func SetFlags() {
if err == nil {
config.ConfigPath = configPath
}

// Set whether multi-thread-streams was set
multiThreadStreamsFlag := pflag.Lookup("multi-thread-streams")
fs.Config.MultiThreadSet = multiThreadStreamsFlag != nil && multiThreadStreamsFlag.Changed

}
5 changes: 4 additions & 1 deletion fs/operations/operations.go
Expand Up @@ -310,7 +310,10 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
}
// If can't server side copy, do it manually
if err == fs.ErrorCantCopy {
if doOpenWriterAt := f.Features().OpenWriterAt; doOpenWriterAt != nil && src.Size() >= int64(fs.Config.MultiThreadCutoff) && fs.Config.MultiThreadStreams > 1 {
// Disable multithread if --multi-thread-streams not in use and source is local
disableMultiThread := !fs.Config.MultiThreadSet && src.Fs().Features().OpenWriterAt != nil
doOpenWriterAt := f.Features().OpenWriterAt
if !disableMultiThread && doOpenWriterAt != nil && src.Size() >= int64(fs.Config.MultiThreadCutoff) && fs.Config.MultiThreadStreams > 1 {
// Number of streams proportional to size
streams := src.Size() / int64(fs.Config.MultiThreadCutoff)
// With maximum
Expand Down

0 comments on commit 784e544

Please sign in to comment.