Skip to content

Commit

Permalink
rc: add _config parameter to set global config for just this rc call
Browse files Browse the repository at this point in the history
  • Loading branch information
ncw committed Feb 4, 2021
1 parent 1562524 commit 56f5a2c
Show file tree
Hide file tree
Showing 3 changed files with 76 additions and 2 deletions.
34 changes: 32 additions & 2 deletions docs/content/rc.md
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,36 @@ $ rclone rc job/list
}
```

### Setting config flags with _config

If you wish to set config (the equivalent of the global flags) for the
duration of an rc call only then pass in the `_config` parameter.

This should be in the same format as the `config` key returned by
[options/get](#options-get).

For example, if you wished to run a sync with the `--checksum`
parameter, you would pass this parameter in your JSON blob.

"_config":{"CheckSum": true}

If using `rclone rc` this could be passed as

rclone rc operations/sync ... _config='{"CheckSum": true}'

Any config parameters you don't set will inherit the global defaults
which were set with command line flags or environment variables.

Note that it is possible to set some values as strings or integers -
see [data types](/#data-types) for more info. Here is an example
setting the equivalent of `--buffer-size` in string or integer format.

"_config":{"BufferSize": "42M"}
"_config":{"BufferSize": 44040192}

If you wish to check the `_config` assignment has worked properly then
calling `options/local` will show what the value got set to.

### Assigning operations to groups with _group = value

Each rc call has its own stats group for tracking its metrics. By default
Expand All @@ -292,14 +322,14 @@ $ rclone rc --json '{ "group": "job/1" }' core/stats
}
```

## Data types
## Data types {#data-types}

When the API returns types, these will mostly be straight forward
integer, string or boolean types.

However some of the types returned by the [options/get](#options-get)
call and taken by the [options/set](#options-set) calls as well as the
`vfsOpt` and the `mountOpt` are as follows:
`vfsOpt`, `mountOpt` and the `_config` parameters.

- `Duration` - these are returned as an integer duration in
nanoseconds. They may be set as an integer, or they may be set with
Expand Down
20 changes: 20 additions & 0 deletions fs/rc/jobs/job.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"time"

"github.com/pkg/errors"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/rc"
)
Expand Down Expand Up @@ -201,6 +202,20 @@ func getAsync(ctx context.Context, in rc.Params) (context.Context, bool, error)
return ctx, isAsync, nil
}

// See if _config is set and if so adjust ctx to include it
func getConfig(ctx context.Context, in rc.Params) (context.Context, error) {
if _, ok := in["_config"]; !ok {
return ctx, nil
}
ctx, ci := fs.AddConfig(ctx)
err := in.GetStruct("_config", ci)
if err != nil {
return ctx, err
}
delete(in, "_config") // remove the parameter
return ctx, nil
}

// NewJob creates a Job and executes it, possibly in the background if _async is set
func (jobs *Jobs) NewJob(ctx context.Context, fn rc.Func, in rc.Params) (job *Job, out rc.Params, err error) {
id := atomic.AddInt64(&jobID, 1)
Expand All @@ -211,6 +226,11 @@ func (jobs *Jobs) NewJob(ctx context.Context, fn rc.Func, in rc.Params) (job *Jo
return nil, nil, err
}

ctx, err = getConfig(ctx, in)
if err != nil {
return nil, nil, err
}

ctx, group, err := getGroup(ctx, in, id)
if err != nil {
return nil, nil, err
Expand Down
24 changes: 24 additions & 0 deletions fs/rc/jobs/job_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"time"

"github.com/pkg/errors"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/fs/rc/rcflags"
"github.com/rclone/rclone/fstest/testy"
Expand Down Expand Up @@ -248,6 +249,29 @@ func TestExecuteJob(t *testing.T) {
assert.Equal(t, rc.Params{}, out)
}

func TestExecuteJobWithConfig(t *testing.T) {
ctx := context.Background()
jobID = 0
jobFn := func(ctx context.Context, in rc.Params) (rc.Params, error) {
ci := fs.GetConfig(ctx)
assert.Equal(t, 42*fs.MebiByte, ci.BufferSize)
return nil, nil
}
_, _, err := NewJob(context.Background(), jobFn, rc.Params{
"_config": rc.Params{
"BufferSize": "42M",
},
})
require.NoError(t, err)
jobID = 0
_, _, err = NewJob(ctx, jobFn, rc.Params{
"_config": `{"BufferSize": "42M"}`,
})
require.NoError(t, err)
ci := fs.GetConfig(ctx)
assert.NotEqual(t, 42*fs.MebiByte, ci.BufferSize)
}

func TestExecuteJobErrorPropagation(t *testing.T) {
ctx := context.Background()
jobID = 0
Expand Down

0 comments on commit 56f5a2c

Please sign in to comment.