-
-
Notifications
You must be signed in to change notification settings - Fork 4.1k
/
sync.go
248 lines (210 loc) · 8.4 KB
/
sync.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
// Package sync provides the sync command.
package sync
import (
"context"
"io"
"os"
mutex "sync" // renamed as "sync" already in use
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fs/operations/operationsflags"
"github.com/rclone/rclone/fs/sync"
"github.com/spf13/cobra"
)
var (
createEmptySrcDirs = false
opt = operations.LoggerOpt{}
loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
)
func init() {
cmd.Root.AddCommand(commandDefinition)
cmdFlags := commandDefinition.Flags()
flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after sync", "")
operationsflags.AddLoggerFlags(cmdFlags, &opt, &loggerFlagsOpt)
// TODO: add same flags to move and copy
}
var lock mutex.Mutex
func syncLoggerFn(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEntry, err error) {
lock.Lock()
defer lock.Unlock()
if err == fs.ErrorIsDir && !opt.FilesOnly && opt.DestAfter != nil {
opt.PrintDestAfter(ctx, sigil, src, dst, err)
return
}
_, srcOk := src.(fs.Object)
_, dstOk := dst.(fs.Object)
var filename string
if !srcOk && !dstOk {
return
} else if srcOk && !dstOk {
filename = src.String()
} else {
filename = dst.String()
}
if sigil.Writer(opt) != nil {
operations.SyncFprintf(sigil.Writer(opt), "%s\n", filename)
}
if opt.Combined != nil {
operations.SyncFprintf(opt.Combined, "%c %s\n", sigil, filename)
fs.Debugf(nil, "Sync Logger: %s: %c %s\n", sigil.String(), sigil, filename)
}
if opt.DestAfter != nil {
opt.PrintDestAfter(ctx, sigil, src, dst, err)
}
}
// GetSyncLoggerOpt gets the options corresponding to the logger flags
func GetSyncLoggerOpt(ctx context.Context, fdst fs.Fs, command *cobra.Command) (operations.LoggerOpt, func(), error) {
closers := []io.Closer{}
opt.LoggerFn = syncLoggerFn
if opt.TimeFormat == "max" {
opt.TimeFormat = operations.FormatForLSFPrecision(fdst.Precision())
}
opt.SetListFormat(ctx, command.Flags())
opt.NewListJSON(ctx, fdst, "")
open := func(name string, pout *io.Writer) error {
if name == "" {
return nil
}
if name == "-" {
*pout = os.Stdout
return nil
}
out, err := os.Create(name)
if err != nil {
return err
}
*pout = out
closers = append(closers, out)
return nil
}
if err := open(loggerFlagsOpt.Combined, &opt.Combined); err != nil {
return opt, nil, err
}
if err := open(loggerFlagsOpt.MissingOnSrc, &opt.MissingOnSrc); err != nil {
return opt, nil, err
}
if err := open(loggerFlagsOpt.MissingOnDst, &opt.MissingOnDst); err != nil {
return opt, nil, err
}
if err := open(loggerFlagsOpt.Match, &opt.Match); err != nil {
return opt, nil, err
}
if err := open(loggerFlagsOpt.Differ, &opt.Differ); err != nil {
return opt, nil, err
}
if err := open(loggerFlagsOpt.ErrFile, &opt.Error); err != nil {
return opt, nil, err
}
if err := open(loggerFlagsOpt.DestAfter, &opt.DestAfter); err != nil {
return opt, nil, err
}
close := func() {
for _, closer := range closers {
err := closer.Close()
if err != nil {
fs.Errorf(nil, "Failed to close report output: %v", err)
}
}
}
return opt, close, nil
}
func anyNotBlank(s ...string) bool {
for _, x := range s {
if x != "" {
return true
}
}
return false
}
var commandDefinition = &cobra.Command{
Use: "sync source:path dest:path",
Short: `Make source and dest identical, modifying destination only.`,
Long: `Sync the source to the destination, changing the destination
only. Doesn't transfer files that are identical on source and
destination, testing by size and modification time or MD5SUM.
Destination is updated to match source, including deleting files
if necessary (except duplicate objects, see below). If you don't
want to delete files from destination, use the
[copy](/commands/rclone_copy/) command instead.
**Important**: Since this can cause data loss, test first with the
` + "`--dry-run` or the `--interactive`/`-i`" + ` flag.
rclone sync --interactive SOURCE remote:DESTINATION
Note that files in the destination won't be deleted if there were any
errors at any point. Duplicate objects (files with the same name, on
those providers that support it) are also not yet handled.
It is always the contents of the directory that is synced, not the
directory itself. So when source:path is a directory, it's the contents of
source:path that are copied, not the directory name and contents. See
extended explanation in the [copy](/commands/rclone_copy/) command if unsure.
If dest:path doesn't exist, it is created and the source:path contents
go there.
It is not possible to sync overlapping remotes. However, you may exclude
the destination from the sync with a filter rule or by putting an
exclude-if-present file inside the destination directory and sync to a
destination that is inside the source directory.
Rclone will sync the modification times of files and directories if
the backend supports it. If metadata syncing is required then use the
` + "`--metadata`" + ` flag.
Note that the modification time and metadata for the root directory
will **not** be synced. See https://github.com/rclone/rclone/issues/7652
for more info.
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
**Note**: Use the ` + "`rclone dedupe`" + ` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.
See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14372) for more info.
## Logger Flags
The ` + "`--differ`" + `, ` + "`--missing-on-dst`" + `, ` + "`--missing-on-src`" + `, ` +
"`--match`" + ` and ` + "`--error`" + ` flags write paths, one per line, to the file name (or
stdout if it is ` + "`-`" + `) supplied. What they write is described in the
help below. For example ` + "`--differ`" + ` will write all paths which are
present on both the source and destination but different.
The ` + "`--combined`" + ` flag will write a file (or stdout) which contains all
file paths with a symbol and then a space and then the path to tell
you what happened to it. These are reminiscent of diff files.
- ` + "`= path`" + ` means path was found in source and destination and was identical
- ` + "`- path`" + ` means path was missing on the source, so only in the destination
- ` + "`+ path`" + ` means path was missing on the destination, so only in the source
- ` + "`* path`" + ` means path was present in source and destination but different.
- ` + "`! path`" + ` means there was an error reading or hashing the source or dest.
The ` + "`--dest-after`" + ` flag writes a list file using the same format flags
as [` + "`lsf`" + `](/commands/rclone_lsf/#synopsis) (including [customizable options
for hash, modtime, etc.](/commands/rclone_lsf/#synopsis))
Conceptually it is similar to rsync's ` + "`--itemize-changes`" + `, but not identical
-- it should output an accurate list of what will be on the destination
after the sync.
Note that these logger flags have a few limitations, and certain scenarios
are not currently supported:
- ` + "`--max-duration`" + ` / ` + "`CutoffModeHard`" + `
- ` + "`--compare-dest`" + ` / ` + "`--copy-dest`" + `
- server-side moves of an entire dir at once
- High-level retries, because there would be duplicates (use ` + "`--retries 1`" + ` to disable)
- Possibly some unusual error scenarios
Note also that each file is logged during the sync, as opposed to after, so it
is most useful as a predictor of what SHOULD happen to each file
(which may or may not match what actually DID.)
`,
Annotations: map[string]string{
"groups": "Sync,Copy,Filter,Listing,Important",
},
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(2, 2, command, args)
fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
cmd.Run(true, true, command, func() error {
ctx := context.Background()
opt, close, err := GetSyncLoggerOpt(ctx, fdst, command)
if err != nil {
return err
}
defer close()
if anyNotBlank(loggerFlagsOpt.Combined, loggerFlagsOpt.MissingOnSrc, loggerFlagsOpt.MissingOnDst,
loggerFlagsOpt.Match, loggerFlagsOpt.Differ, loggerFlagsOpt.ErrFile, loggerFlagsOpt.DestAfter) {
ctx = operations.WithSyncLogger(ctx, opt)
}
if srcFileName == "" {
return sync.Sync(ctx, fdst, fsrc, createEmptySrcDirs)
}
return operations.CopyFile(ctx, fdst, fsrc, srcFileName, srcFileName)
})
},
}