-
Notifications
You must be signed in to change notification settings - Fork 100
/
sync.go
979 lines (828 loc) · 32.5 KB
/
sync.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
package lib
import (
"bytes"
"fmt"
oss "github.com/aliyun/aliyun-oss-go-sdk/oss"
"io"
"os"
"path/filepath"
"sort"
"strings"
)
/*
* Put same type variables together to make them 64bits alignment to avoid
* atomic.AddInt64() panic
* Please guarantee the alignment if you add new filed
*/
var MaxSyncNumbers int = 1000000
type syncOptionType struct {
bDelete bool
encodingType string
backupDir string
force bool
enableSymlinkDir bool
onlyCurrentDir bool
disableDirObject bool
disableAllSymlink bool
cpDir string
removeCount int
filters []filterOptionType
payerOptions []oss.Option
}
var specChineseSync = SpecText{
synopsisText: "将本地文件目录或者oss prefix从源端同步到目的端",
paramText: "src dest [options]",
syntaxText: `
ossutil sync local_dir cloud_url [-f] [-u] [--delete] [--backup-dir] [--enable-symlink-dir] [--disable-all-symlink] [--disable-ignore-error] [--only-current-dir] [--output-dir=odir] [--bigfile-threshold=size] [--checkpoint-dir=cdir] [--snapshot-path=sdir] [--payer requester]
ossutil sync cloud_url local_dir [-f] [-u] [--delete] [--backup-dir] [--only-current-dir] [--disable-ignore-error] [--output-dir=odir] [--bigfile-threshold=size] [--checkpoint-dir=cdir] [--range=x-y] [--payer requester]
ossutil sync cloud_url cloud_url [-f] [-u] [--delete] [--backup-dir] [--only-current-dir] [--disable-ignore-error] [--output-dir=odir] [--bigfile-threshold=size] [--checkpoint-dir=cdir] [--payer requester]
`,
detailHelpText: `
该命令和cp命令类似:支持从本地文件系统上传文件到oss，从oss下载object到本地文件系统，在oss
上进行object拷贝; 用于源端和目的端数据同步.
sync命令和cp命令不同之处如下:
1、当输入--delete选项时,该命令会自动删除目的端存在而源端不存在的object或者移走本地文件
如果目的端是oss, 则会删除多余的object; 如果目的端是本地文件, 则会移走本地多余的文件
警告: 在没有完全搞清楚sync命令的行为之前, 请慎用--delete
2、sync强制是以recursive方式遍历文件或者object, 所以不用输入-r --recursive
3、当源端是oss://bucket/prefix, sync命令会自动在prefix后面加上字符 '/', 但是cp命令不会
当目的端是oss://bucket/prefix, sync命令会在prefix后面加上字符 '/'; cp命令在--recursive被输入时也会在prefix后面加上字符'/'
--delete选项
表示需要删除或者移走目的端存在而源端不存在的object或者文件
--backup-dir
该选项表示用于备份目的端文件的目录, 不能是目的端目录的子目录,如果输入了--delete, 该选项必须输入
其他选项说明、用法和cp命令相同
`,
sampleText: `
1) 从本地文件上传到oss
前置条件如下:
[root]# ls -al ./test_sync
total 16
drwxr-xr-x 2 root root 4096 Oct 5 13:09 .
drwxr-xr-x 5 root root 4096 Oct 5 13:10 ..
-rw-r--r-- 1 root root 38 Oct 5 12:48 a.txt
-rw-r--r-- 1 root root 118 Oct 5 12:48 b.txt
[root]# ./ossutil ls oss://wangtw-test-sync2
LastModifiedTime Size(B) StorageClass ETAG ObjectName
2020-10-05 13:10:23 +0800 CST 156 Standard D282A160AB960C11ED797F009858D41D oss://wangtw-test-sync2/a.txt
2020-10-05 13:09:06 +0800 CST 21029 Standard 975F9F8EC2B34B15936FDAE1C85E71C9 oss://wangtw-test-sync2/c.txt
Object Number is: 2
# run ossutil sync command
[root]# ./ossutil64 sync ./test_sync oss://wangtw-test-sync2/ --delete -f
total file(directory) count:2
oss://wangtw-test-sync2,total oss object count:2
object will be deleted count:1
Succeed: Total num: 2, size: 156. OK num: 2(upload 2 files).
average speed 6000(byte/s)
delete object count:1
0.111655(s) elapsed
[root]# ./ossutil64 ls oss://wangtw-test-sync2
LastModifiedTime Size(B) StorageClass ETAG ObjectName
2020-10-05 13:13:21 +0800 CST 38 Standard 533F2D421ED096DBB4CF458AADA64F8E oss://wangtw-test-sync2/a.txt
2020-10-05 13:13:21 +0800 CST 118 Standard C6639C466B70758C82596413C2BB050A oss://wangtw-test-sync2/b.txt
Object Number is: 2
执行sync命令之后, 本地文件a.txt, b.txt被上传到oss, oss object key c.txt 被删除
2) 从oss上下载
前置条件如下:
[root]# ls -al ./test_sync
total 36
drwxr-xr-x 2 root root 4096 Oct 5 12:42 .
drwxr-xr-x 4 root root 4096 Oct 5 12:42 ..
-rw-r--r-- 1 root root 38 Oct 5 11:22 a.txt
-rw-r--r-- 1 root root 21029 Oct 5 11:22 d.txt
[root]# ./ossutil64 ls oss://wangtw-test-sync2
LastModifiedTime Size(B) StorageClass ETAG ObjectName
2020-10-05 11:55:47 +0800 CST 38 Standard 533F2D421ED096DBB4CF458AADA64F8E oss://wangtw-test-sync2/a.txt
2020-10-05 11:55:47 +0800 CST 118 Standard C6639C466B70758C82596413C2BB050A oss://wangtw-test-sync2/b.txt
2020-10-05 11:55:47 +0800 CST 21029 Standard 975F9F8EC2B34B15936FDAE1C85E71C9 oss://wangtw-test-sync2/c.txt
Object Number is: 3
# run ossutil sync command
[root]# ./ossutil64 sync oss://wangtw-test-sync2 ./test_sync --delete -f --backup-dir backup
oss://wangtw-test-sync2,total oss object count:3
total file(directory) count:2
file(directory) will be removed count:1
Succeed: Total num: 3, size: 21,185. OK num: 3(download 3 objects).
average speed 243000(byte/s)
remove file(dir) count:1
0.172867(s) elapsed
[root]# ls -al ./test_sync/
total 40
drwxr-xr-x 2 root root 4096 Oct 5 12:48 .
drwxr-xr-x 4 root root 4096 Oct 5 12:48 ..
-rw-r--r-- 1 root root 38 Oct 5 12:48 a.txt
-rw-r--r-- 1 root root 118 Oct 5 12:48 b.txt
-rw-r--r-- 1 root root 21029 Oct 5 12:48 c.txt
[root]# ls -al ./backup/
total 32
drwxr-xr-x 2 root root 4096 Oct 5 12:48 .
drwxr-xr-x 4 root root 4096 Oct 5 12:48 ..
-rw-r--r-- 1 root root 21029 Oct 5 11:22 d.txt
执行sync命令之后, oss object keys a.txt, b.txt, c.txt被下载下来
本地文件d.txt被移动到备份目录backup
3) 在oss之间copy
前置条件如下:
[root]# ./ossutil64 ls oss://wangtw-test-sync2
LastModifiedTime Size(B) StorageClass ETAG ObjectName
2020-10-05 12:55:47 +0800 CST 38 Standard 533F2D421ED096DBB4CF458AADA64F8E oss://wangtw-test-sync2/prefix1/a.txt
2020-10-05 12:55:47 +0800 CST 118 Standard C6639C466B70758C82596413C2BB050A oss://wangtw-test-sync2/prefix1/b.txt
2020-10-05 13:00:58 +0800 CST 65 Standard 77F476710889A7FAA2603093B08EB31B oss://wangtw-test-sync2/prefix2/a.txt
2020-10-05 12:56:23 +0800 CST 21029 Standard 975F9F8EC2B34B15936FDAE1C85E71C9 oss://wangtw-test-sync2/prefix2/c.txt
Object Number is: 4
0.067659(s) elapsed
[root]# ./ossutil64 sync oss://wangtw-test-sync2/prefix1 oss://wangtw-test-sync2/prefix2 --delete -f
oss://wangtw-test-sync2/prefix1/,total oss object count:2
oss://wangtw-test-sync2/prefix2/,total oss object count:2
object will be deleted count:1
Succeed: Total num: 2, size: 156. OK num: 2(copy 2 objects).
average speed 5000(byte/s)
delete object count:1
0.071010(s) elapsed
[root]# ./ossutil64 ls oss://wangtw-test-sync2
LastModifiedTime Size(B) StorageClass ETAG ObjectName
2020-10-05 12:55:47 +0800 CST 38 Standard 533F2D421ED096DBB4CF458AADA64F8E oss://wangtw-test-sync2/prefix1/a.txt
2020-10-05 12:55:47 +0800 CST 118 Standard C6639C466B70758C82596413C2BB050A oss://wangtw-test-sync2/prefix1/b.txt
2020-10-05 13:02:02 +0800 CST 38 Standard 533F2D421ED096DBB4CF458AADA64F8E oss://wangtw-test-sync2/prefix2/a.txt
2020-10-05 13:02:02 +0800 CST 118 Standard C6639C466B70758C82596413C2BB050A oss://wangtw-test-sync2/prefix2/b.txt
Object Number is: 4
执行sync命令之后, oss object key prefix1/a.txt, prefix1/b.txt 被拷贝到
prefix2/a.txt, prefix2/b.txt 并且 prefix2/c.txt 被删除
`,
}
var specEnglishSync = SpecText{
synopsisText: "Sync the local file directory or oss prefix from the source to the destination",
paramText: "src dest [options]",
syntaxText: `
ossutil sync local_dir cloud_url [-f] [-u] [--delete] [--backup-dir] [--enable-symlink-dir] [--disable-all-symlink] [--disable-ignore-error] [--only-current-dir] [--output-dir=odir] [--bigfile-threshold=size] [--checkpoint-dir=cdir] [--snapshot-path=sdir] [--payer requester]
ossutil sync cloud_url local_dir [-f] [-u] [--delete] [--backup-dir] [--only-current-dir] [--disable-ignore-error] [--output-dir=odir] [--bigfile-threshold=size] [--checkpoint-dir=cdir] [--range=x-y] [--payer requester]
ossutil sync cloud_url cloud_url [-f] [-u] [--delete] [--backup-dir] [--only-current-dir] [--disable-ignore-error] [--output-dir=odir] [--bigfile-threshold=size] [--checkpoint-dir=cdir] [--payer requester]
`,
detailHelpText: `
This command is similar to the cp command: it supports uploading files from the local file
system to oss, and downloading objects from oss to the local file system, Copying objects between
oss; It's used for data synchronization between source and destination.
The differences between the sync command and the cp command are as follows:
1、When the --delete option is entered, the command will automatically delete objects or remove files
that exist on the destination but not exist on the source.
If the destination is oss, the redundant objects will be deleted;
If the destination is local directory, the redundant files will be removed to back up directory.
Warning: Before you fully understand the behavior of the sync command, please use --delete carefully
2、Sync is forced to traverse files or objects recursively, so there is no need to enter -r or --recursive
3、When the src is oss://bucket/prefix, sync command will add character '/' after the prefix, but cp command doesn't add
When the destination is oss://bucket/prefix, sync command will add character '/' after the prefix; The cp command will also add '/' after the prefix with --recusive is entered
--delete
Indicates that the objects or files which exist on destination
and not exist on src needs to be deleted or removed to backup dir
--backup-dir
This option indicates the directory used to back up the destination files needed to be removed,
It cannot be a subdirectory of the destination directory.
If you enter --delete, this option must be entered
Other options descriptions and usage are the same as the cp command
`,
sampleText: `
1) Upload to oss
The preconditions are as follows:
[root]# ls -al ./test_sync
total 16
drwxr-xr-x 2 root root 4096 Oct 5 13:09 .
drwxr-xr-x 5 root root 4096 Oct 5 13:10 ..
-rw-r--r-- 1 root root 38 Oct 5 12:48 a.txt
-rw-r--r-- 1 root root 118 Oct 5 12:48 b.txt
[root]# ./ossutil64 ls oss://wangtw-test-sync2
LastModifiedTime Size(B) StorageClass ETAG ObjectName
2020-10-05 13:10:23 +0800 CST 156 Standard D282A160AB960C11ED797F009858D41D oss://wangtw-test-sync2/a.txt
2020-10-05 13:09:06 +0800 CST 21029 Standard 975F9F8EC2B34B15936FDAE1C85E71C9 oss://wangtw-test-sync2/c.txt
Object Number is: 2
# run ossutil sync command
[root]# ./ossutil64 sync ./test_sync oss://wangtw-test-sync2/ --delete -f
total file(directory) count:2
oss://wangtw-test-sync2,total oss object count:2
object will be deleted count:1
Succeed: Total num: 2, size: 156. OK num: 2(upload 2 files).
average speed 6000(byte/s)
delete object count:1
0.111655(s) elapsed
[root]# ./ossutil64 ls oss://wangtw-test-sync2/
LastModifiedTime Size(B) StorageClass ETAG ObjectName
2020-10-05 13:13:21 +0800 CST 38 Standard 533F2D421ED096DBB4CF458AADA64F8E oss://wangtw-test-sync2/a.txt
2020-10-05 13:13:21 +0800 CST 118 Standard C6639C466B70758C82596413C2BB050A oss://wangtw-test-sync2/b.txt
Object Number is: 2
After the sync command is executed, local files a.txt, b.txt are uploaded and oss object
key c.txt is deleted
2) download from oss
The preconditions are as follows:
[root]# ls -al ./test_sync
total 36
drwxr-xr-x 2 root root 4096 Oct 5 12:42 .
drwxr-xr-x 4 root root 4096 Oct 5 12:42 ..
-rw-r--r-- 1 root root 38 Oct 5 11:22 a.txt
-rw-r--r-- 1 root root 21029 Oct 5 11:22 d.txt
[root]# ./ossutil64 ls oss://wangtw-test-sync2
LastModifiedTime Size(B) StorageClass ETAG ObjectName
2020-10-05 11:55:47 +0800 CST 38 Standard 533F2D421ED096DBB4CF458AADA64F8E oss://wangtw-test-sync2/a.txt
2020-10-05 11:55:47 +0800 CST 118 Standard C6639C466B70758C82596413C2BB050A oss://wangtw-test-sync2/b.txt
2020-10-05 11:55:47 +0800 CST 21029 Standard 975F9F8EC2B34B15936FDAE1C85E71C9 oss://wangtw-test-sync2/c.txt
Object Number is: 3
# run ossutil sync command
[root]# ./ossutil64 sync oss://wangtw-test-sync2 ./test_sync --delete -f --backup-dir backup
oss://wangtw-test-sync2,total oss object count:3
total file(directory) count:2
file(directory) will be removed count:1
Succeed: Total num: 3, size: 21,185. OK num: 3(download 3 objects).
average speed 243000(byte/s)
remove file(dir) count:1
0.172867(s) elapsed
[root]# ls -al ./test_sync/
total 40
drwxr-xr-x 2 root root 4096 Oct 5 12:48 .
drwxr-xr-x 4 root root 4096 Oct 5 12:48 ..
-rw-r--r-- 1 root root 38 Oct 5 12:48 a.txt
-rw-r--r-- 1 root root 118 Oct 5 12:48 b.txt
-rw-r--r-- 1 root root 21029 Oct 5 12:48 c.txt
[root]# ls -al ./backup/
total 32
drwxr-xr-x 2 root root 4096 Oct 5 12:48 .
drwxr-xr-x 4 root root 4096 Oct 5 12:48 ..
-rw-r--r-- 1 root root 21029 Oct 5 11:22 d.txt
After the sync command is executed, oss keys a.txt, b.txt, c.txt are downloaded
and local file d.txt is removed to backup dir
3) copy between oss
The preconditions are as follows:
[root]# ./ossutil64 ls oss://wangtw-test-sync2
LastModifiedTime Size(B) StorageClass ETAG ObjectName
2020-10-05 12:55:47 +0800 CST 38 Standard 533F2D421ED096DBB4CF458AADA64F8E oss://wangtw-test-sync2/prefix1/a.txt
2020-10-05 12:55:47 +0800 CST 118 Standard C6639C466B70758C82596413C2BB050A oss://wangtw-test-sync2/prefix1/b.txt
2020-10-05 13:00:58 +0800 CST 65 Standard 77F476710889A7FAA2603093B08EB31B oss://wangtw-test-sync2/prefix2/a.txt
2020-10-05 12:56:23 +0800 CST 21029 Standard 975F9F8EC2B34B15936FDAE1C85E71C9 oss://wangtw-test-sync2/prefix2/c.txt
Object Number is: 4
0.067659(s) elapsed
[root]# ./ossutil64 sync oss://wangtw-test-sync2/prefix1 oss://wangtw-test-sync2/prefix2 --delete -f
oss://wangtw-test-sync2/prefix1/,total oss object count:2
oss://wangtw-test-sync2/prefix2/,total oss object count:2
object will be deleted count:1
Succeed: Total num: 2, size: 156. OK num: 2(copy 2 objects).
average speed 5000(byte/s)
delete object count:1
0.071010(s) elapsed
[root]# ./ossutil ls oss://wangtw-test-sync2
LastModifiedTime Size(B) StorageClass ETAG ObjectName
2020-10-05 12:55:47 +0800 CST 38 Standard 533F2D421ED096DBB4CF458AADA64F8E oss://wangtw-test-sync2/prefix1/a.txt
2020-10-05 12:55:47 +0800 CST 118 Standard C6639C466B70758C82596413C2BB050A oss://wangtw-test-sync2/prefix1/b.txt
2020-10-05 13:02:02 +0800 CST 38 Standard 533F2D421ED096DBB4CF458AADA64F8E oss://wangtw-test-sync2/prefix2/a.txt
2020-10-05 13:02:02 +0800 CST 118 Standard C6639C466B70758C82596413C2BB050A oss://wangtw-test-sync2/prefix2/b.txt
Object Number is: 4
After the sync command is executed, oss keys prefix1/a.txt, prefix1/b.txt are copied to
prefix2/a.txt, prefix2/b.txt and prefix2/c.txt is deleted
`,
}
// SyncCommand is the command upload, download and copy objects
type SyncCommand struct {
command Command
syncOption syncOptionType
}
var syncCommand = SyncCommand{
command: Command{
name: "sync",
nameAlias: []string{"sync"},
minArgc: 2,
maxArgc: 2,
specChinese: specChineseSync,
specEnglish: specEnglishSync,
group: GroupTypeNormalCommand,
validOptionNames: []string{
// The following options are supported by sc command and cp command
//OptionRecursion,
OptionForce,
OptionUpdate,
OptionContinue,
OptionOutputDir,
OptionBigFileThreshold,
OptionPartSize,
OptionCheckpointDir,
OptionRange,
OptionEncodingType,
OptionInclude,
OptionExclude,
OptionMeta,
OptionACL,
OptionConfigFile,
OptionEndpoint,
OptionAccessKeyID,
OptionAccessKeySecret,
OptionSTSToken,
OptionProxyHost,
OptionProxyUser,
OptionProxyPwd,
OptionRetryTimes,
OptionRoutines,
OptionParallel,
OptionSnapshotPath,
OptionDisableCRC64,
OptionRequestPayer,
OptionLogLevel,
OptionMaxUpSpeed,
//OptionPartitionDownload,
//OptionVersionId,
OptionLocalHost,
OptionEnableSymlinkDir,
OptionOnlyCurrentDir,
OptionDisableDirObject,
OptionDisableAllSymlink,
OptionDisableIgnoreError,
OptionTagging,
OptionPassword,
OptionMode,
OptionECSRoleName,
OptionTokenTimeout,
OptionRamRoleArn,
OptionRoleSessionName,
OptionReadTimeout,
OptionConnectTimeout,
OptionSTSRegion,
OptionSkipVerifyCert,
OptionMaxDownSpeed,
OptionUserAgent,
OptionSignVersion,
OptionRegion,
OptionCloudBoxID,
// The following options are only supported by sc command, not supported by cp command
OptionDelete,
OptionBackupDir,
},
},
}
// function for FormatHelper interface
func (sc *SyncCommand) formatHelpForWhole() string {
return sc.command.formatHelpForWhole()
}
func (sc *SyncCommand) formatIndependHelp() string {
return sc.command.formatIndependHelp()
}
// Init simulate inheritance, and polymorphism
func (sc *SyncCommand) Init(args []string, options OptionMapType) error {
recursive := true
bakupOptions := make(OptionMapType)
for k, v := range options {
bakupOptions[k] = v
}
// force to recursive and delete unsurpported options
bakupOptions[OptionRecursion] = &recursive
delete(bakupOptions, OptionDelete)
delete(bakupOptions, OptionBackupDir)
copyCommand.cpOption.bSyncCommand = true
err := (©Command).Init(args, bakupOptions)
if err != nil {
return err
}
return sc.command.Init(args, options, sc)
}
// RunCommand simulate inheritance, and polymorphism
func (sc *SyncCommand) RunCommand() error {
sc.syncOption.bDelete, _ = GetBool(OptionDelete, sc.command.options)
sc.syncOption.encodingType, _ = GetString(OptionEncodingType, sc.command.options)
sc.syncOption.backupDir, _ = GetString(OptionBackupDir, sc.command.options)
// for list file
sc.syncOption.enableSymlinkDir, _ = GetBool(OptionEnableSymlinkDir, sc.command.options)
sc.syncOption.onlyCurrentDir, _ = GetBool(OptionOnlyCurrentDir, sc.command.options)
sc.syncOption.disableDirObject, _ = GetBool(OptionDisableDirObject, sc.command.options)
sc.syncOption.disableAllSymlink, _ = GetBool(OptionDisableAllSymlink, sc.command.options)
sc.syncOption.force, _ = GetBool(OptionForce, sc.command.options)
// check point dir
sc.syncOption.cpDir, _ = GetString(OptionCheckpointDir, sc.command.options)
// payer
payer, _ := GetString(OptionRequestPayer, sc.command.options)
if payer != "" {
if payer != strings.ToLower(string(oss.Requester)) {
return fmt.Errorf("invalid request payer: %s, please check", payer)
}
sc.syncOption.payerOptions = append(sc.syncOption.payerOptions, oss.RequestPayer(oss.PayerType(payer)))
}
// filters
var res bool
res, sc.syncOption.filters = getFilter(os.Args)
if !res {
return fmt.Errorf("--include or --exclude does not support format containing dir info")
}
for k, v := range sc.syncOption.filters {
LogInfo("filter %d,name:%s,pattern:%s\n", k, v.name, v.pattern)
}
srcURL, err := StorageURLFromString(sc.command.args[0], sc.syncOption.encodingType)
if err != nil {
return err
}
destURL, err := StorageURLFromString(sc.command.args[1], sc.syncOption.encodingType)
if err != nil {
return err
}
if srcURL.IsFileURL() && destURL.IsFileURL() {
return fmt.Errorf("not support sync between local directory")
}
if srcURL.IsFileURL() {
f, err := os.Stat(srcURL.ToString())
if err != nil {
return err
}
if !f.IsDir() {
return fmt.Errorf("src %s is not directory", srcURL.ToString())
}
}
if !sc.syncOption.bDelete {
return copyCommand.RunCommand()
}
// sync command add '/' afert cloud prefix
// cp command must have the same action when run as sync command
srcURL = sc.adjustCloudUrl(srcURL)
destURL = sc.adjustCloudUrl(destURL)
// check backup dir
if destURL.IsFileURL() {
err = sc.CheckDestBackupDir(destURL)
if err != nil {
return err
}
}
opType := sc.getCommandType(srcURL, destURL)
// get file list or object key list
srcKeys := make(map[string]string)
destKeys := make(map[string]string)
if srcURL.IsFileURL() {
err = sc.GetLocalFileKeys(srcURL, srcKeys)
} else {
err = sc.GetOssKeys(srcURL, srcKeys)
}
if err != nil {
return err
}
if destURL.IsFileURL() {
err = sc.GetLocalFileKeys(destURL, destKeys)
} else {
err = sc.GetOssKeys(destURL, destKeys)
}
if err != nil {
return err
}
// Get keys to be deleted
bSame := (string(os.PathSeparator) == "/")
for k, _ := range srcKeys {
if bSame || opType == operationTypeCopy {
delete(destKeys, k)
} else if opType == operationTypePut {
delete(destKeys, strings.Replace(k, "\\", "/", -1))
} else {
delete(destKeys, strings.Replace(k, "/", "\\", -1))
}
}
if destURL.IsFileURL() {
fmt.Printf("\nfile(directory) will be removed count:%d\n", len(destKeys))
} else {
fmt.Printf("\nobject will be deleted count:%d\n", len(destKeys))
}
err = copyCommand.RunCommand()
if err != nil {
return err
}
// move dest files or rm dest objects which not exist in src
if opType == operationTypeCopy || opType == operationTypePut {
err = sc.DeleteExtraObjects(destKeys, destURL)
} else {
err = sc.RemoveExtraFiles(destKeys, destURL)
}
return err
}
func (sc *SyncCommand) adjustCloudUrl(sUrl StorageURLer) StorageURLer {
if sUrl.IsFileURL() {
return sUrl
}
cloudUrl := sUrl.(CloudURL)
if len(cloudUrl.object) > 0 && !strings.HasSuffix(cloudUrl.object, "/") {
cloudUrl.object += "/"
}
return cloudUrl
}
func (sc *SyncCommand) DeleteExtraObjects(keys map[string]string, sUrl StorageURLer) error {
bucketName := sUrl.(CloudURL).bucket
bucket, err := sc.command.ossBucket(bucketName)
if err != nil {
return err
}
deleteCount := 0
rmOptions := append(sc.syncOption.payerOptions, oss.DeleteObjectsQuiet(true))
objects := []string{}
for k, v := range keys {
if len(objects) >= MaxBatchCount {
if sc.confirm(objects) {
err := sc.BatchRmObjects(bucket, objects, rmOptions)
if err != nil {
return err
}
}
objects = []string{}
deleteCount += MaxBatchCount
fmt.Printf("\rdelete object count:%d", deleteCount)
}
// prefix + relativeKey
objects = append(objects, v+k)
}
if len(objects) > 0 && sc.confirm(objects) {
err := sc.BatchRmObjects(bucket, objects, rmOptions)
if err != nil {
return err
}
deleteCount += len(objects)
fmt.Printf("\rdelete object count:%d", deleteCount)
}
return nil
}
func (sc *SyncCommand) RemoveExtraFiles(keys map[string]string, sUrl StorageURLer) error {
var sortList []string
for k, _ := range keys {
sortList = append(sortList, k)
}
// remove files first,then remove dir
sort.Sort(sort.Reverse(sort.StringSlice(sortList)))
absDirName, err := sc.GetAbsPath(sUrl.ToString())
if err != nil {
return err
}
nowFatherDirName := ""
for _, k := range sortList {
if strings.HasSuffix(k, string(os.PathSeparator)) {
// is dir
dirName := k[0 : len(k)-1]
readerInfos, _ := sc.readDirLimit(absDirName+dirName, 10)
if len(readerInfos) > 0 {
continue
} else {
//empty dir,need to remove or delete
f, err := os.Stat(sc.syncOption.backupDir + dirName)
if err != nil {
sc.movePath(absDirName+dirName, sc.syncOption.backupDir+dirName)
} else {
if !f.IsDir() {
return fmt.Errorf("backup %s is already exist,but is file", sc.syncOption.backupDir+dirName)
} else {
// delete the dir
os.RemoveAll(absDirName + dirName)
}
}
}
} else {
// is file
fatherDir := absDirName
index := strings.LastIndex(k, string(os.PathSeparator))
if index >= 0 {
fatherDir = k[:index]
}
if fatherDir != nowFatherDirName && fatherDir != absDirName {
os.MkdirAll(sc.syncOption.backupDir+fatherDir, 0755)
nowFatherDirName = fatherDir
}
err := sc.movePath(absDirName+k, sc.syncOption.backupDir+k)
if err != nil {
return err
}
}
}
return nil
}
func (sc *SyncCommand) BatchRmObjects(bucket *oss.Bucket, objects []string, options []oss.Option) error {
delRes, err := bucket.DeleteObjects(objects, options...)
if err != nil {
return err
}
if len(delRes.DeletedObjects) > 0 {
errMsg := ""
for _, objectKey := range delRes.DeletedObjects {
errMsg += (" " + objectKey)
}
LogError("delete erro %s\n", errMsg)
return fmt.Errorf("delete %s error", errMsg)
}
for _, v := range objects {
LogInfo("delete object success %s\n", v)
}
return nil
}
func (sc *SyncCommand) getCommandType(srcURL StorageURLer, destURL StorageURLer) operationType {
if srcURL.IsCloudURL() {
if destURL.IsFileURL() {
return operationTypeGet
}
return operationTypeCopy
}
return operationTypePut
}
func (sc *SyncCommand) GetLocalFileKeys(sUrl StorageURLer, keys map[string]string) error {
strPath := sUrl.ToString()
if !strings.HasSuffix(strPath, string(os.PathSeparator)) {
// for symlink dir
strPath += string(os.PathSeparator)
}
chFiles := make(chan fileInfoType, ChannelBuf)
chFinish := make(chan error, 2)
go sc.ReadLocalFileKeys(chFiles, chFinish, keys)
go sc.GetFileList(strPath, chFiles, chFinish)
select {
case err := <-chFinish:
if err != nil {
return err
}
}
return nil
}
func (sc *SyncCommand) GetFileList(strPath string, chFiles chan<- fileInfoType, chFinish chan<- error) {
err := getFileListCommon(strPath, chFiles, sc.syncOption.onlyCurrentDir,
sc.syncOption.disableAllSymlink, sc.syncOption.enableSymlinkDir, sc.syncOption.filters)
if err != nil {
chFinish <- err
}
}
func (sc *SyncCommand) ReadLocalFileKeys(chFiles <-chan fileInfoType, chFinish chan<- error, keys map[string]string) {
totalCount := 0
fmt.Printf("\n")
for fileInfo := range chFiles {
if copyCommand.filterFile(fileInfo, sc.syncOption.cpDir) { // exclude checkpoint files
totalCount++
fmt.Printf("\rtotal file(directory) count:%d", totalCount)
keys[fileInfo.filePath] = ""
if len(keys) > MaxSyncNumbers {
fmt.Printf("\n")
chFinish <- fmt.Errorf("over max sync numbers %d", MaxSyncNumbers)
break
}
}
}
fmt.Printf("\rtotal file(directory) count:%d", totalCount)
chFinish <- nil
}
func (sc *SyncCommand) GetAbsPath(strPath string) (string, error) {
if filepath.IsAbs(strPath) {
return strPath, nil
}
currentDir, err := os.Getwd()
if err != nil {
return "", err
}
if !strings.HasSuffix(strPath, string(os.PathSeparator)) {
strPath += string(os.PathSeparator)
}
strPath = currentDir + string(os.PathSeparator) + strPath
absPath, err := filepath.Abs(strPath)
if err != nil {
return "", err
}
if !strings.HasSuffix(absPath, string(os.PathSeparator)) {
absPath += string(os.PathSeparator)
}
return absPath, err
}
func (sc *SyncCommand) CheckDestBackupDir(sUrl StorageURLer) error {
// create bacup dir
createDirectory := false
f, err := os.Stat(sUrl.ToString())
if err != nil {
if err := os.MkdirAll(sUrl.ToString(), 0755); err != nil {
return err
}
createDirectory = true
} else if !f.IsDir() {
return fmt.Errorf("dest dir %s is file,is not directory", sUrl.ToString())
}
if createDirectory && sc.syncOption.backupDir == "" {
return nil
}
if sc.syncOption.backupDir == "" {
return fmt.Errorf("dest backup dir is empty string,please use --backup-dir")
}
if !strings.HasSuffix(sc.syncOption.backupDir, string(os.PathSeparator)) {
sc.syncOption.backupDir += string(os.PathSeparator)
}
// check backup dir is Subdirectories or not
absfilePath, errF := sc.GetAbsPath(sUrl.ToString())
if errF != nil {
return errF
}
absBackPath, errB := sc.GetAbsPath(sc.syncOption.backupDir)
if errB != nil {
return errB
}
if strings.Index(absBackPath, absfilePath) >= 0 {
return fmt.Errorf("backup dir %s is subdirectory of %s", sc.syncOption.backupDir, sUrl.ToString())
}
// create bacup dir
f, err = os.Stat(sc.syncOption.backupDir)
if err != nil {
if err := os.MkdirAll(sc.syncOption.backupDir, 0755); err != nil {
return err
}
} else if !f.IsDir() {
return fmt.Errorf("dest backup dir %s is file,is not directory", sc.syncOption.backupDir)
}
return nil
}
func (sc *SyncCommand) GetOssKeys(sUrl StorageURLer, keys map[string]string) error {
bucketName := sUrl.(CloudURL).bucket
bucket, err := sc.command.ossBucket(bucketName)
if err != nil {
return err
}
chFiles := make(chan objectInfoType, ChannelBuf)
chFinish := make(chan error, 2)
go sc.ReadOssKeys(keys, sUrl, chFiles, chFinish)
go sc.GetOssKeyList(bucket, sUrl, chFiles, chFinish)
select {
case err := <-chFinish:
if err != nil {
return err
}
}
return nil
}
func (sc *SyncCommand) GetOssKeyList(bucket *oss.Bucket, sURL StorageURLer, chObjects chan<- objectInfoType, chFinish chan<- error) {
cloudURL := sURL.(CloudURL)
err := getObjectListCommon(bucket, cloudURL, chObjects, sc.syncOption.onlyCurrentDir,
sc.syncOption.filters, sc.syncOption.payerOptions)
if err != nil {
chFinish <- err
}
}
func (sc *SyncCommand) ReadOssKeys(keys map[string]string, sURL StorageURLer, chObjects <-chan objectInfoType, chFinish chan<- error) {
totalCount := 0
fmt.Printf("\n")
for objectInfo := range chObjects {
totalCount++
fmt.Printf("\r%s,total oss object count:%d", sURL.ToString(), totalCount)
keys[objectInfo.relativeKey] = objectInfo.prefix
if len(keys) > MaxSyncNumbers {
fmt.Printf("\n")
chFinish <- fmt.Errorf("over max sync numbers %d", MaxSyncNumbers)
break
}
}
fmt.Printf("\r%s,total oss object count:%d", sURL.ToString(), totalCount)
chFinish <- nil
}
func (sc *SyncCommand) confirm(keys []string) bool {
if sc.syncOption.force {
return true
}
var logBuffer bytes.Buffer
logBuffer.WriteString("\n")
for _, v := range keys {
logBuffer.WriteString(fmt.Sprintf("%s\n", v))
}
logBuffer.WriteString(fmt.Sprintf("sync:delete above objects(y or N)? "))
fmt.Printf(logBuffer.String())
var val string
if _, err := fmt.Scanln(&val); err != nil || (strings.ToLower(val) != "yes" && strings.ToLower(val) != "y") {
return false
}
return true
}
func (sc *SyncCommand) readDirLimit(dirName string, limitCount int) ([]os.FileInfo, error) {
f, err := os.Open(dirName)
if err != nil {
return nil, err
}
list, err := f.Readdir(limitCount)
f.Close()
if err != nil {
return nil, err
}
return list, nil
}
func (sc *SyncCommand) movePath(srcName, destName string) error {
err := sc.moveFileToPath(srcName,destName)
if err != nil {
LogError("rename %s %s error,%s\n", srcName, destName, err.Error())
} else {
sc.syncOption.removeCount += 1
fmt.Printf("\rremove file(directory) count:%d", sc.syncOption.removeCount)
LogInfo("rename success %s %s\n", srcName, destName)
}
return err
}
func (sc *SyncCommand)moveFileToPath(srcName, destName string) error {
err := os.Rename(srcName,destName)
if err == nil {
return nil
}else{
inputFile, err := os.Open(srcName)
defer inputFile.Close()
if err != nil {
return err
}
outputFile, err := os.Create(destName)
defer outputFile.Close()
if err != nil {
return err
}
_, err = io.Copy(outputFile, inputFile)
if err != nil {
return err
}
err = os.Remove(srcName)
if err != nil {
return err
}
return nil
}
}